From aa3f79d4c96f7d2da6b8b7ca977d4e6d5f7caa42 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Fri, 12 Dec 2025 08:44:48 +0100 Subject: [PATCH] DRA upgrade/downgrade: add DeviceTaints test This automatically tests a few scenarios across cluster upgrade/downgrade. --- test/e2e/dra/utils/builder.go | 34 ++++--- test/e2e/dra/utils/deploy.go | 1 + test/e2e_dra/devicetaints_test.go | 137 ++++++++++++++++++++++++++ test/e2e_dra/upgradedowngrade_test.go | 11 ++- 4 files changed, 164 insertions(+), 19 deletions(-) create mode 100644 test/e2e_dra/devicetaints_test.go diff --git a/test/e2e/dra/utils/builder.go b/test/e2e/dra/utils/builder.go index 47adf5f0d7f..82c6cabff91 100644 --- a/test/e2e/dra/utils/builder.go +++ b/test/e2e/dra/utils/builder.go @@ -60,7 +60,7 @@ func (b *Builder) ExtendedResourceName(i int) string { case SingletonIndex: return e2enode.SampleDeviceResourceName default: - return b.driver.Name + "/resource" + fmt.Sprintf("-%d", i) + return b.Driver.Name + "/resource" + fmt.Sprintf("-%d", i) } } @@ -68,7 +68,7 @@ func (b *Builder) ExtendedResourceName(i int) string { // namespace. type Builder struct { namespace string - driver *Driver + Driver *Driver UseExtendedResourceName bool podCounter int @@ -79,7 +79,7 @@ type Builder struct { // ClassName returns the default device class name. func (b *Builder) ClassName() string { - return b.namespace + b.driver.NameSuffix + "-class" + return b.namespace + b.Driver.NameSuffix + "-class" } // SingletonIndex causes Builder.Class and ExtendedResourceName to create a @@ -115,14 +115,14 @@ func (b *Builder) Class(i int) *resourceapi.DeviceClass { } class.Spec.Selectors = []resourceapi.DeviceSelector{{ CEL: &resourceapi.CELDeviceSelector{ - Expression: fmt.Sprintf(`device.driver == "%s"`, b.driver.Name), + Expression: fmt.Sprintf(`device.driver == "%s"`, b.Driver.Name), }, }} if b.ClassParameters != "" { class.Spec.Config = []resourceapi.DeviceClassConfiguration{{ DeviceConfiguration: resourceapi.DeviceConfiguration{ Opaque: &resourceapi.OpaqueDeviceConfiguration{ - Driver: b.driver.Name, + Driver: b.Driver.Name, Parameters: runtime.RawExtension{Raw: []byte(b.ClassParameters)}, }, }, @@ -135,7 +135,7 @@ func (b *Builder) Class(i int) *resourceapi.DeviceClass { // that test pods can reference func (b *Builder) ExternalClaim() *resourceapi.ResourceClaim { b.claimCounter++ - name := "external-claim" + b.driver.NameSuffix // This is what podExternal expects. + name := "external-claim" + b.Driver.NameSuffix // This is what podExternal expects. if b.claimCounter > 1 { name += fmt.Sprintf("-%d", b.claimCounter) } @@ -160,7 +160,7 @@ func (b *Builder) claimSpecWithV1beta1() resourcev1beta1.ResourceClaimSpec { Config: []resourcev1beta1.DeviceClaimConfiguration{{ DeviceConfiguration: resourcev1beta1.DeviceConfiguration{ Opaque: &resourcev1beta1.OpaqueDeviceConfiguration{ - Driver: b.driver.Name, + Driver: b.Driver.Name, Parameters: runtime.RawExtension{ Raw: []byte(parameters), }, @@ -188,7 +188,7 @@ func (b *Builder) claimSpecWithV1beta2() resourcev1beta2.ResourceClaimSpec { Config: []resourcev1beta2.DeviceClaimConfiguration{{ DeviceConfiguration: resourcev1beta2.DeviceConfiguration{ Opaque: &resourcev1beta2.OpaqueDeviceConfiguration{ - Driver: b.driver.Name, + Driver: b.Driver.Name, Parameters: runtime.RawExtension{ Raw: []byte(parameters), }, @@ -216,7 +216,7 @@ func (b *Builder) ClaimSpec() resourceapi.ResourceClaimSpec { Config: []resourceapi.DeviceClaimConfiguration{{ DeviceConfiguration: resourceapi.DeviceConfiguration{ Opaque: &resourceapi.OpaqueDeviceConfiguration{ - Driver: b.driver.Name, + Driver: b.Driver.Name, Parameters: runtime.RawExtension{ Raw: []byte(parameters), }, @@ -249,7 +249,7 @@ func (b *Builder) Pod() *v1.Pod { pod.Spec.RestartPolicy = v1.RestartPolicyNever pod.GenerateName = "" b.podCounter++ - pod.Name = fmt.Sprintf("tester%s-%d", b.driver.NameSuffix, b.podCounter) + pod.Name = fmt.Sprintf("tester%s-%d", b.Driver.NameSuffix, b.podCounter) return pod } @@ -314,11 +314,15 @@ func (b *Builder) PodInlineMultiple() (*v1.Pod, *resourceapi.ResourceClaimTempla } // PodExternal adds a pod that references external resource claim with default class name and parameters. +// +// Note that this references *the initial* result of ExternalClaim. When generating multiple such +// external claims, pod.Spec.ResourceClaims[0].ResourceClaimName must be adapted by the caller, +// if desired. func (b *Builder) PodExternal() *v1.Pod { pod := b.Pod() pod.Spec.Containers[0].Name = "with-resource" podClaimName := "resource-claim" - externalClaimName := "external-claim" + b.driver.NameSuffix + externalClaimName := "external-claim" + b.Driver.NameSuffix pod.Spec.ResourceClaims = []v1.PodResourceClaim{ { Name: podClaimName, @@ -417,7 +421,7 @@ func (b *Builder) DeletePodAndWaitForNotFound(tCtx ktesting.TContext, pod *v1.Po func (b *Builder) TestPod(tCtx ktesting.TContext, pod *v1.Pod, env ...string) { tCtx.Helper() - if !b.driver.WithKubelet { + if !b.Driver.WithKubelet { // Less testing when we cannot rely on the kubelet to actually run the pod. err := e2epod.WaitForPodScheduled(tCtx, tCtx.Client(), pod.Namespace, pod.Name) tCtx.ExpectNoError(err, "schedule pod") @@ -474,7 +478,7 @@ func TestContainerEnv(tCtx ktesting.TContext, pod *v1.Pod, containerName string, } func NewBuilder(f *framework.Framework, driver *Driver) *Builder { - b := &Builder{driver: driver} + b := &Builder{Driver: driver} ginkgo.BeforeEach(func() { b.setUp(f.TContext(context.Background())) }) @@ -482,7 +486,7 @@ func NewBuilder(f *framework.Framework, driver *Driver) *Builder { } func NewBuilderNow(tCtx ktesting.TContext, driver *Driver) *Builder { - b := &Builder{driver: driver} + b := &Builder{Driver: driver} b.setUp(tCtx) return b } @@ -542,7 +546,7 @@ func (b *Builder) tearDown(tCtx ktesting.TContext) { } } - for host, plugin := range b.driver.Nodes { + for host, plugin := range b.Driver.Nodes { tCtx.Logf("Waiting for resources on %s to be unprepared", host) tCtx.Eventually(func(ktesting.TContext) []app.ClaimID { return plugin.GetPreparedResources() }).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "prepared claims on host %s", host) } diff --git a/test/e2e/dra/utils/deploy.go b/test/e2e/dra/utils/deploy.go index becde2315f8..5e0b16bb777 100644 --- a/test/e2e/dra/utils/deploy.go +++ b/test/e2e/dra/utils/deploy.go @@ -1038,6 +1038,7 @@ func (d *Driver) TearDown(tCtx ktesting.TContext) { // // Only use this in tests where kubelet support for DRA is guaranteed. func (d *Driver) IsGone(tCtx ktesting.TContext) { + tCtx.Helper() tCtx.Logf("Waiting for ResourceSlices of driver %s to be removed...", d.Name) tCtx.Eventually(d.NewGetSlices()).WithTimeout(2 * time.Minute).Should(gomega.HaveField("Items", gomega.BeEmpty())) } diff --git a/test/e2e_dra/devicetaints_test.go b/test/e2e_dra/devicetaints_test.go new file mode 100644 index 00000000000..503337cd2b7 --- /dev/null +++ b/test/e2e_dra/devicetaints_test.go @@ -0,0 +1,137 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2edra + +import ( + "time" + + "github.com/onsi/gomega" + resourceapi "k8s.io/api/resource/v1" + resourcealpha "k8s.io/api/resource/v1alpha3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + drautils "k8s.io/kubernetes/test/e2e/dra/utils" + e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/utils/ktesting" + "k8s.io/utils/ptr" +) + +// deviceTaints checks that: +// - A pod which gets scheduled on the previous release because of a toleration is kept running after an upgrade. +// - A DeviceTaintRule created to evict the pod before a downgrade prevents pod scheduling after a downgrade. +func deviceTaints(tCtx ktesting.TContext, b *drautils.Builder) upgradedTestFunc { + namespace := tCtx.Namespace() + taintKey := "devicetaints" + taintValueFromSlice := "from-slice" + taintValueFromRule := "from-rule" + taintedDevice := "tainted-device" + + // We need additional devices which are only used by this test. + // We achieve that with cluster-scoped devices that start out with + // a taint. + slice := &resourceapi.ResourceSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "devicetaints", + }, + Spec: resourceapi.ResourceSliceSpec{ + Driver: b.Driver.Name, + Pool: resourceapi.ResourcePool{ + Name: "devicetaints", + ResourceSliceCount: 1, + }, + AllNodes: ptr.To(true), + Devices: []resourceapi.Device{{ + Name: taintedDevice, + Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + "example.com/type": { + StringValue: ptr.To("devicetaints"), + }, + }, + Taints: []resourceapi.DeviceTaint{{ + Key: taintKey, + Value: taintValueFromSlice, + Effect: resourceapi.DeviceTaintEffectNoSchedule, + }}, + }}, + }, + } + _, err := tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, slice, metav1.CreateOptions{}) + tCtx.ExpectNoError(err) + + tCtx.Log("The pod wants exactly the tainted device -> not schedulable.") + claim := b.ExternalClaim() + pod := b.PodExternal() + pod.Spec.ResourceClaims[0].ResourceClaimName = &claim.Name + claim.Spec.Devices.Requests[0].Exactly.Selectors = []resourceapi.DeviceSelector{{ + CEL: &resourceapi.CELDeviceSelector{ + Expression: `device.attributes["example.com"].?type.orValue("") == "devicetaints"`, + }, + }} + b.Create(tCtx, claim, pod) + tCtx.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(tCtx, tCtx.Client(), pod.Name, namespace)) + + tCtx.Log("Adding a toleration makes the pod schedulable.") + claim.Spec.Devices.Requests[0].Exactly.Tolerations = []resourceapi.DeviceToleration{{ + Key: taintKey, + Value: taintValueFromSlice, + Effect: resourceapi.DeviceTaintEffectNoSchedule, + }} + tCtx.ExpectNoError(tCtx.Client().ResourceV1().ResourceClaims(namespace).Delete(tCtx, claim.Name, metav1.DeleteOptions{})) + _, err = tCtx.Client().ResourceV1().ResourceClaims(namespace).Create(tCtx, claim, metav1.CreateOptions{}) + tCtx.ExpectNoError(err) + b.TestPod(tCtx, pod) + + return func(tCtx ktesting.TContext) downgradedTestFunc { + tCtx.Log("Pod running consistently after upgrade.") + tCtx.Consistently(func(tCtx ktesting.TContext) error { + return e2epod.WaitForPodRunningInNamespace(tCtx, tCtx.Client(), pod) + }).WithTimeout(30 * time.Second).WithPolling(5 * time.Second).Should(gomega.Succeed()) + + tCtx.Logf("Evict pod through DeviceTaintRule.") + rule := &resourcealpha.DeviceTaintRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: "device-taint-rule", + }, + Spec: resourcealpha.DeviceTaintRuleSpec{ + DeviceSelector: &resourcealpha.DeviceTaintSelector{ + Driver: &b.Driver.Name, + Pool: &slice.Spec.Pool.Name, + Device: &taintedDevice, + }, + Taint: resourcealpha.DeviceTaint{ + Key: taintKey, + Value: taintValueFromRule, + Effect: resourcealpha.DeviceTaintEffectNoExecute, + }, + }, + } + _, err := tCtx.Client().ResourceV1alpha3().DeviceTaintRules().Create(tCtx, rule, metav1.CreateOptions{}) + tCtx.ExpectNoError(err) + tCtx.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(tCtx, tCtx.Client(), pod.Name, namespace, 5*time.Minute)) + + return func(tCtx ktesting.TContext) { + tCtx.Log("DeviceTaintRule still in effect.") + b.Create(tCtx, pod) + tCtx.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(tCtx, tCtx.Client(), pod.Name, namespace)) + + // We must clean up manually, otherwise the code which checks for ResourceSlice deletion after + // driver removal gets stuck waiting for the removal of special ResourceSlice. + // This cannot be scheduled via tCtx.Cleanup after creating it because then it would be removed + // after the first sub-test. + tCtx.ExpectNoError(tCtx.Client().ResourceV1().ResourceSlices().Delete(tCtx, slice.Name, metav1.DeleteOptions{})) + } + } +} diff --git a/test/e2e_dra/upgradedowngrade_test.go b/test/e2e_dra/upgradedowngrade_test.go index b53c06090f5..2abc6e77893 100644 --- a/test/e2e_dra/upgradedowngrade_test.go +++ b/test/e2e_dra/upgradedowngrade_test.go @@ -66,7 +66,8 @@ func init() { // sub-test. That function then returns the next piece of code, which then // returns the final code. Each callback function is executed as a sub-test. // The builder is configured to not delete objects when that sub-test ends, -// so objects persist until the entire test is done. +// so objects persist until the entire test is done. The same DRA driver +// is used for all sub-tests. // // Each sub-test must be self-contained. They intentionally run in a random // order. However, they share the same cluster and the 8 devices which are @@ -74,6 +75,7 @@ func init() { var subTests = map[string]initialTestFunc{ "core DRA": coreDRA, "ResourceClaim device status": resourceClaimDeviceStatus, + "DeviceTaints": deviceTaints, } type initialTestFunc func(tCtx ktesting.TContext, builder *drautils.Builder) upgradedTestFunc @@ -210,8 +212,8 @@ func testUpgradeDowngrade(tCtx ktesting.TContext) { tCtx.Step(fmt.Sprintf("bring up v%d.%d", major, previousMinor), func(tCtx ktesting.TContext) { cluster = localupcluster.New(tCtx) localUpClusterEnv := map[string]string{ - "RUNTIME_CONFIG": "resource.k8s.io/v1beta1,resource.k8s.io/v1beta2", - "FEATURE_GATES": "DynamicResourceAllocation=true", + "RUNTIME_CONFIG": "resource.k8s.io/v1beta1,resource.k8s.io/v1beta2,resource.k8s.io/v1alpha3", + "FEATURE_GATES": "DynamicResourceAllocation=true,DRADeviceTaintRules=true,DRADeviceTaints=true", // *not* needed because driver will run in "local filesystem" mode (= driver.IsLocal): "ALLOW_PRIVILEGED": "1", } cluster.Start(tCtx, binDir, localUpClusterEnv) @@ -247,6 +249,7 @@ func testUpgradeDowngrade(tCtx ktesting.TContext) { }) } }) + numSlices := len(driver.NewGetSlices()(tCtx).Items) // We could split this up into first updating the apiserver, then control plane components, then restarting kubelet. // For the purpose of this test here we we primarily care about full before/after comparisons, so not done yet. @@ -255,7 +258,7 @@ func testUpgradeDowngrade(tCtx ktesting.TContext) { // The kubelet wipes all ResourceSlices on a restart because it doesn't know which drivers were running. // Wait for the ResourceSlice controller in the driver to notice and recreate the ResourceSlices. - tCtx.WithStep("wait for ResourceSlices").Eventually(driver.NewGetSlices()).WithTimeout(5 * time.Minute).Should(gomega.HaveField("Items", gomega.HaveLen(len(nodes.NodeNames)))) + tCtx.WithStep("wait for ResourceSlices").Eventually(driver.NewGetSlices()).WithTimeout(5 * time.Minute).Should(gomega.HaveField("Items", gomega.HaveLen(numSlices))) downgradedTestFuncs := make(map[string]downgradedTestFunc, len(subTests)) tCtx.Run("after-cluster-upgrade", func(tCtx ktesting.TContext) {