diff --git a/cmd/kube-controller-manager/app/resource.go b/cmd/kube-controller-manager/app/resource.go index 03270641fa4..21eab4e6609 100644 --- a/cmd/kube-controller-manager/app/resource.go +++ b/cmd/kube-controller-manager/app/resource.go @@ -53,7 +53,7 @@ func newDeviceTaintEvictionController(ctx context.Context, controllerContext Con controllerContext.InformerFactory.Core().V1().Pods(), controllerContext.InformerFactory.Resource().V1().ResourceClaims(), controllerContext.InformerFactory.Resource().V1().ResourceSlices(), - controllerContext.InformerFactory.Resource().V1beta2().DeviceTaintRules(), + controllerContext.InformerFactory.Resource().V1().DeviceTaintRules(), controllerContext.InformerFactory.Resource().V1().DeviceClasses(), controllerName, utilfeature.DefaultFeatureGate.Enabled(features.DRAWorkloadResourceClaims), diff --git a/pkg/controller/devicetainteviction/device_taint_eviction.go b/pkg/controller/devicetainteviction/device_taint_eviction.go index 1b3e381d3c8..1da0fb3494a 100644 --- a/pkg/controller/devicetainteviction/device_taint_eviction.go +++ b/pkg/controller/devicetainteviction/device_taint_eviction.go @@ -30,7 +30,6 @@ import ( v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" - resourcbeta "k8s.io/api/resource/v1beta2" schedulingapi "k8s.io/api/scheduling/v1alpha3" apiequality "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -42,15 +41,14 @@ import ( "k8s.io/apimachinery/pkg/util/sets" utilfeature "k8s.io/apiserver/pkg/util/feature" metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" - resourceac "k8s.io/client-go/applyconfigurations/resource/v1beta2" + resourceac "k8s.io/client-go/applyconfigurations/resource/v1" coreinformers "k8s.io/client-go/informers/core/v1" resourceinformers "k8s.io/client-go/informers/resource/v1" - resourcealphainformers "k8s.io/client-go/informers/resource/v1beta2" clientset "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" v1core "k8s.io/client-go/kubernetes/typed/core/v1" corelisters "k8s.io/client-go/listers/core/v1" - resourcealphalisters "k8s.io/client-go/listers/resource/v1beta2" + resourcelisters "k8s.io/client-go/listers/resource/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" @@ -102,9 +100,9 @@ type Controller struct { podLister corelisters.PodLister claimInformer resourceinformers.ResourceClaimInformer sliceInformer resourceinformers.ResourceSliceInformer - ruleInformer resourcealphainformers.DeviceTaintRuleInformer + ruleInformer resourceinformers.DeviceTaintRuleInformer classInformer resourceinformers.DeviceClassInformer - ruleLister resourcealphalisters.DeviceTaintRuleLister + ruleLister resourcelisters.DeviceTaintRuleLister haveSynced []cache.DoneChecker hasSynced atomic.Int32 metrics metrics.Metrics @@ -139,7 +137,7 @@ type Controller struct { pools map[poolID]pool // evictingRules tracks all DeviceTaintRules by name which cause pod eviction. - evictingRules map[string]*resourcbeta.DeviceTaintRule + evictingRules map[string]*resourceapi.DeviceTaintRule // taintRuleStats tracks information about work that was done for a specific DeviceTaintRule instance. // @@ -294,7 +292,7 @@ func (er evictionReason) String() string { // trackedTaint augments a DeviceTaint with a pointer to its origin. // rule and slice are mutually exclusive. Exactly one of them is always set. type trackedTaint struct { - rule *resourcbeta.DeviceTaintRule + rule *resourceapi.DeviceTaintRule slice sliceDeviceTaint } @@ -377,7 +375,7 @@ type workItem struct { ruleRef tainteviction.NamespacedObject } -func workItemForRule(rule *resourcbeta.DeviceTaintRule) workItem { +func workItemForRule(rule *resourceapi.DeviceTaintRule) workItem { return workItem{ruleRef: tainteviction.NamespacedObject{NamespacedName: types.NamespacedName{Name: rule.Name}, UID: rule.UID}} } @@ -525,7 +523,7 @@ func (tc *Controller) maybeUpdateRuleStatus(ctx context.Context, ruleRef taintev // Already set? index := slices.IndexFunc(rule.Status.Conditions, func(condition metav1.Condition) bool { - return condition.Type == resourcbeta.DeviceTaintConditionEvictionInProgress + return condition.Type == resourceapi.DeviceTaintConditionEvictionInProgress }) // LastTransitionTime gets bumped each time we make any change to the condition, @@ -551,7 +549,7 @@ func (tc *Controller) maybeUpdateRuleStatus(ctx context.Context, ruleRef taintev // Checking all pods might be expensive. Only do it if really needed. var numTaintedSliceDevices, numTaintedAllocatedDevices, numPendingPods, numPendingNamespaces int64 switch rule.Spec.Taint.Effect { - case resourcbeta.DeviceTaintEffectNone: + case resourceapi.DeviceTaintEffectNone: // Temporarily change the effect from None to NoExecute to simulate. // We pretend to do that through informer events. We hold the lock, // so there is no race with real informer events or other goroutines. @@ -559,7 +557,7 @@ func (tc *Controller) maybeUpdateRuleStatus(ctx context.Context, ruleRef taintev // To avoid having a lasting impact on the real controller instance // we make a temporary copy. ruleEvict := rule.DeepCopy() - ruleEvict.Spec.Taint.Effect = resourcbeta.DeviceTaintEffectNoExecute + ruleEvict.Spec.Taint.Effect = resourceapi.DeviceTaintEffectNoExecute tc := &Controller{ logger: klog.LoggerWithName(logger, "simulation"), podInformer: tc.podInformer, @@ -568,7 +566,7 @@ func (tc *Controller) maybeUpdateRuleStatus(ctx context.Context, ruleRef taintev deletePodAt: make(map[tainteviction.NamespacedObject]evictionAndReason), allocatedClaims: maps.Clone(tc.allocatedClaims), pools: tc.pools, - evictingRules: make(map[string]*resourcbeta.DeviceTaintRule), + evictingRules: make(map[string]*resourceapi.DeviceTaintRule), workqueue: &NOPQueue[workItem]{}, workloadResourceClaimsEnabled: tc.workloadResourceClaimsEnabled, } @@ -577,7 +575,7 @@ func (tc *Controller) maybeUpdateRuleStatus(ctx context.Context, ruleRef taintev tc.handleRuleChange(rule, ruleEvict) numPendingPods, numPendingNamespaces, err = tc.countPendingPods(rule) numTaintedSliceDevices, numTaintedAllocatedDevices = tc.countTaintedDevices(rule) - case resourcbeta.DeviceTaintEffectNoExecute: + case resourceapi.DeviceTaintEffectNoExecute: numPendingPods, numPendingNamespaces, err = tc.countPendingPods(rule) default: err = nil @@ -588,14 +586,14 @@ func (tc *Controller) maybeUpdateRuleStatus(ctx context.Context, ruleRef taintev // Some fields are tentative and get updated below. newCondition := metav1.Condition{ - Type: resourcbeta.DeviceTaintConditionEvictionInProgress, + Type: resourceapi.DeviceTaintConditionEvictionInProgress, Status: metav1.ConditionFalse, Reason: string("Effect" + rule.Spec.Taint.Effect), ObservedGeneration: rule.Generation, LastTransitionTime: existingCondition.LastTransitionTime, // To avoid a false "is different" in the comparison, gets updated later. } switch rule.Spec.Taint.Effect { - case resourcbeta.DeviceTaintEffectNoExecute: + case resourceapi.DeviceTaintEffectNoExecute: switch { case numPendingPods > 0: newCondition.Reason = "PodsPendingEviction" @@ -615,7 +613,7 @@ func (tc *Controller) maybeUpdateRuleStatus(ctx context.Context, ruleRef taintev default: newCondition.Reason = "NotStarted" } - case resourcbeta.DeviceTaintEffectNone: + case resourceapi.DeviceTaintEffectNone: newCondition.Reason = "NoEffect" if numTaintedSliceDevices == 1 { newCondition.Message += "1 published device selected. " @@ -667,7 +665,7 @@ func (tc *Controller) maybeUpdateRuleStatus(ctx context.Context, ruleRef taintev ObservedGeneration: &newCondition.ObservedGeneration, LastTransitionTime: &newCondition.LastTransitionTime, })) - if _, err := tc.client.ResourceV1beta2().DeviceTaintRules().ApplyStatus(ctx, ruleAC, metav1.ApplyOptions{FieldManager: tc.name, Force: true}); err != nil { + if _, err := tc.client.ResourceV1().DeviceTaintRules().ApplyStatus(ctx, ruleAC, metav1.ApplyOptions{FieldManager: tc.name, Force: true}); err != nil { return 0, fmt.Errorf("add condition to DeviceTaintRule status: %w", err) } } @@ -676,7 +674,7 @@ func (tc *Controller) maybeUpdateRuleStatus(ctx context.Context, ruleRef taintev return 0, nil } -func (tc *Controller) countPendingPods(rule *resourcbeta.DeviceTaintRule) (int64, int64, error) { +func (tc *Controller) countPendingPods(rule *resourceapi.DeviceTaintRule) (int64, int64, error) { pods, err := tc.podLister.List(labels.Everything()) if err != nil { return -1, -1, fmt.Errorf("list pod: %w", err) @@ -706,7 +704,7 @@ func (tc *Controller) countPendingPods(rule *resourcbeta.DeviceTaintRule) (int64 // countTaintedDevices determines the number of devices in slices matching the rule and // the number of allocated devices matching the rule. -func (tc *Controller) countTaintedDevices(rule *resourcbeta.DeviceTaintRule) (numTaintedSliceDevices int64, numTaintedAllocatedDevices int64) { +func (tc *Controller) countTaintedDevices(rule *resourceapi.DeviceTaintRule) (numTaintedSliceDevices int64, numTaintedAllocatedDevices int64) { for poolID, pool := range tc.pools { for _, slice := range pool.slices { if slice.Spec.Pool.Generation != pool.maxGeneration { @@ -733,7 +731,7 @@ func (tc *Controller) countTaintedDevices(rule *resourcbeta.DeviceTaintRule) (nu // New creates a new Controller that will use passed clientset to communicate with the API server. // Spawns no goroutines. That happens in Run. -func New(c clientset.Interface, podInformer coreinformers.PodInformer, claimInformer resourceinformers.ResourceClaimInformer, sliceInformer resourceinformers.ResourceSliceInformer, ruleInformer resourcealphainformers.DeviceTaintRuleInformer, classInformer resourceinformers.DeviceClassInformer, controllerName string, workloadResourceClaimsEnabled bool) *Controller { +func New(c clientset.Interface, podInformer coreinformers.PodInformer, claimInformer resourceinformers.ResourceClaimInformer, sliceInformer resourceinformers.ResourceSliceInformer, ruleInformer resourceinformers.DeviceTaintRuleInformer, classInformer resourceinformers.DeviceClassInformer, controllerName string, workloadResourceClaimsEnabled bool) *Controller { metrics.Register() // It would be nicer to pass the controller name here, but that probably would break generating https://kubernetes.io/docs/reference/instrumentation/metrics. tc := &Controller{ @@ -748,7 +746,7 @@ func New(c clientset.Interface, podInformer coreinformers.PodInformer, claimInfo deletePodAt: make(map[tainteviction.NamespacedObject]evictionAndReason), allocatedClaims: make(map[types.NamespacedName]allocatedClaim), pools: make(map[poolID]pool), - evictingRules: make(map[string]*resourcbeta.DeviceTaintRule), + evictingRules: make(map[string]*resourceapi.DeviceTaintRule), taintRuleStats: make(map[types.UID]taintRuleStats), // Instantiate all informers now to ensure that they get started. haveSynced: []cache.DoneChecker{ @@ -929,7 +927,7 @@ func (tc *Controller) Run(ctx context.Context, numWorkers int) error { if tc.ruleInformer != nil { ruleHandler, err := tc.ruleInformer.Informer().AddEventHandlerWithOptions(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj any) { - rule, ok := obj.(*resourcbeta.DeviceTaintRule) + rule, ok := obj.(*resourceapi.DeviceTaintRule) if !ok { logger.Error(nil, "Expected DeviceTaintRule", "actual", fmt.Sprintf("%T", obj)) return @@ -939,12 +937,12 @@ func (tc *Controller) Run(ctx context.Context, numWorkers int) error { tc.handleRuleChange(nil, rule) }, UpdateFunc: func(oldObj, newObj any) { - oldRule, ok := oldObj.(*resourcbeta.DeviceTaintRule) + oldRule, ok := oldObj.(*resourceapi.DeviceTaintRule) if !ok { logger.Error(nil, "Expected DeviceTaintRule", "actual", fmt.Sprintf("%T", oldObj)) return } - newRule, ok := newObj.(*resourcbeta.DeviceTaintRule) + newRule, ok := newObj.(*resourceapi.DeviceTaintRule) if !ok { logger.Error(nil, "Expected DeviceTaintRule", "actual", fmt.Sprintf("%T", newObj)) } @@ -956,7 +954,7 @@ func (tc *Controller) Run(ctx context.Context, numWorkers int) error { if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { obj = tombstone.Obj } - rule, ok := obj.(*resourcbeta.DeviceTaintRule) + rule, ok := obj.(*resourceapi.DeviceTaintRule) if !ok { logger.Error(nil, "Expected DeviceTaintRule", "actual", fmt.Sprintf("%T", obj)) return @@ -1305,7 +1303,7 @@ func (tc *Controller) allEvictingDeviceTaints(allocatedDevice resourceapi.Device } } -func ruleMatchesDevice(rule *resourcbeta.DeviceTaintRule, driverName, poolName, deviceName string) bool { +func ruleMatchesDevice(rule *resourceapi.DeviceTaintRule, driverName, poolName, deviceName string) bool { selector := rule.Spec.DeviceSelector if selector == nil { return false @@ -1318,7 +1316,7 @@ func ruleMatchesDevice(rule *resourcbeta.DeviceTaintRule, driverName, poolName, return true } -func (tc *Controller) handleRuleChange(oldRule, newRule *resourcbeta.DeviceTaintRule) { +func (tc *Controller) handleRuleChange(oldRule, newRule *resourceapi.DeviceTaintRule) { rule := newRule if rule == nil { rule = oldRule @@ -1356,10 +1354,10 @@ func (tc *Controller) handleRuleChange(oldRule, newRule *resourcbeta.DeviceTaint // Rule spec changes should be rare. Simply do a brute-force re-evaluation of all allocated claims. // Same with trying to avoid delete+add in evictingRules, the logic just becomes unnecessarily complex. - if oldRule != nil && oldRule.Spec.Taint.Effect == resourcbeta.DeviceTaintEffectNoExecute { + if oldRule != nil && oldRule.Spec.Taint.Effect == resourceapi.DeviceTaintEffectNoExecute { delete(tc.evictingRules, oldRule.Name) } - if newRule != nil && newRule.Spec.Taint.Effect == resourcbeta.DeviceTaintEffectNoExecute { + if newRule != nil && newRule.Spec.Taint.Effect == resourceapi.DeviceTaintEffectNoExecute { tc.evictingRules[newRule.Name] = newRule } for name, oldAllocatedClaim := range tc.allocatedClaims { diff --git a/pkg/controller/devicetainteviction/device_taint_eviction_test.go b/pkg/controller/devicetainteviction/device_taint_eviction_test.go index 8b82990a75d..96ee37fac09 100644 --- a/pkg/controller/devicetainteviction/device_taint_eviction_test.go +++ b/pkg/controller/devicetainteviction/device_taint_eviction_test.go @@ -37,7 +37,6 @@ import ( v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" - resourcebeta "k8s.io/api/resource/v1beta2" schedulingapi "k8s.io/api/scheduling/v1alpha3" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -119,7 +118,7 @@ func setup(tCtx ktesting.TContext, workloadResourceClaimsEnabled bool) *testCont informerFactory.Core().V1().Pods(), informerFactory.Resource().V1().ResourceClaims(), informerFactory.Resource().V1().ResourceSlices(), - informerFactory.Resource().V1beta2().DeviceTaintRules(), + informerFactory.Resource().V1().DeviceTaintRules(), informerFactory.Resource().V1().DeviceClasses(), "device-taint-eviction", workloadResourceClaimsEnabled, @@ -155,7 +154,7 @@ type state struct { pods []*v1.Pod allocatedClaims []allocatedClaim slices []*resourceapi.ResourceSlice - rules []*resourcebeta.DeviceTaintRule + rules []*resourceapi.DeviceTaintRule ruleStats map[types.UID]taintRuleStats // Pods might have been queued in the past and then not removed when removing from deletePodAt. @@ -168,7 +167,7 @@ type state struct { // step describes a state after handling ready work items and how much to move time forward. type step struct { pods []*v1.Pod - rules []*resourcebeta.DeviceTaintRule + rules []*resourceapi.DeviceTaintRule ruleStats map[types.UID]taintRuleStats deletePodAt evictMap @@ -382,95 +381,95 @@ var ( slice.Spec.Pool.Generation++ return slice }() - ruleEvict = &resourcebeta.DeviceTaintRule{ + ruleEvict = &resourceapi.DeviceTaintRule{ ObjectMeta: metav1.ObjectMeta{ Name: "evict", UID: "1234", }, - Spec: resourcebeta.DeviceTaintRuleSpec{ - DeviceSelector: &resourcebeta.DeviceTaintSelector{ + Spec: resourceapi.DeviceTaintRuleSpec{ + DeviceSelector: &resourceapi.DeviceTaintSelector{ Driver: ptr.To(driver), }, - Taint: resourcebeta.DeviceTaint{ + Taint: resourceapi.DeviceTaint{ Key: taint.Key, Value: taint.Value, - Effect: resourcebeta.DeviceTaintEffect(taint.Effect), + Effect: resourceapi.DeviceTaintEffect(taint.Effect), TimeAdded: taint.TimeAdded, }, }, } - ruleEvictInstance1 = &resourcebeta.DeviceTaintRule{ + ruleEvictInstance1 = &resourceapi.DeviceTaintRule{ ObjectMeta: metav1.ObjectMeta{ Name: "evict-instance", UID: "1234", }, - Spec: resourcebeta.DeviceTaintRuleSpec{ - DeviceSelector: &resourcebeta.DeviceTaintSelector{ + Spec: resourceapi.DeviceTaintRuleSpec{ + DeviceSelector: &resourceapi.DeviceTaintSelector{ Driver: ptr.To(driver), Device: ptr.To("instance"), }, - Taint: resourcebeta.DeviceTaint{ + Taint: resourceapi.DeviceTaint{ Key: taint.Key, Value: taint.Value, - Effect: resourcebeta.DeviceTaintEffect(taint.Effect), + Effect: resourceapi.DeviceTaintEffect(taint.Effect), TimeAdded: taintTime, }, }, } taintTimeLater = metav1Time(taintTime.Add(40 * time.Second)) - ruleEvictInstance2Later = &resourcebeta.DeviceTaintRule{ + ruleEvictInstance2Later = &resourceapi.DeviceTaintRule{ ObjectMeta: metav1.ObjectMeta{ Name: "evict-instance-no-execute", UID: "5678", }, - Spec: resourcebeta.DeviceTaintRuleSpec{ - DeviceSelector: &resourcebeta.DeviceTaintSelector{ + Spec: resourceapi.DeviceTaintRuleSpec{ + DeviceSelector: &resourceapi.DeviceTaintSelector{ Driver: ptr.To(driver), Device: ptr.To("instance-no-execute"), }, - Taint: resourcebeta.DeviceTaint{ + Taint: resourceapi.DeviceTaint{ Key: taint.Key, Value: taint.Value, - Effect: resourcebeta.DeviceTaintEffect(taint.Effect), + Effect: resourceapi.DeviceTaintEffect(taint.Effect), TimeAdded: taintTimeLater, }, }, } - ruleNone = &resourcebeta.DeviceTaintRule{ + ruleNone = &resourceapi.DeviceTaintRule{ ObjectMeta: metav1.ObjectMeta{ Name: "evict", UID: "1234", }, - Spec: resourcebeta.DeviceTaintRuleSpec{ - DeviceSelector: &resourcebeta.DeviceTaintSelector{ + Spec: resourceapi.DeviceTaintRuleSpec{ + DeviceSelector: &resourceapi.DeviceTaintSelector{ Driver: ptr.To(driver), }, - Taint: resourcebeta.DeviceTaint{ + Taint: resourceapi.DeviceTaint{ Key: taint.Key, Value: taint.Value, - Effect: resourcebeta.DeviceTaintEffectNone, + Effect: resourceapi.DeviceTaintEffectNone, TimeAdded: taint.TimeAdded, }, }, } - ruleEvictOther = &resourcebeta.DeviceTaintRule{ + ruleEvictOther = &resourceapi.DeviceTaintRule{ ObjectMeta: metav1.ObjectMeta{ Name: "evict-other", UID: "1234-other", }, - Spec: resourcebeta.DeviceTaintRuleSpec{ - DeviceSelector: &resourcebeta.DeviceTaintSelector{ + Spec: resourceapi.DeviceTaintRuleSpec{ + DeviceSelector: &resourceapi.DeviceTaintSelector{ Device: ptr.To("instance"), }, - Taint: resourcebeta.DeviceTaint{ + Taint: resourceapi.DeviceTaint{ Key: taint.Key, Value: taint.Value, - Effect: resourcebeta.DeviceTaintEffect(taint.Effect), + Effect: resourceapi.DeviceTaintEffect(taint.Effect), TimeAdded: taint.TimeAdded, }, }, @@ -699,7 +698,7 @@ func newEvictionTime(when *metav1.Time, args ...any) *evictionAndReason { case *resourceapi.ResourceSlice: reason = append(reason, trackedTaint{slice: sliceDeviceTaint{slice: obj, deviceName: args[i+1].(string), taintIndex: args[i+2].(int)}}) i += 3 - case *resourcebeta.DeviceTaintRule: + case *resourceapi.DeviceTaintRule: reason = append(reason, trackedTaint{rule: obj}) i++ default: @@ -725,7 +724,7 @@ func newWorkItem(obj metav1.Object) workItem { ref := newObject(obj) var item workItem switch obj.(type) { - case *resourcebeta.DeviceTaintRule: + case *resourceapi.DeviceTaintRule: item.ruleRef = ref case *v1.Pod: item.podRef = ref @@ -775,10 +774,10 @@ func listEvents(tCtx ktesting.TContext) []v1.Event { return events.Items } -func inProgress(rule *resourcebeta.DeviceTaintRule, status bool, reason, message string, when *metav1.Time) *resourcebeta.DeviceTaintRule { +func inProgress(rule *resourceapi.DeviceTaintRule, status bool, reason, message string, when *metav1.Time) *resourceapi.DeviceTaintRule { rule = rule.DeepCopy() condition := metav1.Condition{ - Type: resourcebeta.DeviceTaintConditionEvictionInProgress, + Type: resourceapi.DeviceTaintConditionEvictionInProgress, Status: metav1.ConditionFalse, Reason: reason, Message: message, @@ -2031,7 +2030,7 @@ func testHandlers(tContext *testContext, tc testCase) { } return false, nil, nil }) - ruleStore := tContext.informerFactory.Resource().V1beta2().DeviceTaintRules().Informer().GetStore() + ruleStore := tContext.informerFactory.Resource().V1().DeviceTaintRules().Informer().GetStore() for _, rule := range tc.initialState.rules { tContext.ExpectNoError(ruleStore.Add(rule)) tContext.ExpectNoError(tContext.client.Tracker().Add(rule)) @@ -2084,7 +2083,7 @@ func testHandlers(tContext *testContext, tc testCase) { pods, err := tContext.client.CoreV1().Pods("").List(tContext, metav1.ListOptions{}) tContext.ExpectNoError(err, prefix+"list pods") assertEqual(tContext, state.pods, trimPods(pods.Items), prefix+"pods after flushing work queue") - rules, err := tContext.client.ResourceV1beta2().DeviceTaintRules().List(tContext, metav1.ListOptions{}) + rules, err := tContext.client.ResourceV1().DeviceTaintRules().List(tContext, metav1.ListOptions{}) tContext.ExpectNoError(err, prefix+"list rules") actualRules := trimRules(rules.Items) assertEqual(tContext, state.rules, actualRules, prefix+"rules after flushing work queue") @@ -2153,16 +2152,16 @@ func applyEventPair(tContext *testContext, event any) { tContext.ExpectNoError(tContext.client.Tracker().Add(obj)) } tContext.handlePodChange(pair[0], pair[1]) - case [2]*resourcebeta.DeviceTaintRule: - store := tContext.informerFactory.Resource().V1beta2().DeviceTaintRules().Informer().GetStore() + case [2]*resourceapi.DeviceTaintRule: + store := tContext.informerFactory.Resource().V1().DeviceTaintRules().Informer().GetStore() switch { case pair[0] != nil && pair[1] != nil: obj := pair[1].DeepCopy() tContext.ExpectNoError(store.Update(obj)) - tContext.ExpectNoError(tContext.client.Tracker().Update(resourcebeta.SchemeGroupVersion.WithResource("devicetaintrules"), obj, pair[1].Namespace)) + tContext.ExpectNoError(tContext.client.Tracker().Update(resourceapi.SchemeGroupVersion.WithResource("devicetaintrules"), obj, pair[1].Namespace)) case pair[0] != nil: tContext.ExpectNoError(store.Delete(pair[0])) - tContext.ExpectNoError(tContext.client.Tracker().Delete(resourcebeta.SchemeGroupVersion.WithResource("devicetaintrules"), pair[0].Namespace, pair[0].Name)) + tContext.ExpectNoError(tContext.client.Tracker().Delete(resourceapi.SchemeGroupVersion.WithResource("devicetaintrules"), pair[0].Namespace, pair[0].Name)) default: obj := pair[1].DeepCopy() tContext.ExpectNoError(store.Add(obj)) @@ -2183,7 +2182,7 @@ func trimPods(objs []v1.Pod) (trimmed []*v1.Pod) { return trimmed } -func trimRules(objs []resourcebeta.DeviceTaintRule) (trimmed []*resourcebeta.DeviceTaintRule) { +func trimRules(objs []resourceapi.DeviceTaintRule) (trimmed []*resourceapi.DeviceTaintRule) { for _, in := range objs { out := in.DeepCopy() out.ManagedFields = nil @@ -2200,7 +2199,7 @@ func newTestController(tCtx ktesting.TContext) *Controller { informerFactory.Core().V1().Pods(), informerFactory.Resource().V1().ResourceClaims(), informerFactory.Resource().V1().ResourceSlices(), - informerFactory.Resource().V1beta2().DeviceTaintRules(), + informerFactory.Resource().V1().DeviceTaintRules(), informerFactory.Resource().V1().DeviceClasses(), "device-taint-eviction", utilfeature.DefaultFeatureGate.Enabled(features.DRAWorkloadResourceClaims), @@ -2513,11 +2512,11 @@ func synctestDeviceTaintRule(tCtx ktesting.TContext, toleration, slowDelete bool time.Sleep(20 * time.Second) updated := metav1.Now() rule = rule.DeepCopy() // fake.NewClientset does not copy! Perhaps it should?! - rule.Spec.Taint.Effect = resourcebeta.DeviceTaintEffectNoExecute + rule.Spec.Taint.Effect = resourceapi.DeviceTaintEffectNoExecute // The real apiserver is going to bump this automatically in 1.36, // but in a unit test we have to do it manually. rule.Spec.Taint.TimeAdded = &updated - rule, err := tCtx.Client().ResourceV1beta2().DeviceTaintRules().Update(tCtx, rule, metav1.UpdateOptions{}) + rule, err := tCtx.Client().ResourceV1().DeviceTaintRules().Update(tCtx, rule, metav1.UpdateOptions{}) tCtx.ExpectNoError(err, "update rule") // Wait for eviction. @@ -2557,7 +2556,7 @@ func synctestDeviceTaintRule(tCtx ktesting.TContext, toleration, slowDelete bool assertEqual(tCtx, map[types.UID]taintRuleStats{rule.UID: {numEvictedPods: 1}}, controller.taintRuleStats, "taint rule statistics should have counted the pod") // Delete the rule and verify that we don't leak memory by still tracking it. - err = tCtx.Client().ResourceV1beta2().DeviceTaintRules().Delete(tCtx, rule.Name, metav1.DeleteOptions{}) + err = tCtx.Client().ResourceV1().DeviceTaintRules().Delete(tCtx, rule.Name, metav1.DeleteOptions{}) tCtx.ExpectNoError(err, "delete rule") tCtx.Wait() deleted := metav1.Now() @@ -2569,7 +2568,7 @@ func synctestDeviceTaintRule(tCtx ktesting.TContext, toleration, slowDelete bool tCtx.ExpectNoError(testPodDeletionsMetrics(controller, slowDeleteDelay)) } -func check(tCtx ktesting.TContext, prefix string, expectRules []*resourcebeta.DeviceTaintRule, expectPods []*v1.Pod) { +func check(tCtx ktesting.TContext, prefix string, expectRules []*resourceapi.DeviceTaintRule, expectPods []*v1.Pod) { tCtx.Helper() opts := []cmp.Option{ @@ -2584,7 +2583,7 @@ func check(tCtx ktesting.TContext, prefix string, expectRules []*resourcebeta.De actualPods, err := tCtx.Client().CoreV1().Pods("").List(tCtx, metav1.ListOptions{}) tCtx.ExpectNoError(err, prefix+"list pods") assertEqual(tCtx, expectPods, trimPods(actualPods.Items), prefix+"pods", opts...) - rules, err := tCtx.Client().ResourceV1beta2().DeviceTaintRules().List(tCtx, metav1.ListOptions{}) + rules, err := tCtx.Client().ResourceV1().DeviceTaintRules().List(tCtx, metav1.ListOptions{}) tCtx.ExpectNoError(err, prefix+"list rules") assertEqual(tCtx, expectRules, trimRules(rules.Items), prefix+"rules", opts...) } diff --git a/pkg/controlplane/instance_test.go b/pkg/controlplane/instance_test.go index 6c2075bf405..78bdb13d331 100644 --- a/pkg/controlplane/instance_test.go +++ b/pkg/controlplane/instance_test.go @@ -765,6 +765,7 @@ var gvrToStorageVersionHash = map[string]string{ "rbac.authorization.k8s.io/v1/rolebindings": "eGsCzGH6b1g=", "rbac.authorization.k8s.io/v1/roles": "7FuwZcIIItM=", "resource.k8s.io/v1/deviceclasses": "Yk2PTc1Ybxk=", + "resource.k8s.io/v1/devicetaintrules": "i+85+TcIKpA=", "resource.k8s.io/v1/resourceclaims": "wgAZaHcZxUg=", "resource.k8s.io/v1/resourceclaimtemplates": "TuzjC49aUfM=", "resource.k8s.io/v1/resourceslices": "KsC072WgaEY=", diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index a82d6c6bb73..4a777f5ff63 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -1307,12 +1307,14 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate DRADeviceTaintRules: { {Version: version.MustParse("1.35"), Default: false, PreRelease: featuregate.Alpha}, - {Version: version.MustParse("1.36"), Default: false, PreRelease: featuregate.Beta}, // Depends on an off-by-default beta API. + {Version: version.MustParse("1.36"), Default: false, PreRelease: featuregate.Beta}, // Depends on an off-by-default beta API. + {Version: version.MustParse("1.37"), Default: true, PreRelease: featuregate.GA, LockToDefault: false}, // LockToDefault: true in 1.38; remove in 1.41 }, DRADeviceTaints: { {Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.36"), Default: true, PreRelease: featuregate.Beta}, + {Version: version.MustParse("1.37"), Default: true, PreRelease: featuregate.GA, LockToDefault: false}, // LockToDefault: true in 1.38; remove in 1.41 }, DRAExtendedResource: { diff --git a/pkg/registry/resource/resourceclaim/strategy_test.go b/pkg/registry/resource/resourceclaim/strategy_test.go index 1623ebe67c5..53a5162347f 100644 --- a/pkg/registry/resource/resourceclaim/strategy_test.go +++ b/pkg/registry/resource/resourceclaim/strategy_test.go @@ -628,12 +628,13 @@ func TestStrategyCreate(t *testing.T) { t.Run(name, func(t *testing.T) { fakeClient := fake.NewSimpleClientset(ns1, ns2) mockNSClient := fakeClient.CoreV1().Namespaces() - if !tc.adminAccess { + if !tc.adminAccess || !tc.deviceTaints { featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.35")) } featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ features.DRAAdminAccess: tc.adminAccess, features.DRADeviceTaints: tc.deviceTaints, + features.DRADeviceTaintRules: tc.deviceTaints, features.DRAPrioritizedList: tc.prioritizedList, features.DRAConsumableCapacity: tc.consumableCapacity, }) @@ -962,12 +963,13 @@ func TestStrategyUpdate(t *testing.T) { fakeClient := fake.NewSimpleClientset(ns1, ns2) mockNSClient := fakeClient.CoreV1().Namespaces() - if !tc.adminAccess { + if !tc.adminAccess || !tc.deviceTaints { featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.35")) } featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ features.DRAAdminAccess: tc.adminAccess, features.DRADeviceTaints: tc.deviceTaints, + features.DRADeviceTaintRules: tc.deviceTaints, features.DRAPrioritizedList: tc.prioritizedList, features.DRAConsumableCapacity: tc.consumableCapacity, }) diff --git a/pkg/registry/resource/resourceclaimtemplate/strategy_test.go b/pkg/registry/resource/resourceclaimtemplate/strategy_test.go index 777d9650373..e5086a8365c 100644 --- a/pkg/registry/resource/resourceclaimtemplate/strategy_test.go +++ b/pkg/registry/resource/resourceclaimtemplate/strategy_test.go @@ -478,12 +478,13 @@ func TestClaimTemplateStrategyCreate(t *testing.T) { t.Run(name, func(t *testing.T) { fakeClient := fake.NewSimpleClientset(ns1, ns2) mockNSClient := fakeClient.CoreV1().Namespaces() - if !tc.adminAccess { + if !tc.adminAccess || !tc.deviceTaints { featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.35")) } featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ features.DRAAdminAccess: tc.adminAccess, features.DRADeviceTaints: tc.deviceTaints, + features.DRADeviceTaintRules: tc.deviceTaints, features.DRAPrioritizedList: tc.prioritizedList, features.DRAConsumableCapacity: tc.consumableCapacity, }) @@ -853,13 +854,16 @@ func TestStrategyUpdate(t *testing.T) { fakeClient := fake.NewSimpleClientset(ns1, ns2) mockNSClient := fakeClient.CoreV1().Namespaces() - if !tc.adminAccess { + if !tc.adminAccess || !tc.deviceTaints { featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.35")) } - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DRAAdminAccess, tc.adminAccess) - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DRADeviceTaints, tc.deviceTaints) - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DRAPrioritizedList, tc.prioritizedList) - featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.DRAConsumableCapacity, tc.consumableCapacity) + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.DRAAdminAccess: tc.adminAccess, + features.DRADeviceTaints: tc.deviceTaints, + features.DRADeviceTaintRules: tc.deviceTaints, + features.DRAPrioritizedList: tc.prioritizedList, + features.DRAConsumableCapacity: tc.consumableCapacity, + }) strategy := NewStrategy(mockNSClient) oldObj := tc.oldObj.DeepCopy() diff --git a/pkg/registry/resource/resourceslice/strategy_test.go b/pkg/registry/resource/resourceslice/strategy_test.go index 73f6cbda509..dfb2e868a05 100644 --- a/pkg/registry/resource/resourceslice/strategy_test.go +++ b/pkg/registry/resource/resourceslice/strategy_test.go @@ -26,6 +26,7 @@ import ( v1 "k8s.io/api/core/v1" k8sresource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/version" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" @@ -222,16 +223,11 @@ func TestResourceSliceStrategy(t *testing.T) { func TestResourceSliceStrategyCreate(t *testing.T) { ctx := genericapirequest.NewDefaultContext() testCases := map[string]struct { - obj *resource.ResourceSlice - deviceTaints bool - partitionableDevices bool - bindingConditions bool - deviceStatus bool - consumableCapacity bool - draNodeAllocatableResources bool - listTypeAttributes bool - expectedValidationError bool - expectObj *resource.ResourceSlice + obj *resource.ResourceSlice + featureOverrides featuregatetesting.FeatureOverrides + emulatedVersion string + expectedValidationError bool + expectObj *resource.ResourceSlice }{ "simple": { obj: slice, @@ -250,8 +246,9 @@ func TestResourceSliceStrategyCreate(t *testing.T) { expectedValidationError: true, }, "drop-fields-device-taints": { - obj: sliceWithDeviceTaints, - deviceTaints: false, + obj: sliceWithDeviceTaints, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRADeviceTaints: false}, + emulatedVersion: "1.35", expectObj: func() *resource.ResourceSlice { obj := slice.DeepCopy() obj.Generation = 1 @@ -259,8 +256,8 @@ func TestResourceSliceStrategyCreate(t *testing.T) { }(), }, "keep-fields-device-taints": { - obj: sliceWithDeviceTaints, - deviceTaints: true, + obj: sliceWithDeviceTaints, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRADeviceTaints: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithDeviceTaints.DeepCopy() obj.ObjectMeta.Generation = 1 @@ -272,7 +269,7 @@ func TestResourceSliceStrategyCreate(t *testing.T) { obj := sliceWithPartitionableDevicesConsumesCounters.DeepCopy() return obj }(), - partitionableDevices: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: false}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesConsumesCounters.DeepCopy() obj.ObjectMeta.Generation = 1 @@ -285,7 +282,7 @@ func TestResourceSliceStrategyCreate(t *testing.T) { // dropped. "drop-fields-partitionable-devices-with-per-device-node-selection": { obj: sliceWithPartitionableDevicesPerDeviceNodeSelection, - partitionableDevices: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: false}, expectedValidationError: true, }, "drop-fields-partitionable-devices-with-shared-counters": { @@ -293,7 +290,7 @@ func TestResourceSliceStrategyCreate(t *testing.T) { obj := sliceWithPartitionableDevicesSharedCounters.DeepCopy() return obj }(), - partitionableDevices: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: false}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesSharedCounters.DeepCopy() obj.ObjectMeta.Generation = 1 @@ -302,8 +299,8 @@ func TestResourceSliceStrategyCreate(t *testing.T) { }(), }, "keep-fields-partitionable-devices-with-consumes-counters": { - obj: sliceWithPartitionableDevicesConsumesCounters, - partitionableDevices: true, + obj: sliceWithPartitionableDevicesConsumesCounters, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesConsumesCounters.DeepCopy() obj.Generation = 1 @@ -311,8 +308,8 @@ func TestResourceSliceStrategyCreate(t *testing.T) { }(), }, "keep-fields-partitionable-devices-with-per-device-node-selection": { - obj: sliceWithPartitionableDevicesPerDeviceNodeSelection, - partitionableDevices: true, + obj: sliceWithPartitionableDevicesPerDeviceNodeSelection, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesPerDeviceNodeSelection.DeepCopy() obj.Generation = 1 @@ -320,8 +317,8 @@ func TestResourceSliceStrategyCreate(t *testing.T) { }(), }, "keep-fields-partitionable-devices-with-shared-counters": { - obj: sliceWithPartitionableDevicesSharedCounters, - partitionableDevices: true, + obj: sliceWithPartitionableDevicesSharedCounters, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesSharedCounters.DeepCopy() obj.Generation = 1 @@ -329,9 +326,8 @@ func TestResourceSliceStrategyCreate(t *testing.T) { }(), }, "drop-fields-binding-conditions": { - obj: sliceWithBindingConditions, - bindingConditions: false, - deviceStatus: false, + obj: sliceWithBindingConditions, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRADeviceBindingConditions: false, features.DRAResourceClaimDeviceStatus: false}, expectObj: func() *resource.ResourceSlice { obj := slice.DeepCopy() obj.Generation = 1 @@ -339,9 +335,8 @@ func TestResourceSliceStrategyCreate(t *testing.T) { }(), }, "drop-fields-binding-conditions-with-binding-conditions": { - obj: sliceWithBindingConditions, - bindingConditions: false, - deviceStatus: true, + obj: sliceWithBindingConditions, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRADeviceBindingConditions: false, features.DRAResourceClaimDeviceStatus: true}, expectObj: func() *resource.ResourceSlice { obj := slice.DeepCopy() obj.Generation = 1 @@ -349,9 +344,8 @@ func TestResourceSliceStrategyCreate(t *testing.T) { }(), }, "keep-fields-binding-conditions": { - obj: sliceWithBindingConditions, - bindingConditions: true, - deviceStatus: true, + obj: sliceWithBindingConditions, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRADeviceBindingConditions: true, features.DRAResourceClaimDeviceStatus: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithBindingConditions.DeepCopy() obj.Generation = 1 @@ -359,8 +353,8 @@ func TestResourceSliceStrategyCreate(t *testing.T) { }(), }, "keep-fields-consumable-capacity": { - obj: sliceWithConsumableCapacity, - consumableCapacity: true, + obj: sliceWithConsumableCapacity, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAConsumableCapacity: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithConsumableCapacity.DeepCopy() obj.Generation = 1 @@ -368,8 +362,8 @@ func TestResourceSliceStrategyCreate(t *testing.T) { }(), }, "drop-fields-consumable-capacity-disabled-feature": { - obj: sliceWithConsumableCapacity, - consumableCapacity: false, + obj: sliceWithConsumableCapacity, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAConsumableCapacity: false}, expectObj: func() *resource.ResourceSlice { obj := sliceWithCapacity.DeepCopy() obj.Generation = 1 @@ -377,8 +371,8 @@ func TestResourceSliceStrategyCreate(t *testing.T) { }(), }, "keep-fields-node-allocatable-dra-claims": { - obj: sliceWithNodeAllocatableResources, - draNodeAllocatableResources: true, + obj: sliceWithNodeAllocatableResources, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRANodeAllocatableResources: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithNodeAllocatableResources.DeepCopy() obj.Generation = 1 @@ -386,8 +380,8 @@ func TestResourceSliceStrategyCreate(t *testing.T) { }(), }, "drop-fields-node-allocatable-dra-claims-disabled-feature": { - obj: sliceWithNodeAllocatableResources, - draNodeAllocatableResources: false, + obj: sliceWithNodeAllocatableResources, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRANodeAllocatableResources: false}, expectObj: func() *resource.ResourceSlice { obj := slice.DeepCopy() obj.Generation = 1 @@ -395,8 +389,8 @@ func TestResourceSliceStrategyCreate(t *testing.T) { }(), }, "keep-fields-list-type-attributes": { - obj: sliceWithListTypeAttributes, - listTypeAttributes: true, + obj: sliceWithListTypeAttributes, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAListTypeAttributes: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithListTypeAttributes.DeepCopy() obj.Generation = 1 @@ -405,22 +399,17 @@ func TestResourceSliceStrategyCreate(t *testing.T) { }, "drop-fields-list-type-attributes": { obj: sliceWithListTypeAttributes, - listTypeAttributes: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAListTypeAttributes: false}, expectedValidationError: true, }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { - featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ - features.DRADeviceTaints: tc.deviceTaints, - features.DRAPartitionableDevices: tc.partitionableDevices, - features.DRADeviceBindingConditions: tc.bindingConditions, - features.DRAResourceClaimDeviceStatus: tc.deviceStatus, - features.DRAConsumableCapacity: tc.consumableCapacity, - features.DRANodeAllocatableResources: tc.draNodeAllocatableResources, - features.DRAListTypeAttributes: tc.listTypeAttributes, - }) + if tc.emulatedVersion != "" { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse(tc.emulatedVersion)) + } + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, tc.featureOverrides) obj := tc.obj.DeepCopy() @@ -446,12 +435,8 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { testcases := map[string]struct { oldObj *resource.ResourceSlice newObj *resource.ResourceSlice - deviceTaints bool - partitionableDevices bool - deviceStatus bool - bindingConditions bool - consumableCapacity bool - listTypeAttributes bool + featureOverrides featuregatetesting.FeatureOverrides + emulatedVersion string expectValidationError bool expectObj *resource.ResourceSlice }{ @@ -485,7 +470,8 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - deviceTaints: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRADeviceTaints: false}, + emulatedVersion: "1.35", expectObj: func() *resource.ResourceSlice { obj := slice.DeepCopy() obj.ResourceVersion = "4" @@ -499,7 +485,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - deviceTaints: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRADeviceTaints: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithDeviceTaints.DeepCopy() obj.ResourceVersion = "4" @@ -514,7 +500,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - deviceTaints: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRADeviceTaints: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithDeviceTaints.DeepCopy() obj.ResourceVersion = "4" @@ -528,7 +514,8 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - deviceTaints: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRADeviceTaints: false}, + emulatedVersion: "1.35", expectObj: func() *resource.ResourceSlice { obj := sliceWithDeviceTaints.DeepCopy() obj.ResourceVersion = "4" @@ -542,7 +529,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - partitionableDevices: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: false}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesConsumesCounters.DeepCopy() obj.ResourceVersion = "4" @@ -558,7 +545,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - partitionableDevices: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: false}, expectValidationError: true, }, "drop-fields-partitionable-devices-with-shared-counters": { @@ -568,7 +555,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - partitionableDevices: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: false}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesSharedCounters.DeepCopy() obj.ResourceVersion = "4" @@ -584,7 +571,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - partitionableDevices: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesConsumesCounters.DeepCopy() obj.ResourceVersion = "4" @@ -599,7 +586,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - partitionableDevices: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: true}, expectValidationError: true, // Spec.NodeName is immutable. }, "keep-fields-partitionable-devices-with-shared-counters": { @@ -609,7 +596,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - partitionableDevices: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesSharedCounters.DeepCopy() obj.ResourceVersion = "4" @@ -624,7 +611,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - partitionableDevices: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesConsumesCounters.DeepCopy() obj.ResourceVersion = "4" @@ -638,7 +625,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - partitionableDevices: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesPerDeviceNodeSelection.DeepCopy() obj.ResourceVersion = "4" @@ -652,7 +639,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - partitionableDevices: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesSharedCounters.DeepCopy() obj.ResourceVersion = "4" @@ -666,7 +653,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - partitionableDevices: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: false}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesConsumesCounters.DeepCopy() obj.ResourceVersion = "4" @@ -680,7 +667,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - partitionableDevices: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: false}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesPerDeviceNodeSelection.DeepCopy() obj.ResourceVersion = "4" @@ -694,7 +681,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - partitionableDevices: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAPartitionableDevices: false}, expectObj: func() *resource.ResourceSlice { obj := sliceWithPartitionableDevicesSharedCounters.DeepCopy() obj.ResourceVersion = "4" @@ -714,8 +701,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.Generation = 1 return obj }(), - bindingConditions: false, - deviceStatus: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRADeviceBindingConditions: false, features.DRAResourceClaimDeviceStatus: false}, }, "drop-fields-binding-conditions-with-binding-conditions": { oldObj: slice, @@ -730,8 +716,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.Generation = 1 return obj }(), - bindingConditions: false, - deviceStatus: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRADeviceBindingConditions: false, features.DRAResourceClaimDeviceStatus: true}, }, "keep-fields-binding-conditions": { oldObj: slice, @@ -746,8 +731,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.Generation = 1 return obj }(), - bindingConditions: true, - deviceStatus: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRADeviceBindingConditions: true, features.DRAResourceClaimDeviceStatus: true}, }, "keep-existing-fields-binding-conditions-without-featuregate-enabled": { oldObj: sliceWithBindingConditions, @@ -763,17 +747,16 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.Generation = 1 return obj }(), - bindingConditions: false, - deviceStatus: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRADeviceBindingConditions: false, features.DRAResourceClaimDeviceStatus: true}, }, - "keep-existing-fields-consumable-capacity": { - oldObj: sliceWithCapacity, + "keep-consumable-capacity": { + oldObj: slice, newObj: func() *resource.ResourceSlice { obj := sliceWithConsumableCapacity.DeepCopy() obj.ResourceVersion = "4" return obj }(), - consumableCapacity: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAConsumableCapacity: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithConsumableCapacity.DeepCopy() obj.ResourceVersion = "4" @@ -788,7 +771,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - consumableCapacity: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAConsumableCapacity: false}, expectObj: func() *resource.ResourceSlice { obj := sliceWithConsumableCapacity.DeepCopy() obj.ResourceVersion = "4" @@ -802,7 +785,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - consumableCapacity: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAConsumableCapacity: false}, expectObj: func() *resource.ResourceSlice { obj := sliceWithCapacity.DeepCopy() obj.ResourceVersion = "4" @@ -811,13 +794,13 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { }(), }, "drop-list-type-attributes": { - oldObj: slice.DeepCopy(), + oldObj: slice, newObj: func() *resource.ResourceSlice { obj := sliceWithListTypeAttributes.DeepCopy() obj.ResourceVersion = "4" return obj }(), - listTypeAttributes: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAListTypeAttributes: false}, expectValidationError: true, }, "keep-fields-list-type-attributes": { @@ -827,7 +810,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - listTypeAttributes: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAListTypeAttributes: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithListTypeAttributes.DeepCopy() obj.ResourceVersion = "4" @@ -842,7 +825,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - listTypeAttributes: true, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAListTypeAttributes: true}, expectObj: func() *resource.ResourceSlice { obj := sliceWithListTypeAttributes.DeepCopy() obj.ResourceVersion = "4" @@ -856,7 +839,7 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { obj.ResourceVersion = "4" return obj }(), - listTypeAttributes: false, + featureOverrides: featuregatetesting.FeatureOverrides{features.DRAListTypeAttributes: false}, expectObj: func() *resource.ResourceSlice { obj := sliceWithListTypeAttributes.DeepCopy() obj.ResourceVersion = "4" @@ -867,14 +850,10 @@ func TestResourceSliceStrategyUpdate(t *testing.T) { for name, tc := range testcases { t.Run(name, func(t *testing.T) { - featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ - features.DRADeviceTaints: tc.deviceTaints, - features.DRAPartitionableDevices: tc.partitionableDevices, - features.DRADeviceBindingConditions: tc.bindingConditions, - features.DRAResourceClaimDeviceStatus: tc.deviceStatus, - features.DRAConsumableCapacity: tc.consumableCapacity, - features.DRAListTypeAttributes: tc.listTypeAttributes, - }) + if tc.emulatedVersion != "" { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse(tc.emulatedVersion)) + } + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, tc.featureOverrides) oldObj := tc.oldObj.DeepCopy() newObj := tc.newObj.DeepCopy() diff --git a/pkg/scheduler/eventhandlers_test.go b/pkg/scheduler/eventhandlers_test.go index f98076c056a..5838f11a122 100644 --- a/pkg/scheduler/eventhandlers_test.go +++ b/pkg/scheduler/eventhandlers_test.go @@ -28,7 +28,6 @@ import ( batchv1 "k8s.io/api/batch/v1" v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" - resourcebetaapi "k8s.io/api/resource/v1beta2" schedulingapi "k8s.io/api/scheduling/v1alpha3" storagev1 "k8s.io/api/storage/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -280,74 +279,86 @@ func withPodName(pod *v1.Pod, name string) *v1.Pod { // test for informers of resources we care about is registered func TestAddAllEventHandlers(t *testing.T) { tests := []struct { - name string - gvkMap map[fwk.EventResource]fwk.ActionType - enableDRA bool - enableDRADeviceTaints bool - enableDRADeviceTaintRules bool - enableDRAExtendedResource bool - enableGenericWorkload bool - expectStaticInformers map[reflect.Type]bool - expectDynamicInformers map[schema.GroupVersionResource]bool + name string + gvkMap map[fwk.EventResource]fwk.ActionType + emulatedVersion string + overrides featuregatetesting.FeatureOverrides + expectStaticInformers map[reflect.Type]bool + expectDynamicInformers map[schema.GroupVersionResource]bool }{ { name: "default handlers in framework", gvkMap: map[fwk.EventResource]fwk.ActionType{}, expectStaticInformers: map[reflect.Type]bool{ - reflect.TypeOf(&v1.Pod{}): true, - reflect.TypeOf(&v1.Node{}): true, - reflect.TypeOf(&v1.Namespace{}): true, + reflect.TypeFor[*v1.Pod](): true, + reflect.TypeFor[*v1.Node](): true, + reflect.TypeFor[*v1.Namespace](): true, + reflect.TypeFor[*resourceapi.ResourceClaim](): true, + reflect.TypeFor[*resourceapi.ResourceSlice](): true, + reflect.TypeFor[*resourceapi.DeviceTaintRule](): true, + reflect.TypeFor[*resourceapi.DeviceClass](): true, }, expectDynamicInformers: map[schema.GroupVersionResource]bool{}, }, { - name: "DRA events disabled", + name: "DRA events disabled", + emulatedVersion: "1.34", + overrides: featuregatetesting.FeatureOverrides{ + features.DynamicResourceAllocation: false, + }, gvkMap: map[fwk.EventResource]fwk.ActionType{ fwk.ResourceClaim: fwk.Add, fwk.ResourceSlice: fwk.Add, fwk.DeviceClass: fwk.Add, }, expectStaticInformers: map[reflect.Type]bool{ - reflect.TypeOf(&v1.Pod{}): true, - reflect.TypeOf(&v1.Node{}): true, - reflect.TypeOf(&v1.Namespace{}): true, + reflect.TypeFor[*v1.Pod](): true, + reflect.TypeFor[*v1.Node](): true, + reflect.TypeFor[*v1.Namespace](): true, }, expectDynamicInformers: map[schema.GroupVersionResource]bool{}, }, { - name: "core DRA events enabled", + name: "core DRA events enabled", + emulatedVersion: "1.35", + overrides: featuregatetesting.FeatureOverrides{ + features.DRADeviceTaints: false, + features.DRADeviceTaintRules: false, + features.DRAExtendedResource: false, + }, gvkMap: map[fwk.EventResource]fwk.ActionType{ fwk.ResourceClaim: fwk.Add, fwk.ResourceSlice: fwk.Add, fwk.DeviceClass: fwk.Add, }, - enableDRA: true, expectStaticInformers: map[reflect.Type]bool{ - reflect.TypeOf(&v1.Pod{}): true, - reflect.TypeOf(&v1.Node{}): true, - reflect.TypeOf(&v1.Namespace{}): true, - reflect.TypeOf(&resourceapi.ResourceClaim{}): true, - reflect.TypeOf(&resourceapi.ResourceSlice{}): true, - reflect.TypeOf(&resourceapi.DeviceClass{}): true, + reflect.TypeFor[*v1.Pod](): true, + reflect.TypeFor[*v1.Node](): true, + reflect.TypeFor[*v1.Namespace](): true, + reflect.TypeFor[*resourceapi.ResourceClaim](): true, + reflect.TypeFor[*resourceapi.ResourceSlice](): true, + reflect.TypeFor[*resourceapi.DeviceClass](): true, }, expectDynamicInformers: map[schema.GroupVersionResource]bool{}, }, { - name: "device taints partially enabled", + name: "DRA device taints partially enabled", + emulatedVersion: "1.35", + overrides: featuregatetesting.FeatureOverrides{ + features.DRADeviceTaintRules: false, + }, gvkMap: map[fwk.EventResource]fwk.ActionType{ fwk.ResourceClaim: fwk.Add, fwk.ResourceSlice: fwk.Add, fwk.DeviceClass: fwk.Add, }, - enableDRA: true, - enableDRADeviceTaints: true, expectStaticInformers: map[reflect.Type]bool{ - reflect.TypeOf(&v1.Pod{}): true, - reflect.TypeOf(&v1.Node{}): true, - reflect.TypeOf(&v1.Namespace{}): true, - reflect.TypeOf(&resourceapi.ResourceClaim{}): true, - reflect.TypeOf(&resourceapi.ResourceSlice{}): true, - reflect.TypeOf(&resourceapi.DeviceClass{}): true, + reflect.TypeFor[*v1.Pod](): true, + reflect.TypeFor[*v1.Node](): true, + reflect.TypeFor[*v1.Namespace](): true, + reflect.TypeFor[*resourceapi.ResourceClaim](): true, + reflect.TypeFor[*resourceapi.ResourceSlice](): true, + reflect.TypeFor[*resourceapi.DeviceClass](): true, }, expectDynamicInformers: map[schema.GroupVersionResource]bool{}, }, @@ -358,17 +369,14 @@ func TestAddAllEventHandlers(t *testing.T) { fwk.ResourceSlice: fwk.Add, fwk.DeviceClass: fwk.Add, }, - enableDRA: true, - enableDRADeviceTaints: true, - enableDRADeviceTaintRules: true, expectStaticInformers: map[reflect.Type]bool{ - reflect.TypeOf(&v1.Pod{}): true, - reflect.TypeOf(&v1.Node{}): true, - reflect.TypeOf(&v1.Namespace{}): true, - reflect.TypeOf(&resourceapi.ResourceClaim{}): true, - reflect.TypeOf(&resourceapi.ResourceSlice{}): true, - reflect.TypeOf(&resourcebetaapi.DeviceTaintRule{}): true, - reflect.TypeOf(&resourceapi.DeviceClass{}): true, + reflect.TypeFor[*v1.Pod](): true, + reflect.TypeFor[*v1.Node](): true, + reflect.TypeFor[*v1.Namespace](): true, + reflect.TypeFor[*resourceapi.ResourceClaim](): true, + reflect.TypeFor[*resourceapi.ResourceSlice](): true, + reflect.TypeFor[*resourceapi.DeviceTaintRule](): true, + reflect.TypeFor[*resourceapi.DeviceClass](): true, }, expectDynamicInformers: map[schema.GroupVersionResource]bool{}, }, @@ -378,26 +386,33 @@ func TestAddAllEventHandlers(t *testing.T) { fwk.PodGroup: fwk.Add, }, expectStaticInformers: map[reflect.Type]bool{ - reflect.TypeOf(&v1.Pod{}): true, - reflect.TypeOf(&v1.Node{}): true, - reflect.TypeOf(&v1.Namespace{}): true, + reflect.TypeFor[*v1.Pod](): true, + reflect.TypeFor[*v1.Node](): true, + reflect.TypeFor[*v1.Namespace](): true, + reflect.TypeFor[*resourceapi.ResourceClaim](): true, + reflect.TypeFor[*resourceapi.ResourceSlice](): true, + reflect.TypeFor[*resourceapi.DeviceTaintRule](): true, + reflect.TypeFor[*resourceapi.DeviceClass](): true, }, expectDynamicInformers: map[schema.GroupVersionResource]bool{}, }, { name: "PodGroup events enabled", + overrides: featuregatetesting.FeatureOverrides{ + features.GenericWorkload: true, + }, gvkMap: map[fwk.EventResource]fwk.ActionType{ fwk.PodGroup: fwk.Add, }, - enableDRA: true, - enableGenericWorkload: true, expectStaticInformers: map[reflect.Type]bool{ - reflect.TypeOf(&v1.Pod{}): true, - reflect.TypeOf(&v1.Node{}): true, - reflect.TypeOf(&v1.Namespace{}): true, - reflect.TypeOf(&resourceapi.ResourceClaim{}): true, - reflect.TypeOf(&resourceapi.ResourceSlice{}): true, - reflect.TypeOf(&schedulingapi.PodGroup{}): true, + reflect.TypeFor[*v1.Pod](): true, + reflect.TypeFor[*v1.Node](): true, + reflect.TypeFor[*v1.Namespace](): true, + reflect.TypeFor[*resourceapi.ResourceClaim](): true, + reflect.TypeFor[*resourceapi.ResourceSlice](): true, + reflect.TypeFor[*resourceapi.DeviceTaintRule](): true, + reflect.TypeFor[*resourceapi.DeviceClass](): true, + reflect.TypeFor[*schedulingapi.PodGroup](): true, }, expectDynamicInformers: map[schema.GroupVersionResource]bool{}, }, @@ -409,11 +424,15 @@ func TestAddAllEventHandlers(t *testing.T) { "storage.k8s.io/CSIStorageCapacity": fwk.Update, }, expectStaticInformers: map[reflect.Type]bool{ - reflect.TypeOf(&v1.Pod{}): true, - reflect.TypeOf(&v1.Node{}): true, - reflect.TypeOf(&v1.Namespace{}): true, - reflect.TypeOf(&v1.PersistentVolume{}): true, - reflect.TypeOf(&storagev1.CSIStorageCapacity{}): true, + reflect.TypeFor[*v1.Pod](): true, + reflect.TypeFor[*v1.Node](): true, + reflect.TypeFor[*v1.Namespace](): true, + reflect.TypeFor[*v1.PersistentVolume](): true, + reflect.TypeFor[*storagev1.CSIStorageCapacity](): true, + reflect.TypeFor[*resourceapi.ResourceClaim](): true, + reflect.TypeFor[*resourceapi.ResourceSlice](): true, + reflect.TypeFor[*resourceapi.DeviceTaintRule](): true, + reflect.TypeFor[*resourceapi.DeviceClass](): true, }, expectDynamicInformers: map[schema.GroupVersionResource]bool{}, }, @@ -424,9 +443,13 @@ func TestAddAllEventHandlers(t *testing.T) { "cronjobs.v1.batch": fwk.Delete, }, expectStaticInformers: map[reflect.Type]bool{ - reflect.TypeOf(&v1.Pod{}): true, - reflect.TypeOf(&v1.Node{}): true, - reflect.TypeOf(&v1.Namespace{}): true, + reflect.TypeFor[*v1.Pod](): true, + reflect.TypeFor[*v1.Node](): true, + reflect.TypeFor[*v1.Namespace](): true, + reflect.TypeFor[*resourceapi.ResourceClaim](): true, + reflect.TypeFor[*resourceapi.ResourceSlice](): true, + reflect.TypeFor[*resourceapi.DeviceTaintRule](): true, + reflect.TypeFor[*resourceapi.DeviceClass](): true, }, expectDynamicInformers: map[schema.GroupVersionResource]bool{ {Group: "apps", Version: "v1", Resource: "daemonsets"}: true, @@ -440,9 +463,13 @@ func TestAddAllEventHandlers(t *testing.T) { "custommetrics.v1beta1": fwk.Update, }, expectStaticInformers: map[reflect.Type]bool{ - reflect.TypeOf(&v1.Pod{}): true, - reflect.TypeOf(&v1.Node{}): true, - reflect.TypeOf(&v1.Namespace{}): true, + reflect.TypeFor[*v1.Pod](): true, + reflect.TypeFor[*v1.Node](): true, + reflect.TypeFor[*v1.Namespace](): true, + reflect.TypeFor[*resourceapi.ResourceClaim](): true, + reflect.TypeFor[*resourceapi.ResourceSlice](): true, + reflect.TypeFor[*resourceapi.DeviceTaintRule](): true, + reflect.TypeFor[*resourceapi.DeviceClass](): true, }, expectDynamicInformers: map[schema.GroupVersionResource]bool{ {Group: "apps", Version: "v1", Resource: "daemonsets"}: true, @@ -459,22 +486,10 @@ func TestAddAllEventHandlers(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - overrides := featuregatetesting.FeatureOverrides{ - features.DynamicResourceAllocation: tt.enableDRA, - features.DRADeviceTaints: tt.enableDRADeviceTaints, - features.DRAExtendedResource: tt.enableDRAExtendedResource, + if tt.emulatedVersion != "" { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse(tt.emulatedVersion)) } - if !tt.enableDRA { - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.34")) - } else { - if !tt.enableDRAExtendedResource { - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.36")) - } - // Making this depend on the emulated version avoids "cannot set feature gate DRADeviceTaintRules to false, feature is PreAlpha at emulated version 1.34". - overrides[features.DRADeviceTaintRules] = tt.enableDRADeviceTaintRules - overrides[features.GenericWorkload] = tt.enableGenericWorkload - } - featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, overrides) + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, tt.overrides) logger, ctx := ktesting.NewTestContext(t) ctx, cancel := context.WithCancel(ctx) @@ -502,15 +517,14 @@ func TestAddAllEventHandlers(t *testing.T) { SliceInformer: informerFactory.Resource().V1().ResourceSlices(), } if opts.EnableDeviceTaintRules { - opts.TaintInformer = informerFactory.Resource().V1beta2().DeviceTaintRules() - + opts.TaintInformer = informerFactory.Resource().V1().DeviceTaintRules() } resourceSliceTracker, err = resourceslicetracker.StartTracker(ctx, opts) if err != nil { t.Fatalf("couldn't start resource slice tracker: %v", err) } - if tt.enableDRAExtendedResource { + if utilfeature.DefaultFeatureGate.Enabled(features.DRAExtendedResource) { draManager = dynamicresources.NewDRAManager(ctx, resourceClaimCache, resourceSliceTracker, informerFactory) } } diff --git a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go index ee26ec482d4..1eacfd20752 100644 --- a/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go +++ b/pkg/scheduler/framework/plugins/dynamicresources/dynamicresources_test.go @@ -4057,7 +4057,7 @@ func setup(tCtx ktesting.TContext, args *config.DynamicResourcesArgs, nodes []*v resourceSliceTrackerOpts := resourceslicetracker.Options{ EnableDeviceTaintRules: true, SliceInformer: tc.informerFactory.Resource().V1().ResourceSlices(), - TaintInformer: tc.informerFactory.Resource().V1beta2().DeviceTaintRules(), + TaintInformer: tc.informerFactory.Resource().V1().DeviceTaintRules(), KubeClient: tc.client, } resourceSliceTracker, err := resourceslicetracker.StartTracker(tCtx, resourceSliceTrackerOpts) diff --git a/pkg/scheduler/framework/plugins/noderesources/resource_allocation_test.go b/pkg/scheduler/framework/plugins/noderesources/resource_allocation_test.go index b077fc81b85..9b1b21edc16 100644 --- a/pkg/scheduler/framework/plugins/noderesources/resource_allocation_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/resource_allocation_test.go @@ -595,7 +595,6 @@ func newTestDRAManager(tCtx ktesting.TContext, objects ...apiruntime.Object) *dy informerFactory := informers.NewSharedInformerFactory(client, 0) resourceSliceTrackerOpts := tracker.Options{ SliceInformer: informerFactory.Resource().V1().ResourceSlices(), - TaintInformer: informerFactory.Resource().V1beta2().DeviceTaintRules(), KubeClient: client, } resourceSliceTracker, err := tracker.StartTracker(tCtx, resourceSliceTrackerOpts) diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go index d20d6720850..eb07cefcf97 100644 --- a/pkg/scheduler/scheduler.go +++ b/pkg/scheduler/scheduler.go @@ -344,7 +344,7 @@ func New(ctx context.Context, // If device taint rules are disabled, the additional informers are not needed and // the tracker turns into a simple wrapper around the slice informer. if resourceSliceTrackerOpts.EnableDeviceTaintRules { - resourceSliceTrackerOpts.TaintInformer = informerFactory.Resource().V1beta2().DeviceTaintRules() + resourceSliceTrackerOpts.TaintInformer = informerFactory.Resource().V1().DeviceTaintRules() } resourceSliceTracker, err = resourceslicetracker.StartTracker(ctx, resourceSliceTrackerOpts) if err != nil { diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml index cc52f84d218..71208070028 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/cluster-roles.yaml @@ -951,6 +951,14 @@ items: verbs: - create - delete + - apiGroups: + - resource.k8s.io + resources: + - devicetaintrules + verbs: + - get + - list + - watch - apiGroups: - resource.k8s.io resources: diff --git a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml index 9d9baff51b5..6a3a2ee493c 100644 --- a/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml +++ b/plugin/pkg/auth/authorizer/rbac/bootstrappolicy/testdata/controller-roles.yaml @@ -410,6 +410,21 @@ items: - create - patch - update + - apiGroups: + - resource.k8s.io + resources: + - devicetaintrules/status + verbs: + - patch + - update + - apiGroups: + - resource.k8s.io + resources: + - devicetaintrules + verbs: + - get + - list + - watch - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/staging/src/k8s.io/dynamic-resource-allocation/resourceslice/tracker/tracker.go b/staging/src/k8s.io/dynamic-resource-allocation/resourceslice/tracker/tracker.go index 6d9e36d1c22..1dd40a0533b 100644 --- a/staging/src/k8s.io/dynamic-resource-allocation/resourceslice/tracker/tracker.go +++ b/staging/src/k8s.io/dynamic-resource-allocation/resourceslice/tracker/tracker.go @@ -25,13 +25,11 @@ import ( v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" - resourcebetaapi "k8s.io/api/resource/v1beta2" labels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/diff" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" resourceinformers "k8s.io/client-go/informers/resource/v1" - resourcebetainformers "k8s.io/client-go/informers/resource/v1beta2" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" v1core "k8s.io/client-go/kubernetes/typed/core/v1" @@ -117,7 +115,7 @@ type Options struct { EnableConsumableCapacity bool SliceInformer resourceinformers.ResourceSliceInformer - TaintInformer resourcebetainformers.DeviceTaintRuleInformer + TaintInformer resourceinformers.DeviceTaintRuleInformer // KubeClient is used to generate Events when CEL expressions // encounter runtime errors. @@ -380,15 +378,15 @@ func sliceDriverPoolDeviceIndexFunc(obj any) ([]string, error) { return indexValues, nil } -func driverPoolDeviceIndexPatchKey(patch *resourcebetaapi.DeviceTaintRule) string { - deviceSelector := ptr.Deref(patch.Spec.DeviceSelector, resourcebetaapi.DeviceTaintSelector{}) +func driverPoolDeviceIndexPatchKey(patch *resourceapi.DeviceTaintRule) string { + deviceSelector := ptr.Deref(patch.Spec.DeviceSelector, resourceapi.DeviceTaintSelector{}) driverKey := ptr.Deref(deviceSelector.Driver, anyDriver) poolKey := ptr.Deref(deviceSelector.Pool, anyPool) deviceKey := ptr.Deref(deviceSelector.Device, anyDevice) return deviceID(driverKey, poolKey, deviceKey) } -func (t *Tracker) sliceNamesForPatch(ctx context.Context, patch *resourcebetaapi.DeviceTaintRule) []string { +func (t *Tracker) sliceNamesForPatch(ctx context.Context, patch *resourceapi.DeviceTaintRule) []string { patchKey := driverPoolDeviceIndexPatchKey(patch) sliceNames, err := t.resourceSlices.GetIndexer().IndexKeys(driverPoolDeviceIndexName, patchKey) if err != nil { @@ -454,7 +452,7 @@ func (t *Tracker) resourceSliceDelete(ctx context.Context) func(obj any) { func (t *Tracker) deviceTaintAdd(ctx context.Context) func(obj any) { logger := klog.FromContext(ctx) return func(obj any) { - rule, ok := obj.(*resourcebetaapi.DeviceTaintRule) + rule, ok := obj.(*resourceapi.DeviceTaintRule) if !ok { return } @@ -472,11 +470,11 @@ func (t *Tracker) deviceTaintAdd(ctx context.Context) func(obj any) { func (t *Tracker) deviceTaintUpdate(ctx context.Context) func(oldObj, newObj any) { logger := klog.FromContext(ctx) return func(oldObj, newObj any) { - oldRule, ok := oldObj.(*resourcebetaapi.DeviceTaintRule) + oldRule, ok := oldObj.(*resourceapi.DeviceTaintRule) if !ok { return } - newRule, ok := newObj.(*resourcebetaapi.DeviceTaintRule) + newRule, ok := newObj.(*resourceapi.DeviceTaintRule) if !ok { return } @@ -504,7 +502,7 @@ func (t *Tracker) deviceTaintDelete(ctx context.Context) func(obj any) { if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { obj = tombstone.Obj } - patch, ok := obj.(*resourcebetaapi.DeviceTaintRule) + patch, ok := obj.(*resourceapi.DeviceTaintRule) if !ok { return } @@ -563,7 +561,7 @@ func (t *Tracker) syncSlice(ctx context.Context, name string, sendEvent bool) { return } - patches := typedSlice[*resourcebetaapi.DeviceTaintRule](t.deviceTaints.GetIndexer().List()) + patches := typedSlice[*resourceapi.DeviceTaintRule](t.deviceTaints.GetIndexer().List()) patchedSlice, err := t.applyPatches(ctx, slice, patches) if err != nil { t.handleError(ctx, err, "failed to apply patches to ResourceSlice", "resourceslice", klog.KObj(slice)) @@ -598,7 +596,7 @@ func (t *Tracker) syncSlice(ctx context.Context, name string, sendEvent bool) { } } -func (t *Tracker) applyPatches(ctx context.Context, slice *resourceapi.ResourceSlice, taintRules []*resourcebetaapi.DeviceTaintRule) (*resourceapi.ResourceSlice, error) { +func (t *Tracker) applyPatches(ctx context.Context, slice *resourceapi.ResourceSlice, taintRules []*resourceapi.DeviceTaintRule) (*resourceapi.ResourceSlice, error) { logger := klog.FromContext(ctx) // slice will be DeepCopied just-in-time, only when necessary. @@ -631,20 +629,10 @@ func (t *Tracker) applyPatches(ctx context.Context, slice *resourceapi.ResourceS } logger.V(6).Info("applying matching DeviceTaintRule") - - // TODO: remove conversion once taint is already in the right API package. - ta := resourceapi.DeviceTaint{ - Key: taintRule.Spec.Taint.Key, - Value: taintRule.Spec.Taint.Value, - Effect: resourceapi.DeviceTaintEffect(taintRule.Spec.Taint.Effect), - TimeAdded: taintRule.Spec.Taint.TimeAdded, - } - if patchedSlice == slice { patchedSlice = slice.DeepCopy() } - - patchedSlice.Spec.Devices[dIndex].Taints = append(patchedSlice.Spec.Devices[dIndex].Taints, ta) + patchedSlice.Spec.Devices[dIndex].Taints = append(patchedSlice.Spec.Devices[dIndex].Taints, taintRule.Spec.Taint) } } diff --git a/staging/src/k8s.io/dynamic-resource-allocation/resourceslice/tracker/tracker_test.go b/staging/src/k8s.io/dynamic-resource-allocation/resourceslice/tracker/tracker_test.go index 347f01e6b98..b2588ca5a6b 100644 --- a/staging/src/k8s.io/dynamic-resource-allocation/resourceslice/tracker/tracker_test.go +++ b/staging/src/k8s.io/dynamic-resource-allocation/resourceslice/tracker/tracker_test.go @@ -30,7 +30,6 @@ import ( "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" - resourcebetaapi "k8s.io/api/resource/v1beta2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" @@ -116,7 +115,7 @@ func applyEventPair(tCtx *testContext, event any) { require.NoError(tCtx, err) tCtx.resourceSliceAdd(tCtx.Context)(pair[1]) } - case [2]*resourcebetaapi.DeviceTaintRule: + case [2]*resourceapi.DeviceTaintRule: store := tCtx.deviceTaints.GetStore() switch { case pair[0] != nil && pair[1] != nil: @@ -250,39 +249,39 @@ var ( slice2 = sliceWithDevices(slice2NoDevices, devices2) slice2Tainted = sliceWithDevices(slice2, taintedDevices2) - alphaDeviceTaint = func(taint resourceapi.DeviceTaint) resourcebetaapi.DeviceTaint { - return resourcebetaapi.DeviceTaint{ + deviceTaint = func(taint resourceapi.DeviceTaint) resourceapi.DeviceTaint { + return resourceapi.DeviceTaint{ Key: taint.Key, Value: taint.Value, - Effect: resourcebetaapi.DeviceTaintEffect(taint.Effect), + Effect: resourceapi.DeviceTaintEffect(taint.Effect), TimeAdded: taint.TimeAdded, } } - taintAllDevicesRule = &resourcebetaapi.DeviceTaintRule{ + taintAllDevicesRule = &resourceapi.DeviceTaintRule{ ObjectMeta: metav1.ObjectMeta{ Name: "rule", }, - Spec: resourcebetaapi.DeviceTaintRuleSpec{ - Taint: alphaDeviceTaint(deviceTaint1), + Spec: resourceapi.DeviceTaintRuleSpec{ + Taint: deviceTaint(deviceTaint1), }, } - taintPoolDevicesRule = func(rule *resourcebetaapi.DeviceTaintRule, pool string) *resourcebetaapi.DeviceTaintRule { + taintPoolDevicesRule = func(rule *resourceapi.DeviceTaintRule, pool string) *resourceapi.DeviceTaintRule { rule = rule.DeepCopy() - rule.Spec.DeviceSelector = &resourcebetaapi.DeviceTaintSelector{ + rule.Spec.DeviceSelector = &resourceapi.DeviceTaintSelector{ Pool: &pool, } return rule } - taintDriverDevicesRule = func(rule *resourcebetaapi.DeviceTaintRule, driver string) *resourcebetaapi.DeviceTaintRule { + taintDriverDevicesRule = func(rule *resourceapi.DeviceTaintRule, driver string) *resourceapi.DeviceTaintRule { rule = rule.DeepCopy() - rule.Spec.DeviceSelector = &resourcebetaapi.DeviceTaintSelector{ + rule.Spec.DeviceSelector = &resourceapi.DeviceTaintSelector{ Driver: &driver, } return rule } - taintNamedDevicesRule = func(rule *resourcebetaapi.DeviceTaintRule, name string) *resourcebetaapi.DeviceTaintRule { + taintNamedDevicesRule = func(rule *resourceapi.DeviceTaintRule, name string) *resourceapi.DeviceTaintRule { rule = rule.DeepCopy() - rule.Spec.DeviceSelector = &resourcebetaapi.DeviceTaintSelector{ + rule.Spec.DeviceSelector = &resourceapi.DeviceTaintSelector{ Device: &name, } return rule @@ -522,7 +521,7 @@ func TestListPatchedResourceSlices(t *testing.T) { opts := Options{ EnableDeviceTaintRules: true, SliceInformer: informerFactory.Resource().V1().ResourceSlices(), - TaintInformer: informerFactory.Resource().V1beta2().DeviceTaintRules(), + TaintInformer: informerFactory.Resource().V1().DeviceTaintRules(), KubeClient: kubeClient, } tracker, err := newTracker(ctx, opts) @@ -688,8 +687,8 @@ func BenchmarkEventHandlers(b *testing.B) { now := time.Now() benchmarks := map[string]struct { resourceSlices []*resourceapi.ResourceSlice - taintRules []*resourcebetaapi.DeviceTaintRule - loop func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, taintRules []*resourcebetaapi.DeviceTaintRule, i int) + taintRules []*resourceapi.DeviceTaintRule + loop func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, taintRules []*resourceapi.DeviceTaintRule, i int) }{ "resource-slice-add-no-taint-rules": { resourceSlices: func() []*resourceapi.ResourceSlice { @@ -706,7 +705,7 @@ func BenchmarkEventHandlers(b *testing.B) { } return resourceSlices }(), - loop: func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, _ []*resourcebetaapi.DeviceTaintRule, i int) { + loop: func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, _ []*resourceapi.DeviceTaintRule, i int) { tracker.resourceSliceAdd(ctx)(resourceSlices[i%len(resourceSlices)]) }, }, @@ -725,23 +724,23 @@ func BenchmarkEventHandlers(b *testing.B) { } return resourceSlices }(), - taintRules: []*resourcebetaapi.DeviceTaintRule{ + taintRules: []*resourceapi.DeviceTaintRule{ { ObjectMeta: metav1.ObjectMeta{ Name: "taintRule", }, - Spec: resourcebetaapi.DeviceTaintRuleSpec{ + Spec: resourceapi.DeviceTaintRuleSpec{ DeviceSelector: nil, // all slices - Taint: resourcebetaapi.DeviceTaint{ + Taint: resourceapi.DeviceTaint{ Key: "example.com/taint", Value: "tainted", - Effect: resourcebetaapi.DeviceTaintEffectNoExecute, + Effect: resourceapi.DeviceTaintEffectNoExecute, TimeAdded: &metav1.Time{Time: now}, }, }, }, }, - loop: func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, taintRules []*resourcebetaapi.DeviceTaintRule, i int) { + loop: func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, taintRules []*resourceapi.DeviceTaintRule, i int) { tracker.deviceTaintAdd(ctx)(taintRules[i%len(taintRules)]) }, }, @@ -760,23 +759,23 @@ func BenchmarkEventHandlers(b *testing.B) { } return resourceSlices }(), - taintRules: []*resourcebetaapi.DeviceTaintRule{ + taintRules: []*resourceapi.DeviceTaintRule{ { ObjectMeta: metav1.ObjectMeta{ Name: "taintRule", }, - Spec: resourcebetaapi.DeviceTaintRuleSpec{ + Spec: resourceapi.DeviceTaintRuleSpec{ DeviceSelector: nil, // all slices - Taint: resourcebetaapi.DeviceTaint{ + Taint: resourceapi.DeviceTaint{ Key: "example.com/taint", Value: "tainted", - Effect: resourcebetaapi.DeviceTaintEffectNoExecute, + Effect: resourceapi.DeviceTaintEffectNoExecute, TimeAdded: &metav1.Time{Time: now}, }, }, }, }, - loop: func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, _ []*resourcebetaapi.DeviceTaintRule, i int) { + loop: func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, _ []*resourceapi.DeviceTaintRule, i int) { tracker.resourceSliceAdd(ctx)(resourceSlices[i%len(resourceSlices)]) }, }, @@ -809,25 +808,25 @@ func BenchmarkEventHandlers(b *testing.B) { resourceSlices[nSlices/2].Spec.Devices[nDevices/2].Name = "patchme" return resourceSlices }(), - taintRules: []*resourcebetaapi.DeviceTaintRule{ + taintRules: []*resourceapi.DeviceTaintRule{ { ObjectMeta: metav1.ObjectMeta{ Name: "taintRule", }, - Spec: resourcebetaapi.DeviceTaintRuleSpec{ - DeviceSelector: &resourcebetaapi.DeviceTaintSelector{ + Spec: resourceapi.DeviceTaintRuleSpec{ + DeviceSelector: &resourceapi.DeviceTaintSelector{ Device: ptr.To("patchme"), }, - Taint: resourcebetaapi.DeviceTaint{ + Taint: resourceapi.DeviceTaint{ Key: "example.com/taint", Value: "tainted", - Effect: resourcebetaapi.DeviceTaintEffectNoExecute, + Effect: resourceapi.DeviceTaintEffectNoExecute, TimeAdded: &metav1.Time{Time: now}, }, }, }, }, - loop: func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, taintRules []*resourcebetaapi.DeviceTaintRule, i int) { + loop: func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, taintRules []*resourceapi.DeviceTaintRule, i int) { tracker.deviceTaintAdd(ctx)(taintRules[i%len(taintRules)]) }, }, @@ -854,26 +853,26 @@ func BenchmarkEventHandlers(b *testing.B) { } return resourceSlices }(), - taintRules: []*resourcebetaapi.DeviceTaintRule{ + taintRules: []*resourceapi.DeviceTaintRule{ { ObjectMeta: metav1.ObjectMeta{ Name: "patch", }, - Spec: resourcebetaapi.DeviceTaintRuleSpec{ - DeviceSelector: &resourcebetaapi.DeviceTaintSelector{ + Spec: resourceapi.DeviceTaintRuleSpec{ + DeviceSelector: &resourceapi.DeviceTaintSelector{ Pool: ptr.To("pool-250"), Device: ptr.To("patchme"), }, - Taint: resourcebetaapi.DeviceTaint{ + Taint: resourceapi.DeviceTaint{ Key: "example.com/taint", Value: "tainted", - Effect: resourcebetaapi.DeviceTaintEffectNoExecute, + Effect: resourceapi.DeviceTaintEffectNoExecute, TimeAdded: &metav1.Time{Time: now}, }, }, }, }, - loop: func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, patches []*resourcebetaapi.DeviceTaintRule, i int) { + loop: func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, patches []*resourceapi.DeviceTaintRule, i int) { tracker.resourceSliceAdd(ctx)(resourceSlices[250]) // the slice affected by the patch }, }, @@ -895,21 +894,21 @@ func BenchmarkEventHandlers(b *testing.B) { } return resourceSlices }(), - taintRules: func() []*resourcebetaapi.DeviceTaintRule { - patches := make([]*resourcebetaapi.DeviceTaintRule, 500) + taintRules: func() []*resourceapi.DeviceTaintRule { + patches := make([]*resourceapi.DeviceTaintRule, 500) for i := range patches { - patches[i] = &resourcebetaapi.DeviceTaintRule{ + patches[i] = &resourceapi.DeviceTaintRule{ ObjectMeta: metav1.ObjectMeta{ Name: "taint-rule-" + strconv.Itoa(i), }, - Spec: resourcebetaapi.DeviceTaintRuleSpec{ - DeviceSelector: &resourcebetaapi.DeviceTaintSelector{ + Spec: resourceapi.DeviceTaintRuleSpec{ + DeviceSelector: &resourceapi.DeviceTaintSelector{ Pool: ptr.To("pool-" + strconv.Itoa(i)), }, - Taint: resourcebetaapi.DeviceTaint{ + Taint: resourceapi.DeviceTaint{ Key: "example.com/taint", Value: "tainted", - Effect: resourcebetaapi.DeviceTaintEffectNoExecute, + Effect: resourceapi.DeviceTaintEffectNoExecute, TimeAdded: &metav1.Time{Time: now}, }, }, @@ -917,7 +916,7 @@ func BenchmarkEventHandlers(b *testing.B) { } return patches }(), - loop: func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, taintRules []*resourcebetaapi.DeviceTaintRule, i int) { + loop: func(ctx context.Context, b *testing.B, tracker *Tracker, resourceSlices []*resourceapi.ResourceSlice, taintRules []*resourceapi.DeviceTaintRule, i int) { tracker.deviceTaintAdd(ctx)(taintRules[i%len(taintRules)]) }, }, @@ -929,7 +928,7 @@ func BenchmarkEventHandlers(b *testing.B) { opts := Options{ EnableDeviceTaintRules: true, SliceInformer: informerFactory.Resource().V1().ResourceSlices(), - TaintInformer: informerFactory.Resource().V1beta2().DeviceTaintRules(), + TaintInformer: informerFactory.Resource().V1().DeviceTaintRules(), KubeClient: kubeClient, } tracker, err := newTracker(ctx, opts) diff --git a/test/compatibility_lifecycle/reference/feature_list.md b/test/compatibility_lifecycle/reference/feature_list.md index e60b7d72770..bc884bc67cc 100644 --- a/test/compatibility_lifecycle/reference/feature_list.md +++ b/test/compatibility_lifecycle/reference/feature_list.md @@ -52,8 +52,8 @@ | DRAAdminAccess | :ballot_box_with_check: 1.34+ | :closed_lock_with_key: 1.36+ | 1.32–1.33 | 1.34–1.35 | 1.36– | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRAAdminAccess%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRAAdminAccess%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | DRAConsumableCapacity | :ballot_box_with_check: 1.36+ | | 1.34–1.35 | 1.36– | | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRAConsumableCapacity%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRAConsumableCapacity%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | DRADeviceBindingConditions | :ballot_box_with_check: 1.36+ | | 1.34–1.35 | 1.36– | | | DRAResourceClaimDeviceStatus
DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRADeviceBindingConditions%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRADeviceBindingConditions%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | -| DRADeviceTaintRules | | | 1.35 | 1.36– | | | DRADeviceTaints | [code](https://cs.k8s.io/?q=%5CbDRADeviceTaintRules%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRADeviceTaintRules%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | -| DRADeviceTaints | :ballot_box_with_check: 1.36+ | | 1.33–1.35 | 1.36– | | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRADeviceTaints%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRADeviceTaints%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | +| DRADeviceTaintRules | :ballot_box_with_check: 1.37+ | | 1.35 | 1.36 | 1.37– | | DRADeviceTaints | [code](https://cs.k8s.io/?q=%5CbDRADeviceTaintRules%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRADeviceTaintRules%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | +| DRADeviceTaints | :ballot_box_with_check: 1.36+ | | 1.33–1.35 | 1.36 | 1.37– | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRADeviceTaints%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRADeviceTaints%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | DRAExtendedResource | :ballot_box_with_check: 1.36+ | :closed_lock_with_key: 1.37+ | 1.34–1.35 | 1.36 | 1.37– | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRAExtendedResource%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRAExtendedResource%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | DRAListTypeAttributes | | | 1.36– | | | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRAListTypeAttributes%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRAListTypeAttributes%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | DRANodeAllocatableResources | | | 1.36– | | | | DynamicResourceAllocation | [code](https://cs.k8s.io/?q=%5CbDRANodeAllocatableResources%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbDRANodeAllocatableResources%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | diff --git a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml index 9912f1fcc9c..034ed140023 100644 --- a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml +++ b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml @@ -499,6 +499,10 @@ lockToDefault: false preRelease: Beta version: "1.36" + - default: true + lockToDefault: false + preRelease: GA + version: "1.37" - name: DRADeviceTaints versionedSpecs: - default: false @@ -509,6 +513,10 @@ lockToDefault: false preRelease: Beta version: "1.36" + - default: true + lockToDefault: false + preRelease: GA + version: "1.37" - name: DRAExtendedResource versionedSpecs: - default: false diff --git a/test/e2e/dra/dra.go b/test/e2e/dra/dra.go index 06bf7526fcb..f06a83ff5bd 100644 --- a/test/e2e/dra/dra.go +++ b/test/e2e/dra/dra.go @@ -115,29 +115,29 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { Testname: CRUD operations for devicetaintrules Description: kube-apiserver must support create/update/list/patch/delete operations for resource.k8s.io/v1beta2 DeviceTaintRule. */ - f.It("resource.k8s.io/v1beta2 DeviceTaintRule", f.WithFeatureGate(features.DRADeviceTaintRules), func(ctx context.Context) { + f.It("resource.k8s.io/v1 DeviceTaintRule", f.WithFeatureGate(features.DRADeviceTaintRules), func(ctx context.Context) { lastTransitionTime := metav1.Now() lastTransitionTimeEncoded, err := lastTransitionTime.MarshalJSON() framework.ExpectNoError(err) e2econformance.TestResource(ctx, f, - &e2econformance.ResourceTestcase[*resourcev1beta2.DeviceTaintRule]{ - GVR: resourcev1beta2.SchemeGroupVersion.WithResource("devicetaintrules"), + &e2econformance.ResourceTestcase[*resourceapi.DeviceTaintRule]{ + GVR: resourceapi.SchemeGroupVersion.WithResource("devicetaintrules"), Namespaced: ptr.To(false), - InitialSpec: &resourcev1beta2.DeviceTaintRule{ - Spec: resourcev1beta2.DeviceTaintRuleSpec{ + InitialSpec: &resourceapi.DeviceTaintRule{ + Spec: resourceapi.DeviceTaintRuleSpec{ // Empty DeviceSelector => no devices selected, so this test is safe. - Taint: resourcev1beta2.DeviceTaint{ + Taint: resourceapi.DeviceTaint{ Key: "testing", - Effect: resourcev1beta2.DeviceTaintEffectNone, + Effect: resourceapi.DeviceTaintEffectNone, }, }, }, - UpdateSpec: func(obj *resourcev1beta2.DeviceTaintRule) *resourcev1beta2.DeviceTaintRule { - obj.Spec.Taint.Effect = resourcev1beta2.DeviceTaintEffectNoExecute + UpdateSpec: func(obj *resourceapi.DeviceTaintRule) *resourceapi.DeviceTaintRule { + obj.Spec.Taint.Effect = resourceapi.DeviceTaintEffectNoExecute return obj }, StrategicMergePatchSpec: `{"spec": {"taint": {"effect": "NoExecute"}}}`, - UpdateStatus: func(obj *resourcev1beta2.DeviceTaintRule) *resourcev1beta2.DeviceTaintRule { + UpdateStatus: func(obj *resourceapi.DeviceTaintRule) *resourceapi.DeviceTaintRule { obj.Status.Conditions = append(obj.Status.Conditions, metav1.Condition{ Type: "Testing", Status: metav1.ConditionTrue, @@ -2322,17 +2322,17 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // Now evict it. ginkgo.By("Evicting pod...") - taint := &resourcev1beta2.DeviceTaintRule{ + taint := &resourceapi.DeviceTaintRule{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "device-taint-rule-" + f.UniqueName + "-", }, - Spec: resourcev1beta2.DeviceTaintRuleSpec{ + Spec: resourceapi.DeviceTaintRuleSpec{ // All devices of the current driver instance. - DeviceSelector: &resourcev1beta2.DeviceTaintSelector{ + DeviceSelector: &resourceapi.DeviceTaintSelector{ Driver: &driver.Name, }, - Taint: resourcev1beta2.DeviceTaint{ - Effect: resourcev1beta2.DeviceTaintEffectNoExecute, + Taint: resourceapi.DeviceTaint{ + Effect: resourceapi.DeviceTaintEffectNoExecute, Key: "test.example.com/evict", Value: "now", // No TimeAdded, gets defaulted. @@ -2340,7 +2340,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }, } createdTaint := b.Create(tCtx, taint) - taint = createdTaint[0].(*resourcev1beta2.DeviceTaintRule) + taint = createdTaint[0].(*resourceapi.DeviceTaintRule) gomega.Expect(*taint).Should(gomega.HaveField("Spec.Taint.TimeAdded.Time", gomega.BeTemporally("~", time.Now(), time.Minute /* allow for some clock drift and delays */))) framework.ExpectNoError(e2epod.WaitForPodTerminatingInNamespaceTimeout(ctx, f.ClientSet, pod.Name, f.Namespace.Name, f.Timeouts.PodStart)) pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod.Name, metav1.GetOptions{}) @@ -2411,17 +2411,17 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // Now evict it. ginkgo.By("Evicting pod...") - taint := &resourcev1beta2.DeviceTaintRule{ + taint := &resourceapi.DeviceTaintRule{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "device-taint-rule-" + f.UniqueName + "-", }, - Spec: resourcev1beta2.DeviceTaintRuleSpec{ + Spec: resourceapi.DeviceTaintRuleSpec{ // All devices of the current driver instance. - DeviceSelector: &resourcev1beta2.DeviceTaintSelector{ + DeviceSelector: &resourceapi.DeviceTaintSelector{ Driver: &driver.Name, }, - Taint: resourcev1beta2.DeviceTaint{ - Effect: resourcev1beta2.DeviceTaintEffectNoExecute, + Taint: resourceapi.DeviceTaint{ + Effect: resourceapi.DeviceTaintEffectNoExecute, Key: "test.example.com/evict", Value: "now", // No TimeAdded, gets defaulted. @@ -2429,7 +2429,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }, } createdTaint := b.Create(tCtx, taint) - taint = createdTaint[0].(*resourcev1beta2.DeviceTaintRule) + taint = createdTaint[0].(*resourceapi.DeviceTaintRule) gomega.Expect(*taint).Should(gomega.HaveField("Spec.Taint.TimeAdded.Time", gomega.BeTemporally("~", time.Now(), time.Minute /* allow for some clock drift and delays */))) framework.ExpectNoError(e2epod.WaitForPodTerminatingInNamespaceTimeout(ctx, f.ClientSet, pod.Name, f.Namespace.Name, f.Timeouts.PodStart)) pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod.Name, metav1.GetOptions{}) diff --git a/test/e2e/dra/test-driver/deploy/example/devicetaintrule.yaml b/test/e2e/dra/test-driver/deploy/example/devicetaintrule.yaml index 4e393484d97..23707014a34 100644 --- a/test/e2e/dra/test-driver/deploy/example/devicetaintrule.yaml +++ b/test/e2e/dra/test-driver/deploy/example/devicetaintrule.yaml @@ -1,4 +1,4 @@ -apiVersion: resource.k8s.io/v1beta2 +apiVersion: resource.k8s.io/v1 kind: DeviceTaintRule metadata: name: example diff --git a/test/e2e/dra/utils/builder.go b/test/e2e/dra/utils/builder.go index ef89642573f..c4c9cfaac2b 100644 --- a/test/e2e/dra/utils/builder.go +++ b/test/e2e/dra/utils/builder.go @@ -491,6 +491,12 @@ func (b *Builder) Create(tCtx ktesting.TContext, objs ...klog.KMetadata) []klog. err := tCtx.Client().ResourceV1beta2().DeviceTaintRules().Delete(tCtx, createdObj.GetName(), metav1.DeleteOptions{}) tCtx.ExpectNoError(err, "delete DeviceTaintRule") }) + case *resourceapi.DeviceTaintRule: + createdObj, err = tCtx.Client().ResourceV1().DeviceTaintRules().Create(tCtx, obj, metav1.CreateOptions{}) + cleanupCtx(func(tCtx ktesting.TContext) { + err := tCtx.Client().ResourceV1().DeviceTaintRules().Delete(tCtx, createdObj.GetName(), metav1.DeleteOptions{}) + tCtx.ExpectNoError(err, "delete DeviceTaintRule") + }) case *appsv1.DaemonSet: createdObj, err = tCtx.Client().AppsV1().DaemonSets(b.namespace).Create(tCtx, obj, metav1.CreateOptions{}) // Cleanup not really needed, but speeds up namespace shutdown. diff --git a/test/integration/dra/api/api_test.go b/test/integration/dra/api/api_test.go index 980ed2fb162..dee176da295 100644 --- a/test/integration/dra/api/api_test.go +++ b/test/integration/dra/api/api_test.go @@ -24,5 +24,5 @@ import ( func TestDRA(t *testing.T) { // Various special configurations with few tests. - dra.Run(t, "^(disabled|v1beta1|v1beta2|slice-taints|GA-opt-out)$") + dra.Run(t, "^(disabled|v1beta1|v1beta2|slice-taints|GA-opt-out|GA-1.*)$") } diff --git a/test/integration/dra/device_taints.go b/test/integration/dra/device_taints.go index 7048c823e9b..f8215cfa6c3 100644 --- a/test/integration/dra/device_taints.go +++ b/test/integration/dra/device_taints.go @@ -33,7 +33,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/informers" - resourcebetainformers "k8s.io/client-go/informers/resource/v1beta2" + resourceinformers "k8s.io/client-go/informers/resource/v1" "k8s.io/client-go/tools/cache" "k8s.io/kubernetes/pkg/controller/devicetainteviction" "k8s.io/kubernetes/pkg/features" @@ -48,6 +48,7 @@ const ( useNoRule = useRuleMode(iota) useV1alpha3Rule useV1beta2Rule + useV1Rule ) // testEvictCluster simulates a cluster with many scheduled pods where each @@ -141,9 +142,9 @@ func testEvictCluster(tCtx ktesting.TContext, useRule useRuleMode) { // Create a new factory and sync it so that when the controller starts, it is up-to-date. // This works as long as this is the only test running it. informerFactory := informers.NewSharedInformerFactory(tCtx.Client(), 0) - var ruleInformer resourcebetainformers.DeviceTaintRuleInformer + var ruleInformer resourceinformers.DeviceTaintRuleInformer if utilfeature.DefaultFeatureGate.Enabled(features.DRADeviceTaintRules) { - ruleInformer = informerFactory.Resource().V1beta2().DeviceTaintRules() + ruleInformer = informerFactory.Resource().V1().DeviceTaintRules() } controller := devicetainteviction.New(tCtx.Client(), informerFactory.Core().V1().Pods(), @@ -200,6 +201,9 @@ func testEvictCluster(tCtx ktesting.TContext, useRule useRuleMode) { getV1beta2Rule := func(tCtx ktesting.TContext) *resourcebeta.DeviceTaintRule { return must(tCtx, tCtx.Client().ResourceV1beta2().DeviceTaintRules().Get, ruleName, metav1.GetOptions{}) } + getV1Rule := func(tCtx ktesting.TContext) *resourceapi.DeviceTaintRule { + return must(tCtx, tCtx.Client().ResourceV1().DeviceTaintRules().Get, ruleName, metav1.GetOptions{}) + } var getRule any switch useRule { @@ -322,7 +326,7 @@ func testEvictCluster(tCtx ktesting.TContext, useRule useRuleMode) { rule = getV1beta2Rule(tCtx) tCtx.Expect(rule).To(gomega.HaveField("Status.Conditions", gomega.ConsistOf(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ "ObservedGeneration": gomega.Equal(rule.Generation), - "Type": gomega.Equal(resourcealpha.DeviceTaintConditionEvictionInProgress), + "Type": gomega.Equal(resourcebeta.DeviceTaintConditionEvictionInProgress), "Status": gomega.Equal(metav1.ConditionFalse), "Message": gomega.Equal(fmt.Sprintf("%[1]d published devices selected. %[1]d allocated devices selected. %[1]d pods would be evicted in 1 namespace if the effect was NoExecute. This information will not be updated again. Recreate the DeviceTaintRule to trigger an update.", numPods)), })))) @@ -363,6 +367,86 @@ func testEvictCluster(tCtx ktesting.TContext, useRule useRuleMode) { if newTime.Equal(rule.Spec.Taint.TimeAdded) { tCtx.Error("TimeAdded should have been updated automatically") } + case useV1Rule: + getRule = getV1Rule + // Evict through DeviceTaintRule. + // + // Let's also test some of the semantics around DeviceTaintRule. + // We start with NoEffect, check the report, change TimeAdded manually, + // then turn on eviction. + rule := &resourceapi.DeviceTaintRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: ruleName, + }, + Spec: resourceapi.DeviceTaintRuleSpec{ + DeviceSelector: &resourceapi.DeviceTaintSelector{ + Driver: &driverName, + }, + Taint: resourceapi.DeviceTaint{ + Key: "testing", + Effect: resourceapi.DeviceTaintEffectNone, + }, + }, + } + rule = must(tCtx, tCtx.Client().ResourceV1().DeviceTaintRules().Create, rule, metav1.CreateOptions{}) + tCtx.CleanupCtx(func(tCtx ktesting.TContext) { + err := tCtx.Client().ResourceV1().DeviceTaintRules().Delete(tCtx, ruleName, metav1.DeleteOptions{}) + if apierrors.IsNotFound(err) { + return + } + tCtx.ExpectNoError(err) + }) + tCtx.Expect(rule).To(gomega.HaveField("ObjectMeta.Generation", gomega.Equal(int64(1)))) + + tCtx.Eventually(getV1Rule). + WithPolling(10*time.Second). + Should(gomega.HaveField("Status.Conditions", gomega.ConsistOf(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "ObservedGeneration": gomega.Equal(rule.Generation), + }))), "waiting for simulation result") + rule = getV1Rule(tCtx) + tCtx.Expect(rule).To(gomega.HaveField("Status.Conditions", gomega.ConsistOf(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "ObservedGeneration": gomega.Equal(rule.Generation), + "Type": gomega.Equal(resourceapi.DeviceTaintConditionEvictionInProgress), + "Status": gomega.Equal(metav1.ConditionFalse), + "Message": gomega.Equal(fmt.Sprintf("%[1]d published devices selected. %[1]d allocated devices selected. %[1]d pods would be evicted in 1 namespace if the effect was NoExecute. This information will not be updated again. Recreate the DeviceTaintRule to trigger an update.", numPods)), + })))) + + var newTime *metav1.Time + for { + newTime = &metav1.Time{Time: time.Now().Truncate(time.Second)} + if !newTime.Equal(rule.Spec.Taint.TimeAdded) { + break + } + time.Sleep(10 * time.Millisecond) + } + rule.Spec.Taint.TimeAdded = newTime + rule = must(tCtx, tCtx.Client().ResourceV1().DeviceTaintRules().Update, rule, metav1.UpdateOptions{}) + if !newTime.Equal(rule.Spec.Taint.TimeAdded) { + tCtx.Error("our TimeAdded should have been stored") + } + tCtx.Expect(rule).To(gomega.HaveField("ObjectMeta.Generation", gomega.Equal(int64(2)))) + tCtx.Eventually(getV1Rule). + WithPolling(10*time.Second). + Should(gomega.HaveField("Status.Conditions", gomega.ConsistOf(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "ObservedGeneration": gomega.Equal(rule.Generation), + }))), "waiting for reaction to TimeAdded bump") + rule = getV1Rule(tCtx) + tCtx.Expect(rule).To(gomega.HaveField("Status.Conditions", gomega.ConsistOf(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{ + "ObservedGeneration": gomega.Equal(rule.Generation), + "Type": gomega.Equal(resourcealpha.DeviceTaintConditionEvictionInProgress), + "Status": gomega.Equal(metav1.ConditionFalse), + "Message": gomega.Equal(fmt.Sprintf("%[1]d published devices selected. %[1]d allocated devices selected. %[1]d pods would be evicted in 1 namespace if the effect was NoExecute. This information will not be updated again. Recreate the DeviceTaintRule to trigger an update.", numPods)), + })))) + + rule.Spec.Taint.Effect = resourceapi.DeviceTaintEffectNoExecute + // Must roll over to next second, that's the resolution of TimeAdded. + // We could rely on the delay for updating the status, but that + // could change. + time.Sleep(time.Second) + rule = must(tCtx, tCtx.Client().ResourceV1().DeviceTaintRules().Update, rule, metav1.UpdateOptions{}) + if newTime.Equal(rule.Spec.Taint.TimeAdded) { + tCtx.Error("TimeAdded should have been updated automatically") + } case useNoRule: // Evict by tainting each device. for i, slice := range slices { @@ -463,6 +547,29 @@ func testNoScheduleRule(tCtx ktesting.TContext, useRule useRuleMode) { } tCtx.ExpectNoError(err) }) + case useV1Rule: + rule := &resourceapi.DeviceTaintRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: ruleName, + }, + Spec: resourceapi.DeviceTaintRuleSpec{ + DeviceSelector: &resourceapi.DeviceTaintSelector{ + Driver: &driverName, + }, + Taint: resourceapi.DeviceTaint{ + Key: taintKey, + Effect: resourceapi.DeviceTaintEffectNoSchedule, + }, + }, + } + _ = must(tCtx, tCtx.Client().ResourceV1().DeviceTaintRules().Create, rule, metav1.CreateOptions{}) + tCtx.CleanupCtx(func(tCtx ktesting.TContext) { + err := tCtx.Client().ResourceV1().DeviceTaintRules().Delete(tCtx, ruleName, metav1.DeleteOptions{}) + if apierrors.IsNotFound(err) { + return + } + tCtx.ExpectNoError(err) + }) case useNoRule: slice.Spec.Devices[0].Taints = []resourceapi.DeviceTaint{ { @@ -470,6 +577,8 @@ func testNoScheduleRule(tCtx ktesting.TContext, useRule useRuleMode) { Effect: resourceapi.DeviceTaintEffectNoSchedule, }, } + default: + tCtx.Fatalf("unsupported useRule %d", useRule) } // Creating the ResourceSlice after the DeviceTaintRule exercises additional diff --git a/test/integration/dra/dra.go b/test/integration/dra/dra.go index 077f04a19ed..7b6ba6f4937 100644 --- a/test/integration/dra/dra.go +++ b/test/integration/dra/dra.go @@ -115,12 +115,18 @@ func run(tCtx ktesting.TContext, whatRE string) { // and no ResourceSlices. To test scheduling, a sub-test must create ResourceSlices. // createTestNamespace can be used to create a unique per-test namespace. The name of that // namespace then can be used to create cluster-scoped objects without conflicts between tests. + // + // Some tests run in different configuration even though the test itself is always + // executed the same way. This increases test coverage because the code under test + // might be different. for name, tc := range map[string]struct { apis map[schema.GroupVersion]bool + version string features map[featuregate.Feature]bool f func(tCtx ktesting.TContext) }{ "disabled": { + version: "1.34", // In 1.34 it was still possible to disable DRA. apis: map[schema.GroupVersion]bool{resourceapi.SchemeGroupVersion: false}, features: map[featuregate.Feature]bool{features.DynamicResourceAllocation: false}, f: func(tCtx ktesting.TContext) { @@ -133,6 +139,7 @@ func run(tCtx ktesting.TContext, whatRE string) { features: map[featuregate.Feature]bool{}, f: func(tCtx ktesting.TContext) { runSubTest(tCtx, "Pod", func(tCtx ktesting.TContext) { testPod(tCtx, true) }) + runSubTest(tCtx, "EvictClusterWithV1Rule", func(tCtx ktesting.TContext) { testEvictCluster(tCtx, useV1Rule) }) runSubTest(tCtx, "EvictClusterWithSlices", func(tCtx ktesting.TContext) { testEvictCluster(tCtx, useNoRule) }) runSubTest(tCtx, "NoScheduleWithSlices", func(tCtx ktesting.TContext) { testNoScheduleRule(tCtx, useNoRule) }) // Number of devices per slice is chosen so that Filter takes a few seconds: @@ -141,6 +148,9 @@ func run(tCtx ktesting.TContext, whatRE string) { runSubTest(tCtx, "UsesAllResources", testUsesAllResources) }, }, + // This covers the *current* Kubernetes version with only GA features enabled. + // The GA scenario(s) for other version(s) verify that version emulation works. + // Just some key tests get replicated there to keep the overall runtime low enough. "GA": { apis: map[schema.GroupVersion]bool{}, features: map[featuregate.Feature]bool{ @@ -148,11 +158,15 @@ func run(tCtx ktesting.TContext, whatRE string) { }, f: func(tCtx ktesting.TContext) { runSubTest(tCtx, "AdminAccess", func(tCtx ktesting.TContext) { testAdminAccess(tCtx, false) }) + runSubTest(tCtx, "EvictClusterWithV1Rule", func(tCtx ktesting.TContext) { testEvictCluster(tCtx, useV1Rule) }) + runSubTest(tCtx, "EvictClusterWithSlices", func(tCtx ktesting.TContext) { testEvictCluster(tCtx, useNoRule) }) + runSubTest(tCtx, "NoScheduleWithV1Rule", func(tCtx ktesting.TContext) { testNoScheduleRule(tCtx, useV1Rule) }) + runSubTest(tCtx, "NoScheduleWithSlices", func(tCtx ktesting.TContext) { testNoScheduleRule(tCtx, useNoRule) }) runSubTest(tCtx, "PartitionableDevices", func(tCtx ktesting.TContext) { testPartitionableDevices(tCtx, false) }) runSubTest(tCtx, "PrioritizedList", func(tCtx ktesting.TContext) { testPrioritizedList(tCtx, true) }) runSubTest(tCtx, "Pod", func(tCtx ktesting.TContext) { testPod(tCtx, true) }) runSubTest(tCtx, "PublishResourceSlices", func(tCtx ktesting.TContext) { - testPublishResourceSlices(tCtx, true, features.DRADeviceTaints, features.DRAPartitionableDevices, features.DRADeviceBindingConditions) + testPublishResourceSlices(tCtx, true, features.DRAPartitionableDevices, features.DRADeviceBindingConditions) }) runSubTest(tCtx, "ExplicitExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, true, true) }) runSubTest(tCtx, "ImplicitExtendedResource", func(tCtx ktesting.TContext) { testExtendedResource(tCtx, true, false) }) @@ -168,6 +182,36 @@ func run(tCtx ktesting.TContext, whatRE string) { runSubTest(tCtx, "WorkloadResourceClaims", func(tCtx ktesting.TContext) { testWorkloadResourceClaims(tCtx, false, false) }) }, }, + "GA-1.36": { + version: "1.36", + apis: map[schema.GroupVersion]bool{}, + features: map[featuregate.Feature]bool{ + featuregate.Feature("AllBeta"): false, + }, + f: func(tCtx ktesting.TContext) { + runSubTest(tCtx, "AdminAccess", func(tCtx ktesting.TContext) { testAdminAccess(tCtx, false) }) + runSubTest(tCtx, "PrioritizedList", func(tCtx ktesting.TContext) { testPrioritizedList(tCtx, true) }) + runSubTest(tCtx, "Pod", func(tCtx ktesting.TContext) { testPod(tCtx, true) }) + runSubTest(tCtx, "PublishResourceSlices", func(tCtx ktesting.TContext) { + testPublishResourceSlices(tCtx, true, features.DRADeviceTaints, features.DRAPartitionableDevices, features.DRADeviceBindingConditions) + }) + }, + }, + "GA-1.35": { + version: "1.35", + apis: map[schema.GroupVersion]bool{}, + features: map[featuregate.Feature]bool{ + featuregate.Feature("AllBeta"): false, + }, + f: func(tCtx ktesting.TContext) { + runSubTest(tCtx, "AdminAccess", func(tCtx ktesting.TContext) { testAdminAccess(tCtx, false) }) + runSubTest(tCtx, "PrioritizedList", func(tCtx ktesting.TContext) { testPrioritizedList(tCtx, false) }) + runSubTest(tCtx, "Pod", func(tCtx ktesting.TContext) { testPod(tCtx, true) }) + runSubTest(tCtx, "PublishResourceSlices", func(tCtx ktesting.TContext) { + testPublishResourceSlices(tCtx, true, features.DRADeviceTaints, features.DRAPartitionableDevices, features.DRADeviceBindingConditions) + }) + }, + }, // This scenario verifies that features which have graduated to GA can // still be explicitly disabled via feature gates. "GA-opt-out": { @@ -248,9 +292,11 @@ func run(tCtx ktesting.TContext, whatRE string) { runSubTest(tCtx, "MaxResourceSlice", testMaxResourceSlice) runSubTest(tCtx, "EvictClusterWithV1alpha3Rule", func(tCtx ktesting.TContext) { testEvictCluster(tCtx, useV1alpha3Rule) }) runSubTest(tCtx, "EvictClusterWithV1beta2Rule", func(tCtx ktesting.TContext) { testEvictCluster(tCtx, useV1beta2Rule) }) + runSubTest(tCtx, "EvictClusterWithV1Rule", func(tCtx ktesting.TContext) { testEvictCluster(tCtx, useV1Rule) }) runSubTest(tCtx, "EvictClusterWithSlices", func(tCtx ktesting.TContext) { testEvictCluster(tCtx, useNoRule) }) runSubTest(tCtx, "NoScheduleWithV1alpha3Rule", func(tCtx ktesting.TContext) { testNoScheduleRule(tCtx, useV1alpha3Rule) }) runSubTest(tCtx, "NoScheduleWithV1beta2Rule", func(tCtx ktesting.TContext) { testNoScheduleRule(tCtx, useV1beta2Rule) }) + runSubTest(tCtx, "NoScheduleWithV1Rule", func(tCtx ktesting.TContext) { testNoScheduleRule(tCtx, useV1Rule) }) runSubTest(tCtx, "NoScheduleWithSlices", func(tCtx ktesting.TContext) { testNoScheduleRule(tCtx, useNoRule) }) runSubTest(tCtx, "InvalidResourceSlices", testInvalidResourceSlices) // Number of devices per slice is chosen so that Filter takes a few seconds: The allocator @@ -280,9 +326,8 @@ func run(tCtx ktesting.TContext, whatRE string) { sort.Strings(entries) tCtx.Logf("Config: %s", strings.Join(entries, ",")) - // We need to set emulation version for DynamicResourceAllocation feature gate, which is locked at 1.35. - if draEnabled, draExists := tc.features[features.DynamicResourceAllocation]; draExists && !draEnabled { - featuregatetesting.SetFeatureGateEmulationVersionDuringTest(tCtx, utilfeature.DefaultFeatureGate, version.MustParse("1.34")) + if tc.version != "" { + featuregatetesting.SetFeatureGateEmulationVersionDuringTest(tCtx, utilfeature.DefaultFeatureGate, version.MustParse(tc.version)) } featuregatetesting.SetFeatureGatesDuringTest(tCtx, utilfeature.DefaultFeatureGate, tc.features) diff --git a/test/integration/dra/prioritizedlist.go b/test/integration/dra/prioritizedlist.go index 88947f70357..029f2ce64c6 100644 --- a/test/integration/dra/prioritizedlist.go +++ b/test/integration/dra/prioritizedlist.go @@ -49,6 +49,7 @@ func testPrioritizedList(tCtx ktesting.TContext, enabled bool) { return } + require.NoError(tCtx, err, "create ResourceClaim") require.NotEmpty(tCtx, claim.Spec.Devices.Requests[0].FirstAvailable, "should store FirstAvailable") startScheduler(tCtx) diff --git a/test/integration/scheduler_perf/dra.go b/test/integration/scheduler_perf/dra.go index 9e36959cff4..098e13ba9bc 100644 --- a/test/integration/scheduler_perf/dra.go +++ b/test/integration/scheduler_perf/dra.go @@ -29,7 +29,6 @@ import ( v1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" - resourcebeta "k8s.io/api/resource/v1beta2" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -291,7 +290,7 @@ func (op *allocResourceClaimsOp) run(tCtx ktesting.TContext) { KubeClient: tCtx.Client(), } if resourceSliceTrackerOpts.EnableDeviceTaintRules { - resourceSliceTrackerOpts.TaintInformer = informerFactory.Resource().V1beta2().DeviceTaintRules() + resourceSliceTrackerOpts.TaintInformer = informerFactory.Resource().V1().DeviceTaintRules() } resourceSliceTracker, err := resourceslicetracker.StartTracker(tCtx, resourceSliceTrackerOpts) tCtx.ExpectNoError(err, "start resource slice tracker") @@ -312,7 +311,7 @@ func (op *allocResourceClaimsOp) run(tCtx ktesting.TContext) { }, } if utilfeature.DefaultFeatureGate.Enabled(features.DRADeviceTaintRules) { - expectSyncResult.Synced[reflect.TypeFor[*resourcebeta.DeviceTaintRule]()] = true + expectSyncResult.Synced[reflect.TypeFor[*resourceapi.DeviceTaintRule]()] = true } if diff := cmp.Diff(expectSyncResult, syncResult, cmp.Transformer("TypeOf", func(t reflect.Type) string { diff --git a/test/integration/scheduler_perf/dra/templates/devicetaintrule.yaml b/test/integration/scheduler_perf/dra/templates/devicetaintrule.yaml index 4e4be2d4624..21105bf14f3 100644 --- a/test/integration/scheduler_perf/dra/templates/devicetaintrule.yaml +++ b/test/integration/scheduler_perf/dra/templates/devicetaintrule.yaml @@ -1,4 +1,4 @@ -apiVersion: resource.k8s.io/v1alpha3 +apiVersion: resource.k8s.io/v1 kind: DeviceTaintRule metadata: name: taint-all-devices