From 14bfad1a70eb2a1843e197729678b4aba3b9600b Mon Sep 17 00:00:00 2001 From: Antoni Basista Date: Mon, 25 May 2026 13:51:22 +0000 Subject: [PATCH] Make scheduler integration podgroup preemption test to check NNN information without seting clearingNominatedNodeNameAfterBinding to false --- .../preemption/podgrouppreemption_test.go | 97 +++++++++++++++---- 1 file changed, 80 insertions(+), 17 deletions(-) diff --git a/test/integration/scheduler/preemption/podgrouppreemption_test.go b/test/integration/scheduler/preemption/podgrouppreemption_test.go index c9c8c8f9440..d653d1559a8 100644 --- a/test/integration/scheduler/preemption/podgrouppreemption_test.go +++ b/test/integration/scheduler/preemption/podgrouppreemption_test.go @@ -19,6 +19,7 @@ package preemption import ( "context" "strings" + "sync" "testing" "time" @@ -27,26 +28,32 @@ import ( schedulingapi "k8s.io/api/scheduling/v1alpha3" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" + configv1 "k8s.io/kube-scheduler/config/v1" + fwk "k8s.io/kube-scheduler/framework" podutil "k8s.io/kubernetes/pkg/api/v1/pod" "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/scheduler" + configtesting "k8s.io/kubernetes/pkg/scheduler/apis/config/testing" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/defaultbinder" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" + frameworkruntime "k8s.io/kubernetes/pkg/scheduler/framework/runtime" st "k8s.io/kubernetes/pkg/scheduler/testing" testutils "k8s.io/kubernetes/test/integration/util" + "k8s.io/utils/ptr" ) // TestPodGroupPreemption tests preemption scenarios involving pod groups. func TestPodGroupPreemption(t *testing.T) { featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ - features.GenericWorkload: true, - features.GangScheduling: true, - features.WorkloadAwarePreemption: true, - features.ClearingNominatedNodeNameAfterBinding: false, + features.GenericWorkload: true, + features.GangScheduling: true, + features.WorkloadAwarePreemption: true, }) - tests := []struct { name string nodes []*v1.Node @@ -730,8 +737,49 @@ func TestPodGroupPreemption(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + registry := make(frameworkruntime.Registry) + + // Register mock bind plugin that will register NNN information during binding. + mockBindPluginName := "mockBindPlugin" + var bindPlugin = mockBindPlugin{ + name: mockBindPluginName, + realPlugin: nil, + nnnInfo: sync.Map{}, + } + err := registry.Register(mockBindPluginName, func(ctx context.Context, o runtime.Object, fh fwk.Handle) (fwk.Plugin, error) { + db, err := defaultbinder.New(ctx, o, fh) + if err != nil { + t.Fatalf("Error creating a default binder plugin: %v", err) + } + bindPlugin.realPlugin = db.(fwk.BindPlugin) + return &bindPlugin, nil + }) + if err != nil { + t.Fatalf("Error registering a bind plugin: %v", err) + } + + cfg := configtesting.V1ToInternalWithDefaults(t, configv1.KubeSchedulerConfiguration{ + Profiles: []configv1.KubeSchedulerProfile{{ + SchedulerName: ptr.To(v1.DefaultSchedulerName), + Plugins: &configv1.Plugins{ + MultiPoint: configv1.PluginSet{ + Enabled: []configv1.Plugin{ + {Name: mockBindPluginName}, + }, + Disabled: []configv1.Plugin{ + {Name: names.DefaultBinder}, + }, + }, + }, + }}, + }) + + // Set PodMaxBackoff to 1 second to turn on backoff and allow apiCacher to get information about + // pod NNN. Without this we might have a race between starting binding and update of apiCacher. testCtx := testutils.InitTestSchedulerWithNS(t, "podgroup-preemption", - scheduler.WithPodMaxBackoffSeconds(0), + scheduler.WithProfiles(cfg.Profiles...), + scheduler.WithFrameworkOutOfTreeRegistry(registry), + scheduler.WithPodMaxBackoffSeconds(1), scheduler.WithPodInitialBackoffSeconds(0)) cs, ns := testCtx.ClientSet, testCtx.NS.Name @@ -766,7 +814,7 @@ func TestPodGroupPreemption(t *testing.T) { } } - // Wait for initial pods to be scheduled + // 3. Wait for initial pods to be scheduled for _, p := range tt.initialPods { if err := wait.PollUntilContextTimeout(testCtx.Ctx, 100*time.Millisecond, 10*time.Second, false, testutils.PodScheduled(cs, ns, p.Name)); err != nil { @@ -774,7 +822,7 @@ func TestPodGroupPreemption(t *testing.T) { } } - // 3. Create preemptor pods + // 4. Create preemptor pods for _, p := range tt.preemptorPods { p.Namespace = ns if _, err := cs.CoreV1().Pods(ns).Create(testCtx.Ctx, p, metav1.CreateOptions{}); err != nil { @@ -782,7 +830,7 @@ func TestPodGroupPreemption(t *testing.T) { } } - // 4. Wait for preemption to complete if WAP calls are expected + // 5. Wait for preemption to complete if WAP calls are expected if tt.expectedPodsPreemptedByWAP > 0 { err := wait.PollUntilContextTimeout(testCtx.Ctx, 100*time.Millisecond, 10*time.Second, false, func(ctx context.Context) (bool, error) { wapCalls := 0 @@ -811,7 +859,7 @@ func TestPodGroupPreemption(t *testing.T) { for _, podName := range tt.expectedUnschedulable { if err := wait.PollUntilContextTimeout(testCtx.Ctx, 100*time.Millisecond, 10*time.Second, false, testutils.PodUnschedulable(cs, ns, podName)); err != nil { - t.Errorf("Pod %s was expected to be unschedulableso but wasn't: %v", podName, err) + t.Errorf("Pod %s was expected to be unschedulable but wasn't: %v", podName, err) } } @@ -843,15 +891,30 @@ func TestPodGroupPreemption(t *testing.T) { // 9. Verify preemptor pods have nominated node name for _, podName := range tt.expectedToHaveNNNInfo { - pod, err := cs.CoreV1().Pods(ns).Get(testCtx.Ctx, podName, metav1.GetOptions{}) - if err != nil { - t.Errorf("Error getting pod %s: %v", podName, err) - continue - } - if err := testutils.WaitForNominatedNodeName(testCtx.Ctx, cs, pod); err != nil { - t.Errorf("Error waiting for nominated node name for pod %s: %v", podName, err) + if node, ok := bindPlugin.nnnInfo.Load(podName); !ok || node.(string) == "" { + t.Errorf("Pod %s was expected to have nominated node name but didn't", podName) } } }) } } + +// mockBindPlugin is a fake plugin that registers NNN information during binding. +type mockBindPlugin struct { + name string + realPlugin fwk.BindPlugin + nnnInfo sync.Map +} + +func (bp *mockBindPlugin) Name() string { + return bp.name +} + +func (bp *mockBindPlugin) Bind(ctx context.Context, state fwk.CycleState, p *v1.Pod, nodeName string) *fwk.Status { + if p.Status.NominatedNodeName != "" { + bp.nnnInfo.Store(p.Name, p.Status.NominatedNodeName) + } + return bp.realPlugin.Bind(ctx, state, p, nodeName) +} + +var _ fwk.BindPlugin = &mockBindPlugin{}