From 070cb9ec4875f5bdaaa73fd293cfb5a809e8780d Mon Sep 17 00:00:00 2001 From: Travis O'Neal Date: Wed, 8 Apr 2026 14:00:25 -0400 Subject: [PATCH 01/10] Add PlacementCycleState to WAS scheduler framework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Added PlacementCycleState as a third state scope for WAS, alongside pod-level CycleState and PodGroupCycleState. This is foundational plumbing only — plugin adoption is a follow-up. Signed-off-by: Travis O'Neal --- pkg/scheduler/framework/cycle_state.go | 13 ++ pkg/scheduler/framework/cycle_state_test.go | 86 ++++++++ pkg/scheduler/schedule_one_podgroup.go | 25 ++- pkg/scheduler/schedule_one_podgroup_test.go | 193 +++++++++++++++++- .../kube-scheduler/framework/cycle_state.go | 33 +++ .../k8s.io/kube-scheduler/framework/types.go | 3 + 6 files changed, 343 insertions(+), 10 deletions(-) diff --git a/pkg/scheduler/framework/cycle_state.go b/pkg/scheduler/framework/cycle_state.go index 31b78725356..32bc015c5a8 100644 --- a/pkg/scheduler/framework/cycle_state.go +++ b/pkg/scheduler/framework/cycle_state.go @@ -46,6 +46,10 @@ type CycleState struct { // or doesn't belong to any pod group. // This field can only be non-nil when GenericWorkload feature flag is enabled. podGroupCycleState fwk.PodGroupCycleState + // placementCycleState contains the CycleState for the current Placement being evaluated. + // If set to nil, it means this pod is not being scheduled within a placement context. + // This field can only be non-nil when GenericWorkload feature flag is enabled. + placementCycleState fwk.PlacementCycleState } // NewCycleState initializes a new CycleState and returns its pointer. @@ -113,6 +117,14 @@ func (c *CycleState) GetPodGroupSchedulingCycle() fwk.PodGroupCycleState { return c.podGroupCycleState } +func (c *CycleState) GetPlacementCycleState() fwk.PlacementCycleState { + return c.placementCycleState +} + +func (c *CycleState) SetPlacementCycleState(placementCycleState fwk.PlacementCycleState) { + c.placementCycleState = placementCycleState +} + func (c *CycleState) SetSkipAllPostFilterPlugins(flag bool) { c.skipAllPostFilterPlugins = flag } @@ -140,6 +152,7 @@ func (c *CycleState) Clone() fwk.CycleState { copy.skipPreBindPlugins = c.skipPreBindPlugins copy.parallelPreBindPlugins = c.parallelPreBindPlugins copy.podGroupCycleState = c.podGroupCycleState + copy.placementCycleState = c.placementCycleState copy.skipAllPostFilterPlugins = c.skipAllPostFilterPlugins return copy diff --git a/pkg/scheduler/framework/cycle_state_test.go b/pkg/scheduler/framework/cycle_state_test.go index 70b70085fdf..e72b220e94d 100644 --- a/pkg/scheduler/framework/cycle_state_test.go +++ b/pkg/scheduler/framework/cycle_state_test.go @@ -193,3 +193,89 @@ func TestCycleStateClone(t *testing.T) { }) } } + +func TestPlacementCycleState(t *testing.T) { + t.Run("nil by default", func(t *testing.T) { + state := NewCycleState() + if state.GetPlacementCycleState() != nil { + t.Errorf("expected nil PlacementCycleState on fresh CycleState") + } + }) + + t.Run("set and get", func(t *testing.T) { + state := NewCycleState() + placementState := NewCycleState() + placementState.Write("testkey", &fakeData{data: "placementdata"}) + + state.SetPlacementCycleState(placementState) + + got := state.GetPlacementCycleState() + if got == nil { + t.Fatal("expected non-nil PlacementCycleState after Set") + } + + data, err := got.Read("testkey") + if err != nil { + t.Fatalf("unexpected error reading from PlacementCycleState: %v", err) + } + if data.(*fakeData).data != "placementdata" { + t.Errorf("expected 'placementdata', got %q", data.(*fakeData).data) + } + }) + + t.Run("set to nil clears", func(t *testing.T) { + state := NewCycleState() + state.SetPlacementCycleState(NewCycleState()) + state.SetPlacementCycleState(nil) + + if state.GetPlacementCycleState() != nil { + t.Errorf("expected nil PlacementCycleState after setting to nil") + } + }) + + t.Run("clone preserves reference", func(t *testing.T) { + state := NewCycleState() + state.Write(key, &fakeData{data: "pod-data"}) + + placementState := NewCycleState() + placementState.Write("pkey", &fakeData{data: "placement-data"}) + state.SetPlacementCycleState(placementState) + + cloned := state.Clone().(*CycleState) + + // The cloned state should reference the same PlacementCycleState. + if cloned.GetPlacementCycleState() == nil { + t.Fatal("cloned state should have non-nil PlacementCycleState") + } + + data, err := cloned.GetPlacementCycleState().Read("pkey") + if err != nil { + t.Fatalf("unexpected error reading from cloned PlacementCycleState: %v", err) + } + if data.(*fakeData).data != "placement-data" { + t.Errorf("expected 'placement-data', got %q", data.(*fakeData).data) + } + + // Writes to the PlacementCycleState via the clone should be visible from the original, + // since it's a shared reference (same as podGroupCycleState behavior). + cloned.GetPlacementCycleState().Write("newkey", &fakeData{data: "new"}) + newData, err := state.GetPlacementCycleState().Read("newkey") + if err != nil { + t.Fatalf("write via clone's PlacementCycleState should be visible from original: %v", err) + } + if newData.(*fakeData).data != "new" { + t.Errorf("expected 'new', got %q", newData.(*fakeData).data) + } + }) + + t.Run("clone with nil placement state", func(t *testing.T) { + state := NewCycleState() + state.Write(key, &fakeData{data: "data"}) + // Do not set PlacementCycleState — leave nil. + + cloned := state.Clone().(*CycleState) + if cloned.GetPlacementCycleState() != nil { + t.Errorf("cloned state should have nil PlacementCycleState when original has nil") + } + }) +} diff --git a/pkg/scheduler/schedule_one_podgroup.go b/pkg/scheduler/schedule_one_podgroup.go index 78c363ad428..50440c5b10d 100644 --- a/pkg/scheduler/schedule_one_podgroup.go +++ b/pkg/scheduler/schedule_one_podgroup.go @@ -182,7 +182,7 @@ type podSchedulingContext struct { } // initPodSchedulingContext initializes the scheduling context of a single pod for pod group scheduling cycle. -func initPodSchedulingContext(ctx context.Context, pod *v1.Pod, podGroupState *framework.CycleState, postFilterMode podGroupPostFilterMode) *podSchedulingContext { +func initPodSchedulingContext(ctx context.Context, pod *v1.Pod, podGroupState *framework.CycleState, placementCycleState fwk.PlacementCycleState, postFilterMode podGroupPostFilterMode) *podSchedulingContext { logger := klog.FromContext(ctx) // TODO(knelasevero): Remove duplicated keys from log entry calls // When contextualized logging hits GA @@ -202,6 +202,8 @@ func initPodSchedulingContext(ctx context.Context, pod *v1.Pod, podGroupState *f // Marks this cycle as a pod group scheduling cycle. state.SetPodGroupSchedulingCycle(podGroupState) + // Set the placement cycle state so per-pod plugins can access placement-scoped data. + state.SetPlacementCycleState(placementCycleState) // Skip post filters if requested. switch postFilterMode { @@ -358,6 +360,9 @@ type podGroupAlgorithmResult struct { // waitingOnPreemption indicates whether this pod group requires or is waiting for preemption to complete. // This can only be set to true when the status is Unschedulable. waitingOnPreemption bool + // placementCycleState holds the state accumulated during simulation of a specific placement. + // Only set for successful placements that proceed to the scoring phase. + placementCycleState fwk.PlacementCycleState } // podGroupSchedulingDefaultAlgorithm runs the default algorithm for scheduling a pod group. @@ -365,23 +370,24 @@ type podGroupAlgorithmResult struct { // If a pod requires preemption to be schedulable, subsequent pods in the algorithm // treat that pod as already scheduled on that node with victims being already removed in memory. func (sched *Scheduler) podGroupSchedulingDefaultAlgorithm(ctx context.Context, schedFwk framework.Framework, podGroupCycleState *framework.CycleState, podGroupInfo *framework.QueuedPodGroupInfo, postFilterMode podGroupPostFilterMode) podGroupAlgorithmResult { + placementCycleState := framework.NewCycleState() + placementCycleState.SetRecordPluginMetrics(true) + placementCycleState.SetPodGroupSchedulingCycle(podGroupCycleState) + result := podGroupAlgorithmResult{ podResults: make([]algorithmResult, 0, len(podGroupInfo.QueuedPodInfos)), status: fwk.NewStatus(fwk.Unschedulable).WithError(errPodGroupUnschedulable), waitingOnPreemption: false, + placementCycleState: placementCycleState, } - placementCycleState := framework.NewCycleState() - placementCycleState.SetRecordPluginMetrics(true) - placementCycleState.SetPodGroupSchedulingCycle(podGroupCycleState) - logger := klog.FromContext(ctx) logger.V(5).Info("Running a pod group scheduling algorithm", "podGroup", klog.KObj(podGroupInfo), "unscheduledPodsCount", len(podGroupInfo.QueuedPodInfos)) requiresPreemption := false anyScheduled := false for _, podInfo := range podGroupInfo.QueuedPodInfos { - podResult, revertFn := sched.podGroupPodSchedulingAlgorithm(ctx, schedFwk, podGroupCycleState, podGroupInfo, podInfo, postFilterMode) + podResult, revertFn := sched.podGroupPodSchedulingAlgorithm(ctx, schedFwk, podGroupCycleState, placementCycleState, podGroupInfo, podInfo, postFilterMode) result.podResults = append(result.podResults, podResult) if revertFn != nil { // We unreserve the pod at the end of the whole algorithm (via defer) because it should be ultimately returned to the queue, @@ -435,9 +441,9 @@ func (sched *Scheduler) podGroupSchedulingDefaultAlgorithm(ctx context.Context, // podGroupPodSchedulingAlgorithm runs a scheduling algorithm for individual pod from a pod group. // It returns the algorithm result together with the revert function. -func (sched *Scheduler) podGroupPodSchedulingAlgorithm(ctx context.Context, schedFwk framework.Framework, podGroupCycleState *framework.CycleState, podGroupInfo *framework.QueuedPodGroupInfo, podInfo *framework.QueuedPodInfo, postFilterMode podGroupPostFilterMode) (algorithmResult, func()) { +func (sched *Scheduler) podGroupPodSchedulingAlgorithm(ctx context.Context, schedFwk framework.Framework, podGroupCycleState *framework.CycleState, placementCycleState fwk.PlacementCycleState, podGroupInfo *framework.QueuedPodGroupInfo, podInfo *framework.QueuedPodInfo, postFilterMode podGroupPostFilterMode) (algorithmResult, func()) { pod := podInfo.Pod - podCtx := initPodSchedulingContext(ctx, pod, podGroupCycleState, postFilterMode) + podCtx := initPodSchedulingContext(ctx, pod, podGroupCycleState, placementCycleState, postFilterMode) logger := podCtx.logger ctx = klog.NewContext(ctx, logger) start := time.Now() @@ -506,7 +512,7 @@ func completePodGroupAlgorithmResult(ctx context.Context, podGroupInfo *framewor pInfo := podGroupInfo.QueuedPodInfos[i] newResults[i] = algorithmResult{ pod: pInfo.Pod, - podCtx: initPodSchedulingContext(ctx, pInfo.Pod, podGroupState, postFilterMode), + podCtx: initPodSchedulingContext(ctx, pInfo.Pod, podGroupState, nil, postFilterMode), status: podGroupResult.status.Clone(), } } @@ -783,6 +789,7 @@ func makePodGroupAssignments(successfulResults map[*fwk.Placement]*podGroupAlgor placementPodGroupAssignments = append(placementPodGroupAssignments, &fwk.PodGroupAssignments{ Placement: placement, ProposedAssignments: proposedAssignments, + PlacementCycleState: result.placementCycleState, }) } return placementPodGroupAssignments diff --git a/pkg/scheduler/schedule_one_podgroup_test.go b/pkg/scheduler/schedule_one_podgroup_test.go index 330abad102e..6d827007811 100644 --- a/pkg/scheduler/schedule_one_podgroup_test.go +++ b/pkg/scheduler/schedule_one_podgroup_test.go @@ -1751,7 +1751,7 @@ func TestSubmitPodGroupAlgorithmResult(t *testing.T) { for i := range tt.algorithmResult.podResults { pod := podGroupInfo.QueuedPodInfos[i].Pod - podCtx := initPodSchedulingContext(ctx, pod, podGroupCycleState, runAllPostFilters) + podCtx := initPodSchedulingContext(ctx, pod, podGroupCycleState, nil, runAllPostFilters) tt.algorithmResult.podResults[i].podCtx = podCtx } @@ -2538,6 +2538,7 @@ func TestPodGroupSchedulingPlacementAlgorithm(t *testing.T) { ScheduleResult{}, fwk.Status{}), cmpopts.IgnoreFields(algorithmResult{}, "podCtx", "schedulingDuration"), + cmpopts.IgnoreFields(podGroupAlgorithmResult{}, "placementCycleState"), statusCmpOpt, } @@ -2702,6 +2703,196 @@ func TestPodGroupSchedulingPlacementAlgorithm_Scoring(t *testing.T) { } } +// placementStateTracker is a fake plugin that writes to PlacementCycleState during Filter +// and reads from it during ScorePlacement, to verify the lifecycle of PlacementCycleState. +type placementStateTracker struct { + name string + mu sync.Mutex + // scoreReadValues records what value was read from PlacementCycleState + // during each ScorePlacement call, keyed by placement name. + scoreReadValues map[string]string + // generatePlacementsResult defines the placements to generate. + generatePlacementsResult map[string][]string +} + +type placementStateData struct { + value string +} + +func (d *placementStateData) Clone() fwk.StateData { return d } + +var placementStateKey fwk.StateKey = "placementStateTracker" + +var _ fwk.FilterPlugin = &placementStateTracker{} +var _ fwk.PlacementGeneratePlugin = &placementStateTracker{} +var _ fwk.PlacementScorePlugin = &placementStateTracker{} + +func (p *placementStateTracker) Name() string { return p.name } + +func (p *placementStateTracker) Filter(ctx context.Context, state fwk.CycleState, pod *v1.Pod, nodeInfo fwk.NodeInfo) *fwk.Status { + placementState := state.GetPlacementCycleState() + if placementState == nil { + return fwk.NewStatus(fwk.Error, "PlacementCycleState is nil during Filter") + } + + // Write the node name as a marker so ScorePlacement can verify + // which placement's state it received. + placementState.Write(placementStateKey, &placementStateData{value: nodeInfo.Node().Name}) + return nil +} + +func (p *placementStateTracker) ScorePlacement(ctx context.Context, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { + if placement.PlacementCycleState == nil { + return 0, fwk.NewStatus(fwk.Error, "PlacementCycleState is nil during ScorePlacement") + } + + data, err := placement.PlacementCycleState.Read(placementStateKey) + p.mu.Lock() + defer p.mu.Unlock() + if err != nil { + p.scoreReadValues[placement.Name] = "" + } else { + p.scoreReadValues[placement.Name] = data.(*placementStateData).value + } + return 1, nil +} + +func (p *placementStateTracker) PlacementScoreExtensions() fwk.PlacementScoreExtensions { + return nil +} + +func (p *placementStateTracker) GeneratePlacements(ctx context.Context, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo, parentPlacement *fwk.Placement) (*fwk.GeneratePlacementsResult, *fwk.Status) { + parentNodes := map[string]fwk.NodeInfo{} + for _, node := range parentPlacement.Nodes { + parentNodes[node.Node().Name] = node + } + + placements := make([]*fwk.Placement, 0, len(p.generatePlacementsResult)) + for placementName, nodeNames := range p.generatePlacementsResult { + placement := &fwk.Placement{Name: placementName} + for _, nodeName := range nodeNames { + placement.Nodes = append(placement.Nodes, parentNodes[nodeName]) + } + placements = append(placements, placement) + } + return &fwk.GeneratePlacementsResult{Placements: placements}, nil +} + +func TestPlacementCycleStateLifecycle(t *testing.T) { + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.TopologyAwareWorkloadScheduling: true, + features.GenericWorkload: true, + }) + + // A single scenario exercises both isolation and continuity: + // - Filter writes a node-name marker into PlacementCycleState during each placement's simulation. + // - ScorePlacement reads from PodGroupAssignments.PlacementCycleState after all simulations. + // Assertions verify: + // 1. Each placement's scorer reads only the value its own simulation wrote (isolation). + // 2. The value is readable at all during scoring (continuity from simulation to scoring). + + nodes := []*v1.Node{ + st.MakeNode().Name("node1").Obj(), + st.MakeNode().Name("node2").Obj(), + } + podGroupPod := st.MakePod().Name("foo").UID("foo").PodGroupName("pg").Obj() + + logger, ctx := ktesting.NewTestContext(t) + + informerFactory := informers.NewSharedInformerFactory(clientsetfake.NewClientset(), 0) + queue := internalqueue.NewSchedulingQueue(nil, informerFactory) + + tracker := &placementStateTracker{ + name: "StateTracker", + scoreReadValues: make(map[string]string), + generatePlacementsResult: map[string][]string{ + "placementA": {nodes[0].Name}, + "placementB": {nodes[1].Name}, + }, + } + + registry := []tf.RegisterPluginFunc{ + tf.RegisterPlacementGeneratePlugin(tracker.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) { + return tracker, nil + }), + tf.RegisterPlacementScorePlugin(tracker.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) { + return tracker, nil + }, 1), + tf.RegisterFilterPlugin(tracker.Name(), func(_ context.Context, _ runtime.Object, _ fwk.Handle) (fwk.Plugin, error) { + return tracker, nil + }), + } + + snapshot := internalcache.NewEmptySnapshot() + schedFwk, err := tf.NewFramework(ctx, + append(registry, + tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), + tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), + ), + "test-scheduler", + frameworkruntime.WithInformerFactory(informerFactory), + frameworkruntime.WithSnapshotSharedLister(snapshot), + frameworkruntime.WithPodNominator(queue), + ) + if err != nil { + t.Fatalf("Failed to create framework: %v", err) + } + + cache := internalcache.New(ctx, nil, true) + for _, node := range nodes { + cache.AddNode(logger, node) + } + + sched := &Scheduler{ + Cache: cache, + nodeInfoSnapshot: snapshot, + SchedulingQueue: queue, + Profiles: profile.Map{"test-scheduler": schedFwk}, + } + sched.SchedulePod = sched.schedulePod + + if err := sched.Cache.UpdateSnapshot(logger, sched.nodeInfoSnapshot); err != nil { + t.Fatalf("Failed to update snapshot: %v", err) + } + + pgInfo := &framework.QueuedPodGroupInfo{ + QueuedPodInfos: []*framework.QueuedPodInfo{ + {PodInfo: &framework.PodInfo{Pod: podGroupPod}}, + }, + PodGroupInfo: &framework.PodGroupInfo{ + UnscheduledPods: []*v1.Pod{podGroupPod}, + }, + } + + result := sched.podGroupSchedulingPlacementAlgorithm(ctx, schedFwk, framework.NewCycleState(), pgInfo, runAllPostFilters) + if !result.status.IsSuccess() { + t.Fatalf("Expected success, got: %v", result.status) + } + + tracker.mu.Lock() + defer tracker.mu.Unlock() + + // Continuity: ScorePlacement must have been called and able to read simulation data. + for _, placementName := range []string{"placementA", "placementB"} { + readValue, ok := tracker.scoreReadValues[placementName] + if !ok { + t.Fatalf("placement %s: ScorePlacement was not called", placementName) + } + if readValue == "" { + t.Fatalf("placement %s: ScorePlacement could not read state written during simulation", placementName) + } + } + + // Isolation: each placement's scorer must read only what its own simulation wrote. + // placementA simulated on node1, placementB simulated on node2. + if v := tracker.scoreReadValues["placementA"]; v != "node1" { + t.Errorf("placementA: scorer read %q, want %q (isolation violation if it read placementB's value)", v, "node1") + } + if v := tracker.scoreReadValues["placementB"]; v != "node2" { + t.Errorf("placementB: scorer read %q, want %q (isolation violation if it read placementA's value)", v, "node2") + } +} + type fakeDefaultPreemption struct { *fakePodGroupPlugin } diff --git a/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go b/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go index 4d103b2e39c..fdf41701ba6 100644 --- a/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go +++ b/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go @@ -104,6 +104,39 @@ type CycleState interface { // SetPodGroupSchedulingCycle sets the cycle state of the PodGroup for a Pod. // This should be only used when GenericWorkload feature flag is enabled. SetPodGroupSchedulingCycle(PodGroupCycleState) + // GetPlacementCycleState gets the cycle state of the current Placement for a Pod. + // Returns nil if this pod is not being scheduled within a placement context. + // This should be only used when GenericWorkload feature flag is enabled. + GetPlacementCycleState() PlacementCycleState + // SetPlacementCycleState sets the cycle state of the current Placement for a Pod. + // This should be only used when GenericWorkload feature flag is enabled. + SetPlacementCycleState(PlacementCycleState) +} + +// PlacementCycleState provides a mechanism for plugins to store and retrieve arbitrary data +// scoped to a single placement candidate within a pod group scheduling cycle. +// Data stored in PlacementCycleState is shared across all pods scheduled within the same +// placement iteration and is preserved through to the placement scoring phase via PodGroupAssignments. +// PlacementCycleState does not provide any data protection, as all plugins are assumed to be +// trusted. +type PlacementCycleState interface { + // ShouldRecordPluginMetrics returns whether metrics.PluginExecutionDuration metrics + // should be recorded. + // This function is mostly for the scheduling framework runtime, plugins usually don't have to use it. + ShouldRecordPluginMetrics() bool + // Read retrieves data with the given "key" from PlacementCycleState. If the key is not + // present, ErrNotFound is returned. + // + // See PlacementCycleState for notes on concurrency. + Read(key StateKey) (StateData, error) + // Write stores the given "val" in PlacementCycleState with the given "key". + // + // See PlacementCycleState for notes on concurrency. + Write(key StateKey, val StateData) + // Delete deletes data with the given key from PlacementCycleState. + // + // See PlacementCycleState for notes on concurrency. + Delete(key StateKey) } // PodGroupCycleState provides a mechanism for plugins that operate on pod groups to store and retrieve arbitrary data. diff --git a/staging/src/k8s.io/kube-scheduler/framework/types.go b/staging/src/k8s.io/kube-scheduler/framework/types.go index c5df461b3d7..d6e2f062bf5 100644 --- a/staging/src/k8s.io/kube-scheduler/framework/types.go +++ b/staging/src/k8s.io/kube-scheduler/framework/types.go @@ -682,6 +682,9 @@ type PodGroupAssignments struct { // during the pod group scheduling cycle. // The pods are guaranteed to also be present in the PodGroupInfo. ProposedAssignments []ProposedAssignment + // PlacementCycleState holds the state that was accumulated during the simulation of this placement. + // Placement score plugins can use this to access per-placement cached data. + PlacementCycleState PlacementCycleState } // NodeAllocatableDRAClaimState holds information about a node allocatable resource DRA claim's allocation on a node. From e428177311c4d349ed50d1eef68187678e56d1d2 Mon Sep 17 00:00:00 2001 From: wtravO Date: Wed, 13 May 2026 09:40:38 -0400 Subject: [PATCH 02/10] Addressed review feedback --- pkg/scheduler/schedule_one_podgroup.go | 4 ++-- pkg/scheduler/schedule_one_podgroup_test.go | 25 +++++++++------------ 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/pkg/scheduler/schedule_one_podgroup.go b/pkg/scheduler/schedule_one_podgroup.go index 50440c5b10d..31fc93d9a8a 100644 --- a/pkg/scheduler/schedule_one_podgroup.go +++ b/pkg/scheduler/schedule_one_podgroup.go @@ -182,7 +182,7 @@ type podSchedulingContext struct { } // initPodSchedulingContext initializes the scheduling context of a single pod for pod group scheduling cycle. -func initPodSchedulingContext(ctx context.Context, pod *v1.Pod, podGroupState *framework.CycleState, placementCycleState fwk.PlacementCycleState, postFilterMode podGroupPostFilterMode) *podSchedulingContext { +func initPodSchedulingContext(ctx context.Context, pod *v1.Pod, podGroupState *framework.CycleState, placementCycleState *framework.CycleState, postFilterMode podGroupPostFilterMode) *podSchedulingContext { logger := klog.FromContext(ctx) // TODO(knelasevero): Remove duplicated keys from log entry calls // When contextualized logging hits GA @@ -441,7 +441,7 @@ func (sched *Scheduler) podGroupSchedulingDefaultAlgorithm(ctx context.Context, // podGroupPodSchedulingAlgorithm runs a scheduling algorithm for individual pod from a pod group. // It returns the algorithm result together with the revert function. -func (sched *Scheduler) podGroupPodSchedulingAlgorithm(ctx context.Context, schedFwk framework.Framework, podGroupCycleState *framework.CycleState, placementCycleState fwk.PlacementCycleState, podGroupInfo *framework.QueuedPodGroupInfo, podInfo *framework.QueuedPodInfo, postFilterMode podGroupPostFilterMode) (algorithmResult, func()) { +func (sched *Scheduler) podGroupPodSchedulingAlgorithm(ctx context.Context, schedFwk framework.Framework, podGroupCycleState *framework.CycleState, placementCycleState *framework.CycleState, podGroupInfo *framework.QueuedPodGroupInfo, podInfo *framework.QueuedPodInfo, postFilterMode podGroupPostFilterMode) (algorithmResult, func()) { pod := podInfo.Pod podCtx := initPodSchedulingContext(ctx, pod, podGroupCycleState, placementCycleState, postFilterMode) logger := podCtx.logger diff --git a/pkg/scheduler/schedule_one_podgroup_test.go b/pkg/scheduler/schedule_one_podgroup_test.go index 6d827007811..3dc87d8b9c6 100644 --- a/pkg/scheduler/schedule_one_podgroup_test.go +++ b/pkg/scheduler/schedule_one_podgroup_test.go @@ -2747,13 +2747,12 @@ func (p *placementStateTracker) ScorePlacement(ctx context.Context, state fwk.Po } data, err := placement.PlacementCycleState.Read(placementStateKey) + if err != nil { + return 0, fwk.NewStatus(fwk.Error, fmt.Sprintf("failed to read PlacementCycleState for %s: %v", placement.Name, err)) + } p.mu.Lock() defer p.mu.Unlock() - if err != nil { - p.scoreReadValues[placement.Name] = "" - } else { - p.scoreReadValues[placement.Name] = data.(*placementStateData).value - } + p.scoreReadValues[placement.Name] = data.(*placementStateData).value return 1, nil } @@ -2767,15 +2766,15 @@ func (p *placementStateTracker) GeneratePlacements(ctx context.Context, state fw parentNodes[node.Node().Name] = node } - placements := make([]*fwk.Placement, 0, len(p.generatePlacementsResult)) + resultPlacements := make([]*fwk.Placement, 0, len(p.generatePlacementsResult)) for placementName, nodeNames := range p.generatePlacementsResult { placement := &fwk.Placement{Name: placementName} for _, nodeName := range nodeNames { placement.Nodes = append(placement.Nodes, parentNodes[nodeName]) } - placements = append(placements, placement) + resultPlacements = append(resultPlacements, placement) } - return &fwk.GeneratePlacementsResult{Placements: placements}, nil + return &fwk.GeneratePlacementsResult{Placements: resultPlacements}, nil } func TestPlacementCycleStateLifecycle(t *testing.T) { @@ -2789,7 +2788,7 @@ func TestPlacementCycleStateLifecycle(t *testing.T) { // - ScorePlacement reads from PodGroupAssignments.PlacementCycleState after all simulations. // Assertions verify: // 1. Each placement's scorer reads only the value its own simulation wrote (isolation). - // 2. The value is readable at all during scoring (continuity from simulation to scoring). + // 2. Data written during each placement's simulation remains readable during its scoring (continuity from simulation to scoring). nodes := []*v1.Node{ st.MakeNode().Name("node1").Obj(), @@ -2873,14 +2872,12 @@ func TestPlacementCycleStateLifecycle(t *testing.T) { defer tracker.mu.Unlock() // Continuity: ScorePlacement must have been called and able to read simulation data. + // ScorePlacement returns an error status if the read fails, so a failed read would + // already have been caught by the result.status check above. for _, placementName := range []string{"placementA", "placementB"} { - readValue, ok := tracker.scoreReadValues[placementName] - if !ok { + if _, ok := tracker.scoreReadValues[placementName]; !ok { t.Fatalf("placement %s: ScorePlacement was not called", placementName) } - if readValue == "" { - t.Fatalf("placement %s: ScorePlacement could not read state written during simulation", placementName) - } } // Isolation: each placement's scorer must read only what its own simulation wrote. From bd97e3f190713ebcce2b636e8b3e68f8e214e827 Mon Sep 17 00:00:00 2001 From: wtravO Date: Sat, 16 May 2026 21:10:21 -0400 Subject: [PATCH 03/10] expose PodGroupCycleState via PlacementCycleState --- pkg/scheduler/framework/cycle_state_test.go | 5 ++ pkg/scheduler/framework/interface.go | 1 + .../framework/plugins/noderesources/fit.go | 2 +- .../podgrouppodscount/podgroup_pods_count.go | 2 +- pkg/scheduler/framework/runtime/framework.go | 4 +- .../framework/runtime/framework_test.go | 10 ++-- pkg/scheduler/schedule_one_podgroup.go | 59 +++++++++++-------- pkg/scheduler/schedule_one_podgroup_test.go | 15 ++--- .../kube-scheduler/framework/cycle_state.go | 3 + .../kube-scheduler/framework/interface.go | 5 +- 10 files changed, 66 insertions(+), 40 deletions(-) diff --git a/pkg/scheduler/framework/cycle_state_test.go b/pkg/scheduler/framework/cycle_state_test.go index e72b220e94d..df61747ca1a 100644 --- a/pkg/scheduler/framework/cycle_state_test.go +++ b/pkg/scheduler/framework/cycle_state_test.go @@ -204,7 +204,9 @@ func TestPlacementCycleState(t *testing.T) { t.Run("set and get", func(t *testing.T) { state := NewCycleState() + podGroupState := NewCycleState() placementState := NewCycleState() + placementState.SetPodGroupSchedulingCycle(podGroupState) placementState.Write("testkey", &fakeData{data: "placementdata"}) state.SetPlacementCycleState(placementState) @@ -221,6 +223,9 @@ func TestPlacementCycleState(t *testing.T) { if data.(*fakeData).data != "placementdata" { t.Errorf("expected 'placementdata', got %q", data.(*fakeData).data) } + if got.GetPodGroupSchedulingCycle() != podGroupState { + t.Errorf("expected PlacementCycleState to expose its PodGroupCycleState") + } }) t.Run("set to nil clears", func(t *testing.T) { diff --git a/pkg/scheduler/framework/interface.go b/pkg/scheduler/framework/interface.go index 267f65cba8f..faf519658b5 100644 --- a/pkg/scheduler/framework/interface.go +++ b/pkg/scheduler/framework/interface.go @@ -308,6 +308,7 @@ type Framework interface { // It returns a list that stores scores from each plugin and total score for each Placement. // It also returns *Status, which is set to non-success if any of the plugins returns // a non-success status. + // Each PodGroupAssignments carries the PlacementCycleState passed to ScorePlacement for that Placement. RunPlacementScorePlugins(ctx context.Context, state fwk.PodGroupCycleState, podGroupInfo fwk.PodGroupInfo, placements []*fwk.PodGroupAssignments) (ns []fwk.PlacementPluginScores, status *fwk.Status) // HasFilterPlugins returns true if at least one Filter plugin is defined. diff --git a/pkg/scheduler/framework/plugins/noderesources/fit.go b/pkg/scheduler/framework/plugins/noderesources/fit.go index ce437246e5e..111f07eaa90 100644 --- a/pkg/scheduler/framework/plugins/noderesources/fit.go +++ b/pkg/scheduler/framework/plugins/noderesources/fit.go @@ -760,6 +760,6 @@ func (f *Fit) PlacementScoreExtensions() fwk.PlacementScoreExtensions { } // ScorePlacement scores capacity ratio on all nodes in the placement including all assigned pod group pods' resource requests. -func (f *Fit) ScorePlacement(ctx context.Context, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { +func (f *Fit) ScorePlacement(ctx context.Context, state fwk.PlacementCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { return f.placementScorer.scorePlacement(ctx, podGroup, placement) } diff --git a/pkg/scheduler/framework/plugins/podgrouppodscount/podgroup_pods_count.go b/pkg/scheduler/framework/plugins/podgrouppodscount/podgroup_pods_count.go index 143e66d7636..508300f8e27 100644 --- a/pkg/scheduler/framework/plugins/podgrouppodscount/podgroup_pods_count.go +++ b/pkg/scheduler/framework/plugins/podgrouppodscount/podgroup_pods_count.go @@ -49,7 +49,7 @@ func (pl *PodGroupPodsCount) Name() string { // Both scheduled (assumed/assigned) pods and the proposed assignments are taken into consideration // when computing the score. This ensures that the relative difference between choices is reduced, // and small changes to the total count result in small changes to the score. -func (pl *PodGroupPodsCount) ScorePlacement(ctx context.Context, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { +func (pl *PodGroupPodsCount) ScorePlacement(ctx context.Context, state fwk.PlacementCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { pgState, err := pl.handle.SnapshotSharedLister().PodGroupStates().Get(podGroup.GetNamespace(), podGroup.GetName()) if err != nil { return 0, fwk.AsStatus(err) diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index 7c6b0e6d0f6..bf408adb8ab 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -1520,7 +1520,7 @@ func (f *frameworkImpl) RunPlacementScorePlugins(ctx context.Context, state fwk. logger := klog.LoggerWithName(logger, pl.Name()) ctx = klog.NewContext(ctx, logger) } - s, status := f.runPlacementScorePlugin(ctx, pl, state, podGroupInfo, pga) + s, status := f.runPlacementScorePlugin(ctx, pl, pga.PlacementCycleState, podGroupInfo, pga) if !status.IsSuccess() { err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError()) errCh.SendWithCancel(err, cancel) @@ -1589,7 +1589,7 @@ func (f *frameworkImpl) RunPlacementScorePlugins(ctx context.Context, state fwk. return allPlacementPluginScores, nil } -func (f *frameworkImpl) runPlacementScorePlugin(ctx context.Context, pl fwk.PlacementScorePlugin, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { +func (f *frameworkImpl) runPlacementScorePlugin(ctx context.Context, pl fwk.PlacementScorePlugin, state fwk.PlacementCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { if !state.ShouldRecordPluginMetrics() { return pl.ScorePlacement(ctx, state, podGroup, placement) } diff --git a/pkg/scheduler/framework/runtime/framework_test.go b/pkg/scheduler/framework/runtime/framework_test.go index d11ae059b7f..1ec08e63f05 100644 --- a/pkg/scheduler/framework/runtime/framework_test.go +++ b/pkg/scheduler/framework/runtime/framework_test.go @@ -278,7 +278,7 @@ func (pl *TestPlugin) GeneratePlacements(ctx context.Context, state fwk.PodGroup return &fwk.GeneratePlacementsResult{Placements: pl.inj.GeneratePlacementsResult}, fwk.NewStatus(fwk.Code(pl.inj.GeneratePlacementsStatus), injectReason) } -func (pl *TestPlugin) ScorePlacement(ctx context.Context, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { +func (pl *TestPlugin) ScorePlacement(ctx context.Context, state fwk.PlacementCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { return 0, fwk.NewStatus(fwk.Code(pl.inj.PlacementScoreStatus), injectReason) } @@ -3666,7 +3666,7 @@ func TestRecordingMetrics(t *testing.T) { { name: "PlacementScore - Success", action: func(ctx context.Context, f framework.Framework) { - f.RunPlacementScorePlugins(ctx, state, nil, []*fwk.PodGroupAssignments{{Placement: &fwk.Placement{}}}) + f.RunPlacementScorePlugins(ctx, state, nil, []*fwk.PodGroupAssignments{{Placement: &fwk.Placement{}, PlacementCycleState: state}}) }, wantExtensionPoint: "PlacementScore", wantStatus: fwk.Success, @@ -3750,7 +3750,7 @@ func TestRecordingMetrics(t *testing.T) { { name: "PlacementScore - Error", action: func(ctx context.Context, f framework.Framework) { - f.RunPlacementScorePlugins(ctx, state, nil, []*fwk.PodGroupAssignments{{Placement: &fwk.Placement{}}}) + f.RunPlacementScorePlugins(ctx, state, nil, []*fwk.PodGroupAssignments{{Placement: &fwk.Placement{}, PlacementCycleState: state}}) }, inject: injectedResult{PlacementScoreStatus: int(fwk.Error)}, wantExtensionPoint: "PlacementScore", @@ -4440,7 +4440,7 @@ func (pl *testPlacementScorePlugin) Name() string { return pl.name } -func (pl *testPlacementScorePlugin) ScorePlacement(ctx context.Context, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { +func (pl *testPlacementScorePlugin) ScorePlacement(ctx context.Context, state fwk.PlacementCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { r := pl.results[placement.Placement] return r.score, r.status } @@ -4683,7 +4683,7 @@ func TestRunPlacementScorePlugins(t *testing.T) { } assumedPlacements := make([]*fwk.PodGroupAssignments, len(placements)) for i := range placements { - assumedPlacements[i] = &fwk.PodGroupAssignments{Placement: placements[i]} + assumedPlacements[i] = &fwk.PodGroupAssignments{Placement: placements[i], PlacementCycleState: framework.NewCycleState()} } result, status := fw.RunPlacementScorePlugins(ctx, framework.NewCycleState(), nil, assumedPlacements) diff --git a/pkg/scheduler/schedule_one_podgroup.go b/pkg/scheduler/schedule_one_podgroup.go index 31fc93d9a8a..0b3ea8ece71 100644 --- a/pkg/scheduler/schedule_one_podgroup.go +++ b/pkg/scheduler/schedule_one_podgroup.go @@ -182,7 +182,7 @@ type podSchedulingContext struct { } // initPodSchedulingContext initializes the scheduling context of a single pod for pod group scheduling cycle. -func initPodSchedulingContext(ctx context.Context, pod *v1.Pod, podGroupState *framework.CycleState, placementCycleState *framework.CycleState, postFilterMode podGroupPostFilterMode) *podSchedulingContext { +func initPodSchedulingContext(ctx context.Context, pod *v1.Pod, placementCycleState *framework.CycleState, postFilterMode podGroupPostFilterMode) *podSchedulingContext { logger := klog.FromContext(ctx) // TODO(knelasevero): Remove duplicated keys from log entry calls // When contextualized logging hits GA @@ -200,8 +200,12 @@ func initPodSchedulingContext(ctx context.Context, pod *v1.Pod, podGroupState *f podsToActivate := framework.NewPodsToActivate() state.Write(framework.PodsToActivateKey, podsToActivate) + var podGroupCycleState fwk.PodGroupCycleState + if placementCycleState != nil { + podGroupCycleState = placementCycleState.GetPodGroupSchedulingCycle() + } // Marks this cycle as a pod group scheduling cycle. - state.SetPodGroupSchedulingCycle(podGroupState) + state.SetPodGroupSchedulingCycle(podGroupCycleState) // Set the placement cycle state so per-pod plugins can access placement-scoped data. state.SetPlacementCycleState(placementCycleState) @@ -360,8 +364,10 @@ type podGroupAlgorithmResult struct { // waitingOnPreemption indicates whether this pod group requires or is waiting for preemption to complete. // This can only be set to true when the status is Unschedulable. waitingOnPreemption bool - // placementCycleState holds the state accumulated during simulation of a specific placement. - // Only set for successful placements that proceed to the scoring phase. +} + +type placementAlgorithmResult struct { + result podGroupAlgorithmResult placementCycleState fwk.PlacementCycleState } @@ -369,16 +375,11 @@ type podGroupAlgorithmResult struct { // It tries to schedule each pod using standard filtering and scoring logic in a fixed order. // If a pod requires preemption to be schedulable, subsequent pods in the algorithm // treat that pod as already scheduled on that node with victims being already removed in memory. -func (sched *Scheduler) podGroupSchedulingDefaultAlgorithm(ctx context.Context, schedFwk framework.Framework, podGroupCycleState *framework.CycleState, podGroupInfo *framework.QueuedPodGroupInfo, postFilterMode podGroupPostFilterMode) podGroupAlgorithmResult { - placementCycleState := framework.NewCycleState() - placementCycleState.SetRecordPluginMetrics(true) - placementCycleState.SetPodGroupSchedulingCycle(podGroupCycleState) - +func (sched *Scheduler) podGroupSchedulingDefaultAlgorithm(ctx context.Context, schedFwk framework.Framework, placementCycleState *framework.CycleState, podGroupInfo *framework.QueuedPodGroupInfo, postFilterMode podGroupPostFilterMode) podGroupAlgorithmResult { result := podGroupAlgorithmResult{ podResults: make([]algorithmResult, 0, len(podGroupInfo.QueuedPodInfos)), status: fwk.NewStatus(fwk.Unschedulable).WithError(errPodGroupUnschedulable), waitingOnPreemption: false, - placementCycleState: placementCycleState, } logger := klog.FromContext(ctx) @@ -387,7 +388,7 @@ func (sched *Scheduler) podGroupSchedulingDefaultAlgorithm(ctx context.Context, requiresPreemption := false anyScheduled := false for _, podInfo := range podGroupInfo.QueuedPodInfos { - podResult, revertFn := sched.podGroupPodSchedulingAlgorithm(ctx, schedFwk, podGroupCycleState, placementCycleState, podGroupInfo, podInfo, postFilterMode) + podResult, revertFn := sched.podGroupPodSchedulingAlgorithm(ctx, schedFwk, placementCycleState, podGroupInfo, podInfo, postFilterMode) result.podResults = append(result.podResults, podResult) if revertFn != nil { // We unreserve the pod at the end of the whole algorithm (via defer) because it should be ultimately returned to the queue, @@ -441,9 +442,9 @@ func (sched *Scheduler) podGroupSchedulingDefaultAlgorithm(ctx context.Context, // podGroupPodSchedulingAlgorithm runs a scheduling algorithm for individual pod from a pod group. // It returns the algorithm result together with the revert function. -func (sched *Scheduler) podGroupPodSchedulingAlgorithm(ctx context.Context, schedFwk framework.Framework, podGroupCycleState *framework.CycleState, placementCycleState *framework.CycleState, podGroupInfo *framework.QueuedPodGroupInfo, podInfo *framework.QueuedPodInfo, postFilterMode podGroupPostFilterMode) (algorithmResult, func()) { +func (sched *Scheduler) podGroupPodSchedulingAlgorithm(ctx context.Context, schedFwk framework.Framework, placementCycleState *framework.CycleState, podGroupInfo *framework.QueuedPodGroupInfo, podInfo *framework.QueuedPodInfo, postFilterMode podGroupPostFilterMode) (algorithmResult, func()) { pod := podInfo.Pod - podCtx := initPodSchedulingContext(ctx, pod, podGroupCycleState, placementCycleState, postFilterMode) + podCtx := initPodSchedulingContext(ctx, pod, placementCycleState, postFilterMode) logger := podCtx.logger ctx = klog.NewContext(ctx, logger) start := time.Now() @@ -510,9 +511,11 @@ func completePodGroupAlgorithmResult(ctx context.Context, podGroupInfo *framewor copy(newResults, podGroupResult.podResults) for i := numInResult; i < numInQueue; i++ { pInfo := podGroupInfo.QueuedPodInfos[i] + placementCycleState := framework.NewCycleState() + placementCycleState.SetPodGroupSchedulingCycle(podGroupState) newResults[i] = algorithmResult{ pod: pInfo.Pod, - podCtx: initPodSchedulingContext(ctx, pInfo.Pod, podGroupState, nil, postFilterMode), + podCtx: initPodSchedulingContext(ctx, pInfo.Pod, placementCycleState, postFilterMode), status: podGroupResult.status.Clone(), } } @@ -700,7 +703,7 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context } var anyResult *podGroupAlgorithmResult - successfulResults := make(map[*fwk.Placement]*podGroupAlgorithmResult) + successfulResults := make(map[*fwk.Placement]*placementAlgorithmResult) for _, placement := range placements { logger.V(4).Info("Assuming placement in snapshot", "placement", placement.Name) @@ -710,7 +713,9 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context status: fwk.AsStatus(fmt.Errorf("failed to assume pod group placement: %w", err)), } } - result := sched.podGroupSchedulingDefaultAlgorithm(ctx, schedFwk, podGroupCycleState, podGroupInfo, postFilterMode) + placementCycleState := framework.NewCycleState() + placementCycleState.SetPodGroupSchedulingCycle(podGroupCycleState) + result := sched.podGroupSchedulingDefaultAlgorithm(ctx, schedFwk, placementCycleState, podGroupInfo, postFilterMode) sched.nodeInfoSnapshot.ForgetPlacement() if result.status.IsError() { return result @@ -721,7 +726,10 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context } if result.status.IsSuccess() || result.waitingOnPreemption { - successfulResults[placement] = &result + successfulResults[placement] = &placementAlgorithmResult{ + result: result, + placementCycleState: placementCycleState, + } } } @@ -734,7 +742,7 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context if len(successfulResults) == 1 { for _, result := range successfulResults { - return *result + return result.result } } @@ -745,11 +753,11 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context } } - return *successfulResults[bestPlacement] + return successfulResults[bestPlacement].result } // findBestPlacement uses PlacementScore plugins to determine the best placement based on the scheduling results. -func (sched *Scheduler) findBestPlacement(ctx context.Context, schedFwk framework.Framework, podGroupCycleState fwk.PodGroupCycleState, podGroupInfo *framework.QueuedPodGroupInfo, successfulResults map[*fwk.Placement]*podGroupAlgorithmResult) (*fwk.Placement, *fwk.Status) { +func (sched *Scheduler) findBestPlacement(ctx context.Context, schedFwk framework.Framework, podGroupCycleState fwk.PodGroupCycleState, podGroupInfo *framework.QueuedPodGroupInfo, successfulResults map[*fwk.Placement]*placementAlgorithmResult) (*fwk.Placement, *fwk.Status) { placementPodGroupAssignments := makePodGroupAssignments(successfulResults) scores, status := schedFwk.RunPlacementScorePlugins(ctx, podGroupCycleState, podGroupInfo, placementPodGroupAssignments) @@ -782,10 +790,10 @@ func (sched *Scheduler) findBestPlacement(ctx context.Context, schedFwk framewor return bestScore.Placement, nil } -func makePodGroupAssignments(successfulResults map[*fwk.Placement]*podGroupAlgorithmResult) []*fwk.PodGroupAssignments { +func makePodGroupAssignments(successfulResults map[*fwk.Placement]*placementAlgorithmResult) []*fwk.PodGroupAssignments { placementPodGroupAssignments := make([]*fwk.PodGroupAssignments, 0, len(successfulResults)) for placement, result := range successfulResults { - proposedAssignments := makeProposedAssignments(result) + proposedAssignments := makeProposedAssignments(&result.result) placementPodGroupAssignments = append(placementPodGroupAssignments, &fwk.PodGroupAssignments{ Placement: placement, ProposedAssignments: proposedAssignments, @@ -814,5 +822,10 @@ func (sched *Scheduler) podGroupSchedulingAlgorithm(ctx context.Context, schedFw if utilfeature.DefaultFeatureGate.Enabled(features.TopologyAwareWorkloadScheduling) { return sched.podGroupSchedulingPlacementAlgorithm(podGroupCycleCtx, schedFwk, podGroupCycleState, podGroupInfo, postFilterMode) } - return sched.podGroupSchedulingDefaultAlgorithm(podGroupCycleCtx, schedFwk, podGroupCycleState, podGroupInfo, postFilterMode) + // The non-TAS default algorithm does not evaluate placement candidates, but it + // still runs in a single implicit placement context so placement-scoped + // extension points can use the same state plumbing as TAS. + placementCycleState := framework.NewCycleState() + placementCycleState.SetPodGroupSchedulingCycle(podGroupCycleState) + return sched.podGroupSchedulingDefaultAlgorithm(podGroupCycleCtx, schedFwk, placementCycleState, podGroupInfo, postFilterMode) } diff --git a/pkg/scheduler/schedule_one_podgroup_test.go b/pkg/scheduler/schedule_one_podgroup_test.go index 3dc87d8b9c6..32ffa552885 100644 --- a/pkg/scheduler/schedule_one_podgroup_test.go +++ b/pkg/scheduler/schedule_one_podgroup_test.go @@ -1751,7 +1751,9 @@ func TestSubmitPodGroupAlgorithmResult(t *testing.T) { for i := range tt.algorithmResult.podResults { pod := podGroupInfo.QueuedPodInfos[i].Pod - podCtx := initPodSchedulingContext(ctx, pod, podGroupCycleState, nil, runAllPostFilters) + placementCycleState := framework.NewCycleState() + placementCycleState.SetPodGroupSchedulingCycle(podGroupCycleState) + podCtx := initPodSchedulingContext(ctx, pod, placementCycleState, runAllPostFilters) tt.algorithmResult.podResults[i].podCtx = podCtx } @@ -2200,7 +2202,7 @@ func (mp *fakePlacementPlugin) PlacementScoreExtensions() fwk.PlacementScoreExte return nil } -func (mp *fakePlacementPlugin) ScorePlacement(ctx context.Context, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { +func (mp *fakePlacementPlugin) ScorePlacement(ctx context.Context, state fwk.PlacementCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { return mp.scorePlacementsResult[placement.Name], mp.scorePlacementsStatus[placement.Name] } @@ -2538,7 +2540,6 @@ func TestPodGroupSchedulingPlacementAlgorithm(t *testing.T) { ScheduleResult{}, fwk.Status{}), cmpopts.IgnoreFields(algorithmResult{}, "podCtx", "schedulingDuration"), - cmpopts.IgnoreFields(podGroupAlgorithmResult{}, "placementCycleState"), statusCmpOpt, } @@ -2741,12 +2742,12 @@ func (p *placementStateTracker) Filter(ctx context.Context, state fwk.CycleState return nil } -func (p *placementStateTracker) ScorePlacement(ctx context.Context, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { - if placement.PlacementCycleState == nil { +func (p *placementStateTracker) ScorePlacement(ctx context.Context, state fwk.PlacementCycleState, podGroup fwk.PodGroupInfo, placement *fwk.PodGroupAssignments) (int64, *fwk.Status) { + if state == nil { return 0, fwk.NewStatus(fwk.Error, "PlacementCycleState is nil during ScorePlacement") } - data, err := placement.PlacementCycleState.Read(placementStateKey) + data, err := state.Read(placementStateKey) if err != nil { return 0, fwk.NewStatus(fwk.Error, fmt.Sprintf("failed to read PlacementCycleState for %s: %v", placement.Name, err)) } @@ -2785,7 +2786,7 @@ func TestPlacementCycleStateLifecycle(t *testing.T) { // A single scenario exercises both isolation and continuity: // - Filter writes a node-name marker into PlacementCycleState during each placement's simulation. - // - ScorePlacement reads from PodGroupAssignments.PlacementCycleState after all simulations. + // - ScorePlacement reads from the placement state after all simulations. // Assertions verify: // 1. Each placement's scorer reads only the value its own simulation wrote (isolation). // 2. Data written during each placement's simulation remains readable during its scoring (continuity from simulation to scoring). diff --git a/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go b/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go index fdf41701ba6..9e4402730d9 100644 --- a/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go +++ b/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go @@ -124,6 +124,9 @@ type PlacementCycleState interface { // should be recorded. // This function is mostly for the scheduling framework runtime, plugins usually don't have to use it. ShouldRecordPluginMetrics() bool + // GetPodGroupSchedulingCycle gets the cycle state of the PodGroup for this Placement. + // This should be only used when GenericWorkload feature flag is enabled. + GetPodGroupSchedulingCycle() PodGroupCycleState // Read retrieves data with the given "key" from PlacementCycleState. If the key is not // present, ErrNotFound is returned. // diff --git a/staging/src/k8s.io/kube-scheduler/framework/interface.go b/staging/src/k8s.io/kube-scheduler/framework/interface.go index 08d8ec0481b..80b2f2ff388 100644 --- a/staging/src/k8s.io/kube-scheduler/framework/interface.go +++ b/staging/src/k8s.io/kube-scheduler/framework/interface.go @@ -785,6 +785,8 @@ type PlacementScoreExtensions interface { // NormalizePlacementScore is called for all placement scores produced by the same plugin's "ScorePlacement" // method. A successful run of NormalizePlacementScore will update the scores list and return // a success status. + // It receives PodGroupCycleState because normalization operates across all candidate placements, + // not within a single PlacementCycleState. NormalizePlacementScore(ctx context.Context, state PodGroupCycleState, podGroup PodGroupInfo, placementScores []PlacementScore) *Status } @@ -795,9 +797,10 @@ type PlacementScorePlugin interface { // ScorePlacement calculates a score for a given Placement. // This function is called only for Placements that have been deemed feasible for the sufficient number of pods in the PodGroup scheduling cycle. // The PodGroupAssignments indicates the node assigned to each pod within this Placement. + // The state is scoped to the Placement being scored and can be used to access the owning PodGroup cycle state. // The returned score is a int64 with higher scores generally indicating more preferable Placements. // Plugins can implement various scoring strategies, such as bin packing to minimize resource fragmentation. - ScorePlacement(ctx context.Context, state PodGroupCycleState, podGroup PodGroupInfo, placement *PodGroupAssignments) (int64, *Status) + ScorePlacement(ctx context.Context, state PlacementCycleState, podGroup PodGroupInfo, placement *PodGroupAssignments) (int64, *Status) // PlacementScoreExtensions returns a PlacementScoreExtensions interface if it implements one, or nil if does not. PlacementScoreExtensions() PlacementScoreExtensions From 4103e030ebc0883edd2368f9c2c8b21c6cd5a8ef Mon Sep 17 00:00:00 2001 From: wtravO Date: Sat, 16 May 2026 21:33:57 -0400 Subject: [PATCH 04/10] removed dead defensive if not nil check for placementCycleState --- pkg/scheduler/schedule_one_podgroup.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/scheduler/schedule_one_podgroup.go b/pkg/scheduler/schedule_one_podgroup.go index 0b3ea8ece71..e35e5d1e730 100644 --- a/pkg/scheduler/schedule_one_podgroup.go +++ b/pkg/scheduler/schedule_one_podgroup.go @@ -200,10 +200,7 @@ func initPodSchedulingContext(ctx context.Context, pod *v1.Pod, placementCycleSt podsToActivate := framework.NewPodsToActivate() state.Write(framework.PodsToActivateKey, podsToActivate) - var podGroupCycleState fwk.PodGroupCycleState - if placementCycleState != nil { - podGroupCycleState = placementCycleState.GetPodGroupSchedulingCycle() - } + podGroupCycleState := placementCycleState.GetPodGroupSchedulingCycle() // Marks this cycle as a pod group scheduling cycle. state.SetPodGroupSchedulingCycle(podGroupCycleState) // Set the placement cycle state so per-pod plugins can access placement-scoped data. From 8370c09875b11366a76c5fcb36d0c7bff4af6856 Mon Sep 17 00:00:00 2001 From: wtravO Date: Fri, 22 May 2026 21:29:14 -0400 Subject: [PATCH 05/10] remove the explanatory NormalizePlacementScore comment --- staging/src/k8s.io/kube-scheduler/framework/interface.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/staging/src/k8s.io/kube-scheduler/framework/interface.go b/staging/src/k8s.io/kube-scheduler/framework/interface.go index 80b2f2ff388..e90744b7088 100644 --- a/staging/src/k8s.io/kube-scheduler/framework/interface.go +++ b/staging/src/k8s.io/kube-scheduler/framework/interface.go @@ -785,8 +785,6 @@ type PlacementScoreExtensions interface { // NormalizePlacementScore is called for all placement scores produced by the same plugin's "ScorePlacement" // method. A successful run of NormalizePlacementScore will update the scores list and return // a success status. - // It receives PodGroupCycleState because normalization operates across all candidate placements, - // not within a single PlacementCycleState. NormalizePlacementScore(ctx context.Context, state PodGroupCycleState, podGroup PodGroupInfo, placementScores []PlacementScore) *Status } From 150f03cb2f9b14285b5f098741de3f42bf86a46c Mon Sep 17 00:00:00 2001 From: wtravO Date: Fri, 22 May 2026 21:47:01 -0400 Subject: [PATCH 06/10] Removed SetPodGroupSchedulingCycle from CycleState interface --- staging/src/k8s.io/kube-scheduler/framework/cycle_state.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go b/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go index 9e4402730d9..d0b28a216be 100644 --- a/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go +++ b/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go @@ -96,14 +96,11 @@ type CycleState interface { // IsPodGroupSchedulingCycle returns true if this cycle is a pod group scheduling cycle. // If set to false, it means that the pod referencing this CycleState either passed the pod group cycle // or doesn't belong to any pod group. - // This field can only be set to true when GenericWorkload feature flag is enabled. + // This field can only be true when GenericWorkload feature flag is enabled. IsPodGroupSchedulingCycle() bool // GetPodGroupSchedulingCycle gets the cycle state of the PodGroup for a Pod. // This should be only used when GenericWorkload feature flag is enabled. GetPodGroupSchedulingCycle() PodGroupCycleState - // SetPodGroupSchedulingCycle sets the cycle state of the PodGroup for a Pod. - // This should be only used when GenericWorkload feature flag is enabled. - SetPodGroupSchedulingCycle(PodGroupCycleState) // GetPlacementCycleState gets the cycle state of the current Placement for a Pod. // Returns nil if this pod is not being scheduled within a placement context. // This should be only used when GenericWorkload feature flag is enabled. From 101af7309c73dedb2ba6d93cadc26731d7f47061 Mon Sep 17 00:00:00 2001 From: wtravO Date: Fri, 22 May 2026 22:07:09 -0400 Subject: [PATCH 07/10] Keep PlacementCycleState out of PodGroupAssignments --- pkg/scheduler/framework/interface.go | 4 +-- pkg/scheduler/framework/runtime/framework.go | 17 ++++++++-- .../framework/runtime/framework_test.go | 10 +++--- pkg/scheduler/schedule_one_podgroup.go | 32 ++++++++----------- pkg/scheduler/schedule_one_podgroup_test.go | 1 + .../kube-scheduler/framework/cycle_state.go | 2 +- .../k8s.io/kube-scheduler/framework/types.go | 3 -- 7 files changed, 39 insertions(+), 30 deletions(-) diff --git a/pkg/scheduler/framework/interface.go b/pkg/scheduler/framework/interface.go index faf519658b5..9708c8c74ba 100644 --- a/pkg/scheduler/framework/interface.go +++ b/pkg/scheduler/framework/interface.go @@ -308,8 +308,8 @@ type Framework interface { // It returns a list that stores scores from each plugin and total score for each Placement. // It also returns *Status, which is set to non-success if any of the plugins returns // a non-success status. - // Each PodGroupAssignments carries the PlacementCycleState passed to ScorePlacement for that Placement. - RunPlacementScorePlugins(ctx context.Context, state fwk.PodGroupCycleState, podGroupInfo fwk.PodGroupInfo, placements []*fwk.PodGroupAssignments) (ns []fwk.PlacementPluginScores, status *fwk.Status) + // Each PlacementCycleState is passed to ScorePlacement for the PodGroupAssignments at the same index. + RunPlacementScorePlugins(ctx context.Context, state fwk.PodGroupCycleState, podGroupInfo fwk.PodGroupInfo, placements []*fwk.PodGroupAssignments, placementStates []fwk.PlacementCycleState) (ns []fwk.PlacementPluginScores, status *fwk.Status) // HasFilterPlugins returns true if at least one Filter plugin is defined. HasFilterPlugins() bool diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index bf408adb8ab..46fba1bdbb3 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -1483,12 +1483,25 @@ func (f *frameworkImpl) runScoreExtension(ctx context.Context, pl fwk.ScorePlugi // a non-success status. // // This function mostly duplicates RunScorePlugins. Any changes to it should likely be reflected in both places. -func (f *frameworkImpl) RunPlacementScorePlugins(ctx context.Context, state fwk.PodGroupCycleState, podGroupInfo fwk.PodGroupInfo, podGroupAssignments []*fwk.PodGroupAssignments) (ps []fwk.PlacementPluginScores, status *fwk.Status) { +func (f *frameworkImpl) RunPlacementScorePlugins(ctx context.Context, state fwk.PodGroupCycleState, podGroupInfo fwk.PodGroupInfo, podGroupAssignments []*fwk.PodGroupAssignments, placementStates []fwk.PlacementCycleState) (ps []fwk.PlacementPluginScores, status *fwk.Status) { startTime := time.Now() defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(metrics.PlacementScore, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) }() + if len(podGroupAssignments) != len(placementStates) { + return nil, fwk.AsStatus(fmt.Errorf("expected one PlacementCycleState per PodGroupAssignments, got %d placement states for %d assignments", len(placementStates), len(podGroupAssignments))) + } + for i, placementState := range placementStates { + if placementState == nil { + placementName := "" + if podGroupAssignments[i] != nil && podGroupAssignments[i].Placement != nil { + placementName = podGroupAssignments[i].Placement.Name + } + return nil, fwk.AsStatus(fmt.Errorf("missing PlacementCycleState for placement %q", placementName)) + } + } + allPlacementPluginScores := make([]fwk.PlacementPluginScores, len(podGroupAssignments)) numPlugins := len(f.placementScorePlugins) plugins := make([]fwk.PlacementScorePlugin, 0, numPlugins) @@ -1520,7 +1533,7 @@ func (f *frameworkImpl) RunPlacementScorePlugins(ctx context.Context, state fwk. logger := klog.LoggerWithName(logger, pl.Name()) ctx = klog.NewContext(ctx, logger) } - s, status := f.runPlacementScorePlugin(ctx, pl, pga.PlacementCycleState, podGroupInfo, pga) + s, status := f.runPlacementScorePlugin(ctx, pl, placementStates[index], podGroupInfo, pga) if !status.IsSuccess() { err := fmt.Errorf("plugin %q failed with: %w", pl.Name(), status.AsError()) errCh.SendWithCancel(err, cancel) diff --git a/pkg/scheduler/framework/runtime/framework_test.go b/pkg/scheduler/framework/runtime/framework_test.go index 1ec08e63f05..8116da4fe35 100644 --- a/pkg/scheduler/framework/runtime/framework_test.go +++ b/pkg/scheduler/framework/runtime/framework_test.go @@ -3666,7 +3666,7 @@ func TestRecordingMetrics(t *testing.T) { { name: "PlacementScore - Success", action: func(ctx context.Context, f framework.Framework) { - f.RunPlacementScorePlugins(ctx, state, nil, []*fwk.PodGroupAssignments{{Placement: &fwk.Placement{}, PlacementCycleState: state}}) + f.RunPlacementScorePlugins(ctx, state, nil, []*fwk.PodGroupAssignments{{Placement: &fwk.Placement{}}}, []fwk.PlacementCycleState{state}) }, wantExtensionPoint: "PlacementScore", wantStatus: fwk.Success, @@ -3750,7 +3750,7 @@ func TestRecordingMetrics(t *testing.T) { { name: "PlacementScore - Error", action: func(ctx context.Context, f framework.Framework) { - f.RunPlacementScorePlugins(ctx, state, nil, []*fwk.PodGroupAssignments{{Placement: &fwk.Placement{}, PlacementCycleState: state}}) + f.RunPlacementScorePlugins(ctx, state, nil, []*fwk.PodGroupAssignments{{Placement: &fwk.Placement{}}}, []fwk.PlacementCycleState{state}) }, inject: injectedResult{PlacementScoreStatus: int(fwk.Error)}, wantExtensionPoint: "PlacementScore", @@ -4682,11 +4682,13 @@ func TestRunPlacementScorePlugins(t *testing.T) { t.Fatalf("Unexpected error during calling NewFramework, got %v", err) } assumedPlacements := make([]*fwk.PodGroupAssignments, len(placements)) + placementStates := make([]fwk.PlacementCycleState, len(placements)) for i := range placements { - assumedPlacements[i] = &fwk.PodGroupAssignments{Placement: placements[i], PlacementCycleState: framework.NewCycleState()} + assumedPlacements[i] = &fwk.PodGroupAssignments{Placement: placements[i]} + placementStates[i] = framework.NewCycleState() } - result, status := fw.RunPlacementScorePlugins(ctx, framework.NewCycleState(), nil, assumedPlacements) + result, status := fw.RunPlacementScorePlugins(ctx, framework.NewCycleState(), nil, assumedPlacements, placementStates) if status.Code() != tt.wantStatusCode { t.Errorf("got status code %s, want %s", status.Code(), tt.wantStatusCode) } diff --git a/pkg/scheduler/schedule_one_podgroup.go b/pkg/scheduler/schedule_one_podgroup.go index e35e5d1e730..1d93a190426 100644 --- a/pkg/scheduler/schedule_one_podgroup.go +++ b/pkg/scheduler/schedule_one_podgroup.go @@ -361,10 +361,7 @@ type podGroupAlgorithmResult struct { // waitingOnPreemption indicates whether this pod group requires or is waiting for preemption to complete. // This can only be set to true when the status is Unschedulable. waitingOnPreemption bool -} - -type placementAlgorithmResult struct { - result podGroupAlgorithmResult + // placementCycleState is used to score this result as one placement candidate. placementCycleState fwk.PlacementCycleState } @@ -377,6 +374,7 @@ func (sched *Scheduler) podGroupSchedulingDefaultAlgorithm(ctx context.Context, podResults: make([]algorithmResult, 0, len(podGroupInfo.QueuedPodInfos)), status: fwk.NewStatus(fwk.Unschedulable).WithError(errPodGroupUnschedulable), waitingOnPreemption: false, + placementCycleState: placementCycleState, } logger := klog.FromContext(ctx) @@ -700,7 +698,7 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context } var anyResult *podGroupAlgorithmResult - successfulResults := make(map[*fwk.Placement]*placementAlgorithmResult) + successfulResults := make(map[*fwk.Placement]*podGroupAlgorithmResult) for _, placement := range placements { logger.V(4).Info("Assuming placement in snapshot", "placement", placement.Name) @@ -723,10 +721,7 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context } if result.status.IsSuccess() || result.waitingOnPreemption { - successfulResults[placement] = &placementAlgorithmResult{ - result: result, - placementCycleState: placementCycleState, - } + successfulResults[placement] = &result } } @@ -739,7 +734,7 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context if len(successfulResults) == 1 { for _, result := range successfulResults { - return result.result + return *result } } @@ -750,14 +745,14 @@ func (sched *Scheduler) podGroupSchedulingPlacementAlgorithm(ctx context.Context } } - return successfulResults[bestPlacement].result + return *successfulResults[bestPlacement] } // findBestPlacement uses PlacementScore plugins to determine the best placement based on the scheduling results. -func (sched *Scheduler) findBestPlacement(ctx context.Context, schedFwk framework.Framework, podGroupCycleState fwk.PodGroupCycleState, podGroupInfo *framework.QueuedPodGroupInfo, successfulResults map[*fwk.Placement]*placementAlgorithmResult) (*fwk.Placement, *fwk.Status) { - placementPodGroupAssignments := makePodGroupAssignments(successfulResults) +func (sched *Scheduler) findBestPlacement(ctx context.Context, schedFwk framework.Framework, podGroupCycleState fwk.PodGroupCycleState, podGroupInfo *framework.QueuedPodGroupInfo, successfulResults map[*fwk.Placement]*podGroupAlgorithmResult) (*fwk.Placement, *fwk.Status) { + placementPodGroupAssignments, placementStates := makePodGroupAssignments(successfulResults) - scores, status := schedFwk.RunPlacementScorePlugins(ctx, podGroupCycleState, podGroupInfo, placementPodGroupAssignments) + scores, status := schedFwk.RunPlacementScorePlugins(ctx, podGroupCycleState, podGroupInfo, placementPodGroupAssignments, placementStates) if !status.IsSuccess() { return nil, status } @@ -787,17 +782,18 @@ func (sched *Scheduler) findBestPlacement(ctx context.Context, schedFwk framewor return bestScore.Placement, nil } -func makePodGroupAssignments(successfulResults map[*fwk.Placement]*placementAlgorithmResult) []*fwk.PodGroupAssignments { +func makePodGroupAssignments(successfulResults map[*fwk.Placement]*podGroupAlgorithmResult) ([]*fwk.PodGroupAssignments, []fwk.PlacementCycleState) { placementPodGroupAssignments := make([]*fwk.PodGroupAssignments, 0, len(successfulResults)) + placementStates := make([]fwk.PlacementCycleState, 0, len(successfulResults)) for placement, result := range successfulResults { - proposedAssignments := makeProposedAssignments(&result.result) + proposedAssignments := makeProposedAssignments(result) placementPodGroupAssignments = append(placementPodGroupAssignments, &fwk.PodGroupAssignments{ Placement: placement, ProposedAssignments: proposedAssignments, - PlacementCycleState: result.placementCycleState, }) + placementStates = append(placementStates, result.placementCycleState) } - return placementPodGroupAssignments + return placementPodGroupAssignments, placementStates } // makeProposedAssignments builds a list of proposedAssignments from the result of a pod group scheduling attempt. diff --git a/pkg/scheduler/schedule_one_podgroup_test.go b/pkg/scheduler/schedule_one_podgroup_test.go index 32ffa552885..bb3705cf946 100644 --- a/pkg/scheduler/schedule_one_podgroup_test.go +++ b/pkg/scheduler/schedule_one_podgroup_test.go @@ -2539,6 +2539,7 @@ func TestPodGroupSchedulingPlacementAlgorithm(t *testing.T) { algorithmResult{}, ScheduleResult{}, fwk.Status{}), + cmpopts.IgnoreFields(podGroupAlgorithmResult{}, "placementCycleState"), cmpopts.IgnoreFields(algorithmResult{}, "podCtx", "schedulingDuration"), statusCmpOpt, } diff --git a/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go b/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go index d0b28a216be..6d3af7646c2 100644 --- a/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go +++ b/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go @@ -113,7 +113,7 @@ type CycleState interface { // PlacementCycleState provides a mechanism for plugins to store and retrieve arbitrary data // scoped to a single placement candidate within a pod group scheduling cycle. // Data stored in PlacementCycleState is shared across all pods scheduled within the same -// placement iteration and is preserved through to the placement scoring phase via PodGroupAssignments. +// placement iteration and is passed to ScorePlacement for that placement. // PlacementCycleState does not provide any data protection, as all plugins are assumed to be // trusted. type PlacementCycleState interface { diff --git a/staging/src/k8s.io/kube-scheduler/framework/types.go b/staging/src/k8s.io/kube-scheduler/framework/types.go index d6e2f062bf5..c5df461b3d7 100644 --- a/staging/src/k8s.io/kube-scheduler/framework/types.go +++ b/staging/src/k8s.io/kube-scheduler/framework/types.go @@ -682,9 +682,6 @@ type PodGroupAssignments struct { // during the pod group scheduling cycle. // The pods are guaranteed to also be present in the PodGroupInfo. ProposedAssignments []ProposedAssignment - // PlacementCycleState holds the state that was accumulated during the simulation of this placement. - // Placement score plugins can use this to access per-placement cached data. - PlacementCycleState PlacementCycleState } // NodeAllocatableDRAClaimState holds information about a node allocatable resource DRA claim's allocation on a node. From bb6664b1e2429b6ee05736f2fecae3a1e1eafa23 Mon Sep 17 00:00:00 2001 From: wtravO Date: Tue, 26 May 2026 15:52:50 -0400 Subject: [PATCH 08/10] Address review feedback --- pkg/scheduler/framework/runtime/framework.go | 13 +++------- pkg/scheduler/schedule_one_podgroup.go | 2 +- pkg/scheduler/schedule_one_podgroup_test.go | 27 ++++++-------------- 3 files changed, 13 insertions(+), 29 deletions(-) diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index 46fba1bdbb3..de3a5dfbf12 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -1482,6 +1482,10 @@ func (f *frameworkImpl) runScoreExtension(ctx context.Context, pl fwk.ScorePlugi // It also returns *Status, which is set to non-success if any of the plugins returns // a non-success status. // +// placementStates holds the per-placement cycle state for each entry in podGroupAssignments +// (1:1 by index): placementStates[i] is the state with which podGroupAssignments[i] was +// processed, and is passed to ScorePlacement for that placement. +// // This function mostly duplicates RunScorePlugins. Any changes to it should likely be reflected in both places. func (f *frameworkImpl) RunPlacementScorePlugins(ctx context.Context, state fwk.PodGroupCycleState, podGroupInfo fwk.PodGroupInfo, podGroupAssignments []*fwk.PodGroupAssignments, placementStates []fwk.PlacementCycleState) (ps []fwk.PlacementPluginScores, status *fwk.Status) { startTime := time.Now() @@ -1492,15 +1496,6 @@ func (f *frameworkImpl) RunPlacementScorePlugins(ctx context.Context, state fwk. if len(podGroupAssignments) != len(placementStates) { return nil, fwk.AsStatus(fmt.Errorf("expected one PlacementCycleState per PodGroupAssignments, got %d placement states for %d assignments", len(placementStates), len(podGroupAssignments))) } - for i, placementState := range placementStates { - if placementState == nil { - placementName := "" - if podGroupAssignments[i] != nil && podGroupAssignments[i].Placement != nil { - placementName = podGroupAssignments[i].Placement.Name - } - return nil, fwk.AsStatus(fmt.Errorf("missing PlacementCycleState for placement %q", placementName)) - } - } allPlacementPluginScores := make([]fwk.PlacementPluginScores, len(podGroupAssignments)) numPlugins := len(f.placementScorePlugins) diff --git a/pkg/scheduler/schedule_one_podgroup.go b/pkg/scheduler/schedule_one_podgroup.go index 1d93a190426..4387babd944 100644 --- a/pkg/scheduler/schedule_one_podgroup.go +++ b/pkg/scheduler/schedule_one_podgroup.go @@ -361,7 +361,7 @@ type podGroupAlgorithmResult struct { // waitingOnPreemption indicates whether this pod group requires or is waiting for preemption to complete. // This can only be set to true when the status is Unschedulable. waitingOnPreemption bool - // placementCycleState is used to score this result as one placement candidate. + // placementCycleState is the state with which this placement was processed. placementCycleState fwk.PlacementCycleState } diff --git a/pkg/scheduler/schedule_one_podgroup_test.go b/pkg/scheduler/schedule_one_podgroup_test.go index bb3705cf946..c1c209232c3 100644 --- a/pkg/scheduler/schedule_one_podgroup_test.go +++ b/pkg/scheduler/schedule_one_podgroup_test.go @@ -2783,6 +2783,7 @@ func TestPlacementCycleStateLifecycle(t *testing.T) { featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ features.TopologyAwareWorkloadScheduling: true, features.GenericWorkload: true, + features.GangScheduling: true, }) // A single scenario exercises both isolation and continuity: @@ -2870,25 +2871,13 @@ func TestPlacementCycleStateLifecycle(t *testing.T) { t.Fatalf("Expected success, got: %v", result.status) } - tracker.mu.Lock() - defer tracker.mu.Unlock() - - // Continuity: ScorePlacement must have been called and able to read simulation data. - // ScorePlacement returns an error status if the read fails, so a failed read would - // already have been caught by the result.status check above. - for _, placementName := range []string{"placementA", "placementB"} { - if _, ok := tracker.scoreReadValues[placementName]; !ok { - t.Fatalf("placement %s: ScorePlacement was not called", placementName) - } - } - - // Isolation: each placement's scorer must read only what its own simulation wrote. - // placementA simulated on node1, placementB simulated on node2. - if v := tracker.scoreReadValues["placementA"]; v != "node1" { - t.Errorf("placementA: scorer read %q, want %q (isolation violation if it read placementB's value)", v, "node1") - } - if v := tracker.scoreReadValues["placementB"]; v != "node2" { - t.Errorf("placementB: scorer read %q, want %q (isolation violation if it read placementA's value)", v, "node2") + // Each placement's scorer must read only what its own simulation wrote + // (placementA simulated on node1, placementB on node2). This proves both: + // - Continuity: data written during a placement's simulation is readable during its scoring. + // - Isolation: a placement's scorer does not see another placement's writes. + expectedScoreReadValues := map[string]string{"placementA": "node1", "placementB": "node2"} + if diff := cmp.Diff(expectedScoreReadValues, tracker.scoreReadValues); diff != "" { + t.Errorf("Unexpected scoreReadValues (-want,+got)\n%s", diff) } } From b968273f03bd1aaa685597f235297c47b62f4b69 Mon Sep 17 00:00:00 2001 From: wtravO Date: Wed, 27 May 2026 10:44:44 -0400 Subject: [PATCH 09/10] Pass PlacementCycleState to PlacementFeasible plugins --- pkg/scheduler/framework/interface.go | 4 ++-- .../framework/plugins/gangscheduling/gangscheduling.go | 4 ++-- pkg/scheduler/framework/runtime/framework.go | 4 ++-- pkg/scheduler/framework/runtime/framework_test.go | 4 ++-- pkg/scheduler/schedule_one_podgroup_test.go | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pkg/scheduler/framework/interface.go b/pkg/scheduler/framework/interface.go index 9708c8c74ba..9437d5a9bde 100644 --- a/pkg/scheduler/framework/interface.go +++ b/pkg/scheduler/framework/interface.go @@ -192,7 +192,7 @@ type PlacementFeasiblePlugin interface { // The scheduler will give up this placement and won't even evaluate remaining pods. The placement will remain eligible for preemption. // Return Success status if the pod group can be scheduled in the current partially evaluated placement. // After returning Success, the plugin should keep returning Success for the remaining pods. - PlacementFeasible(ctx context.Context, placementCycleState fwk.PodGroupCycleState, podGroupInfo fwk.PodGroupInfo) *fwk.Status + PlacementFeasible(ctx context.Context, placementCycleState fwk.PlacementCycleState, podGroupInfo fwk.PodGroupInfo) *fwk.Status } // Framework manages the set of plugins in use by the scheduling framework. @@ -283,7 +283,7 @@ type Framework interface { // If any plugin returns invalid status, the result will be Error and the remaining plugins won't be invoked. // Otherwise, if at least 1 plugin returns UnschedulableAndUnresolvable, the remaining plugins won't be invoked and the result will be UnschdulableAndUnresolvable. The placement will remain eligible for preemption. // Otherwise, if at least 1 plugin returns Unschedulable, the remaining plugins will be invoked and the result will be Unschedulable. - RunPlacementFeasiblePlugins(ctx context.Context, placementCycleState fwk.PodGroupCycleState, podGroupInfo fwk.PodGroupInfo) *fwk.Status + RunPlacementFeasiblePlugins(ctx context.Context, placementCycleState fwk.PlacementCycleState, podGroupInfo fwk.PodGroupInfo) *fwk.Status // AddWaitingPod creates a waiting pod instance and adds it to the framework. // It takes the pluginsWaitTime map returned by the RunPermitPlugins. diff --git a/pkg/scheduler/framework/plugins/gangscheduling/gangscheduling.go b/pkg/scheduler/framework/plugins/gangscheduling/gangscheduling.go index 6a8e8a72cac..cfc12cacb8d 100644 --- a/pkg/scheduler/framework/plugins/gangscheduling/gangscheduling.go +++ b/pkg/scheduler/framework/plugins/gangscheduling/gangscheduling.go @@ -225,7 +225,7 @@ func (s *placementFeasibleState) Clone() fwk.StateData { } } -func getPlacementFeasibleState(placementCycleState fwk.PodGroupCycleState) *placementFeasibleState { +func getPlacementFeasibleState(placementCycleState fwk.PlacementCycleState) *placementFeasibleState { state, err := placementCycleState.Read(placementFeasibleStateKey) if err != nil { state = &placementFeasibleState{} @@ -237,7 +237,7 @@ func getPlacementFeasibleState(placementCycleState fwk.PodGroupCycleState) *plac // PlacementFeasible is responsible for enforcing the gang's MinCount constraint in the pod group scheduling cycle. // The function will only return success once the gang's MinCount is satisfied or if the pod group is not using gang scheduling policy. // In case there are not enough remaining pods to satisfy the gang's MinCount, it returns UnschedulableAndUnresolvable which will terminate the pod group scheduling cycle early. -func (pl *GangScheduling) PlacementFeasible(ctx context.Context, placementCycleState fwk.PodGroupCycleState, podGroupInfo fwk.PodGroupInfo) *fwk.Status { +func (pl *GangScheduling) PlacementFeasible(ctx context.Context, placementCycleState fwk.PlacementCycleState, podGroupInfo fwk.PodGroupInfo) *fwk.Status { pg, err := pl.podGroupLister.PodGroups(podGroupInfo.GetNamespace()).Get(podGroupInfo.GetName()) if err != nil { return fwk.AsStatus(fmt.Errorf("failed to get podGroup %s to compute gang feasibility: %w", klog.KObj(podGroupInfo), err)) diff --git a/pkg/scheduler/framework/runtime/framework.go b/pkg/scheduler/framework/runtime/framework.go index de3a5dfbf12..9763f1ee211 100644 --- a/pkg/scheduler/framework/runtime/framework.go +++ b/pkg/scheduler/framework/runtime/framework.go @@ -2022,7 +2022,7 @@ func (f *frameworkImpl) runPermitPlugin(ctx context.Context, pl fwk.PermitPlugin // If any plugin returns invalid status, the result will be Error and the remaining plugins won't be invoked. // Otherwise, if at least 1 plugin returns UnschedulableAndUnresolvable, the remaining plugins won't be invoked and the result will be UnschdulableAndUnresolvable. // Otherwise, if at least 1 plugin returns Unschedulable, the remaining plugins will be invoked and the result will be Unschedulable. -func (f *frameworkImpl) RunPlacementFeasiblePlugins(ctx context.Context, placementCycleState fwk.PodGroupCycleState, podGroupInfo fwk.PodGroupInfo) (status *fwk.Status) { +func (f *frameworkImpl) RunPlacementFeasiblePlugins(ctx context.Context, placementCycleState fwk.PlacementCycleState, podGroupInfo fwk.PodGroupInfo) (status *fwk.Status) { startTime := time.Now() defer func() { metrics.FrameworkExtensionPointDuration.WithLabelValues(metrics.PlacementFeasible, status.Code().String(), f.profileName).Observe(metrics.SinceInSeconds(startTime)) @@ -2049,7 +2049,7 @@ func (f *frameworkImpl) RunPlacementFeasiblePlugins(ctx context.Context, placeme return status } -func (f *frameworkImpl) runPlacementFeasiblePlugin(ctx context.Context, pl framework.PlacementFeasiblePlugin, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo) *fwk.Status { +func (f *frameworkImpl) runPlacementFeasiblePlugin(ctx context.Context, pl framework.PlacementFeasiblePlugin, state fwk.PlacementCycleState, podGroup fwk.PodGroupInfo) *fwk.Status { if !state.ShouldRecordPluginMetrics() { return pl.PlacementFeasible(ctx, state, podGroup) } diff --git a/pkg/scheduler/framework/runtime/framework_test.go b/pkg/scheduler/framework/runtime/framework_test.go index 8116da4fe35..a569588efb1 100644 --- a/pkg/scheduler/framework/runtime/framework_test.go +++ b/pkg/scheduler/framework/runtime/framework_test.go @@ -228,7 +228,7 @@ func (pl *TestPlugin) PreFilter(ctx context.Context, state fwk.CycleState, p *v1 return pl.inj.PreFilterResult, fwk.NewStatus(fwk.Code(pl.inj.PreFilterStatus), injectReason) } -func (pl *TestPlugin) PlacementFeasible(ctx context.Context, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo) *fwk.Status { +func (pl *TestPlugin) PlacementFeasible(ctx context.Context, state fwk.PlacementCycleState, podGroup fwk.PodGroupInfo) *fwk.Status { return fwk.NewStatus(fwk.Code(pl.inj.PlacementFeasibleStatus), injectReason) } @@ -795,7 +795,7 @@ type mockPlacementFeasiblePlugin struct { func (p *mockPlacementFeasiblePlugin) Name() string { return p.name } -func (p *mockPlacementFeasiblePlugin) PlacementFeasible(ctx context.Context, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo) *fwk.Status { +func (p *mockPlacementFeasiblePlugin) PlacementFeasible(ctx context.Context, state fwk.PlacementCycleState, podGroup fwk.PodGroupInfo) *fwk.Status { p.called = true return p.status } diff --git a/pkg/scheduler/schedule_one_podgroup_test.go b/pkg/scheduler/schedule_one_podgroup_test.go index c1c209232c3..5e2bc641f0e 100644 --- a/pkg/scheduler/schedule_one_podgroup_test.go +++ b/pkg/scheduler/schedule_one_podgroup_test.go @@ -141,7 +141,7 @@ func (mp *fakePlacementFeasiblePlugin) Name() string { // - The outer slice represents distinct placements (e.g., when evaluating multiple topology placements). // - The inner slice represents the pod-by-pod evaluation within a single placement. // It uses placementCycleState to track how many pods have been evaluated in the current placement. -func (mp *fakePlacementFeasiblePlugin) PlacementFeasible(ctx context.Context, placementCycleState fwk.PodGroupCycleState, podGroupInfo fwk.PodGroupInfo) *fwk.Status { +func (mp *fakePlacementFeasiblePlugin) PlacementFeasible(ctx context.Context, placementCycleState fwk.PlacementCycleState, podGroupInfo fwk.PodGroupInfo) *fwk.Status { // If no mock statuses are configured, always succeed. if len(mp.placementFeasibleStatuses) == 0 { return nil From c10f1fd4959a3d9f8582871b6c69791206e03729 Mon Sep 17 00:00:00 2001 From: wtravO Date: Wed, 27 May 2026 10:47:58 -0400 Subject: [PATCH 10/10] Moved GetPodGroupSchedulingCycle to last method for consistency --- staging/src/k8s.io/kube-scheduler/framework/cycle_state.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go b/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go index 6d3af7646c2..3869ca10ce6 100644 --- a/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go +++ b/staging/src/k8s.io/kube-scheduler/framework/cycle_state.go @@ -121,9 +121,6 @@ type PlacementCycleState interface { // should be recorded. // This function is mostly for the scheduling framework runtime, plugins usually don't have to use it. ShouldRecordPluginMetrics() bool - // GetPodGroupSchedulingCycle gets the cycle state of the PodGroup for this Placement. - // This should be only used when GenericWorkload feature flag is enabled. - GetPodGroupSchedulingCycle() PodGroupCycleState // Read retrieves data with the given "key" from PlacementCycleState. If the key is not // present, ErrNotFound is returned. // @@ -137,6 +134,9 @@ type PlacementCycleState interface { // // See PlacementCycleState for notes on concurrency. Delete(key StateKey) + // GetPodGroupSchedulingCycle gets the cycle state of the PodGroup for this Placement. + // This should be only used when GenericWorkload feature flag is enabled. + GetPodGroupSchedulingCycle() PodGroupCycleState } // PodGroupCycleState provides a mechanism for plugins that operate on pod groups to store and retrieve arbitrary data.