From 8f5f69bc70c30c10a8e3cc6b53c9b4a429c8df73 Mon Sep 17 00:00:00 2001 From: Bartosz Date: Thu, 27 Nov 2025 15:11:23 +0000 Subject: [PATCH 1/3] Change scoring function for balanced allocation --- .../noderesources/balanced_allocation.go | 22 +- .../noderesources/balanced_allocation_test.go | 236 ++++++++++-------- .../plugins/noderesources/least_allocated.go | 4 +- .../plugins/noderesources/most_allocated.go | 4 +- .../requested_to_capacity_ratio.go | 6 +- .../noderesources/resource_allocation.go | 39 +-- .../noderesources/resource_allocation_test.go | 36 +-- pkg/scheduler/schedule_one_test.go | 16 +- 8 files changed, 191 insertions(+), 172 deletions(-) diff --git a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go index 035c77b3348..36041fb8d33 100644 --- a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go +++ b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go @@ -191,14 +191,30 @@ func NewBalancedAllocation(_ context.Context, baArgs runtime.Object, h fwk.Handl }, nil } -func balancedResourceScorer(requested, allocable []int64) int64 { +func balancedResourceScorer(requested, allocated, allocatable []int64) int64 { + // Compute the score for the node with pending pod requests + scoreWithPod := balancedResourceScore(requested, allocatable) + // Compute the score for the node without pending pod requests + scoreWithoutPod := balancedResourceScore(allocated, allocatable) + // Score based on the improvement to the node balance (before vs after adding the pod). + // Scores with/without pod are in range [MaxNodeScore/2, MaxNodeScore]. + // Therefore, their difference will be in range [-MaxNodeScore/2, MaxNodeScore/2]. + // This operation brings the range back to [MaxNodeScore/2, MaxNodeScore] + // to be closer to the original implementation that only considered the score after adding the pod. + // If the difference to the balance is small, the score will be around 75. + // If the balance is improved, the score will move towards 100. + // If the balance is decreased, the score will move towards 50. + return fwk.MaxNodeScore/2 + (fwk.MaxNodeScore/2+scoreWithPod-scoreWithoutPod)/2 +} + +func balancedResourceScore(requested, allocatable []int64) int64 { var resourceToFractions []float64 var totalFraction float64 for i := range requested { - if allocable[i] == 0 { + if allocatable[i] == 0 { continue } - fraction := float64(requested[i]) / float64(allocable[i]) + fraction := float64(requested[i]) / float64(allocatable[i]) if fraction > 1 { fraction = 1 } diff --git a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go index 7b0528588a6..2ab523c86b0 100644 --- a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go @@ -84,6 +84,27 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { }, }, } + memoryOnly := v1.PodSpec{ + NodeName: "node1", + Containers: []v1.Container{ + { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("0"), + v1.ResourceMemory: resource.MustParse("2000"), + }, + }, + }, + { + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("0"), + v1.ResourceMemory: resource.MustParse("3000"), + }, + }, + }, + }, + } cpuOnly2 := cpuOnly cpuOnly2.NodeName = "node2" cpuAndMemory := v1.PodSpec{ @@ -137,37 +158,33 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { wantPreScoreStatusCode: fwk.Skip, }, { - // Node1 scores on 0-MaxNodeScore scale - // CPU Fraction: 3000 / 4000= 75% - // Memory Fraction: 5000 / 10000 = 50% - // Node1 std: (0.75 - 0.5) / 2 = 0.125 - // Node1 Score: (1 - 0.125)*MaxNodeScore = 87 - // Node2 scores on 0-MaxNodeScore scale - // CPU Fraction: 3000 / 6000= 50% - // Memory Fraction: 5000/10000 = 50% - // Node2 std: 0 - // Node2 Score: (1-0) * MaxNodeScore = MaxNodeScore + // Node1 + // CPU: 0 -> 3000/4000 (0% -> 75%) + // Memory: 0 -> 5000/10000 (0% -> 50%) + // Score: 68 (100 -> 87) + // Node2 + // CPU: 0 -> 3000/6000 (0% -> 50%) + // Memory: 0 -> 5000/10000 (0% -> 50%) + // Score: 75 (100 -> 100) pod: &v1.Pod{Spec: cpuAndMemory}, nodes: []*v1.Node{makeNode("node1", 4000, 10000, nil), makeNode("node2", 6000, 10000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 87}, {Name: "node2", Score: fwk.MaxNodeScore}}, + expectedList: []fwk.NodeScore{{Name: "node1", Score: 68}, {Name: "node2", Score: 75}}, name: "nothing scheduled, resources requested, differently sized nodes", args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, runPreScore: true, }, { - // Node1 scores on 0-MaxNodeScore scale - // CPU Fraction: 6000 / 10000 = 60% - // Memory Fraction: 5000 / 20000 = 25% - // Node1 std: (0.6 - 0.25) / 2 = 0.175 - // Node1 Score: (1 - 0.175)*MaxNodeScore = 82 - // Node2 scores on 0-MaxNodeScore scale - // CPU Fraction: 6000 / 10000 = 60% - // Memory Fraction: 10000 / 20000 = 50% - // Node2 std: (0.6 - 0.5) / 2 = 0.05 - // Node2 Score: (1 - 0.05)*MaxNodeScore = 95 + // Node1 + // CPU: 3000 -> 6000/10000 (30% -> 60%) + // Memory: 0 -> 5000/20000 (0% -> 25%) + // Score: 73 (85 -> 82) + // Node2 + // CPU: 3000 -> 6000/10000 (30% -> 60%) + // Memory: 5000 -> 10000/20000 (25% -> 50%) + // Score: 74 (97 -> 95) pod: &v1.Pod{Spec: cpuAndMemory}, nodes: []*v1.Node{makeNode("node1", 10000, 20000, nil), makeNode("node2", 10000, 20000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 82}, {Name: "node2", Score: 95}}, + expectedList: []fwk.NodeScore{{Name: "node1", Score: 73}, {Name: "node2", Score: 74}}, name: "resources requested, pods scheduled with resources", pods: []*v1.Pod{ {Spec: cpuOnly}, @@ -177,19 +194,17 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { runPreScore: true, }, { - // Node1 scores on 0-MaxNodeScore scale - // CPU Fraction: 6000 / 10000 = 60% - // Memory Fraction: 5000 / 20000 = 25% - // Node1 std: (0.6 - 0.25) / 2 = 0.175 - // Node1 Score: (1 - 0.175)*MaxNodeScore = 82 - // Node2 scores on 0-MaxNodeScore scale - // CPU Fraction: 6000 / 10000 = 60% - // Memory Fraction: 10000 / 50000 = 20% - // Node2 std: (0.6 - 0.2) / 2 = 0.2 - // Node2 Score: (1 - 0.2)*MaxNodeScore = 80 + // Node1 + // CPU: 3000 -> 6000/10000 (30% -> 60%) + // Memory: 0 -> 5000/20000 (0% -> 25%) + // Score: 73 (85 -> 82) + // Node2 + // CPU: 3000 -> 6000/10000 (30% -> 60%) + // Memory: 5000 -> 10000/50000 (10% -> 20%) + // Score: 70 (90 -> 80) pod: &v1.Pod{Spec: cpuAndMemory}, nodes: []*v1.Node{makeNode("node1", 10000, 20000, nil), makeNode("node2", 10000, 50000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 82}, {Name: "node2", Score: 80}}, + expectedList: []fwk.NodeScore{{Name: "node1", Score: 73}, {Name: "node2", Score: 70}}, name: "resources requested, pods scheduled with resources, differently sized nodes", pods: []*v1.Pod{ {Spec: cpuOnly}, @@ -199,20 +214,36 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { runPreScore: true, }, { - // Node1 scores on 0-MaxNodeScore scale - // CPU Fraction: 6000 / 6000 = 1 - // Memory Fraction: 0 / 10000 = 0 - // Node1 std: (1 - 0) / 2 = 0.5 - // Node1 Score: (1 - 0.5)*MaxNodeScore = 50 - // Node1 Score: MaxNodeScore - (1 - 0) * MaxNodeScore = 0 - // Node2 scores on 0-MaxNodeScore scale - // CPU Fraction: 6000 / 6000 = 1 - // Memory Fraction 5000 / 10000 = 50% - // Node2 std: (1 - 0.5) / 2 = 0.25 - // Node2 Score: (1 - 0.25)*MaxNodeScore = 75 + // Node1 + // CPU: 3000 -> 3000/3000 (100% -> 100%) + // Memory: 0 -> 5000/5000 (0% -> 100%) + // Score: 100 (50 -> 100) + // Node2 + // CPU: 0 -> 0/10000 (0% -> 0%) + // Memory: 0 -> 5000/5000 (0% -> 100%) + // Score: 50 (100 -> 50) + pod: &v1.Pod{Spec: memoryOnly}, + nodes: []*v1.Node{makeNode("node1", 3000, 5000, nil), makeNode("node2", 3000, 5000, nil)}, + expectedList: []fwk.NodeScore{{Name: "node1", Score: 100}, {Name: "node2", Score: 50}}, + name: "resources requested, pods scheduled with resources, nodes to reach min/max score", + pods: []*v1.Pod{ + {Spec: cpuOnly}, + }, + args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, + runPreScore: true, + }, + { + // Node1 + // CPU: 3000 -> 6000/6000 (50% -> 100%) + // Memory: 0 -> 0/10000 (0% -> 0%) + // Score: 62 (75 -> 50) + // Node2 + // CPU: 3000 -> 6000/6000 (50% -> 100%) + // Memory: 5000 -> 5000/10000 (50% -> 50%) + // Score: 62 (100 -> 75) pod: &v1.Pod{Spec: cpuOnly}, nodes: []*v1.Node{makeNode("node1", 6000, 10000, nil), makeNode("node2", 6000, 10000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 50}, {Name: "node2", Score: 75}}, + expectedList: []fwk.NodeScore{{Name: "node1", Score: 62}, {Name: "node2", Score: 62}}, name: "requested resources at node capacity", pods: []*v1.Pod{ {Spec: cpuOnly}, @@ -221,25 +252,23 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, runPreScore: true, }, - // Node1 scores on 0-MaxNodeScore scale - // CPU Fraction: 3000 / 3500 = 85.71% - // Memory Fraction: 5000 / 40000 = 12.5% - // GPU Fraction: 4 / 8 = 0.5% - // Node1 std: sqrt(((0.8571 - 0.503) * (0.8571 - 0.503) + (0.503 - 0.125) * (0.503 - 0.125) + (0.503 - 0.5) * (0.503 - 0.5)) / 3) = 0.3002 - // Node1 Score: (1 - 0.3002)*MaxNodeScore = 70 - // Node2 scores on 0-MaxNodeScore scale - // CPU Fraction: 3000 / 3500 = 85.71% - // Memory Fraction: 5000 / 40000 = 12.5% - // GPU Fraction: 1 / 8 = 12.5% - // Node2 std: sqrt(((0.8571 - 0.378) * (0.8571 - 0.378) + (0.378 - 0.125) * (0.378 - 0.125)) + (0.378 - 0.125) * (0.378 - 0.125)) / 3) = 0.345 - // Node2 Score: (1 - 0.358)*MaxNodeScore = 65 { + // Node1 + // CPU: 3000 -> 3000/3500 (85.7% -> 85.7%) + // Memory: 5000 -> 5000/40000 (12.5% -> 12.5%) + // GPU: 3 -> 4/8 (37.5% -> 50%) + // Score: 75 (69 -> 70) + // Node2 + // CPU: 3000 -> 3000/3500 (85.7% -> 85.7%) + // Memory: 5000 -> 5000/40000 (12.5% -> 12.5%) + // GPU: 0 -> 1/8 (0% -> 12.5%) + // Score: 76 (62 -> 65) pod: st.MakePod().Req(map[v1.ResourceName]string{ v1.ResourceMemory: "0", "nvidia.com/gpu": "1", }).Obj(), nodes: []*v1.Node{makeNode("node1", 3500, 40000, scalarResource), makeNode("node2", 3500, 40000, scalarResource)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 70}, {Name: "node2", Score: 65}}, + expectedList: []fwk.NodeScore{{Name: "node1", Score: 75}, {Name: "node2", Score: 76}}, name: "include scalar resource on a node for balanced resource allocation", pods: []*v1.Pod{ {Spec: cpuAndMemory}, @@ -252,21 +281,19 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { }}, runPreScore: true, }, - // Only one node (node1) has the scalar resource, pod doesn't request the scalar resource and the scalar resource should be skipped for consideration. - // Node1 scores on 0-MaxNodeScore scale - // CPU Fraction: 3000 / 3500 = 85.71% - // Memory Fraction: 5000 / 40000 = 12.5% - // Node1 std: (0.8571 - 0.125) / 2 = 0.36605 - // Node1 Score: (1 - 0.22705)*MaxNodeScore = 63 - // Node2 scores on 0-MaxNodeScore scale - // CPU Fraction: 3000 / 3500 = 85.71% - // Memory Fraction: 5000 / 40000 = 12.5% - // Node2 std: (0.8571 - 0.125) / 2 = 0.36605 - // Node2 Score: (1 - 0.22705)*MaxNodeScore = 63 { + // Node1 + // CPU: 0 -> 3000/3500 (0% -> 85.7%) + // Memory: 0 -> 5000/40000 (0% -> 12.5%) + // GPU: 0 -> 0/8 (scalar resource not requested by pod, ignored) + // Score: 56 (100 -> 63) + // Node2 + // CPU: 0 -> 3000/3500 (0% -> 85.7%) + // Memory: 0 -> 5000/40000 (0% -> 12.5%) + // Score: 56 (100 -> 63) pod: &v1.Pod{Spec: cpuAndMemory}, nodes: []*v1.Node{makeNode("node1", 3500, 40000, scalarResource), makeNode("node2", 3500, 40000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 63}, {Name: "node2", Score: 63}}, + expectedList: []fwk.NodeScore{{Name: "node1", Score: 56}, {Name: "node2", Score: 56}}, name: "node without the scalar resource should skip the scalar resource", pods: []*v1.Pod{}, args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{ @@ -277,19 +304,18 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { runPreScore: true, }, { - // Node1 scores on 0-MaxNodeScore scale - // CPU Fraction: 6000 / 10000 = 60% - // Memory Fraction: 5000 / 20000 = 25% - // Node1 std: (0.6 - 0.25) / 2 = 0.175 - // Node1 Score: (1 - 0.175)*MaxNodeScore = 82 - // Node2 scores on 0-MaxNodeScore scale - // CPU Fraction: 6000 / 10000 = 60% - // Memory Fraction: 10000 / 20000 = 50% - // Node2 std: (0.6 - 0.5) / 2 = 0.05 - // Node2 Score: (1 - 0.05)*MaxNodeScore = 95 + // Whether or not prescore was called, the end result should be the same. + // Node1 + // CPU: 3000 -> 6000/10000 (30% -> 60%) + // Memory: 0 -> 5000/20000 (0% -> 25%) + // Score: 73 (85 -> 82) + // Node2 + // CPU: 3000 -> 6000/10000 (30% -> 60%) + // Memory: 5000 -> 10000/20000 (25% -> 50%) + // Score: 74 (97 -> 95) pod: &v1.Pod{Spec: cpuAndMemory}, nodes: []*v1.Node{makeNode("node1", 10000, 20000, nil), makeNode("node2", 10000, 20000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 82}, {Name: "node2", Score: 95}}, + expectedList: []fwk.NodeScore{{Name: "node1", Score: 73}, {Name: "node2", Score: 74}}, name: "resources requested, pods scheduled with resources if PreScore not called", pods: []*v1.Pod{ {Spec: cpuOnly}, @@ -299,21 +325,19 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { runPreScore: false, }, { - // Node1 scores on 0-MaxNodeScore scale - // CPU Fraction: 3000 / 3500 = 0.8571 - // Memory Fraction: 0 / 40000 = 0 - // DRA Fraction: 1 / 8 = 0.125 - // Fraction mean: (0.8571 + 0 + 0.125) / 3 = 0.3274 - // Node1 std: sqrt(((0.8571 - 0.3274)**2 + (0- 0.3274)**2 + (0.125 - 0.3274)**2) / 3) = 0.378 - // Node1 Score: (1 - 0.378)*MaxNodeScore = 62 - // Node2 scores on 0-MaxNodeScore scale - // CPU Fraction: 3000 / 3500 = 0.8571 - // Memory Fraction: 5000 / 40000 = 0.125 - // Node2 std: (0.8571 - 0.125) / 2 = 0.36605 - // Node2 Score: (1 - 0.36605)*MaxNodeScore = 63 + // Node1 + // CPU: 3000 -> 3000/3500 (86% -> 86%) + // Memory: 0 -> 0/40000 (0% -> 0%) + // DRA: 0 -> 1/8 (0% -> 12%) + // Score: 76 (60 -> 62) + // Node2 + // CPU: 3000 -> 3000/3500 (86% -> 86%) + // Memory: 5000 -> 5000/40000 (12% -> 12%) + // DRA: unsatisfiable + // Score: 75 (63 -> 63) pod: st.MakePod().Req(map[v1.ResourceName]string{extendedResourceDRA: "1"}).Obj(), nodes: []*v1.Node{makeNode("node1", 3500, 40000, nil), makeNode("node2", 3500, 40000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 62}, {Name: "node2", Score: 63}}, + expectedList: []fwk.NodeScore{{Name: "node1", Score: 76}, {Name: "node2", Score: 75}}, name: "include DRA resource on a node for balanced resource allocation", pods: []*v1.Pod{ {Spec: cpuOnly}, @@ -331,21 +355,19 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { runPreScore: true, }, { - // Node1 scores on 0-MaxNodeScore scale - // CPU Fraction: 3000 / 35000 = 0.8571 - // Memory Fraction: 0 / 40000 = 0 - // DRA Fraction: 1 / 8 = 0.125 - // Fraction mean: (0.8571 + 0 + 0.125) / 3 = 0.3274 - // Node1 std: sqrt(((0.8571 - 0.3274)**2 + (0- 0.3274)**2 + (0.125 - 0.3274)**2) / 3) = 0.378 - // Node1 Score: (1 - 0.378)*MaxNodeScore = 62 - // Node2 scores on 0-MaxNodeScore scale - // CPU Fraction: 3000 / 35000 = 0.8571 - // Memory Fraction: 5000 / 40000 = 0.125 - // Node2 std: (0.8571 - 0.125) / 2 = 0.36605 - // Node2 Score: (1 - 0.36605)*MaxNodeScore = 63 + // Node1 + // CPU: 3000 -> 3000/3500 (86% -> 86%) + // Memory: 0 -> 0/40000 (0% -> 0%) + // DRA: 0 -> 1/8 (0% -> 12%) + // Score: 76 (60 -> 62) + // Node2 + // CPU: 3000 -> 3000/3500 (86% -> 86%) + // Memory: 5000 -> 5000/40000 (12% -> 12%) + // DRA: unsatisfiable + // Score: 75 (63 -> 63) pod: st.MakePod().Req(map[v1.ResourceName]string{extendedResourceDRA: "1"}).Obj(), nodes: []*v1.Node{makeNode("node1", 3500, 40000, nil), makeNode("node2", 3500, 40000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 62}, {Name: "node2", Score: 63}}, + expectedList: []fwk.NodeScore{{Name: "node1", Score: 76}, {Name: "node2", Score: 75}}, name: "include DRA resource on a node for balanced resource allocation if PreScore not called", pods: []*v1.Pod{ {Spec: cpuOnly}, diff --git a/pkg/scheduler/framework/plugins/noderesources/least_allocated.go b/pkg/scheduler/framework/plugins/noderesources/least_allocated.go index 20a456d9636..19fa2761d3f 100644 --- a/pkg/scheduler/framework/plugins/noderesources/least_allocated.go +++ b/pkg/scheduler/framework/plugins/noderesources/least_allocated.go @@ -27,8 +27,8 @@ import ( // // Details: // (cpu((capacity-requested)*MaxNodeScore*cpuWeight/capacity) + memory((capacity-requested)*MaxNodeScore*memoryWeight/capacity) + ...)/weightSum -func leastResourceScorer(resources []config.ResourceSpec) func([]int64, []int64) int64 { - return func(requested, allocable []int64) int64 { +func leastResourceScorer(resources []config.ResourceSpec) func([]int64, []int64, []int64) int64 { + return func(requested, _, allocable []int64) int64 { var nodeScore, weightSum int64 for i := range requested { if allocable[i] == 0 { diff --git a/pkg/scheduler/framework/plugins/noderesources/most_allocated.go b/pkg/scheduler/framework/plugins/noderesources/most_allocated.go index 3e694d1b22f..2d514c0540e 100644 --- a/pkg/scheduler/framework/plugins/noderesources/most_allocated.go +++ b/pkg/scheduler/framework/plugins/noderesources/most_allocated.go @@ -27,8 +27,8 @@ import ( // // Details: // (cpu(MaxNodeScore * requested * cpuWeight / capacity) + memory(MaxNodeScore * requested * memoryWeight / capacity) + ...) / weightSum -func mostResourceScorer(resources []config.ResourceSpec) func(requested, allocable []int64) int64 { - return func(requested, allocable []int64) int64 { +func mostResourceScorer(resources []config.ResourceSpec) func(requested, _, allocable []int64) int64 { + return func(requested, _, allocable []int64) int64 { var nodeScore, weightSum int64 for i := range requested { if allocable[i] == 0 { diff --git a/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go b/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go index f6c7838be3e..a15500d4bc3 100644 --- a/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go +++ b/pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go @@ -28,7 +28,7 @@ const maxUtilization = 100 // buildRequestedToCapacityRatioScorerFunction allows users to apply bin packing // on core resources like CPU, Memory as well as extended resources like accelerators. -func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.FunctionShape, resources []config.ResourceSpec) func([]int64, []int64) int64 { +func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.FunctionShape, resources []config.ResourceSpec) func([]int64, []int64, []int64) int64 { rawScoringFunction := helper.BuildBrokenLinearFunction(scoringFunctionShape) resourceScoringFunction := func(requested, capacity int64) int64 { if capacity == 0 || requested > capacity { @@ -37,7 +37,7 @@ func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.Fun return rawScoringFunction(requested * maxUtilization / capacity) } - return func(requested, allocable []int64) int64 { + return func(requested, _, allocable []int64) int64 { var nodeScore, weightSum int64 for i := range requested { if allocable[i] == 0 { @@ -57,7 +57,7 @@ func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.Fun } } -func requestedToCapacityRatioScorer(resources []config.ResourceSpec, shape []config.UtilizationShapePoint) func([]int64, []int64) int64 { +func requestedToCapacityRatioScorer(resources []config.ResourceSpec, shape []config.UtilizationShapePoint) func([]int64, []int64, []int64) int64 { shapes := make([]helper.FunctionShapePoint, 0, len(shape)) for _, point := range shape { shapes = append(shapes, helper.FunctionShapePoint{ diff --git a/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go b/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go index b4f6f16b4bf..a16a01824df 100644 --- a/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go +++ b/pkg/scheduler/framework/plugins/noderesources/resource_allocation.go @@ -58,7 +58,7 @@ type resourceAllocationScorer struct { // used to decide whether to use Requested or NonZeroRequested for // cpu and memory. useRequested bool - scorer func(requested, allocable []int64) int64 + scorer func(requested, allocated, allocatable []int64) int64 resources []config.ResourceSpec draFeatures structured.Features draManager fwk.SharedDRAManager @@ -150,19 +150,27 @@ func (r *resourceAllocationScorer) score( return 0, fwk.NewStatus(fwk.Error, "resources not found") } + allocated := make([]int64, len(r.resources)) requested := make([]int64, len(r.resources)) allocatable := make([]int64, len(r.resources)) for i := range r.resources { - alloc, req := r.calculateResourceAllocatableRequest(ctx, nodeInfo, v1.ResourceName(r.resources[i].Name), podRequests[i], draPreScoreState) - // Only fill the extended resource entry when it's non-zero. - if alloc == 0 { + resource := v1.ResourceName(r.resources[i].Name) + // If it's an extended resource, and the pod doesn't request it. + // We don't fill the resource entry as an implication to bypass scoring on it. + if podRequests[i] == 0 && schedutil.IsScalarResourceName(resource) { continue } - allocatable[i] = alloc - requested[i] = req + nodeAllocatable, nodeAllocated := r.calculateResourceAllocatableRequest(ctx, nodeInfo, resource, draPreScoreState) + // Only fill the extended resource entry when it's non-zero. + if nodeAllocatable == 0 { + continue + } + allocatable[i] = nodeAllocatable + allocated[i] = nodeAllocated + requested[i] = allocated[i] + podRequests[i] } - score := r.scorer(requested, allocatable) + score := r.scorer(requested, allocated, allocatable) if loggerV := logger.V(10); loggerV.Enabled() { // Serializing these maps is costly. loggerV.Info("Listed internal info for allocatable resources, requested resources and score", "pod", @@ -177,12 +185,10 @@ func (r *resourceAllocationScorer) score( // calculateResourceAllocatableRequest returns 2 parameters: // - 1st param: quantity of allocatable resource on the node. // - 2nd param: aggregated quantity of requested resource on the node. -// Note: if it's an extended resource, and the pod doesn't request it, (0, 0) is returned. func (r *resourceAllocationScorer) calculateResourceAllocatableRequest( ctx context.Context, nodeInfo fwk.NodeInfo, resource v1.ResourceName, - podRequest int64, draPreScoreState *draPreScoreState, ) (int64, int64) { requested := nodeInfo.GetNonZeroRequested() @@ -190,18 +196,13 @@ func (r *resourceAllocationScorer) calculateResourceAllocatableRequest( requested = nodeInfo.GetRequested() } - // If it's an extended resource, and the pod doesn't request it. We return (0, 0) - // as an implication to bypass scoring on this resource. - if podRequest == 0 && schedutil.IsScalarResourceName(resource) { - return 0, 0 - } switch resource { case v1.ResourceCPU: - return nodeInfo.GetAllocatable().GetMilliCPU(), (requested.GetMilliCPU() + podRequest) + return nodeInfo.GetAllocatable().GetMilliCPU(), requested.GetMilliCPU() case v1.ResourceMemory: - return nodeInfo.GetAllocatable().GetMemory(), (requested.GetMemory() + podRequest) + return nodeInfo.GetAllocatable().GetMemory(), requested.GetMemory() case v1.ResourceEphemeralStorage: - return nodeInfo.GetAllocatable().GetEphemeralStorage(), (nodeInfo.GetRequested().GetEphemeralStorage() + podRequest) + return nodeInfo.GetAllocatable().GetEphemeralStorage(), nodeInfo.GetRequested().GetEphemeralStorage() default: allocatable, exists := nodeInfo.GetAllocatable().GetScalarResources()[resource] if allocatable == 0 && r.enableDRAExtendedResource && draPreScoreState != nil { @@ -209,11 +210,11 @@ func (r *resourceAllocationScorer) calculateResourceAllocatableRequest( // Calculate allocatable and requested for resources backed by DRA. allocatable, allocated := r.calculateDRAExtendedResourceAllocatableRequest(ctx, nodeInfo.Node(), resource, draPreScoreState) if allocatable > 0 { - return allocatable, allocated + podRequest + return allocatable, allocated } } if exists { - return allocatable, (nodeInfo.GetRequested().GetScalarResources()[resource] + podRequest) + return allocatable, nodeInfo.GetRequested().GetScalarResources()[resource] } } klog.FromContext(ctx).V(10).Info("Requested resource is omitted for node score calculation", "resourceName", resource) diff --git a/pkg/scheduler/framework/plugins/noderesources/resource_allocation_test.go b/pkg/scheduler/framework/plugins/noderesources/resource_allocation_test.go index cd212bb6485..9976a220096 100644 --- a/pkg/scheduler/framework/plugins/noderesources/resource_allocation_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/resource_allocation_test.go @@ -287,25 +287,22 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { node *v1.Node extendedResource v1.ResourceName objects []apiruntime.Object - podRequest int64 expectedAllocatable int64 - expectedRequested int64 + expectedAllocated int64 }{ "device-plugin-resource-feature-disabled": { enableDRAExtendedResource: false, node: st.MakeNode().Name(nodeName).Capacity(map[v1.ResourceName]string{explicitExtendedResource: "4"}).Obj(), extendedResource: explicitExtendedResource, - podRequest: 1, expectedAllocatable: 4, - expectedRequested: 1, + expectedAllocated: 0, }, "device-plugin-resource-feature-enabled": { enableDRAExtendedResource: true, node: st.MakeNode().Name(nodeName).Capacity(map[v1.ResourceName]string{explicitExtendedResource: "4"}).Obj(), extendedResource: explicitExtendedResource, - podRequest: 1, expectedAllocatable: 4, - expectedRequested: 1, + expectedAllocated: 0, }, "DRA-backed-resource-explicit": { enableDRAExtendedResource: true, @@ -315,9 +312,8 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { deviceClassWithExtendResourceName, st.MakeResourceSlice(nodeName, driverName).Device("device-1").Obj(), }, - podRequest: 1, expectedAllocatable: 1, - expectedRequested: 1, + expectedAllocated: 0, }, "DRA-backed-resource-implicit": { enableDRAExtendedResource: true, @@ -331,18 +327,16 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { }, st.MakeResourceSlice(nodeName, driverName).Device("device-1").Obj(), }, - podRequest: 1, expectedAllocatable: 1, - expectedRequested: 1, + expectedAllocated: 0, }, "DRA-backed-resource-no-slices": { enableDRAExtendedResource: true, node: st.MakeNode().Name(nodeName).Obj(), extendedResource: explicitExtendedResource, objects: []apiruntime.Object{deviceClassWithExtendResourceName}, - podRequest: 1, expectedAllocatable: 0, - expectedRequested: 0, + expectedAllocated: 0, }, "DRA-backed-resource-with-allocated-device": { enableDRAExtendedResource: true, @@ -369,9 +363,8 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { }). Obj(), }, - podRequest: 1, expectedAllocatable: 2, - expectedRequested: 2, // 1 allocated + 1 requested + expectedAllocated: 1, // 1 allocated }, "DRA-backed-resource-with-shared-device-allocation": { enableDRAExtendedResource: true, @@ -399,9 +392,8 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { }). Obj(), }, - podRequest: 1, expectedAllocatable: 2, - expectedRequested: 2, // 1 allocated (shared) + 1 requested + expectedAllocated: 1, // 1 allocated (shared) }, "DRA-backed-resource-multiple-devices-mixed-allocation": { enableDRAExtendedResource: true, @@ -447,9 +439,8 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { Obj(), // device-3 remains unallocated }, - podRequest: 1, expectedAllocatable: 3, - expectedRequested: 3, // 2 allocated (1 full + 1 shared) + 1 requested + expectedAllocated: 2, // 2 allocated (1 full + 1 shared) }, "DRA-backed-resource-with-per-device-node-selection": { enableDRAExtendedResource: true, @@ -545,9 +536,8 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { }). Obj(), }, - podRequest: 1, expectedAllocatable: 2, // device-1 matches the test node and device-3 matches via its selector - expectedRequested: 2, // 1 allocated (device-1) + 1 requested + expectedAllocated: 1, // 1 allocated (device-1) }, } @@ -580,12 +570,12 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { } // Test calculateResourceAllocatableRequest API - allocatable, requested := scorer.calculateResourceAllocatableRequest(tCtx, nodeInfo, tc.extendedResource, tc.podRequest, draPreScoreState) + allocatable, allocated := scorer.calculateResourceAllocatableRequest(tCtx, nodeInfo, tc.extendedResource, draPreScoreState) if !cmp.Equal(allocatable, tc.expectedAllocatable) { tCtx.Errorf("Expected allocatable=%v, but got allocatable=%v", tc.expectedAllocatable, allocatable) } - if !cmp.Equal(requested, tc.expectedRequested) { - tCtx.Errorf("Expected requested=%v, but got requested=%v", tc.expectedRequested, requested) + if !cmp.Equal(allocated, tc.expectedAllocated) { + tCtx.Errorf("Expected allocated=%v, but got allocated=%v", tc.expectedAllocated, allocated) } }) } diff --git a/pkg/scheduler/schedule_one_test.go b/pkg/scheduler/schedule_one_test.go index 592ff01a84e..852605f6a27 100644 --- a/pkg/scheduler/schedule_one_test.go +++ b/pkg/scheduler/schedule_one_test.go @@ -3829,13 +3829,8 @@ func TestFindFitPredicateCallCounts(t *testing.T) { } } -// The point of this test is to show that you: -// - get the same priority for a zero-request pod as for a pod with the defaults requests, -// both when the zero-request pod is already on the node and when the zero-request pod -// is the one being scheduled. -// - don't get the same score no matter what we schedule. +// Tests score for pods on nodes with zero-request pods. func TestZeroRequest(t *testing.T) { - // A pod with no resources. We expect spreading to count it as having the default resources. noResources := v1.PodSpec{ Containers: []v1.Container{ {}, @@ -3843,7 +3838,6 @@ func TestZeroRequest(t *testing.T) { } noResources1 := noResources noResources1.NodeName = "node1" - // A pod with the same resources as a 0-request pod gets by default as its resources (for spreading). small := v1.PodSpec{ Containers: []v1.Container{ { @@ -3860,7 +3854,6 @@ func TestZeroRequest(t *testing.T) { } small2 := small small2.NodeName = "node2" - // A larger pod. large := v1.PodSpec{ Containers: []v1.Container{ { @@ -3886,9 +3879,6 @@ func TestZeroRequest(t *testing.T) { name string expectedScore int64 }{ - // The point of these next two tests is to show you get the same priority for a zero-request pod - // as for a pod with the defaults requests, both when the zero-request pod is already on the node - // and when the zero-request pod is the one being scheduled. { pod: &v1.Pod{Spec: noResources}, nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10), makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10)}, @@ -3907,7 +3897,7 @@ func TestZeroRequest(t *testing.T) { {Spec: large1}, {Spec: noResources1}, {Spec: large2}, {Spec: small2}, }, - expectedScore: 150, + expectedScore: 125, }, // The point of this test is to verify that we're not just getting the same score no matter what we schedule. { @@ -3918,7 +3908,7 @@ func TestZeroRequest(t *testing.T) { {Spec: large1}, {Spec: noResources1}, {Spec: large2}, {Spec: small2}, }, - expectedScore: 130, + expectedScore: 105, }, } From 56ca09911f5b0a8ec09b7e36f229039307b9c140 Mon Sep 17 00:00:00 2001 From: Bartosz Date: Mon, 8 Dec 2025 12:12:04 +0000 Subject: [PATCH 2/3] Refactor resource allocation tests to be more readable --- .../noderesources/balanced_allocation_test.go | 345 +++++++----------- .../noderesources/resource_allocation_test.go | 44 +-- pkg/scheduler/schedule_one_test.go | 36 +- 3 files changed, 176 insertions(+), 249 deletions(-) diff --git a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go index 2ab523c86b0..23346916d84 100644 --- a/pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go @@ -21,7 +21,6 @@ import ( "github.com/google/go-cmp/cmp" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" apiruntime "k8s.io/apimachinery/pkg/runtime" utilfeature "k8s.io/apiserver/pkg/util/feature" featuregatetesting "k8s.io/component-base/featuregate/testing" @@ -41,121 +40,30 @@ func TestNodeResourcesBalancedAllocation(t *testing.T) { testNodeResourcesBalancedAllocation(ktesting.Init(t)) } func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { - cpuAndMemoryAndGPU := v1.PodSpec{ - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1000m"), - v1.ResourceMemory: resource.MustParse("2000"), - }, - }, - }, - { - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2000m"), - v1.ResourceMemory: resource.MustParse("3000"), - "nvidia.com/gpu": resource.MustParse("3"), - }, - }, - }, - }, - NodeName: "node1", - } - cpuOnly := v1.PodSpec{ - NodeName: "node1", - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1000m"), - v1.ResourceMemory: resource.MustParse("0"), - }, - }, - }, - { - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2000m"), - v1.ResourceMemory: resource.MustParse("0"), - }, - }, - }, - }, - } - memoryOnly := v1.PodSpec{ - NodeName: "node1", - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("0"), - v1.ResourceMemory: resource.MustParse("2000"), - }, - }, - }, - { - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("0"), - v1.ResourceMemory: resource.MustParse("3000"), - }, - }, - }, - }, - } - cpuOnly2 := cpuOnly - cpuOnly2.NodeName = "node2" - cpuAndMemory := v1.PodSpec{ - NodeName: "node2", - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1000m"), - v1.ResourceMemory: resource.MustParse("2000"), - }, - }, - }, - { - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2000m"), - v1.ResourceMemory: resource.MustParse("3000"), - }, - }, - }, - }, - } - defaultResourceBalancedAllocationSet := []config.ResourceSpec{ {Name: string(v1.ResourceCPU), Weight: 1}, {Name: string(v1.ResourceMemory), Weight: 1}, } - scalarResource := map[string]int64{ - "nvidia.com/gpu": 8, - } tests := []struct { - pod *v1.Pod - pods []*v1.Pod - nodes []*v1.Node - expectedList fwk.NodeScoreList - name string - args config.NodeResourcesBalancedAllocationArgs - runPreScore bool - wantPreScoreStatusCode fwk.Code - draObjects []apiruntime.Object + name string + requestedPod *v1.Pod + nodes []*v1.Node + existingPods []*v1.Pod + expectedScores fwk.NodeScoreList + args config.NodeResourcesBalancedAllocationArgs + runPreScore bool + expectedPreScoreStatusCode fwk.Code + draObjects []apiruntime.Object }{ { // bestEffort pods, skip in PreScore - pod: st.MakePod().Obj(), - nodes: []*v1.Node{makeNode("node1", 4000, 10000, nil), makeNode("node2", 4000, 10000, nil)}, - name: "nothing scheduled, nothing requested, skip in PreScore", - args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, - runPreScore: true, - wantPreScoreStatusCode: fwk.Skip, + name: "nothing scheduled, nothing requested, skip in PreScore", + requestedPod: st.MakePod().Obj(), + nodes: []*v1.Node{makeNode("node1", 4000, 10000, nil), makeNode("node2", 4000, 10000, nil)}, + args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, + runPreScore: true, + expectedPreScoreStatusCode: fwk.Skip, }, { // Node1 @@ -166,12 +74,14 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { // CPU: 0 -> 3000/6000 (0% -> 50%) // Memory: 0 -> 5000/10000 (0% -> 50%) // Score: 75 (100 -> 100) - pod: &v1.Pod{Spec: cpuAndMemory}, - nodes: []*v1.Node{makeNode("node1", 4000, 10000, nil), makeNode("node2", 6000, 10000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 68}, {Name: "node2", Score: 75}}, name: "nothing scheduled, resources requested, differently sized nodes", - args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, - runPreScore: true, + requestedPod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(), + nodes: []*v1.Node{ + makeNode("node1", 4000, 10000, nil), + makeNode("node2", 6000, 10000, nil)}, + expectedScores: []fwk.NodeScore{{Name: "node1", Score: 68}, {Name: "node2", Score: 75}}, + args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, + runPreScore: true, }, { // Node1 @@ -182,16 +92,19 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { // CPU: 3000 -> 6000/10000 (30% -> 60%) // Memory: 5000 -> 10000/20000 (25% -> 50%) // Score: 74 (97 -> 95) - pod: &v1.Pod{Spec: cpuAndMemory}, - nodes: []*v1.Node{makeNode("node1", 10000, 20000, nil), makeNode("node2", 10000, 20000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 73}, {Name: "node2", Score: 74}}, name: "resources requested, pods scheduled with resources", - pods: []*v1.Pod{ - {Spec: cpuOnly}, - {Spec: cpuAndMemory}, + requestedPod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(), + nodes: []*v1.Node{ + makeNode("node1", 10000, 20000, nil), + makeNode("node2", 10000, 20000, nil), }, - args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, - runPreScore: true, + existingPods: []*v1.Pod{ + st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(), + st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(), + }, + expectedScores: []fwk.NodeScore{{Name: "node1", Score: 73}, {Name: "node2", Score: 74}}, + args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, + runPreScore: true, }, { // Node1 @@ -202,16 +115,19 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { // CPU: 3000 -> 6000/10000 (30% -> 60%) // Memory: 5000 -> 10000/50000 (10% -> 20%) // Score: 70 (90 -> 80) - pod: &v1.Pod{Spec: cpuAndMemory}, - nodes: []*v1.Node{makeNode("node1", 10000, 20000, nil), makeNode("node2", 10000, 50000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 73}, {Name: "node2", Score: 70}}, name: "resources requested, pods scheduled with resources, differently sized nodes", - pods: []*v1.Pod{ - {Spec: cpuOnly}, - {Spec: cpuAndMemory}, + requestedPod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(), + nodes: []*v1.Node{ + makeNode("node1", 10000, 20000, nil), + makeNode("node2", 10000, 50000, nil), }, - args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, - runPreScore: true, + existingPods: []*v1.Pod{ + st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(), + st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(), + }, + expectedScores: []fwk.NodeScore{{Name: "node1", Score: 73}, {Name: "node2", Score: 70}}, + args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, + runPreScore: true, }, { // Node1 @@ -222,15 +138,18 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { // CPU: 0 -> 0/10000 (0% -> 0%) // Memory: 0 -> 5000/5000 (0% -> 100%) // Score: 50 (100 -> 50) - pod: &v1.Pod{Spec: memoryOnly}, - nodes: []*v1.Node{makeNode("node1", 3000, 5000, nil), makeNode("node2", 3000, 5000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 100}, {Name: "node2", Score: 50}}, name: "resources requested, pods scheduled with resources, nodes to reach min/max score", - pods: []*v1.Pod{ - {Spec: cpuOnly}, + requestedPod: st.MakePod().Req(map[v1.ResourceName]string{"memory": "2000"}).Req(map[v1.ResourceName]string{"memory": "3000"}).Obj(), + nodes: []*v1.Node{ + makeNode("node1", 3000, 5000, nil), + makeNode("node2", 3000, 5000, nil), }, - args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, - runPreScore: true, + existingPods: []*v1.Pod{ + st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(), + }, + expectedScores: []fwk.NodeScore{{Name: "node1", Score: 100}, {Name: "node2", Score: 50}}, + args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, + runPreScore: true, }, { // Node1 @@ -241,16 +160,19 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { // CPU: 3000 -> 6000/6000 (50% -> 100%) // Memory: 5000 -> 5000/10000 (50% -> 50%) // Score: 62 (100 -> 75) - pod: &v1.Pod{Spec: cpuOnly}, - nodes: []*v1.Node{makeNode("node1", 6000, 10000, nil), makeNode("node2", 6000, 10000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 62}, {Name: "node2", Score: 62}}, name: "requested resources at node capacity", - pods: []*v1.Pod{ - {Spec: cpuOnly}, - {Spec: cpuAndMemory}, + requestedPod: st.MakePod().Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(), + nodes: []*v1.Node{ + makeNode("node1", 6000, 10000, nil), + makeNode("node2", 6000, 10000, nil), }, - args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, - runPreScore: true, + existingPods: []*v1.Pod{ + st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(), + st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(), + }, + expectedScores: []fwk.NodeScore{{Name: "node1", Score: 62}, {Name: "node2", Score: 62}}, + args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, + runPreScore: true, }, { // Node1 @@ -263,17 +185,17 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { // Memory: 5000 -> 5000/40000 (12.5% -> 12.5%) // GPU: 0 -> 1/8 (0% -> 12.5%) // Score: 76 (62 -> 65) - pod: st.MakePod().Req(map[v1.ResourceName]string{ - v1.ResourceMemory: "0", - "nvidia.com/gpu": "1", - }).Obj(), - nodes: []*v1.Node{makeNode("node1", 3500, 40000, scalarResource), makeNode("node2", 3500, 40000, scalarResource)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 75}, {Name: "node2", Score: 76}}, - name: "include scalar resource on a node for balanced resource allocation", - pods: []*v1.Pod{ - {Spec: cpuAndMemory}, - {Spec: cpuAndMemoryAndGPU}, + name: "scalar resource is included in the score computation if pod requests the scalar resource", + requestedPod: st.MakePod().Req(cpuAndMemoryAndGpu("0", "0", "1")).Obj(), + nodes: []*v1.Node{ + makeNode("node1", 3500, 40000, gpu(8)), + makeNode("node2", 3500, 40000, gpu(8)), }, + existingPods: []*v1.Pod{ + st.MakePod().Node("node1").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemoryAndGpu("2000m", "3000", "3")).Obj(), + st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(), + }, + expectedScores: []fwk.NodeScore{{Name: "node1", Score: 75}, {Name: "node2", Score: 76}}, args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{ {Name: string(v1.ResourceCPU), Weight: 1}, {Name: string(v1.ResourceMemory), Weight: 1}, @@ -291,11 +213,14 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { // CPU: 0 -> 3000/3500 (0% -> 85.7%) // Memory: 0 -> 5000/40000 (0% -> 12.5%) // Score: 56 (100 -> 63) - pod: &v1.Pod{Spec: cpuAndMemory}, - nodes: []*v1.Node{makeNode("node1", 3500, 40000, scalarResource), makeNode("node2", 3500, 40000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 56}, {Name: "node2", Score: 56}}, - name: "node without the scalar resource should skip the scalar resource", - pods: []*v1.Pod{}, + name: "scalar resource is not included in the score computation if pod doesn't request the scalar resource", + requestedPod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(), + nodes: []*v1.Node{ + makeNode("node1", 3500, 40000, gpu(8)), + makeNode("node2", 3500, 40000, nil), + }, + existingPods: []*v1.Pod{}, + expectedScores: []fwk.NodeScore{{Name: "node1", Score: 56}, {Name: "node2", Score: 56}}, args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{ {Name: string(v1.ResourceCPU), Weight: 1}, {Name: string(v1.ResourceMemory), Weight: 1}, @@ -313,16 +238,19 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { // CPU: 3000 -> 6000/10000 (30% -> 60%) // Memory: 5000 -> 10000/20000 (25% -> 50%) // Score: 74 (97 -> 95) - pod: &v1.Pod{Spec: cpuAndMemory}, - nodes: []*v1.Node{makeNode("node1", 10000, 20000, nil), makeNode("node2", 10000, 20000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 73}, {Name: "node2", Score: 74}}, name: "resources requested, pods scheduled with resources if PreScore not called", - pods: []*v1.Pod{ - {Spec: cpuOnly}, - {Spec: cpuAndMemory}, + requestedPod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(), + nodes: []*v1.Node{ + makeNode("node1", 10000, 20000, nil), + makeNode("node2", 10000, 20000, nil), }, - args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, - runPreScore: false, + existingPods: []*v1.Pod{ + st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(), + st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(), + }, + expectedScores: []fwk.NodeScore{{Name: "node1", Score: 73}, {Name: "node2", Score: 74}}, + args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet}, + runPreScore: false, }, { // Node1 @@ -335,14 +263,14 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { // Memory: 5000 -> 5000/40000 (12% -> 12%) // DRA: unsatisfiable // Score: 75 (63 -> 63) - pod: st.MakePod().Req(map[v1.ResourceName]string{extendedResourceDRA: "1"}).Obj(), - nodes: []*v1.Node{makeNode("node1", 3500, 40000, nil), makeNode("node2", 3500, 40000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 76}, {Name: "node2", Score: 75}}, name: "include DRA resource on a node for balanced resource allocation", - pods: []*v1.Pod{ - {Spec: cpuOnly}, - {Spec: cpuAndMemory}, + requestedPod: st.MakePod().Req(map[v1.ResourceName]string{extendedResourceDRA: "1"}).Obj(), + nodes: []*v1.Node{makeNode("node1", 3500, 40000, nil), makeNode("node2", 3500, 40000, nil)}, + existingPods: []*v1.Pod{ + st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(), + st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(), }, + expectedScores: []fwk.NodeScore{{Name: "node1", Score: 76}, {Name: "node2", Score: 75}}, args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{ {Name: string(v1.ResourceCPU), Weight: 1}, {Name: string(v1.ResourceMemory), Weight: 1}, @@ -365,14 +293,14 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { // Memory: 5000 -> 5000/40000 (12% -> 12%) // DRA: unsatisfiable // Score: 75 (63 -> 63) - pod: st.MakePod().Req(map[v1.ResourceName]string{extendedResourceDRA: "1"}).Obj(), - nodes: []*v1.Node{makeNode("node1", 3500, 40000, nil), makeNode("node2", 3500, 40000, nil)}, - expectedList: []fwk.NodeScore{{Name: "node1", Score: 76}, {Name: "node2", Score: 75}}, name: "include DRA resource on a node for balanced resource allocation if PreScore not called", - pods: []*v1.Pod{ - {Spec: cpuOnly}, - {Spec: cpuAndMemory}, + requestedPod: st.MakePod().Req(map[v1.ResourceName]string{extendedResourceDRA: "1"}).Obj(), + nodes: []*v1.Node{makeNode("node1", 3500, 40000, nil), makeNode("node2", 3500, 40000, nil)}, + existingPods: []*v1.Pod{ + st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(), + st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(), }, + expectedScores: []fwk.NodeScore{{Name: "node1", Score: 76}, {Name: "node2", Score: 75}}, args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{ {Name: string(v1.ResourceCPU), Weight: 1}, {Name: string(v1.ResourceMemory), Weight: 1}, @@ -389,7 +317,7 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { for _, test := range tests { tCtx.SyncTest(test.name, func(tCtx ktesting.TContext) { featuregatetesting.SetFeatureGateDuringTest(tCtx, utilfeature.DefaultFeatureGate, features.DRAExtendedResource, test.draObjects != nil) - snapshot := cache.NewSnapshot(test.pods, test.nodes) + snapshot := cache.NewSnapshot(test.existingPods, test.nodes) fh, _ := runtime.NewFramework(tCtx, nil, nil, runtime.WithSnapshotSharedLister(snapshot)) defer func() { tCtx.Cancel("test has completed") @@ -404,9 +332,9 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { state := framework.NewCycleState() if test.runPreScore { - status := p.(fwk.PreScorePlugin).PreScore(tCtx, state, test.pod, tf.BuildNodeInfos(test.nodes)) - if status.Code() != test.wantPreScoreStatusCode { - tCtx.Errorf("unexpected status code, want: %v, got: %v", test.wantPreScoreStatusCode, status.Code()) + status := p.(fwk.PreScorePlugin).PreScore(tCtx, state, test.requestedPod, tf.BuildNodeInfos(test.nodes)) + if status.Code() != test.expectedPreScoreStatusCode { + tCtx.Errorf("unexpected status code, want: %v, got: %v", test.expectedPreScoreStatusCode, status.Code()) } if status.Code() == fwk.Skip { tCtx.Log("skipping score test as PreScore returned skip") @@ -418,11 +346,11 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) { if err != nil { tCtx.Errorf("failed to get node %q from snapshot: %v", test.nodes[i].Name, err) } - hostResult, status := p.(fwk.ScorePlugin).Score(tCtx, state, test.pod, nodeInfo) + hostResult, status := p.(fwk.ScorePlugin).Score(tCtx, state, test.requestedPod, nodeInfo) if !status.IsSuccess() { tCtx.Errorf("Score is expected to return success, but didn't. Got status: %v", status) } - if diff := cmp.Diff(test.expectedList[i].Score, hostResult); diff != "" { + if diff := cmp.Diff(test.expectedScores[i].Score, hostResult); diff != "" { tCtx.Errorf("unexpected score for host %v (-want,+got):\n%s", test.nodes[i].Name, diff) } } @@ -439,16 +367,11 @@ func TestBalancedAllocationSignPod(t *testing.T) { expectedStatusCode fwk.Code }{ "pod with CPU and memory requests": { - pod: st.MakePod().Req(map[v1.ResourceName]string{ - v1.ResourceCPU: "1000m", - v1.ResourceMemory: "2000", - }).Obj(), + pod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Obj(), enableDRAExtendedResource: false, expectedFragments: []fwk.SignFragment{ - {Key: fwk.ResourcesSignerName, Value: computePodResourceRequest(st.MakePod().Req(map[v1.ResourceName]string{ - v1.ResourceCPU: "1000m", - v1.ResourceMemory: "2000", - }).Obj(), ResourceRequestsOptions{})}, + {Key: fwk.ResourcesSignerName, Value: computePodResourceRequest( + st.MakePod().Req(cpuAndMemory("1000m", "2000")).Obj(), ResourceRequestsOptions{})}, }, expectedStatusCode: fwk.Success, }, @@ -461,30 +384,19 @@ func TestBalancedAllocationSignPod(t *testing.T) { expectedStatusCode: fwk.Success, }, "pod with multiple containers": { - pod: st.MakePod().Container("container1").Req(map[v1.ResourceName]string{ - v1.ResourceCPU: "500m", - v1.ResourceMemory: "1000", - }).Container("container2").Req(map[v1.ResourceName]string{ - v1.ResourceCPU: "1500m", - v1.ResourceMemory: "3000", - }).Obj(), + pod: st.MakePod(). + Container("container1").Req(cpuAndMemory("500m", "1000")). + Container("container2").Req(cpuAndMemory("1500m", "3000")).Obj(), enableDRAExtendedResource: false, expectedFragments: []fwk.SignFragment{ - {Key: fwk.ResourcesSignerName, Value: computePodResourceRequest(st.MakePod().Container("container1").Req(map[v1.ResourceName]string{ - v1.ResourceCPU: "500m", - v1.ResourceMemory: "1000", - }).Container("container2").Req(map[v1.ResourceName]string{ - v1.ResourceCPU: "1500m", - v1.ResourceMemory: "3000", - }).Obj(), ResourceRequestsOptions{})}, + {Key: fwk.ResourcesSignerName, Value: computePodResourceRequest(st.MakePod(). + Container("container1").Req(cpuAndMemory("500m", "1000")). + Container("container2").Req(cpuAndMemory("1500m", "3000")).Obj(), ResourceRequestsOptions{})}, }, expectedStatusCode: fwk.Success, }, "DRA extended resource enabled - returns unschedulable": { - pod: st.MakePod().Req(map[v1.ResourceName]string{ - v1.ResourceCPU: "1000m", - v1.ResourceMemory: "2000", - }).Obj(), + pod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Obj(), enableDRAExtendedResource: true, expectedFragments: nil, expectedStatusCode: fwk.Unschedulable, @@ -517,3 +429,16 @@ func TestBalancedAllocationSignPod(t *testing.T) { }) } } + +func cpuOnly(req string) map[v1.ResourceName]string { + return map[v1.ResourceName]string{v1.ResourceCPU: req} +} +func cpuAndMemory(cpuReq, memoryReq string) map[v1.ResourceName]string { + return map[v1.ResourceName]string{v1.ResourceCPU: cpuReq, v1.ResourceMemory: memoryReq} +} +func cpuAndMemoryAndGpu(cpuReq, memoryReq, gpuReq string) map[v1.ResourceName]string { + return map[v1.ResourceName]string{v1.ResourceCPU: cpuReq, v1.ResourceMemory: memoryReq, "nvidia.com/gpu": gpuReq} +} +func gpu(count int64) map[string]int64 { + return map[string]int64{"nvidia.com/gpu": count} +} diff --git a/pkg/scheduler/framework/plugins/noderesources/resource_allocation_test.go b/pkg/scheduler/framework/plugins/noderesources/resource_allocation_test.go index 9976a220096..093551f4aca 100644 --- a/pkg/scheduler/framework/plugins/noderesources/resource_allocation_test.go +++ b/pkg/scheduler/framework/plugins/noderesources/resource_allocation_test.go @@ -286,7 +286,7 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { enableDRAExtendedResource bool node *v1.Node extendedResource v1.ResourceName - objects []apiruntime.Object + draObjects []apiruntime.Object expectedAllocatable int64 expectedAllocated int64 }{ @@ -308,18 +308,18 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { enableDRAExtendedResource: true, node: st.MakeNode().Name(nodeName).Obj(), extendedResource: explicitExtendedResource, - objects: []apiruntime.Object{ + draObjects: []apiruntime.Object{ deviceClassWithExtendResourceName, st.MakeResourceSlice(nodeName, driverName).Device("device-1").Obj(), }, - expectedAllocatable: 1, + expectedAllocatable: 1, // 1 device `device-1` expectedAllocated: 0, }, "DRA-backed-resource-implicit": { enableDRAExtendedResource: true, node: st.MakeNode().Name(nodeName).Obj(), extendedResource: implicitExtendedResource, - objects: []apiruntime.Object{ + draObjects: []apiruntime.Object{ &resourceapi.DeviceClass{ ObjectMeta: metav1.ObjectMeta{ Name: deviceClassName, @@ -327,22 +327,24 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { }, st.MakeResourceSlice(nodeName, driverName).Device("device-1").Obj(), }, - expectedAllocatable: 1, + expectedAllocatable: 1, // 1 device `device-1` expectedAllocated: 0, }, "DRA-backed-resource-no-slices": { enableDRAExtendedResource: true, node: st.MakeNode().Name(nodeName).Obj(), extendedResource: explicitExtendedResource, - objects: []apiruntime.Object{deviceClassWithExtendResourceName}, - expectedAllocatable: 0, - expectedAllocated: 0, + draObjects: []apiruntime.Object{ + deviceClassWithExtendResourceName, + }, + expectedAllocatable: 0, + expectedAllocated: 0, }, "DRA-backed-resource-with-allocated-device": { enableDRAExtendedResource: true, node: st.MakeNode().Name(nodeName).Obj(), extendedResource: explicitExtendedResource, - objects: []apiruntime.Object{ + draObjects: []apiruntime.Object{ deviceClassWithExtendResourceName, st.MakeResourceSlice(nodeName, driverName).Devices("device-1", "device-2").Obj(), // Create a resource claim that fully allocates device-1 @@ -363,14 +365,14 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { }). Obj(), }, - expectedAllocatable: 2, - expectedAllocated: 1, // 1 allocated + expectedAllocatable: 2, // 2 allocatable devices `device-1`, `device-2` + expectedAllocated: 1, // 1 allocated by `testClaim` }, "DRA-backed-resource-with-shared-device-allocation": { enableDRAExtendedResource: true, node: st.MakeNode().Name(nodeName).Obj(), extendedResource: explicitExtendedResource, - objects: []apiruntime.Object{ + draObjects: []apiruntime.Object{ deviceClassWithExtendResourceName, st.MakeResourceSlice(nodeName, driverName).Devices("device-1", "device-2").Obj(), // Create a resource claim with shared device allocation (consumable capacity) @@ -392,14 +394,14 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { }). Obj(), }, - expectedAllocatable: 2, - expectedAllocated: 1, // 1 allocated (shared) + expectedAllocatable: 2, // 2 allocatable devices `device-1`, `device-2` + expectedAllocated: 1, // 1 allocated (shared) by `testClaim` }, "DRA-backed-resource-multiple-devices-mixed-allocation": { enableDRAExtendedResource: true, node: st.MakeNode().Name(nodeName).Obj(), extendedResource: explicitExtendedResource, - objects: []apiruntime.Object{ + draObjects: []apiruntime.Object{ deviceClassWithExtendResourceName, st.MakeResourceSlice(nodeName, driverName).Devices("device-1", "device-2", "device-3").Obj(), // Mix of fully allocated and shared device allocations @@ -439,14 +441,14 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { Obj(), // device-3 remains unallocated }, - expectedAllocatable: 3, - expectedAllocated: 2, // 2 allocated (1 full + 1 shared) + expectedAllocatable: 3, // 3 allocatable devices `device-1`, `device-2`, `device-3` + expectedAllocated: 2, // 2 allocated (1 full `test-claim-1` + 1 shared `test-claim-2`) }, "DRA-backed-resource-with-per-device-node-selection": { enableDRAExtendedResource: true, node: st.MakeNode().Name(nodeName).Label("zone", "us-east-1a").Obj(), extendedResource: explicitExtendedResource, - objects: []apiruntime.Object{ + draObjects: []apiruntime.Object{ &resourceapi.DeviceClass{ ObjectMeta: metav1.ObjectMeta{ Name: deviceClassName, @@ -546,7 +548,7 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { // Setup environment, create required objects featuregatetesting.SetFeatureGateDuringTest(tCtx, utilfeature.DefaultFeatureGate, features.DRAExtendedResource, tc.enableDRAExtendedResource) - draManager := newTestDRAManager(tCtx, tc.objects...) + draManager := newTestDRAManager(tCtx, tc.draObjects...) nodeInfo := framework.NewNodeInfo() nodeInfo.SetNode(tc.node) @@ -572,10 +574,10 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) { // Test calculateResourceAllocatableRequest API allocatable, allocated := scorer.calculateResourceAllocatableRequest(tCtx, nodeInfo, tc.extendedResource, draPreScoreState) if !cmp.Equal(allocatable, tc.expectedAllocatable) { - tCtx.Errorf("Expected allocatable=%v, but got allocatable=%v", tc.expectedAllocatable, allocatable) + tCtx.Errorf("Expected allocatable=%v, but got %v", tc.expectedAllocatable, allocatable) } if !cmp.Equal(allocated, tc.expectedAllocated) { - tCtx.Errorf("Expected allocated=%v, but got allocated=%v", tc.expectedAllocated, allocated) + tCtx.Errorf("Expected allocated=%v, but got %v", tc.expectedAllocated, allocated) } }) } diff --git a/pkg/scheduler/schedule_one_test.go b/pkg/scheduler/schedule_one_test.go index 852605f6a27..7375ef51fa0 100644 --- a/pkg/scheduler/schedule_one_test.go +++ b/pkg/scheduler/schedule_one_test.go @@ -3873,27 +3873,27 @@ func TestZeroRequest(t *testing.T) { large2 := large large2.NodeName = "node2" tests := []struct { - pod *v1.Pod - pods []*v1.Pod + requestedPod *v1.Pod + existingPods []*v1.Pod nodes []*v1.Node name string expectedScore int64 }{ { - pod: &v1.Pod{Spec: noResources}, - nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10), makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10)}, - name: "test priority of zero-request pod with node with zero-request pod", - pods: []*v1.Pod{ + name: "test priority of zero-request pod with node with zero-request pod", + requestedPod: &v1.Pod{Spec: noResources}, + nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10), makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10)}, + existingPods: []*v1.Pod{ {Spec: large1}, {Spec: noResources1}, {Spec: large2}, {Spec: small2}, }, expectedScore: 50, }, { - pod: &v1.Pod{Spec: small}, - nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10), makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10)}, - name: "test priority of nonzero-request pod with node with zero-request pod", - pods: []*v1.Pod{ + name: "test priority of nonzero-request pod with node with zero-request pod", + requestedPod: &v1.Pod{Spec: small}, + nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10), makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10)}, + existingPods: []*v1.Pod{ {Spec: large1}, {Spec: noResources1}, {Spec: large2}, {Spec: small2}, }, @@ -3901,10 +3901,10 @@ func TestZeroRequest(t *testing.T) { }, // The point of this test is to verify that we're not just getting the same score no matter what we schedule. { - pod: &v1.Pod{Spec: large}, - nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10), makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10)}, - name: "test priority of larger pod with node with zero-request pod", - pods: []*v1.Pod{ + name: "test priority of larger pod with node with zero-request pod", + requestedPod: &v1.Pod{Spec: large}, + nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10), makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10)}, + existingPods: []*v1.Pod{ {Spec: large1}, {Spec: noResources1}, {Spec: large2}, {Spec: small2}, }, @@ -3917,7 +3917,7 @@ func TestZeroRequest(t *testing.T) { client := clientsetfake.NewClientset() informerFactory := informers.NewSharedInformerFactory(client, 0) - snapshot := internalcache.NewSnapshot(test.pods, test.nodes) + snapshot := internalcache.NewSnapshot(test.existingPods, test.nodes) fts := feature.Features{} pluginRegistrations := []tf.RegisterPluginFunc{ tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), @@ -3947,7 +3947,7 @@ func TestZeroRequest(t *testing.T) { sched.applyDefaultHandlers() state := framework.NewCycleState() - _, _, _, _, err = sched.findNodesThatFitPod(ctx, fwk, state, test.pod) + _, _, _, _, err = sched.findNodesThatFitPod(ctx, fwk, state, test.requestedPod) if err != nil { t.Fatalf("error filtering nodes: %+v", err) } @@ -3955,8 +3955,8 @@ func TestZeroRequest(t *testing.T) { if err != nil { t.Fatalf("failed to list node from snapshot: %v", err) } - fwk.RunPreScorePlugins(ctx, state, test.pod, nodeInfos) - list, err := prioritizeNodes(ctx, nil, fwk, state, test.pod, nodeInfos) + fwk.RunPreScorePlugins(ctx, state, test.requestedPod, nodeInfos) + list, err := prioritizeNodes(ctx, nil, fwk, state, test.requestedPod, nodeInfos) if err != nil { t.Errorf("unexpected error: %v", err) } From 720d648d2f964685ba1c419686678b2e92f09d65 Mon Sep 17 00:00:00 2001 From: Bartosz Date: Wed, 17 Dec 2025 11:24:24 +0000 Subject: [PATCH 3/3] Remove outdated test for scoring zero request pods --- pkg/scheduler/schedule_one_test.go | 140 ----------------------------- 1 file changed, 140 deletions(-) diff --git a/pkg/scheduler/schedule_one_test.go b/pkg/scheduler/schedule_one_test.go index 7375ef51fa0..43a606de145 100644 --- a/pkg/scheduler/schedule_one_test.go +++ b/pkg/scheduler/schedule_one_test.go @@ -3829,146 +3829,6 @@ func TestFindFitPredicateCallCounts(t *testing.T) { } } -// Tests score for pods on nodes with zero-request pods. -func TestZeroRequest(t *testing.T) { - noResources := v1.PodSpec{ - Containers: []v1.Container{ - {}, - }, - } - noResources1 := noResources - noResources1.NodeName = "node1" - small := v1.PodSpec{ - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse( - strconv.FormatInt(schedutil.DefaultMilliCPURequest, 10) + "m"), - v1.ResourceMemory: resource.MustParse( - strconv.FormatInt(schedutil.DefaultMemoryRequest, 10)), - }, - }, - }, - }, - } - small2 := small - small2.NodeName = "node2" - large := v1.PodSpec{ - Containers: []v1.Container{ - { - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse( - strconv.FormatInt(schedutil.DefaultMilliCPURequest*3, 10) + "m"), - v1.ResourceMemory: resource.MustParse( - strconv.FormatInt(schedutil.DefaultMemoryRequest*3, 10)), - }, - }, - }, - }, - } - large1 := large - large1.NodeName = "node1" - large2 := large - large2.NodeName = "node2" - tests := []struct { - requestedPod *v1.Pod - existingPods []*v1.Pod - nodes []*v1.Node - name string - expectedScore int64 - }{ - { - name: "test priority of zero-request pod with node with zero-request pod", - requestedPod: &v1.Pod{Spec: noResources}, - nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10), makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10)}, - existingPods: []*v1.Pod{ - {Spec: large1}, {Spec: noResources1}, - {Spec: large2}, {Spec: small2}, - }, - expectedScore: 50, - }, - { - name: "test priority of nonzero-request pod with node with zero-request pod", - requestedPod: &v1.Pod{Spec: small}, - nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10), makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10)}, - existingPods: []*v1.Pod{ - {Spec: large1}, {Spec: noResources1}, - {Spec: large2}, {Spec: small2}, - }, - expectedScore: 125, - }, - // The point of this test is to verify that we're not just getting the same score no matter what we schedule. - { - name: "test priority of larger pod with node with zero-request pod", - requestedPod: &v1.Pod{Spec: large}, - nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10), makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10)}, - existingPods: []*v1.Pod{ - {Spec: large1}, {Spec: noResources1}, - {Spec: large2}, {Spec: small2}, - }, - expectedScore: 105, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - client := clientsetfake.NewClientset() - informerFactory := informers.NewSharedInformerFactory(client, 0) - - snapshot := internalcache.NewSnapshot(test.existingPods, test.nodes) - fts := feature.Features{} - pluginRegistrations := []tf.RegisterPluginFunc{ - tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New), - tf.RegisterScorePlugin(noderesources.Name, frameworkruntime.FactoryAdapter(fts, noderesources.NewFit), 1), - tf.RegisterScorePlugin(noderesources.BalancedAllocationName, frameworkruntime.FactoryAdapter(fts, noderesources.NewBalancedAllocation), 1), - tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New), - } - _, ctx := ktesting.NewTestContext(t) - ctx, cancel := context.WithCancel(ctx) - defer cancel() - fwk, err := tf.NewFramework( - ctx, - pluginRegistrations, "", - frameworkruntime.WithInformerFactory(informerFactory), - frameworkruntime.WithSnapshotSharedLister(snapshot), - frameworkruntime.WithClientSet(client), - frameworkruntime.WithPodNominator(internalqueue.NewSchedulingQueue(nil, informerFactory)), - ) - if err != nil { - t.Fatalf("error creating framework: %+v", err) - } - - sched := &Scheduler{ - nodeInfoSnapshot: snapshot, - percentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore, - } - sched.applyDefaultHandlers() - - state := framework.NewCycleState() - _, _, _, _, err = sched.findNodesThatFitPod(ctx, fwk, state, test.requestedPod) - if err != nil { - t.Fatalf("error filtering nodes: %+v", err) - } - nodeInfos, err := snapshot.NodeInfos().List() - if err != nil { - t.Fatalf("failed to list node from snapshot: %v", err) - } - fwk.RunPreScorePlugins(ctx, state, test.requestedPod, nodeInfos) - list, err := prioritizeNodes(ctx, nil, fwk, state, test.requestedPod, nodeInfos) - if err != nil { - t.Errorf("unexpected error: %v", err) - } - for _, hp := range list { - if hp.TotalScore != test.expectedScore { - t.Errorf("expected %d for all priorities, got list %#v", test.expectedScore, list) - } - } - }) - } -} - func Test_prioritizeNodes(t *testing.T) { imageStatus1 := []v1.ContainerImage{ {