upstream/kubernetes
1
0
Fork 0
mirror of https://github.com/kubernetes/kubernetes.git synced 2026-06-09 00:34:10 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #135573 from brejman/issue-129733-score-update

Browse source 
Update scoring function for balanced allocation to consider change to the node's balance
This commit is contained in:
Kubernetes Prow Robot 2026-01-26 21:49:52 +05:30 committed by GitHub
commit efc15394a1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 312 additions and 506 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
pkg/scheduler/framework/plugins/noderesources/balanced_allocation.go
View file

@ -191,14 +191,30 @@ func NewBalancedAllocation(_ context.Context, baArgs runtime.Object, h fwk.Handl
}, nil
}
func balancedResourceScorer(requested, allocable []int64) int64 {
func balancedResourceScorer(requested, allocated, allocatable []int64) int64 {
// Compute the score for the node with pending pod requests
scoreWithPod := balancedResourceScore(requested, allocatable)
// Compute the score for the node without pending pod requests
scoreWithoutPod := balancedResourceScore(allocated, allocatable)
// Score based on the improvement to the node balance (before vs after adding the pod).
// Scores with/without pod are in range [MaxNodeScore/2, MaxNodeScore].
// Therefore, their difference will be in range [-MaxNodeScore/2, MaxNodeScore/2].
// This operation brings the range back to [MaxNodeScore/2, MaxNodeScore]
// to be closer to the original implementation that only considered the score after adding the pod.
// If the difference to the balance is small, the score will be around 75.
// If the balance is improved, the score will move towards 100.
// If the balance is decreased, the score will move towards 50.
return fwk.MaxNodeScore/2 + (fwk.MaxNodeScore/2+scoreWithPod-scoreWithoutPod)/2
}
func balancedResourceScore(requested, allocatable []int64) int64 {
var resourceToFractions []float64
var totalFraction float64
for i := range requested {
if allocable[i] == 0 {
if allocatable[i] == 0 {
continue
}
fraction := float64(requested[i]) / float64(allocable[i])
fraction := float64(requested[i]) / float64(allocatable[i])
if fraction > 1 {
fraction = 1
}

523
pkg/scheduler/framework/plugins/noderesources/balanced_allocation_test.go
View file

@ -21,7 +21,6 @@ import (
"github.com/google/go-cmp/cmp"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
apiruntime "k8s.io/apimachinery/pkg/runtime"
utilfeature "k8s.io/apiserver/pkg/util/feature"
featuregatetesting "k8s.io/component-base/featuregate/testing"
@ -41,234 +40,162 @@ func TestNodeResourcesBalancedAllocation(t *testing.T) {
testNodeResourcesBalancedAllocation(ktesting.Init(t))
}
func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) {
cpuAndMemoryAndGPU := v1.PodSpec{
Containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("1000m"),
v1.ResourceMemory: resource.MustParse("2000"),
},
},
},
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2000m"),
v1.ResourceMemory: resource.MustParse("3000"),
"nvidia.com/gpu": resource.MustParse("3"),
},
},
},
},
NodeName: "node1",
}
cpuOnly := v1.PodSpec{
NodeName: "node1",
Containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("1000m"),
v1.ResourceMemory: resource.MustParse("0"),
},
},
},
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2000m"),
v1.ResourceMemory: resource.MustParse("0"),
},
},
},
},
}
cpuOnly2 := cpuOnly
cpuOnly2.NodeName = "node2"
cpuAndMemory := v1.PodSpec{
NodeName: "node2",
Containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("1000m"),
v1.ResourceMemory: resource.MustParse("2000"),
},
},
},
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2000m"),
v1.ResourceMemory: resource.MustParse("3000"),
},
},
},
},
}
defaultResourceBalancedAllocationSet := []config.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: string(v1.ResourceMemory), Weight: 1},
}
scalarResource := map[string]int64{
"nvidia.com/gpu": 8,
}
tests := []struct {
pod *v1.Pod
pods []*v1.Pod
nodes []*v1.Node
expectedList fwk.NodeScoreList
name string
args config.NodeResourcesBalancedAllocationArgs
runPreScore bool
wantPreScoreStatusCode fwk.Code
draObjects []apiruntime.Object
name string
requestedPod *v1.Pod
nodes []*v1.Node
existingPods []*v1.Pod
expectedScores fwk.NodeScoreList
args config.NodeResourcesBalancedAllocationArgs
runPreScore bool
expectedPreScoreStatusCode fwk.Code
draObjects []apiruntime.Object
}{
{
// bestEffort pods, skip in PreScore
pod: st.MakePod().Obj(),
nodes: []*v1.Node{makeNode("node1", 4000, 10000, nil), makeNode("node2", 4000, 10000, nil)},
name: "nothing scheduled, nothing requested, skip in PreScore",
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
runPreScore: true,
wantPreScoreStatusCode: fwk.Skip,
name: "nothing scheduled, nothing requested, skip in PreScore",
requestedPod: st.MakePod().Obj(),
nodes: []*v1.Node{makeNode("node1", 4000, 10000, nil), makeNode("node2", 4000, 10000, nil)},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
runPreScore: true,
expectedPreScoreStatusCode: fwk.Skip,
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 3000 / 4000= 75%
// Memory Fraction: 5000 / 10000 = 50%
// Node1 std: (0.75 - 0.5) / 2 = 0.125
// Node1 Score: (1 - 0.125)*MaxNodeScore = 87
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 3000 / 6000= 50%
// Memory Fraction: 5000/10000 = 50%
// Node2 std: 0
// Node2 Score: (1-0) * MaxNodeScore = MaxNodeScore
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("node1", 4000, 10000, nil), makeNode("node2", 6000, 10000, nil)},
expectedList: []fwk.NodeScore{{Name: "node1", Score: 87}, {Name: "node2", Score: fwk.MaxNodeScore}},
// Node1
// CPU: 0 -> 3000/4000 (0% -> 75%)
// Memory: 0 -> 5000/10000 (0% -> 50%)
// Score: 68 (100 -> 87)
// Node2
// CPU: 0 -> 3000/6000 (0% -> 50%)
// Memory: 0 -> 5000/10000 (0% -> 50%)
// Score: 75 (100 -> 100)
name: "nothing scheduled, resources requested, differently sized nodes",
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
runPreScore: true,
requestedPod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(),
nodes: []*v1.Node{
makeNode("node1", 4000, 10000, nil),
makeNode("node2", 6000, 10000, nil)},
expectedScores: []fwk.NodeScore{{Name: "node1", Score: 68}, {Name: "node2", Score: 75}},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
runPreScore: true,
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 10000 = 60%
// Memory Fraction: 5000 / 20000 = 25%
// Node1 std: (0.6 - 0.25) / 2 = 0.175
// Node1 Score: (1 - 0.175)*MaxNodeScore = 82
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 10000 = 60%
// Memory Fraction: 10000 / 20000 = 50%
// Node2 std: (0.6 - 0.5) / 2 = 0.05
// Node2 Score: (1 - 0.05)*MaxNodeScore = 95
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("node1", 10000, 20000, nil), makeNode("node2", 10000, 20000, nil)},
expectedList: []fwk.NodeScore{{Name: "node1", Score: 82}, {Name: "node2", Score: 95}},
// Node1
// CPU: 3000 -> 6000/10000 (30% -> 60%)
// Memory: 0 -> 5000/20000 (0% -> 25%)
// Score: 73 (85 -> 82)
// Node2
// CPU: 3000 -> 6000/10000 (30% -> 60%)
// Memory: 5000 -> 10000/20000 (25% -> 50%)
// Score: 74 (97 -> 95)
name: "resources requested, pods scheduled with resources",
pods: []*v1.Pod{
{Spec: cpuOnly},
{Spec: cpuAndMemory},
requestedPod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(),
nodes: []*v1.Node{
makeNode("node1", 10000, 20000, nil),
makeNode("node2", 10000, 20000, nil),
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
runPreScore: true,
existingPods: []*v1.Pod{
st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(),
st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(),
},
expectedScores: []fwk.NodeScore{{Name: "node1", Score: 73}, {Name: "node2", Score: 74}},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
runPreScore: true,
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 10000 = 60%
// Memory Fraction: 5000 / 20000 = 25%
// Node1 std: (0.6 - 0.25) / 2 = 0.175
// Node1 Score: (1 - 0.175)*MaxNodeScore = 82
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 10000 = 60%
// Memory Fraction: 10000 / 50000 = 20%
// Node2 std: (0.6 - 0.2) / 2 = 0.2
// Node2 Score: (1 - 0.2)*MaxNodeScore = 80
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("node1", 10000, 20000, nil), makeNode("node2", 10000, 50000, nil)},
expectedList: []fwk.NodeScore{{Name: "node1", Score: 82}, {Name: "node2", Score: 80}},
// Node1
// CPU: 3000 -> 6000/10000 (30% -> 60%)
// Memory: 0 -> 5000/20000 (0% -> 25%)
// Score: 73 (85 -> 82)
// Node2
// CPU: 3000 -> 6000/10000 (30% -> 60%)
// Memory: 5000 -> 10000/50000 (10% -> 20%)
// Score: 70 (90 -> 80)
name: "resources requested, pods scheduled with resources, differently sized nodes",
pods: []*v1.Pod{
{Spec: cpuOnly},
{Spec: cpuAndMemory},
requestedPod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(),
nodes: []*v1.Node{
makeNode("node1", 10000, 20000, nil),
makeNode("node2", 10000, 50000, nil),
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
runPreScore: true,
existingPods: []*v1.Pod{
st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(),
st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(),
},
expectedScores: []fwk.NodeScore{{Name: "node1", Score: 73}, {Name: "node2", Score: 70}},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
runPreScore: true,
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 6000 = 1
// Memory Fraction: 0 / 10000 = 0
// Node1 std: (1 - 0) / 2 = 0.5
// Node1 Score: (1 - 0.5)*MaxNodeScore = 50
// Node1 Score: MaxNodeScore - (1 - 0) * MaxNodeScore = 0
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 6000 = 1
// Memory Fraction 5000 / 10000 = 50%
// Node2 std: (1 - 0.5) / 2 = 0.25
// Node2 Score: (1 - 0.25)*MaxNodeScore = 75
pod: &v1.Pod{Spec: cpuOnly},
nodes: []*v1.Node{makeNode("node1", 6000, 10000, nil), makeNode("node2", 6000, 10000, nil)},
expectedList: []fwk.NodeScore{{Name: "node1", Score: 50}, {Name: "node2", Score: 75}},
// Node1
// CPU: 3000 -> 3000/3000 (100% -> 100%)
// Memory: 0 -> 5000/5000 (0% -> 100%)
// Score: 100 (50 -> 100)
// Node2
// CPU: 0 -> 0/10000 (0% -> 0%)
// Memory: 0 -> 5000/5000 (0% -> 100%)
// Score: 50 (100 -> 50)
name: "resources requested, pods scheduled with resources, nodes to reach min/max score",
requestedPod: st.MakePod().Req(map[v1.ResourceName]string{"memory": "2000"}).Req(map[v1.ResourceName]string{"memory": "3000"}).Obj(),
nodes: []*v1.Node{
makeNode("node1", 3000, 5000, nil),
makeNode("node2", 3000, 5000, nil),
},
existingPods: []*v1.Pod{
st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(),
},
expectedScores: []fwk.NodeScore{{Name: "node1", Score: 100}, {Name: "node2", Score: 50}},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
runPreScore: true,
},
{
// Node1
// CPU: 3000 -> 6000/6000 (50% -> 100%)
// Memory: 0 -> 0/10000 (0% -> 0%)
// Score: 62 (75 -> 50)
// Node2
// CPU: 3000 -> 6000/6000 (50% -> 100%)
// Memory: 5000 -> 5000/10000 (50% -> 50%)
// Score: 62 (100 -> 75)
name: "requested resources at node capacity",
pods: []*v1.Pod{
{Spec: cpuOnly},
{Spec: cpuAndMemory},
requestedPod: st.MakePod().Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(),
nodes: []*v1.Node{
makeNode("node1", 6000, 10000, nil),
makeNode("node2", 6000, 10000, nil),
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
runPreScore: true,
},
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 3000 / 3500 = 85.71%
// Memory Fraction: 5000 / 40000 = 12.5%
// GPU Fraction: 4 / 8 = 0.5%
// Node1 std: sqrt(((0.8571 - 0.503) * (0.8571 - 0.503) + (0.503 - 0.125) * (0.503 - 0.125) + (0.503 - 0.5) * (0.503 - 0.5)) / 3) = 0.3002
// Node1 Score: (1 - 0.3002)*MaxNodeScore = 70
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 3000 / 3500 = 85.71%
// Memory Fraction: 5000 / 40000 = 12.5%
// GPU Fraction: 1 / 8 = 12.5%
// Node2 std: sqrt(((0.8571 - 0.378) * (0.8571 - 0.378) + (0.378 - 0.125) * (0.378 - 0.125)) + (0.378 - 0.125) * (0.378 - 0.125)) / 3) = 0.345
// Node2 Score: (1 - 0.358)*MaxNodeScore = 65
{
pod: st.MakePod().Req(map[v1.ResourceName]string{
v1.ResourceMemory: "0",
"nvidia.com/gpu": "1",
}).Obj(),
nodes: []*v1.Node{makeNode("node1", 3500, 40000, scalarResource), makeNode("node2", 3500, 40000, scalarResource)},
expectedList: []fwk.NodeScore{{Name: "node1", Score: 70}, {Name: "node2", Score: 65}},
name: "include scalar resource on a node for balanced resource allocation",
pods: []*v1.Pod{
{Spec: cpuAndMemory},
{Spec: cpuAndMemoryAndGPU},
existingPods: []*v1.Pod{
st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(),
st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(),
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: string(v1.ResourceMemory), Weight: 1},
{Name: "nvidia.com/gpu", Weight: 1},
}},
runPreScore: true,
expectedScores: []fwk.NodeScore{{Name: "node1", Score: 62}, {Name: "node2", Score: 62}},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
runPreScore: true,
},
// Only one node (node1) has the scalar resource, pod doesn't request the scalar resource and the scalar resource should be skipped for consideration.
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 3000 / 3500 = 85.71%
// Memory Fraction: 5000 / 40000 = 12.5%
// Node1 std: (0.8571 - 0.125) / 2 = 0.36605
// Node1 Score: (1 - 0.22705)*MaxNodeScore = 63
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 3000 / 3500 = 85.71%
// Memory Fraction: 5000 / 40000 = 12.5%
// Node2 std: (0.8571 - 0.125) / 2 = 0.36605
// Node2 Score: (1 - 0.22705)*MaxNodeScore = 63
{
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("node1", 3500, 40000, scalarResource), makeNode("node2", 3500, 40000, nil)},
expectedList: []fwk.NodeScore{{Name: "node1", Score: 63}, {Name: "node2", Score: 63}},
name: "node without the scalar resource should skip the scalar resource",
pods: []*v1.Pod{},
// Node1
// CPU: 3000 -> 3000/3500 (85.7% -> 85.7%)
// Memory: 5000 -> 5000/40000 (12.5% -> 12.5%)
// GPU: 3 -> 4/8 (37.5% -> 50%)
// Score: 75 (69 -> 70)
// Node2
// CPU: 3000 -> 3000/3500 (85.7% -> 85.7%)
// Memory: 5000 -> 5000/40000 (12.5% -> 12.5%)
// GPU: 0 -> 1/8 (0% -> 12.5%)
// Score: 76 (62 -> 65)
name: "scalar resource is included in the score computation if pod requests the scalar resource",
requestedPod: st.MakePod().Req(cpuAndMemoryAndGpu("0", "0", "1")).Obj(),
nodes: []*v1.Node{
makeNode("node1", 3500, 40000, gpu(8)),
makeNode("node2", 3500, 40000, gpu(8)),
},
existingPods: []*v1.Pod{
st.MakePod().Node("node1").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemoryAndGpu("2000m", "3000", "3")).Obj(),
st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(),
},
expectedScores: []fwk.NodeScore{{Name: "node1", Score: 75}, {Name: "node2", Score: 76}},
args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: string(v1.ResourceMemory), Weight: 1},
@ -277,48 +204,73 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) {
runPreScore: true,
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 10000 = 60%
// Memory Fraction: 5000 / 20000 = 25%
// Node1 std: (0.6 - 0.25) / 2 = 0.175
// Node1 Score: (1 - 0.175)*MaxNodeScore = 82
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 6000 / 10000 = 60%
// Memory Fraction: 10000 / 20000 = 50%
// Node2 std: (0.6 - 0.5) / 2 = 0.05
// Node2 Score: (1 - 0.05)*MaxNodeScore = 95
pod: &v1.Pod{Spec: cpuAndMemory},
nodes: []*v1.Node{makeNode("node1", 10000, 20000, nil), makeNode("node2", 10000, 20000, nil)},
expectedList: []fwk.NodeScore{{Name: "node1", Score: 82}, {Name: "node2", Score: 95}},
// Node1
// CPU: 0 -> 3000/3500 (0% -> 85.7%)
// Memory: 0 -> 5000/40000 (0% -> 12.5%)
// GPU: 0 -> 0/8 (scalar resource not requested by pod, ignored)
// Score: 56 (100 -> 63)
// Node2
// CPU: 0 -> 3000/3500 (0% -> 85.7%)
// Memory: 0 -> 5000/40000 (0% -> 12.5%)
// Score: 56 (100 -> 63)
name: "scalar resource is not included in the score computation if pod doesn't request the scalar resource",
requestedPod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(),
nodes: []*v1.Node{
makeNode("node1", 3500, 40000, gpu(8)),
makeNode("node2", 3500, 40000, nil),
},
existingPods: []*v1.Pod{},
expectedScores: []fwk.NodeScore{{Name: "node1", Score: 56}, {Name: "node2", Score: 56}},
args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: string(v1.ResourceMemory), Weight: 1},
{Name: "nvidia.com/gpu", Weight: 1},
}},
runPreScore: true,
},
{
// Whether or not prescore was called, the end result should be the same.
// Node1
// CPU: 3000 -> 6000/10000 (30% -> 60%)
// Memory: 0 -> 5000/20000 (0% -> 25%)
// Score: 73 (85 -> 82)
// Node2
// CPU: 3000 -> 6000/10000 (30% -> 60%)
// Memory: 5000 -> 10000/20000 (25% -> 50%)
// Score: 74 (97 -> 95)
name: "resources requested, pods scheduled with resources if PreScore not called",
pods: []*v1.Pod{
{Spec: cpuOnly},
{Spec: cpuAndMemory},
requestedPod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(),
nodes: []*v1.Node{
makeNode("node1", 10000, 20000, nil),
makeNode("node2", 10000, 20000, nil),
},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
runPreScore: false,
existingPods: []*v1.Pod{
st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(),
st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(),
},
expectedScores: []fwk.NodeScore{{Name: "node1", Score: 73}, {Name: "node2", Score: 74}},
args: config.NodeResourcesBalancedAllocationArgs{Resources: defaultResourceBalancedAllocationSet},
runPreScore: false,
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 3000 / 3500 = 0.8571
// Memory Fraction: 0 / 40000 = 0
// DRA Fraction: 1 / 8 = 0.125
// Fraction mean: (0.8571 + 0 + 0.125) / 3 = 0.3274
// Node1 std: sqrt(((0.8571 - 0.3274)**2 + (0- 0.3274)**2 + (0.125 - 0.3274)**2) / 3) = 0.378
// Node1 Score: (1 - 0.378)*MaxNodeScore = 62
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 3000 / 3500 = 0.8571
// Memory Fraction: 5000 / 40000 = 0.125
// Node2 std: (0.8571 - 0.125) / 2 = 0.36605
// Node2 Score: (1 - 0.36605)*MaxNodeScore = 63
pod: st.MakePod().Req(map[v1.ResourceName]string{extendedResourceDRA: "1"}).Obj(),
nodes: []*v1.Node{makeNode("node1", 3500, 40000, nil), makeNode("node2", 3500, 40000, nil)},
expectedList: []fwk.NodeScore{{Name: "node1", Score: 62}, {Name: "node2", Score: 63}},
// Node1
// CPU: 3000 -> 3000/3500 (86% -> 86%)
// Memory: 0 -> 0/40000 (0% -> 0%)
// DRA: 0 -> 1/8 (0% -> 12%)
// Score: 76 (60 -> 62)
// Node2
// CPU: 3000 -> 3000/3500 (86% -> 86%)
// Memory: 5000 -> 5000/40000 (12% -> 12%)
// DRA: unsatisfiable
// Score: 75 (63 -> 63)
name: "include DRA resource on a node for balanced resource allocation",
pods: []*v1.Pod{
{Spec: cpuOnly},
{Spec: cpuAndMemory},
requestedPod: st.MakePod().Req(map[v1.ResourceName]string{extendedResourceDRA: "1"}).Obj(),
nodes: []*v1.Node{makeNode("node1", 3500, 40000, nil), makeNode("node2", 3500, 40000, nil)},
existingPods: []*v1.Pod{
st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(),
st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(),
},
expectedScores: []fwk.NodeScore{{Name: "node1", Score: 76}, {Name: "node2", Score: 75}},
args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: string(v1.ResourceMemory), Weight: 1},
@ -331,26 +283,24 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) {
runPreScore: true,
},
{
// Node1 scores on 0-MaxNodeScore scale
// CPU Fraction: 3000 / 35000 = 0.8571
// Memory Fraction: 0 / 40000 = 0
// DRA Fraction: 1 / 8 = 0.125
// Fraction mean: (0.8571 + 0 + 0.125) / 3 = 0.3274
// Node1 std: sqrt(((0.8571 - 0.3274)**2 + (0- 0.3274)**2 + (0.125 - 0.3274)**2) / 3) = 0.378
// Node1 Score: (1 - 0.378)*MaxNodeScore = 62
// Node2 scores on 0-MaxNodeScore scale
// CPU Fraction: 3000 / 35000 = 0.8571
// Memory Fraction: 5000 / 40000 = 0.125
// Node2 std: (0.8571 - 0.125) / 2 = 0.36605
// Node2 Score: (1 - 0.36605)*MaxNodeScore = 63
pod: st.MakePod().Req(map[v1.ResourceName]string{extendedResourceDRA: "1"}).Obj(),
nodes: []*v1.Node{makeNode("node1", 3500, 40000, nil), makeNode("node2", 3500, 40000, nil)},
expectedList: []fwk.NodeScore{{Name: "node1", Score: 62}, {Name: "node2", Score: 63}},
// Node1
// CPU: 3000 -> 3000/3500 (86% -> 86%)
// Memory: 0 -> 0/40000 (0% -> 0%)
// DRA: 0 -> 1/8 (0% -> 12%)
// Score: 76 (60 -> 62)
// Node2
// CPU: 3000 -> 3000/3500 (86% -> 86%)
// Memory: 5000 -> 5000/40000 (12% -> 12%)
// DRA: unsatisfiable
// Score: 75 (63 -> 63)
name: "include DRA resource on a node for balanced resource allocation if PreScore not called",
pods: []*v1.Pod{
{Spec: cpuOnly},
{Spec: cpuAndMemory},
requestedPod: st.MakePod().Req(map[v1.ResourceName]string{extendedResourceDRA: "1"}).Obj(),
nodes: []*v1.Node{makeNode("node1", 3500, 40000, nil), makeNode("node2", 3500, 40000, nil)},
existingPods: []*v1.Pod{
st.MakePod().Node("node1").Req(cpuOnly("1000m")).Req(cpuOnly("2000m")).Obj(),
st.MakePod().Node("node2").Req(cpuAndMemory("1000m", "2000")).Req(cpuAndMemory("2000m", "3000")).Obj(),
},
expectedScores: []fwk.NodeScore{{Name: "node1", Score: 76}, {Name: "node2", Score: 75}},
args: config.NodeResourcesBalancedAllocationArgs{Resources: []config.ResourceSpec{
{Name: string(v1.ResourceCPU), Weight: 1},
{Name: string(v1.ResourceMemory), Weight: 1},
@ -367,7 +317,7 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) {
for _, test := range tests {
tCtx.SyncTest(test.name, func(tCtx ktesting.TContext) {
featuregatetesting.SetFeatureGateDuringTest(tCtx, utilfeature.DefaultFeatureGate, features.DRAExtendedResource, test.draObjects != nil)
snapshot := cache.NewSnapshot(test.pods, test.nodes)
snapshot := cache.NewSnapshot(test.existingPods, test.nodes)
fh, _ := runtime.NewFramework(tCtx, nil, nil, runtime.WithSnapshotSharedLister(snapshot))
defer func() {
tCtx.Cancel("test has completed")
@ -382,9 +332,9 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) {
state := framework.NewCycleState()
if test.runPreScore {
status := p.(fwk.PreScorePlugin).PreScore(tCtx, state, test.pod, tf.BuildNodeInfos(test.nodes))
if status.Code() != test.wantPreScoreStatusCode {
tCtx.Errorf("unexpected status code, want: %v, got: %v", test.wantPreScoreStatusCode, status.Code())
status := p.(fwk.PreScorePlugin).PreScore(tCtx, state, test.requestedPod, tf.BuildNodeInfos(test.nodes))
if status.Code() != test.expectedPreScoreStatusCode {
tCtx.Errorf("unexpected status code, want: %v, got: %v", test.expectedPreScoreStatusCode, status.Code())
}
if status.Code() == fwk.Skip {
tCtx.Log("skipping score test as PreScore returned skip")
@ -396,11 +346,11 @@ func testNodeResourcesBalancedAllocation(tCtx ktesting.TContext) {
if err != nil {
tCtx.Errorf("failed to get node %q from snapshot: %v", test.nodes[i].Name, err)
}
hostResult, status := p.(fwk.ScorePlugin).Score(tCtx, state, test.pod, nodeInfo)
hostResult, status := p.(fwk.ScorePlugin).Score(tCtx, state, test.requestedPod, nodeInfo)
if !status.IsSuccess() {
tCtx.Errorf("Score is expected to return success, but didn't. Got status: %v", status)
}
if diff := cmp.Diff(test.expectedList[i].Score, hostResult); diff != "" {
if diff := cmp.Diff(test.expectedScores[i].Score, hostResult); diff != "" {
tCtx.Errorf("unexpected score for host %v (-want,+got):\n%s", test.nodes[i].Name, diff)
}
}
@ -417,16 +367,11 @@ func TestBalancedAllocationSignPod(t *testing.T) {
expectedStatusCode fwk.Code
}{
"pod with CPU and memory requests": {
pod: st.MakePod().Req(map[v1.ResourceName]string{
v1.ResourceCPU: "1000m",
v1.ResourceMemory: "2000",
}).Obj(),
pod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Obj(),
enableDRAExtendedResource: false,
expectedFragments: []fwk.SignFragment{
{Key: fwk.ResourcesSignerName, Value: computePodResourceRequest(st.MakePod().Req(map[v1.ResourceName]string{
v1.ResourceCPU: "1000m",
v1.ResourceMemory: "2000",
}).Obj(), ResourceRequestsOptions{})},
{Key: fwk.ResourcesSignerName, Value: computePodResourceRequest(
st.MakePod().Req(cpuAndMemory("1000m", "2000")).Obj(), ResourceRequestsOptions{})},
},
expectedStatusCode: fwk.Success,
},
@ -439,30 +384,19 @@ func TestBalancedAllocationSignPod(t *testing.T) {
expectedStatusCode: fwk.Success,
},
"pod with multiple containers": {
pod: st.MakePod().Container("container1").Req(map[v1.ResourceName]string{
v1.ResourceCPU: "500m",
v1.ResourceMemory: "1000",
}).Container("container2").Req(map[v1.ResourceName]string{
v1.ResourceCPU: "1500m",
v1.ResourceMemory: "3000",
}).Obj(),
pod: st.MakePod().
Container("container1").Req(cpuAndMemory("500m", "1000")).
Container("container2").Req(cpuAndMemory("1500m", "3000")).Obj(),
enableDRAExtendedResource: false,
expectedFragments: []fwk.SignFragment{
{Key: fwk.ResourcesSignerName, Value: computePodResourceRequest(st.MakePod().Container("container1").Req(map[v1.ResourceName]string{
v1.ResourceCPU: "500m",
v1.ResourceMemory: "1000",
}).Container("container2").Req(map[v1.ResourceName]string{
v1.ResourceCPU: "1500m",
v1.ResourceMemory: "3000",
}).Obj(), ResourceRequestsOptions{})},
{Key: fwk.ResourcesSignerName, Value: computePodResourceRequest(st.MakePod().
Container("container1").Req(cpuAndMemory("500m", "1000")).
Container("container2").Req(cpuAndMemory("1500m", "3000")).Obj(), ResourceRequestsOptions{})},
},
expectedStatusCode: fwk.Success,
},
"DRA extended resource enabled - returns unschedulable": {
pod: st.MakePod().Req(map[v1.ResourceName]string{
v1.ResourceCPU: "1000m",
v1.ResourceMemory: "2000",
}).Obj(),
pod: st.MakePod().Req(cpuAndMemory("1000m", "2000")).Obj(),
enableDRAExtendedResource: true,
expectedFragments: nil,
expectedStatusCode: fwk.Unschedulable,
@ -495,3 +429,16 @@ func TestBalancedAllocationSignPod(t *testing.T) {
})
}
}
func cpuOnly(req string) map[v1.ResourceName]string {
return map[v1.ResourceName]string{v1.ResourceCPU: req}
}
func cpuAndMemory(cpuReq, memoryReq string) map[v1.ResourceName]string {
return map[v1.ResourceName]string{v1.ResourceCPU: cpuReq, v1.ResourceMemory: memoryReq}
}
func cpuAndMemoryAndGpu(cpuReq, memoryReq, gpuReq string) map[v1.ResourceName]string {
return map[v1.ResourceName]string{v1.ResourceCPU: cpuReq, v1.ResourceMemory: memoryReq, "nvidia.com/gpu": gpuReq}
}
func gpu(count int64) map[string]int64 {
return map[string]int64{"nvidia.com/gpu": count}
}

4
pkg/scheduler/framework/plugins/noderesources/least_allocated.go
View file

@ -27,8 +27,8 @@ import (
//
// Details:
// (cpu((capacity-requested)*MaxNodeScore*cpuWeight/capacity) + memory((capacity-requested)*MaxNodeScore*memoryWeight/capacity) + ...)/weightSum
func leastResourceScorer(resources []config.ResourceSpec) func([]int64, []int64) int64 {
return func(requested, allocable []int64) int64 {
func leastResourceScorer(resources []config.ResourceSpec) func([]int64, []int64, []int64) int64 {
return func(requested, _, allocable []int64) int64 {
var nodeScore, weightSum int64
for i := range requested {
if allocable[i] == 0 {

4
pkg/scheduler/framework/plugins/noderesources/most_allocated.go
View file

@ -27,8 +27,8 @@ import (
//
// Details:
// (cpu(MaxNodeScore * requested * cpuWeight / capacity) + memory(MaxNodeScore * requested * memoryWeight / capacity) + ...) / weightSum
func mostResourceScorer(resources []config.ResourceSpec) func(requested, allocable []int64) int64 {
return func(requested, allocable []int64) int64 {
func mostResourceScorer(resources []config.ResourceSpec) func(requested, _, allocable []int64) int64 {
return func(requested, _, allocable []int64) int64 {
var nodeScore, weightSum int64
for i := range requested {
if allocable[i] == 0 {

6
pkg/scheduler/framework/plugins/noderesources/requested_to_capacity_ratio.go
View file

@ -28,7 +28,7 @@ const maxUtilization = 100
// buildRequestedToCapacityRatioScorerFunction allows users to apply bin packing
// on core resources like CPU, Memory as well as extended resources like accelerators.
func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.FunctionShape, resources []config.ResourceSpec) func([]int64, []int64) int64 {
func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.FunctionShape, resources []config.ResourceSpec) func([]int64, []int64, []int64) int64 {
rawScoringFunction := helper.BuildBrokenLinearFunction(scoringFunctionShape)
resourceScoringFunction := func(requested, capacity int64) int64 {
if capacity == 0 || requested > capacity {
@ -37,7 +37,7 @@ func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.Fun
return rawScoringFunction(requested * maxUtilization / capacity)
}
return func(requested, allocable []int64) int64 {
return func(requested, _, allocable []int64) int64 {
var nodeScore, weightSum int64
for i := range requested {
if allocable[i] == 0 {
@ -57,7 +57,7 @@ func buildRequestedToCapacityRatioScorerFunction(scoringFunctionShape helper.Fun
}
}
func requestedToCapacityRatioScorer(resources []config.ResourceSpec, shape []config.UtilizationShapePoint) func([]int64, []int64) int64 {
func requestedToCapacityRatioScorer(resources []config.ResourceSpec, shape []config.UtilizationShapePoint) func([]int64, []int64, []int64) int64 {
shapes := make([]helper.FunctionShapePoint, 0, len(shape))
for _, point := range shape {
shapes = append(shapes, helper.FunctionShapePoint{

39
pkg/scheduler/framework/plugins/noderesources/resource_allocation.go
View file

@ -58,7 +58,7 @@ type resourceAllocationScorer struct {
// used to decide whether to use Requested or NonZeroRequested for
// cpu and memory.
useRequested bool
scorer func(requested, allocable []int64) int64
scorer func(requested, allocated, allocatable []int64) int64
resources []config.ResourceSpec
draFeatures structured.Features
draManager fwk.SharedDRAManager
@ -150,19 +150,27 @@ func (r *resourceAllocationScorer) score(
return 0, fwk.NewStatus(fwk.Error, "resources not found")
}
allocated := make([]int64, len(r.resources))
requested := make([]int64, len(r.resources))
allocatable := make([]int64, len(r.resources))
for i := range r.resources {
alloc, req := r.calculateResourceAllocatableRequest(ctx, nodeInfo, v1.ResourceName(r.resources[i].Name), podRequests[i], draPreScoreState)
// Only fill the extended resource entry when it's non-zero.
if alloc == 0 {
resource := v1.ResourceName(r.resources[i].Name)
// If it's an extended resource, and the pod doesn't request it.
// We don't fill the resource entry as an implication to bypass scoring on it.
if podRequests[i] == 0 && schedutil.IsScalarResourceName(resource) {
continue
}
allocatable[i] = alloc
requested[i] = req
nodeAllocatable, nodeAllocated := r.calculateResourceAllocatableRequest(ctx, nodeInfo, resource, draPreScoreState)
// Only fill the extended resource entry when it's non-zero.
if nodeAllocatable == 0 {
continue
}
allocatable[i] = nodeAllocatable
allocated[i] = nodeAllocated
requested[i] = allocated[i] + podRequests[i]
}
score := r.scorer(requested, allocatable)
score := r.scorer(requested, allocated, allocatable)
if loggerV := logger.V(10); loggerV.Enabled() { // Serializing these maps is costly.
loggerV.Info("Listed internal info for allocatable resources, requested resources and score", "pod",
@ -177,12 +185,10 @@ func (r *resourceAllocationScorer) score(
// calculateResourceAllocatableRequest returns 2 parameters:
// - 1st param: quantity of allocatable resource on the node.
// - 2nd param: aggregated quantity of requested resource on the node.
// Note: if it's an extended resource, and the pod doesn't request it, (0, 0) is returned.
func (r *resourceAllocationScorer) calculateResourceAllocatableRequest(
ctx context.Context,
nodeInfo fwk.NodeInfo,
resource v1.ResourceName,
podRequest int64,
draPreScoreState *draPreScoreState,
) (int64, int64) {
requested := nodeInfo.GetNonZeroRequested()
@ -190,18 +196,13 @@ func (r *resourceAllocationScorer) calculateResourceAllocatableRequest(
requested = nodeInfo.GetRequested()
}
// If it's an extended resource, and the pod doesn't request it. We return (0, 0)
// as an implication to bypass scoring on this resource.
if podRequest == 0 && schedutil.IsScalarResourceName(resource) {
return 0, 0
}
switch resource {
case v1.ResourceCPU:
return nodeInfo.GetAllocatable().GetMilliCPU(), (requested.GetMilliCPU() + podRequest)
return nodeInfo.GetAllocatable().GetMilliCPU(), requested.GetMilliCPU()
case v1.ResourceMemory:
return nodeInfo.GetAllocatable().GetMemory(), (requested.GetMemory() + podRequest)
return nodeInfo.GetAllocatable().GetMemory(), requested.GetMemory()
case v1.ResourceEphemeralStorage:
return nodeInfo.GetAllocatable().GetEphemeralStorage(), (nodeInfo.GetRequested().GetEphemeralStorage() + podRequest)
return nodeInfo.GetAllocatable().GetEphemeralStorage(), nodeInfo.GetRequested().GetEphemeralStorage()
default:
allocatable, exists := nodeInfo.GetAllocatable().GetScalarResources()[resource]
if allocatable == 0 && r.enableDRAExtendedResource && draPreScoreState != nil {
@ -209,11 +210,11 @@ func (r *resourceAllocationScorer) calculateResourceAllocatableRequest(
// Calculate allocatable and requested for resources backed by DRA.
allocatable, allocated := r.calculateDRAExtendedResourceAllocatableRequest(ctx, nodeInfo.Node(), resource, draPreScoreState)
if allocatable > 0 {
return allocatable, allocated + podRequest
return allocatable, allocated
}
}
if exists {
return allocatable, (nodeInfo.GetRequested().GetScalarResources()[resource] + podRequest)
return allocatable, nodeInfo.GetRequested().GetScalarResources()[resource]
}
}
klog.FromContext(ctx).V(10).Info("Requested resource is omitted for node score calculation", "resourceName", resource)

70
pkg/scheduler/framework/plugins/noderesources/resource_allocation_test.go
View file

@ -286,44 +286,40 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) {
enableDRAExtendedResource bool
node *v1.Node
extendedResource v1.ResourceName
objects []apiruntime.Object
podRequest int64
draObjects []apiruntime.Object
expectedAllocatable int64
expectedRequested int64
expectedAllocated int64
}{
"device-plugin-resource-feature-disabled": {
enableDRAExtendedResource: false,
node: st.MakeNode().Name(nodeName).Capacity(map[v1.ResourceName]string{explicitExtendedResource: "4"}).Obj(),
extendedResource: explicitExtendedResource,
podRequest: 1,
expectedAllocatable: 4,
expectedRequested: 1,
expectedAllocated: 0,
},
"device-plugin-resource-feature-enabled": {
enableDRAExtendedResource: true,
node: st.MakeNode().Name(nodeName).Capacity(map[v1.ResourceName]string{explicitExtendedResource: "4"}).Obj(),
extendedResource: explicitExtendedResource,
podRequest: 1,
expectedAllocatable: 4,
expectedRequested: 1,
expectedAllocated: 0,
},
"DRA-backed-resource-explicit": {
enableDRAExtendedResource: true,
node: st.MakeNode().Name(nodeName).Obj(),
extendedResource: explicitExtendedResource,
objects: []apiruntime.Object{
draObjects: []apiruntime.Object{
deviceClassWithExtendResourceName,
st.MakeResourceSlice(nodeName, driverName).Device("device-1").Obj(),
},
podRequest: 1,
expectedAllocatable: 1,
expectedRequested: 1,
expectedAllocatable: 1, // 1 device `device-1`
expectedAllocated: 0,
},
"DRA-backed-resource-implicit": {
enableDRAExtendedResource: true,
node: st.MakeNode().Name(nodeName).Obj(),
extendedResource: implicitExtendedResource,
objects: []apiruntime.Object{
draObjects: []apiruntime.Object{
&resourceapi.DeviceClass{
ObjectMeta: metav1.ObjectMeta{
Name: deviceClassName,
@ -331,24 +327,24 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) {
},
st.MakeResourceSlice(nodeName, driverName).Device("device-1").Obj(),
},
podRequest: 1,
expectedAllocatable: 1,
expectedRequested: 1,
expectedAllocatable: 1, // 1 device `device-1`
expectedAllocated: 0,
},
"DRA-backed-resource-no-slices": {
enableDRAExtendedResource: true,
node: st.MakeNode().Name(nodeName).Obj(),
extendedResource: explicitExtendedResource,
objects: []apiruntime.Object{deviceClassWithExtendResourceName},
podRequest: 1,
expectedAllocatable: 0,
expectedRequested: 0,
draObjects: []apiruntime.Object{
deviceClassWithExtendResourceName,
},
expectedAllocatable: 0,
expectedAllocated: 0,
},
"DRA-backed-resource-with-allocated-device": {
enableDRAExtendedResource: true,
node: st.MakeNode().Name(nodeName).Obj(),
extendedResource: explicitExtendedResource,
objects: []apiruntime.Object{
draObjects: []apiruntime.Object{
deviceClassWithExtendResourceName,
st.MakeResourceSlice(nodeName, driverName).Devices("device-1", "device-2").Obj(),
// Create a resource claim that fully allocates device-1
@ -369,15 +365,14 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) {
}).
Obj(),
},
podRequest: 1,
expectedAllocatable: 2,
expectedRequested: 2, // 1 allocated + 1 requested
expectedAllocatable: 2, // 2 allocatable devices `device-1`, `device-2`
expectedAllocated: 1, // 1 allocated by `testClaim`
},
"DRA-backed-resource-with-shared-device-allocation": {
enableDRAExtendedResource: true,
node: st.MakeNode().Name(nodeName).Obj(),
extendedResource: explicitExtendedResource,
objects: []apiruntime.Object{
draObjects: []apiruntime.Object{
deviceClassWithExtendResourceName,
st.MakeResourceSlice(nodeName, driverName).Devices("device-1", "device-2").Obj(),
// Create a resource claim with shared device allocation (consumable capacity)
@ -399,15 +394,14 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) {
}).
Obj(),
},
podRequest: 1,
expectedAllocatable: 2,
expectedRequested: 2, // 1 allocated (shared) + 1 requested
expectedAllocatable: 2, // 2 allocatable devices `device-1`, `device-2`
expectedAllocated: 1, // 1 allocated (shared) by `testClaim`
},
"DRA-backed-resource-multiple-devices-mixed-allocation": {
enableDRAExtendedResource: true,
node: st.MakeNode().Name(nodeName).Obj(),
extendedResource: explicitExtendedResource,
objects: []apiruntime.Object{
draObjects: []apiruntime.Object{
deviceClassWithExtendResourceName,
st.MakeResourceSlice(nodeName, driverName).Devices("device-1", "device-2", "device-3").Obj(),
// Mix of fully allocated and shared device allocations
@ -447,15 +441,14 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) {
Obj(),
// device-3 remains unallocated
},
podRequest: 1,
expectedAllocatable: 3,
expectedRequested: 3, // 2 allocated (1 full + 1 shared) + 1 requested
expectedAllocatable: 3, // 3 allocatable devices `device-1`, `device-2`, `device-3`
expectedAllocated: 2, // 2 allocated (1 full `test-claim-1` + 1 shared `test-claim-2`)
},
"DRA-backed-resource-with-per-device-node-selection": {
enableDRAExtendedResource: true,
node: st.MakeNode().Name(nodeName).Label("zone", "us-east-1a").Obj(),
extendedResource: explicitExtendedResource,
objects: []apiruntime.Object{
draObjects: []apiruntime.Object{
&resourceapi.DeviceClass{
ObjectMeta: metav1.ObjectMeta{
Name: deviceClassName,
@ -545,9 +538,8 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) {
}).
Obj(),
},
podRequest: 1,
expectedAllocatable: 2, // device-1 matches the test node and device-3 matches via its selector
expectedRequested: 2, // 1 allocated (device-1) + 1 requested
expectedAllocated: 1, // 1 allocated (device-1)
},
}
@ -556,7 +548,7 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) {
// Setup environment, create required objects
featuregatetesting.SetFeatureGateDuringTest(tCtx, utilfeature.DefaultFeatureGate, features.DRAExtendedResource, tc.enableDRAExtendedResource)
draManager := newTestDRAManager(tCtx, tc.objects...)
draManager := newTestDRAManager(tCtx, tc.draObjects...)
nodeInfo := framework.NewNodeInfo()
nodeInfo.SetNode(tc.node)
@ -580,12 +572,12 @@ func testCalculateResourceAllocatableRequest(tCtx ktesting.TContext) {
}
// Test calculateResourceAllocatableRequest API
allocatable, requested := scorer.calculateResourceAllocatableRequest(tCtx, nodeInfo, tc.extendedResource, tc.podRequest, draPreScoreState)
allocatable, allocated := scorer.calculateResourceAllocatableRequest(tCtx, nodeInfo, tc.extendedResource, draPreScoreState)
if !cmp.Equal(allocatable, tc.expectedAllocatable) {
tCtx.Errorf("Expected allocatable=%v, but got allocatable=%v", tc.expectedAllocatable, allocatable)
tCtx.Errorf("Expected allocatable=%v, but got %v", tc.expectedAllocatable, allocatable)
}
if !cmp.Equal(requested, tc.expectedRequested) {
tCtx.Errorf("Expected requested=%v, but got requested=%v", tc.expectedRequested, requested)
if !cmp.Equal(allocated, tc.expectedAllocated) {
tCtx.Errorf("Expected allocated=%v, but got %v", tc.expectedAllocated, allocated)
}
})
}

150
pkg/scheduler/schedule_one_test.go
View file

@ -3747,156 +3747,6 @@ func TestFindFitPredicateCallCounts(t *testing.T) {
}
}
// The point of this test is to show that you:
// - get the same priority for a zero-request pod as for a pod with the defaults requests,
// both when the zero-request pod is already on the node and when the zero-request pod
// is the one being scheduled.
// - don't get the same score no matter what we schedule.
func TestZeroRequest(t *testing.T) {
// A pod with no resources. We expect spreading to count it as having the default resources.
noResources := v1.PodSpec{
Containers: []v1.Container{
{},
},
}
noResources1 := noResources
noResources1.NodeName = "node1"
// A pod with the same resources as a 0-request pod gets by default as its resources (for spreading).
small := v1.PodSpec{
Containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse(
strconv.FormatInt(schedutil.DefaultMilliCPURequest, 10) + "m"),
v1.ResourceMemory: resource.MustParse(
strconv.FormatInt(schedutil.DefaultMemoryRequest, 10)),
},
},
},
},
}
small2 := small
small2.NodeName = "node2"
// A larger pod.
large := v1.PodSpec{
Containers: []v1.Container{
{
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceCPU: resource.MustParse(
strconv.FormatInt(schedutil.DefaultMilliCPURequest*3, 10) + "m"),
v1.ResourceMemory: resource.MustParse(
strconv.FormatInt(schedutil.DefaultMemoryRequest*3, 10)),
},
},
},
},
}
large1 := large
large1.NodeName = "node1"
large2 := large
large2.NodeName = "node2"
tests := []struct {
pod *v1.Pod
pods []*v1.Pod
nodes []*v1.Node
name string
expectedScore int64
}{
// The point of these next two tests is to show you get the same priority for a zero-request pod
// as for a pod with the defaults requests, both when the zero-request pod is already on the node
// and when the zero-request pod is the one being scheduled.
{
pod: &v1.Pod{Spec: noResources},
nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10), makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10)},
name: "test priority of zero-request pod with node with zero-request pod",
pods: []*v1.Pod{
{Spec: large1}, {Spec: noResources1},
{Spec: large2}, {Spec: small2},
},
expectedScore: 50,
},
{
pod: &v1.Pod{Spec: small},
nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10), makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10)},
name: "test priority of nonzero-request pod with node with zero-request pod",
pods: []*v1.Pod{
{Spec: large1}, {Spec: noResources1},
{Spec: large2}, {Spec: small2},
},
expectedScore: 150,
},
// The point of this test is to verify that we're not just getting the same score no matter what we schedule.
{
pod: &v1.Pod{Spec: large},
nodes: []*v1.Node{makeNode("node1", 1000, schedutil.DefaultMemoryRequest*10), makeNode("node2", 1000, schedutil.DefaultMemoryRequest*10)},
name: "test priority of larger pod with node with zero-request pod",
pods: []*v1.Pod{
{Spec: large1}, {Spec: noResources1},
{Spec: large2}, {Spec: small2},
},
expectedScore: 130,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
client := clientsetfake.NewClientset()
informerFactory := informers.NewSharedInformerFactory(client, 0)
snapshot := internalcache.NewSnapshot(test.pods, test.nodes)
fts := feature.Features{}
pluginRegistrations := []tf.RegisterPluginFunc{
tf.RegisterQueueSortPlugin(queuesort.Name, queuesort.New),
tf.RegisterScorePlugin(noderesources.Name, frameworkruntime.FactoryAdapter(fts, noderesources.NewFit), 1),
tf.RegisterScorePlugin(noderesources.BalancedAllocationName, frameworkruntime.FactoryAdapter(fts, noderesources.NewBalancedAllocation), 1),
tf.RegisterBindPlugin(defaultbinder.Name, defaultbinder.New),
}
_, ctx := ktesting.NewTestContext(t)
ctx, cancel := context.WithCancel(ctx)
defer cancel()
fwk, err := tf.NewFramework(
ctx,
pluginRegistrations, "",
frameworkruntime.WithInformerFactory(informerFactory),
frameworkruntime.WithSnapshotSharedLister(snapshot),
frameworkruntime.WithClientSet(client),
frameworkruntime.WithPodNominator(internalqueue.NewSchedulingQueue(nil, informerFactory)),
)
if err != nil {
t.Fatalf("error creating framework: %+v", err)
}
sched := &Scheduler{
nodeInfoSnapshot: snapshot,
percentageOfNodesToScore: schedulerapi.DefaultPercentageOfNodesToScore,
}
sched.applyDefaultHandlers()
state := framework.NewCycleState()
_, _, _, _, err = sched.findNodesThatFitPod(ctx, fwk, state, test.pod)
if err != nil {
t.Fatalf("error filtering nodes: %+v", err)
}
nodeInfos, err := snapshot.NodeInfos().List()
if err != nil {
t.Fatalf("failed to list node from snapshot: %v", err)
}
fwk.RunPreScorePlugins(ctx, state, test.pod, nodeInfos)
list, err := prioritizeNodes(ctx, nil, fwk, state, test.pod, nodeInfos)
if err != nil {
t.Errorf("unexpected error: %v", err)
}
for _, hp := range list {
if hp.TotalScore != test.expectedScore {
t.Errorf("expected %d for all priorities, got list %#v", test.expectedScore, list)
}
}
})
}
}
func Test_prioritizeNodes(t *testing.T) {
imageStatus1 := []v1.ContainerImage{
{