mirror of
https://github.com/kubernetes/kubernetes.git
synced 2026-05-23 18:35:51 -04:00
67800794aa
The client-go variant of ktesting is a superset of the normal ktesting, which makes it possible to get the full original functionality simply by changing the import path.
669 lines
24 KiB
Go
669 lines
24 KiB
Go
/*
|
|
Copyright The Kubernetes Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package dra
|
|
|
|
import (
|
|
"fmt"
|
|
"time"
|
|
|
|
"github.com/onsi/gomega"
|
|
"github.com/onsi/gomega/gstruct"
|
|
gtypes "github.com/onsi/gomega/types"
|
|
v1 "k8s.io/api/core/v1"
|
|
resourceapi "k8s.io/api/resource/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
st "k8s.io/kubernetes/pkg/scheduler/testing"
|
|
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
|
|
"k8s.io/kubernetes/test/utils/client-go/ktesting"
|
|
"k8s.io/utils/ptr"
|
|
)
|
|
|
|
type testEnv struct {
|
|
namespace string
|
|
nodeName string
|
|
class *resourceapi.DeviceClass
|
|
driverName string
|
|
poolName string
|
|
}
|
|
|
|
func testNodeAllocatableResources(tCtx ktesting.TContext, enabled bool) {
|
|
if !enabled {
|
|
return
|
|
}
|
|
tCtx.Run("ConsumablePool", testNodeAllocatableResourcesConsumablePool)
|
|
tCtx.Run("IndividualDevices", testNodeAllocatableResourcesIndividualDevices)
|
|
tCtx.Run("ClaimSharing", testNodeAllocatableResourceClaimSharing)
|
|
tCtx.Run("PodLevelResourceValidation", testPodLevelResourceValidation)
|
|
tCtx.Run("InsufficientNodeResources", testInsufficientNodeResources)
|
|
tCtx.Run("ClaimTemplateBasedAllocation", testNodeAllocatableResourcesWithClaimTemplate)
|
|
tCtx.Run("UnreferencedClaimInPod", testNodeAllocatableUnreferencedClaimInPod)
|
|
}
|
|
|
|
func setupTestEnv(tCtx ktesting.TContext, nodeNum int) *testEnv {
|
|
|
|
nodeName := fmt.Sprintf("worker-%d", nodeNum)
|
|
namespace := createTestNamespace(tCtx, nil)
|
|
class, driverName := createTestClass(tCtx, namespace)
|
|
poolName := namespace + "-pool"
|
|
|
|
return &testEnv{
|
|
namespace: namespace,
|
|
nodeName: nodeName,
|
|
class: class,
|
|
driverName: driverName,
|
|
poolName: poolName,
|
|
}
|
|
}
|
|
|
|
func createSliceAndStartScheduler(tCtx ktesting.TContext, slice *resourceapi.ResourceSlice) {
|
|
tCtx.Helper()
|
|
createSlice(tCtx, slice)
|
|
startScheduler(tCtx)
|
|
}
|
|
|
|
func makeSlice(env *testEnv, devices []resourceapi.Device) *resourceapi.ResourceSlice {
|
|
return &resourceapi.ResourceSlice{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: env.namespace + "-",
|
|
},
|
|
Spec: resourceapi.ResourceSliceSpec{
|
|
NodeName: &env.nodeName,
|
|
Pool: resourceapi.ResourcePool{
|
|
Name: env.poolName,
|
|
ResourceSliceCount: 1,
|
|
},
|
|
Driver: env.driverName,
|
|
Devices: devices,
|
|
},
|
|
}
|
|
}
|
|
|
|
func expectPodUnschedulable(tCtx ktesting.TContext, pod *v1.Pod, reason string) {
|
|
tCtx.Helper()
|
|
tCtx.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(tCtx, tCtx.Client(), pod.Name, pod.Namespace), fmt.Sprintf("expected pod to be unschedulable because %q", reason))
|
|
pod, err := tCtx.Client().CoreV1().Pods(pod.Namespace).Get(tCtx, pod.Name, metav1.GetOptions{})
|
|
tCtx.ExpectNoError(err)
|
|
gomega.NewWithT(tCtx).Expect(pod).To(gomega.HaveField("Status.Conditions", gomega.ContainElement(gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
|
"Type": gomega.Equal(v1.PodScheduled),
|
|
"Status": gomega.Equal(v1.ConditionFalse),
|
|
"Reason": gomega.Equal(v1.PodReasonUnschedulable),
|
|
"Message": gomega.ContainSubstring(reason),
|
|
}))))
|
|
}
|
|
|
|
func verifyPodNodeAllocatableStatus(tCtx ktesting.TContext, namespace, podName string, expectedStatus []v1.NodeAllocatableResourceClaimStatus) {
|
|
tCtx.Helper()
|
|
var statusMatchers []gtypes.GomegaMatcher
|
|
for _, expected := range expectedStatus {
|
|
statusMatchers = append(statusMatchers, gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
|
"ResourceClaimName": gomega.ContainSubstring(expected.ResourceClaimName),
|
|
"Containers": gomega.Equal(expected.Containers),
|
|
"Resources": gomega.Equal(expected.Resources),
|
|
}))
|
|
}
|
|
tCtx.Eventually(func(tCtx ktesting.TContext) []v1.NodeAllocatableResourceClaimStatus {
|
|
pod, err := tCtx.Client().CoreV1().Pods(namespace).Get(tCtx, podName, metav1.GetOptions{})
|
|
if err != nil {
|
|
tCtx.Logf("Error getting pod: %v", err)
|
|
return nil
|
|
}
|
|
return pod.Status.NodeAllocatableResourceClaimStatuses
|
|
}).WithTimeout(30*time.Second).WithPolling(200*time.Millisecond).Should(gomega.ConsistOf(statusMatchers), "pod node allocatable resource claim status")
|
|
}
|
|
|
|
func createPodWithNodeAllocatableClaim(tCtx ktesting.TContext, env *testEnv, claimName, podName string, numContainers int) (*v1.Pod, *resourceapi.ResourceClaim) {
|
|
tCtx.Helper()
|
|
// Create a ResourceClaim for the node allocatable resource class
|
|
claim := st.MakeResourceClaim().
|
|
Name(claimName).
|
|
Namespace(env.namespace).
|
|
Request(env.class.Name).
|
|
Obj()
|
|
claim = createClaim(tCtx, env.namespace, "", env.class, claim)
|
|
|
|
// Create a Pod that uses the node allocatable claim
|
|
pod := st.MakePod().Name(podName).Namespace(env.namespace).Obj()
|
|
|
|
containers := make([]v1.Container, 0, numContainers)
|
|
for i := range numContainers {
|
|
containers = append(containers, v1.Container{
|
|
Name: fmt.Sprintf("my-container-%d", i+1),
|
|
Image: "test-image",
|
|
Resources: v1.ResourceRequirements{
|
|
Claims: []v1.ResourceClaim{
|
|
{Name: claimName},
|
|
},
|
|
},
|
|
})
|
|
}
|
|
pod.Spec.Containers = containers
|
|
pod.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": env.nodeName}
|
|
|
|
// createPod will generate the pod.Spec.ResourceClaims entry
|
|
pod = createPod(tCtx, env.namespace, "", pod, claim)
|
|
return pod, claim
|
|
}
|
|
|
|
func testNodeAllocatableResourcesConsumablePool(tCtx ktesting.TContext) {
|
|
tCtx.Parallel()
|
|
env := setupTestEnv(tCtx, 0)
|
|
|
|
cpuCapacityKey := resourceapi.QualifiedName("dra.example.com/cpu")
|
|
memCapacityKey := resourceapi.QualifiedName("dra.example.com/memory")
|
|
|
|
devices := []resourceapi.Device{
|
|
{
|
|
Name: "node-allocatable-device-0",
|
|
AllowMultipleAllocations: ptr.To(true),
|
|
Capacity: map[resourceapi.QualifiedName]resourceapi.DeviceCapacity{
|
|
cpuCapacityKey: {Value: resource.MustParse(nodeCPUCapacity)},
|
|
memCapacityKey: {Value: resource.MustParse(nodeMemoryCapacity)},
|
|
},
|
|
NodeAllocatableResourceMappings: map[v1.ResourceName]resourceapi.NodeAllocatableResourceMapping{
|
|
v1.ResourceCPU: {CapacityKey: &cpuCapacityKey},
|
|
v1.ResourceMemory: {CapacityKey: &memCapacityKey},
|
|
},
|
|
},
|
|
}
|
|
slice := makeSlice(env, devices)
|
|
env.poolName = env.namespace + "-node-allocatable-pool" // Override poolName for this test
|
|
slice.Spec.Pool.Name = env.poolName
|
|
createSliceAndStartScheduler(tCtx, slice)
|
|
|
|
podName := "test-pod-consumable-claim"
|
|
claimName := "node-allocatable-claim-consumable"
|
|
pod, claim := createPodWithNodeAllocatableClaim(tCtx, env, claimName, podName, 2)
|
|
|
|
waitForPodScheduled(tCtx, env.namespace, pod.Name)
|
|
allocatedClaim := waitForClaimAllocatedToDevice(tCtx, env.namespace, claim.Name, schedulingTimeout)
|
|
|
|
gomega.NewWithT(tCtx).Expect(allocatedClaim).To(
|
|
gomega.HaveField("Status.Allocation", gstruct.PointTo(
|
|
gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
|
"Devices": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
|
"Results": gomega.ConsistOf(
|
|
gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
|
"Request": gomega.Equal(claim.Spec.Devices.Requests[0].Name),
|
|
"Driver": gomega.Equal(env.driverName),
|
|
"Pool": gomega.Equal(env.poolName),
|
|
"Device": gomega.Equal("node-allocatable-device-0"),
|
|
"ConsumedCapacity": gomega.Equal(map[resourceapi.QualifiedName]resource.Quantity{
|
|
cpuCapacityKey: resource.MustParse(nodeCPUCapacity),
|
|
memCapacityKey: resource.MustParse(nodeMemoryCapacity),
|
|
}),
|
|
"ShareID": gomega.Not(gomega.BeNil()),
|
|
}),
|
|
),
|
|
}),
|
|
}),
|
|
)),
|
|
"node allocatable claim allocation",
|
|
)
|
|
|
|
expectedStatus := []v1.NodeAllocatableResourceClaimStatus{{
|
|
ResourceClaimName: claim.Name,
|
|
Containers: []string{"my-container-1", "my-container-2"},
|
|
Resources: map[v1.ResourceName]resource.Quantity{
|
|
v1.ResourceCPU: resource.MustParse(nodeCPUCapacity),
|
|
v1.ResourceMemory: resource.MustParse(nodeMemoryCapacity),
|
|
},
|
|
}}
|
|
verifyPodNodeAllocatableStatus(tCtx, env.namespace, pod.Name, expectedStatus)
|
|
|
|
anotherPod := st.MakePod().Name("another-pod").Namespace(env.namespace).Obj()
|
|
anotherPod.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": env.nodeName}
|
|
anotherPod.Spec.Containers = []v1.Container{
|
|
{
|
|
Name: "test-container",
|
|
Image: "test-image",
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")},
|
|
},
|
|
},
|
|
}
|
|
anotherPod = createPod(tCtx, env.namespace, "", anotherPod)
|
|
expectPodUnschedulable(tCtx, anotherPod, "Insufficient cpu")
|
|
}
|
|
|
|
func testNodeAllocatableResourcesIndividualDevices(tCtx ktesting.TContext) {
|
|
tCtx.Parallel()
|
|
env := setupTestEnv(tCtx, 1)
|
|
|
|
numCPUsPerDevice := resource.MustParse("16")
|
|
devices := []resourceapi.Device{
|
|
{
|
|
Name: "numa-0-cpus",
|
|
NodeAllocatableResourceMappings: map[v1.ResourceName]resourceapi.NodeAllocatableResourceMapping{
|
|
v1.ResourceCPU: {AllocationMultiplier: &numCPUsPerDevice},
|
|
},
|
|
},
|
|
{
|
|
Name: "numa-1-cpus",
|
|
NodeAllocatableResourceMappings: map[v1.ResourceName]resourceapi.NodeAllocatableResourceMapping{
|
|
v1.ResourceCPU: {AllocationMultiplier: &numCPUsPerDevice},
|
|
},
|
|
},
|
|
}
|
|
slice := makeSlice(env, devices)
|
|
env.poolName = env.namespace + "-node-allocatable-pool" // Override poolName for this test
|
|
slice.Spec.Pool.Name = env.poolName
|
|
createSliceAndStartScheduler(tCtx, slice)
|
|
|
|
podName := "test-pod-individual"
|
|
claimName := "node-allocatable-claim-individual"
|
|
pod, claim := createPodWithNodeAllocatableClaim(tCtx, env, claimName, podName, 2)
|
|
|
|
waitForPodScheduled(tCtx, env.namespace, pod.Name)
|
|
allocatedClaim := waitForClaimAllocatedToDevice(tCtx, env.namespace, claim.Name, schedulingTimeout)
|
|
|
|
gomega.NewWithT(tCtx).Expect(allocatedClaim).To(
|
|
gomega.HaveField("Status.Allocation", gstruct.PointTo(
|
|
gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
|
"Devices": gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
|
"Results": gomega.ConsistOf(
|
|
gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
|
"Request": gomega.Equal(claim.Spec.Devices.Requests[0].Name),
|
|
"Driver": gomega.Equal(env.driverName),
|
|
"Pool": gomega.Equal(env.poolName),
|
|
"Device": gomega.MatchRegexp("numa-[0-1]+-cpus"),
|
|
}),
|
|
),
|
|
}),
|
|
}),
|
|
)),
|
|
"node allocatable claim allocation",
|
|
)
|
|
|
|
expectedStatus := []v1.NodeAllocatableResourceClaimStatus{{
|
|
ResourceClaimName: claim.Name,
|
|
Containers: []string{"my-container-1", "my-container-2"},
|
|
Resources: map[v1.ResourceName]resource.Quantity{
|
|
v1.ResourceCPU: numCPUsPerDevice,
|
|
},
|
|
}}
|
|
verifyPodNodeAllocatableStatus(tCtx, env.namespace, pod.Name, expectedStatus)
|
|
}
|
|
|
|
func testNodeAllocatableResourceClaimSharing(tCtx ktesting.TContext) {
|
|
tCtx.Parallel()
|
|
env := setupTestEnv(tCtx, 2)
|
|
|
|
cpuMultiplier := resource.MustParse("1")
|
|
devices := []resourceapi.Device{
|
|
{
|
|
Name: "dev-sharetest",
|
|
NodeAllocatableResourceMappings: map[v1.ResourceName]resourceapi.NodeAllocatableResourceMapping{
|
|
v1.ResourceCPU: {AllocationMultiplier: &cpuMultiplier},
|
|
},
|
|
},
|
|
}
|
|
createSliceAndStartScheduler(tCtx, makeSlice(env, devices))
|
|
|
|
pod1, claim := createPodWithNodeAllocatableClaim(tCtx, env, "node-allocatable-claim", "pod1", 2)
|
|
waitForPodScheduled(tCtx, env.namespace, pod1.Name)
|
|
_ = waitForClaimAllocatedToDevice(tCtx, env.namespace, claim.Name, schedulingTimeout)
|
|
|
|
// Pod 2 - Should NOT schedule as the claim is already used on this node
|
|
pod2 := st.MakePod().Name("pod2").Namespace(env.namespace).Obj()
|
|
container1 := v1.Container{
|
|
Name: "c1",
|
|
Image: "test-image",
|
|
Resources: v1.ResourceRequirements{
|
|
Claims: []v1.ResourceClaim{
|
|
{Name: claim.Name}, // USE EXISTING CLAIM
|
|
},
|
|
},
|
|
}
|
|
pod2.Spec.Containers = []v1.Container{container1}
|
|
pod2.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": env.nodeName}
|
|
pod2 = createPod(tCtx, env.namespace, "", pod2, claim) // Pass the existing claim
|
|
expectPodUnschedulable(tCtx, pod2, "is already used by another pod")
|
|
}
|
|
|
|
func testPodLevelResourceValidation(tCtx ktesting.TContext) {
|
|
tCtx.Parallel()
|
|
env := setupTestEnv(tCtx, 3)
|
|
|
|
cpuMultiplier := resource.MustParse("4")
|
|
devices := []resourceapi.Device{
|
|
{
|
|
Name: "dev0",
|
|
NodeAllocatableResourceMappings: map[v1.ResourceName]resourceapi.NodeAllocatableResourceMapping{
|
|
v1.ResourceCPU: {AllocationMultiplier: &cpuMultiplier},
|
|
},
|
|
},
|
|
{
|
|
Name: "dev1",
|
|
NodeAllocatableResourceMappings: map[v1.ResourceName]resourceapi.NodeAllocatableResourceMapping{
|
|
v1.ResourceCPU: {AllocationMultiplier: &cpuMultiplier},
|
|
},
|
|
},
|
|
}
|
|
createSliceAndStartScheduler(tCtx, makeSlice(env, devices))
|
|
|
|
tCtx.Run("SufficientPodLevelRequest", func(tCtx ktesting.TContext) {
|
|
claimName := "claim-podlevel-sufficient"
|
|
podName := "pod-podlevel-sufficient"
|
|
claim := st.MakeResourceClaim().
|
|
Name(claimName).
|
|
Namespace(env.namespace).
|
|
Request(env.class.Name).
|
|
Obj()
|
|
claim = createClaim(tCtx, env.namespace, "", env.class, claim)
|
|
|
|
pod := st.MakePod().Name(podName).Namespace(env.namespace).Obj()
|
|
pod.Spec.Resources = &v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("10"), // 2 (container) + 4 (DRA) < 10
|
|
},
|
|
}
|
|
container1 := v1.Container{
|
|
Name: "c1",
|
|
Image: "test-image",
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("2"),
|
|
},
|
|
Claims: []v1.ResourceClaim{
|
|
{Name: claimName},
|
|
},
|
|
},
|
|
}
|
|
pod.Spec.Containers = []v1.Container{container1}
|
|
pod.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": env.nodeName}
|
|
pod = createPod(tCtx, env.namespace, "", pod, claim)
|
|
|
|
waitForPodScheduled(tCtx, env.namespace, pod.Name)
|
|
})
|
|
|
|
tCtx.Run("InsufficientPodLevelRequest", func(tCtx ktesting.TContext) {
|
|
claimName := "claim-podlevel-insufficient"
|
|
podName := "pod-podlevel-insufficient"
|
|
claim := st.MakeResourceClaim().
|
|
Name(claimName).
|
|
Namespace(env.namespace).
|
|
Request(env.class.Name).
|
|
Obj()
|
|
claim = createClaim(tCtx, env.namespace, "", env.class, claim)
|
|
|
|
pod := st.MakePod().Name(podName).Namespace(env.namespace).Obj()
|
|
pod.Spec.Resources = &v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("5"), // 2 (container) + 4 (DRA) > 5
|
|
},
|
|
}
|
|
container1 := v1.Container{
|
|
Name: "c1",
|
|
Image: "test-image",
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("2"),
|
|
},
|
|
Claims: []v1.ResourceClaim{
|
|
{Name: claimName},
|
|
},
|
|
},
|
|
}
|
|
pod.Spec.Containers = []v1.Container{container1}
|
|
pod.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": env.nodeName}
|
|
pod = createPod(tCtx, env.namespace, "", pod, claim)
|
|
|
|
expectPodUnschedulable(tCtx, pod, "pod level request for cpu is insufficient to cover the aggregated container and node-allocatable DRA requests")
|
|
})
|
|
}
|
|
|
|
func testInsufficientNodeResources(tCtx ktesting.TContext) {
|
|
tCtx.Parallel()
|
|
env := setupTestEnv(tCtx, 4)
|
|
|
|
tCtx.Run("ExceedNodeResourceCapacityMultiplier", func(tCtx ktesting.TContext) {
|
|
cpuMultiplier := resource.MustParse(nodeCPUCapacity)
|
|
cpuMultiplier.Add(resource.MustParse("1"))
|
|
|
|
devices := []resourceapi.Device{
|
|
{
|
|
Name: "dev-exceed-cpu",
|
|
NodeAllocatableResourceMappings: map[v1.ResourceName]resourceapi.NodeAllocatableResourceMapping{
|
|
v1.ResourceCPU: {AllocationMultiplier: &cpuMultiplier},
|
|
},
|
|
},
|
|
}
|
|
createSliceAndStartScheduler(tCtx, makeSlice(env, devices))
|
|
|
|
pod, _ := createPodWithNodeAllocatableClaim(tCtx, env, "claim-exceed-cpu", "pod-exceed-cpu", 1)
|
|
expectPodUnschedulable(tCtx, pod, "Insufficient cpu")
|
|
})
|
|
|
|
tCtx.Run("ExceedNodeCapacityWithCapacityKey", func(tCtx ktesting.TContext) {
|
|
cpuCapacityKey := resourceapi.QualifiedName("dra.example.com/cpu")
|
|
exceedCPU := resource.MustParse(nodeCPUCapacity)
|
|
exceedCPU.Add(resource.MustParse("1"))
|
|
|
|
devices := []resourceapi.Device{
|
|
{
|
|
Name: "dev-exceed-cpu-capkey",
|
|
Capacity: map[resourceapi.QualifiedName]resourceapi.DeviceCapacity{
|
|
cpuCapacityKey: {Value: exceedCPU},
|
|
},
|
|
NodeAllocatableResourceMappings: map[v1.ResourceName]resourceapi.NodeAllocatableResourceMapping{
|
|
v1.ResourceCPU: {CapacityKey: &cpuCapacityKey},
|
|
},
|
|
AllowMultipleAllocations: ptr.To(true),
|
|
},
|
|
}
|
|
createSliceAndStartScheduler(tCtx, makeSlice(env, devices))
|
|
|
|
pod, _ := createPodWithNodeAllocatableClaim(tCtx, env, "claim-exceed-cpu-capkey", "pod-exceed-cpu-capkey", 1)
|
|
expectPodUnschedulable(tCtx, pod, "Insufficient cpu")
|
|
})
|
|
|
|
tCtx.Run("StandardPodFailsDueToInsufficientResourcesAfterDRAConsumption", func(tCtx ktesting.TContext) {
|
|
cpuMultiplier := resource.MustParse("98") // Consume most of the CPU
|
|
devices := []resourceapi.Device{
|
|
{
|
|
Name: "dev-consume-most",
|
|
NodeAllocatableResourceMappings: map[v1.ResourceName]resourceapi.NodeAllocatableResourceMapping{
|
|
v1.ResourceCPU: {AllocationMultiplier: &cpuMultiplier},
|
|
},
|
|
},
|
|
}
|
|
createSliceAndStartScheduler(tCtx, makeSlice(env, devices))
|
|
|
|
pod1, _ := createPodWithNodeAllocatableClaim(tCtx, env, "claim-consume-most", "pod-consume-most", 1)
|
|
waitForPodScheduled(tCtx, env.namespace, pod1.Name)
|
|
waitForClaimAllocatedToDevice(tCtx, env.namespace, "claim-consume-most", schedulingTimeout)
|
|
|
|
pod2 := st.MakePod().Name("pod-standard-fails").Namespace(env.namespace).Obj()
|
|
pod2.Spec.Containers = []v1.Container{
|
|
{
|
|
Name: "c2",
|
|
Image: "test-image",
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("5"),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
pod2.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": env.nodeName}
|
|
pod2 = createPod(tCtx, env.namespace, "-std-fails", pod2)
|
|
expectPodUnschedulable(tCtx, pod2, "Insufficient cpu")
|
|
})
|
|
|
|
tCtx.Run("StandardPodSucceedsAfterDRA", func(tCtx ktesting.TContext) {
|
|
cpuMultiplier := resource.MustParse("50") // Consume half of the CPU
|
|
devices := []resourceapi.Device{
|
|
{
|
|
Name: "dev-consume-half",
|
|
NodeAllocatableResourceMappings: map[v1.ResourceName]resourceapi.NodeAllocatableResourceMapping{
|
|
v1.ResourceCPU: {AllocationMultiplier: &cpuMultiplier},
|
|
},
|
|
},
|
|
}
|
|
createSliceAndStartScheduler(tCtx, makeSlice(env, devices))
|
|
|
|
pod1, _ := createPodWithNodeAllocatableClaim(tCtx, env, "claim-consume-half", "pod-consume-half", 1)
|
|
waitForPodScheduled(tCtx, env.namespace, pod1.Name)
|
|
waitForClaimAllocatedToDevice(tCtx, env.namespace, "claim-consume-half", schedulingTimeout)
|
|
|
|
pod2 := st.MakePod().Name("pod-standard-succeeds").Namespace(env.namespace).Obj()
|
|
pod2.Spec.Containers = []v1.Container{
|
|
{
|
|
Name: "c2",
|
|
Image: "test-image",
|
|
Resources: v1.ResourceRequirements{
|
|
Requests: v1.ResourceList{
|
|
v1.ResourceCPU: resource.MustParse("5"),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
pod2.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": env.nodeName}
|
|
pod2 = createPod(tCtx, env.namespace, "-std-succeeds", pod2)
|
|
waitForPodScheduled(tCtx, env.namespace, pod2.Name)
|
|
})
|
|
}
|
|
|
|
func testNodeAllocatableResourcesWithClaimTemplate(tCtx ktesting.TContext) {
|
|
tCtx.Parallel()
|
|
env := setupTestEnv(tCtx, 6) // Use a different node index
|
|
|
|
cpuCapacityKey := resourceapi.QualifiedName("dra.example.com/cpu")
|
|
memCapacityKey := resourceapi.QualifiedName("dra.example.com/memory")
|
|
|
|
devices := []resourceapi.Device{
|
|
{
|
|
Name: "dev0",
|
|
AllowMultipleAllocations: ptr.To(true),
|
|
Capacity: map[resourceapi.QualifiedName]resourceapi.DeviceCapacity{
|
|
cpuCapacityKey: {Value: resource.MustParse(nodeCPUCapacity)},
|
|
memCapacityKey: {Value: resource.MustParse(nodeMemoryCapacity)},
|
|
},
|
|
NodeAllocatableResourceMappings: map[v1.ResourceName]resourceapi.NodeAllocatableResourceMapping{
|
|
v1.ResourceCPU: {CapacityKey: &cpuCapacityKey},
|
|
v1.ResourceMemory: {CapacityKey: &memCapacityKey},
|
|
},
|
|
},
|
|
}
|
|
slice := makeSlice(env, devices)
|
|
env.poolName = env.namespace + "-pool-template"
|
|
slice.Spec.Pool.Name = env.poolName
|
|
createSliceAndStartScheduler(tCtx, slice)
|
|
startClaimController(tCtx)
|
|
|
|
templateName := "node-allocatable-resource-template"
|
|
podClaimName := "claim1"
|
|
podName := "pod-with-claim-template"
|
|
|
|
claimTemplate := &resourceapi.ResourceClaimTemplate{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: templateName,
|
|
Namespace: env.namespace,
|
|
},
|
|
Spec: resourceapi.ResourceClaimTemplateSpec{
|
|
Spec: resourceapi.ResourceClaimSpec{
|
|
Devices: resourceapi.DeviceClaim{
|
|
Requests: []resourceapi.DeviceRequest{
|
|
{
|
|
Name: "req",
|
|
Exactly: &resourceapi.ExactDeviceRequest{
|
|
DeviceClassName: env.class.Name,
|
|
Capacity: &resourceapi.CapacityRequirements{
|
|
Requests: map[resourceapi.QualifiedName]resource.Quantity{
|
|
cpuCapacityKey: resource.MustParse("10"),
|
|
memCapacityKey: resource.MustParse("100"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
_, err := tCtx.Client().ResourceV1().ResourceClaimTemplates(env.namespace).Create(tCtx, claimTemplate, metav1.CreateOptions{})
|
|
tCtx.ExpectNoError(err)
|
|
|
|
claimTemplateRef := v1.PodResourceClaim{
|
|
Name: podClaimName,
|
|
ResourceClaimTemplateName: &claimTemplate.Name,
|
|
}
|
|
|
|
pod := st.MakePod().Name(podName).Namespace(env.namespace).PodResourceClaims(claimTemplateRef).
|
|
Containers([]v1.Container{{Name: "c1", Image: "test", Resources: v1.ResourceRequirements{Claims: []v1.ResourceClaim{{Name: podClaimName}}}}}).
|
|
NodeSelector(map[string]string{"kubernetes.io/hostname": env.nodeName}).
|
|
Obj()
|
|
pod = createPod(tCtx, env.namespace, "", pod)
|
|
|
|
waitForPodScheduled(tCtx, env.namespace, pod.Name)
|
|
|
|
expectedStatus := []v1.NodeAllocatableResourceClaimStatus{{
|
|
ResourceClaimName: podName, // The genereate claim based on template contains pod name
|
|
Containers: []string{"c1"},
|
|
Resources: map[v1.ResourceName]resource.Quantity{
|
|
v1.ResourceCPU: resource.MustParse("10"),
|
|
v1.ResourceMemory: resource.MustParse("100"),
|
|
},
|
|
}}
|
|
verifyPodNodeAllocatableStatus(tCtx, env.namespace, pod.Name, expectedStatus)
|
|
}
|
|
|
|
func testNodeAllocatableUnreferencedClaimInPod(tCtx ktesting.TContext) {
|
|
tCtx.Parallel()
|
|
env := setupTestEnv(tCtx, 7) // Use a different node index
|
|
|
|
cpuCapacityKey := resourceapi.QualifiedName("dra.example.com/cpu")
|
|
devices := []resourceapi.Device{
|
|
{
|
|
Name: "dev0",
|
|
AllowMultipleAllocations: ptr.To(true),
|
|
Capacity: map[resourceapi.QualifiedName]resourceapi.DeviceCapacity{
|
|
cpuCapacityKey: {Value: resource.MustParse(nodeCPUCapacity)},
|
|
},
|
|
NodeAllocatableResourceMappings: map[v1.ResourceName]resourceapi.NodeAllocatableResourceMapping{
|
|
v1.ResourceCPU: {CapacityKey: &cpuCapacityKey},
|
|
},
|
|
},
|
|
}
|
|
slice := makeSlice(env, devices)
|
|
env.poolName = env.namespace + "-pool-unused"
|
|
slice.Spec.Pool.Name = env.poolName
|
|
createSliceAndStartScheduler(tCtx, slice)
|
|
startClaimController(tCtx)
|
|
|
|
claimName := "unused-claim"
|
|
podName := "pod-with-unused-claim"
|
|
|
|
// Create a ResourceClaim for the node allocatable resource class
|
|
claim := st.MakeResourceClaim().
|
|
Name(claimName).
|
|
Namespace(env.namespace).
|
|
Request(env.class.Name).
|
|
Obj()
|
|
claim = createClaim(tCtx, env.namespace, "", env.class, claim)
|
|
|
|
// Create a Pod that has the node allocatable claim in spec.ResourceClaims, but no container uses it.
|
|
pod := st.MakePod().Name(podName).Namespace(env.namespace).
|
|
Containers([]v1.Container{{Name: "c1", Image: "test"}}).
|
|
NodeSelector(map[string]string{"kubernetes.io/hostname": env.nodeName}).
|
|
Obj()
|
|
pod = createPod(tCtx, env.namespace, "", pod, claim)
|
|
|
|
expectedErrorMsg := "node-allocatable resource claim not referenced by any container within the pod"
|
|
expectPodUnschedulable(tCtx, pod, expectedErrorMsg)
|
|
}
|