From dec79e1fb25bfffd67e8079880231ceea34ed94f Mon Sep 17 00:00:00 2001 From: Kevin Torres Date: Wed, 22 Oct 2025 06:00:21 +0000 Subject: [PATCH] E2E tests --- test/e2e/feature/feature.go | 5 + test/e2e_node/cpu_manager_test.go | 1249 ++++++++++++++++++++- test/e2e_node/memory_manager_test.go | 1504 ++++++++++++++++++++++++-- 3 files changed, 2633 insertions(+), 125 deletions(-) diff --git a/test/e2e/feature/feature.go b/test/e2e/feature/feature.go index 6d4b3f41b4a..f2dfb4650e5 100644 --- a/test/e2e/feature/feature.go +++ b/test/e2e/feature/feature.go @@ -320,6 +320,11 @@ var ( // TODO: document the feature (owning SIG, when to use this feature for a test) PodGarbageCollector = framework.WithFeature(framework.ValidFeatures.Add("PodGarbageCollector")) + // owner: sig-node + // Marks a test for pod-level resource managers feature that requires + // PodLevelResourceManagers feature gate to be enabled. + PodLevelResourceManagers = framework.WithFeature(framework.ValidFeatures.Add("PodLevelResourceManagers")) + // owner: sig-node // Marks a test for pod-level resources feature that requires // PodLevelResources feature gate to be enabled. diff --git a/test/e2e_node/cpu_manager_test.go b/test/e2e_node/cpu_manager_test.go index 7ca4c52867f..6be123a961a 100644 --- a/test/e2e_node/cpu_manager_test.go +++ b/test/e2e_node/cpu_manager_test.go @@ -41,6 +41,7 @@ import ( kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/cm" "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" + "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" admissionapi "k8s.io/pod-security-admission/api" "k8s.io/utils/cpuset" @@ -185,16 +186,7 @@ var _ = SIGDescribe("CPU Manager", ginkgo.Ordered, ginkgo.ContinueOnFailure, fra // use a closure to minimize the arguments, to make the usage more straightforward skipIfAllocatableCPUsLessThan = func(node *v1.Node, val int) { ginkgo.GinkgoHelper() - cpuReq := int64(val + reservedCPUs.Size()) // reserved CPUs are not usable, need to account them - // the framework is initialized using an injected BeforeEach node, so the - // earliest we can do is to initialize the other objects here - nodeCPUDetails := cpuDetailsFromNode(node) - - msg := fmt.Sprintf("%v full CPUs (detected=%v requested=%v reserved=%v online=%v smt=%v)", cpuReq, nodeCPUDetails.Allocatable, val, reservedCPUs.Size(), onlineCPUs.Size(), smtLevel) - ginkgo.By("Checking if allocatable: " + msg) - if nodeCPUDetails.Allocatable < cpuReq { - e2eskipper.Skipf("Skipping CPU Manager test: not allocatable %s", msg) - } + checkAllocatableCPUs(node, val, reservedCPUs, onlineCPUs, smtLevel) } }) @@ -1967,8 +1959,8 @@ var _ = SIGDescribe("CPU Manager", ginkgo.Ordered, ginkgo.ContinueOnFailure, fra }) }) -var _ = SIGDescribe("CPU Manager Incompatibility Pod Level Resources", ginkgo.Ordered, ginkgo.ContinueOnFailure, framework.WithSerial(), feature.CPUManager, feature.PodLevelResources, framework.WithFeatureGate(features.PodLevelResources), func() { - f := framework.NewDefaultFramework("cpu-manager-incompatibility-pod-level-resources-test") +var _ = SIGDescribe("CPU Manager Pod Level Resources", ginkgo.Ordered, ginkgo.ContinueOnFailure, framework.WithSerial(), feature.CPUManager, feature.PodLevelResources, feature.PodLevelResourceManagers, framework.WithFeatureGate(features.PodLevelResources), framework.WithFeatureGate(features.PodLevelResourceManagers), func() { + f := framework.NewDefaultFramework("cpu-manager-pod-level-resources") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged // original kubeletconfig before the context start, to be restored @@ -1979,6 +1971,9 @@ var _ = SIGDescribe("CPU Manager Incompatibility Pod Level Resources", ginkgo.Or var uncoreGroupSize int // tracks all the pods created by a It() block. Best would be a namespace per It block var podMap map[string]*v1.Pod + var skipIfAllocatableCPUsLessThan func(node *v1.Node, cpuReq int) + + var createPodSync func(ctx context.Context, pod *v1.Pod) *v1.Pod ginkgo.BeforeAll(func(ctx context.Context) { var err error @@ -2018,24 +2013,1130 @@ var _ = SIGDescribe("CPU Manager Incompatibility Pod Level Resources", ginkgo.Or ginkgo.BeforeEach(func(ctx context.Context) { // note intentionally NOT set reservedCPUs - this must be initialized on a test-by-test basis podMap = make(map[string]*v1.Pod) + createPodSync = func(ctx context.Context, pod *v1.Pod) *v1.Pod { + newPod := e2epod.NewPodClient(f).CreateSync(ctx, pod) + podMap[string(newPod.UID)] = newPod + return newPod + } + reservedCPUs = cpuset.New() }) ginkgo.JustBeforeEach(func(ctx context.Context) { if !e2enodeCgroupV2Enabled { e2eskipper.Skipf("Skipping since CgroupV2 not used") } + // use a closure to minimize the arguments, to make the usage more straightforward + skipIfAllocatableCPUsLessThan = func(node *v1.Node, val int) { + ginkgo.GinkgoHelper() + checkAllocatableCPUs(node, val, reservedCPUs, onlineCPUs, smtLevel) + } }) ginkgo.AfterEach(func(ctx context.Context) { deletePodsAsync(ctx, f, podMap) }) - ginkgo.When("running guaranteed pod level resources tests", ginkgo.Label("guaranteed pod level resources", "reserved-cpus"), func() { - ginkgo.It("should let the container access all the online CPUs without a reserved CPUs set", func(ctx context.Context) { + ginkgo.Context("when the topology manager scope is 'pod'", func() { + ginkgo.It("should allocate exclusive CPUs to a guaranteed pod with pod-level resources and guaranteed container, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 1) + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ - policyName: string(cpumanager.PolicyStatic), - reservedSystemCPUs: cpuset.New(), - enablePodLevelResources: true, + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + reservedSystemCPUs: cpuset.New(), + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + pod := makeCPUManagerPod("gu-pod-level-resources", []ctnAttribute{ + { + ctnName: "gu-container", + cpuRequest: "1", + cpuLimit: "1", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the expected cpuset was assigned") + + gomega.Expect(pod).To(HavePodExclusiveCPUs(1)) + + gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", 1)) + gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs)) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs)) + }) + + ginkgo.It("should allocate exclusive CPUs to a guaranteed pod with pod-level resources and non-guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 1) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + pod := makeCPUManagerPod("gu-pod-level-ngu-ctn", []ctnAttribute{ + { + ctnName: "ngu-container", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("200Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("200Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the expected cpuset was assigned") + + gomega.Expect(pod).To(HavePodExclusiveCPUs(1)) + + gomega.Expect(pod).To(HaveContainerCPUsCount("ngu-container", 1)) + gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("ngu-container", onlineCPUs)) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("ngu-container", reservedCPUs)) + }) + + ginkgo.It("should allocate exclusive CPUs to a guaranteed pod with pod-level resources and mix of guaranteed and non-guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 3) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + pod := makeCPUManagerPod("gu-pod-level-mix-ctn", []ctnAttribute{ + { + ctnName: "gu-container-1", + cpuRequest: "1", + cpuLimit: "1", + }, + { + ctnName: "gu-container-2", + cpuRequest: "1", + cpuLimit: "1", + }, + { + ctnName: "ngu-container", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("3"), + v1.ResourceMemory: resource.MustParse("300Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("3"), + v1.ResourceMemory: resource.MustParse("300Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the expected cpuset was assigned") + + gomega.Expect(pod).To(HavePodExclusiveCPUs(3)) + + gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-1", 1)) + gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-1", onlineCPUs)) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-1", reservedCPUs)) + + gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-2", 1)) + gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-2", onlineCPUs)) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-2", reservedCPUs)) + + gomega.Expect(pod).To(HaveContainerCPUsCount("ngu-container", 1)) + gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("ngu-container", onlineCPUs)) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("ngu-container", reservedCPUs)) + + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlap("gu-container-1", "gu-container-2")) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlap("gu-container-1", "ngu-container")) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlap("gu-container-2", "ngu-container")) + }) + + ginkgo.It("should allocate exclusive CPUs to a guaranteed pod with pod-level resources and mix of guaranteed and non-guaranteed standard and init containers, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 2) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + pod := makeCPUManagerPod("gu-pod-level-mix-init-ctn", []ctnAttribute{ + { + ctnName: "gu-container", + cpuRequest: "1", + cpuLimit: "1", + }, + { + ctnName: "ngu-container", + }, + }) + pod.Spec.InitContainers = []v1.Container{ + { + Name: "gu-init-container", + Image: busyboxImage, + Command: []string{"sh", "-c", "echo hello"}, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + }, + } + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("200Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("200Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the expected cpuset was assigned") + + gomega.Expect(pod).To(HavePodExclusiveCPUs(2)) + + gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", 1)) + gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs)) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs)) + + gomega.Expect(pod).To(HaveContainerCPUsCount("ngu-container", 1)) + gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("ngu-container", onlineCPUs)) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("ngu-container", reservedCPUs)) + + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlap("gu-container", "ngu-container")) + }) + + ginkgo.It("should not allocate exclusive CPUs to a non-guaranteed pod with pod-level resources and guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 1) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + pod := makeCPUManagerPod("ngu-pod-level-gu-ctn", []ctnAttribute{ + { + ctnName: "gu-container", + cpuRequest: "1", + cpuLimit: "1", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the expected cpuset was assigned") + + gomega.Expect(pod).To(HaveContainerCPUsEqualTo("gu-container", onlineCPUs)) + }) + + ginkgo.It("should not allocate exclusive CPUs to a non-guaranteed pod with pod-level resources and non-guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 1) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + pod := makeCPUManagerPod("ngu-pod-level-ngu-ctn", []ctnAttribute{ + { + ctnName: "ngu-container", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the expected cpuset was assigned") + + gomega.Expect(pod).To(HaveContainerCPUsEqualTo("ngu-container", onlineCPUs)) + }) + + ginkgo.It("should reject a pod that would result in an empty pod shared pool", ginkgo.Label("pod-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 2) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + pod := makeCPUManagerPod("gu-pod-level-empty-shared", []ctnAttribute{ + { + ctnName: "gu-container-1", + cpuRequest: "1", + cpuLimit: "1", + }, + { + ctnName: "gu-container-2", + cpuRequest: "1", + cpuLimit: "1", + }, + { + ctnName: "ngu-container", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("300Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("300Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = e2epod.NewPodClient(f).Create(ctx, pod) + podMap[string(pod.UID)] = pod + + ginkgo.By("waiting for the pod to fail") + err := e2epod.WaitForPodCondition(ctx, f.ClientSet, f.Namespace.Name, pod.Name, "Failed", 30*time.Second, func(pod *v1.Pod) (bool, error) { + return pod.Status.Phase == v1.PodFailed, nil + }) + framework.ExpectNoError(err) + + pod, err = e2epod.NewPodClient(f).Get(ctx, pod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + gomega.Expect(pod).To(BeAPodInPhase(v1.PodFailed)) + }) + + // This test demonstrates that CPUs are correctly released and re-allocated. + // It runs two pods sequentially that each request a large number of CPUs. + // It verifies that the second pod is allocated the same CPUs as the first, + // proving that the CPU manager freed the resources after the first pod completed. + ginkgo.It("should release and re-allocate CPUs correctly for sequential guaranteed pods with guaranteed and non-guaranteed containers", ginkgo.Label("pod-scope"), func(ctx context.Context) { + node := getLocalNode(ctx, f) + nodeCPUDetails := cpuDetailsFromNode(node) + // Request almost all allocatable CPUs to ensure we get a predictable, large set. + cpuCount := int(nodeCPUDetails.Allocatable - 1) + if cpuCount < 2 { + e2eskipper.Skipf("Skipping test: requires at least 2 allocatable CPUs, but has %d", int(nodeCPUDetails.Allocatable)) + } + framework.Logf("Node has %d allocatable CPUs. Requesting %d for pods.", int(nodeCPUDetails.Allocatable), cpuCount) + + skipIfAllocatableCPUsLessThan(node, cpuCount) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + reservedSystemCPUs: cpuset.New(0), + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + cpuRequest := strconv.Itoa(cpuCount) + podLevelResources := &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(cpuRequest), + v1.ResourceMemory: resource.MustParse("300Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(cpuRequest), + v1.ResourceMemory: resource.MustParse("300Mi"), + }, + } + + guaranteedContainerName := "gu-container" + nonGuaranteedContainerName := "ngu-container" + containers := []ctnAttribute{ + { + ctnName: guaranteedContainerName, cpuRequest: "1", cpuLimit: "1", ctnCommand: "echo 'container done' && sleep 2", + }, + { + ctnName: nonGuaranteedContainerName, ctnCommand: "echo 'container done' && sleep 2", + }, + } + + pod1 := makeCPUManagerPod("gu-pod-sequential-1", containers) + pod1.Spec.Resources = podLevelResources + pod1.Spec.RestartPolicy = v1.RestartPolicyNever + + ginkgo.By("creating the first test pod") + pod1 = createPodSync(ctx, pod1) + + ginkgo.By("verifying CPU allocation for the first pod") + gomega.Expect(pod1).To(HavePodExclusiveCPUs(cpuCount)) + guCpus1, err := getContainerAllowedCPUs(pod1, guaranteedContainerName, false) + framework.ExpectNoError(err) + nguCpus1, err := getContainerAllowedCPUs(pod1, nonGuaranteedContainerName, false) + framework.ExpectNoError(err) + totalCpus1 := guCpus1.Union(nguCpus1) + framework.Logf("Pod 1 allocated total CPUs: %s", totalCpus1.String()) + + ginkgo.By("waiting for the first pod to succeed") + err = e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod1.Name, f.Namespace.Name) + framework.ExpectNoError(err, "first pod failed to complete") + + pod2 := makeCPUManagerPod("gu-pod-sequential-2", containers) + pod2.Spec.Resources = podLevelResources + pod2.Spec.RestartPolicy = v1.RestartPolicyNever + + ginkgo.By("creating the second test pod") + pod2 = createPodSync(ctx, pod2) + + ginkgo.By("verifying CPU allocation for the second pod") + gomega.Expect(pod2).To(HavePodExclusiveCPUs(cpuCount)) + guCpus2, err := getContainerAllowedCPUs(pod2, guaranteedContainerName, false) + framework.ExpectNoError(err) + nguCpus2, err := getContainerAllowedCPUs(pod2, nonGuaranteedContainerName, false) + framework.ExpectNoError(err) + totalCpus2 := guCpus2.Union(nguCpus2) + framework.Logf("Pod 2 allocated total CPUs: %s", totalCpus2.String()) + + ginkgo.By("verifying the second pod reused the CPUs from the first pod") + gomega.Expect(totalCpus2).To(gomega.Equal(totalCpus1), "The CPU set for the second pod should be the same as the first pod's.") + }) + + // This test demonstrates that CPUs are correctly released from the pod shared + // pool, even when no containers are linked to it. This test demonstrates that + // the remove container logic cleans resources properly using the state. + ginkgo.It("should release resources from the pod shared pool when no containers used it", ginkgo.Label("pod-scope"), func(ctx context.Context) { + node := getLocalNode(ctx, f) + nodeCPUDetails := cpuDetailsFromNode(node) + // Request almost all allocatable CPUs to ensure we get a predictable, large set. + cpuCount := int(nodeCPUDetails.Allocatable - 1) + if cpuCount < 2 { + e2eskipper.Skipf("Skipping test: requires at least 2 allocatable CPUs, but has %d", int(nodeCPUDetails.Allocatable)) + } + framework.Logf("Node has %d allocatable CPUs. Requesting %d for pods.", int(nodeCPUDetails.Allocatable), cpuCount) + + skipIfAllocatableCPUsLessThan(node, cpuCount) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + reservedSystemCPUs: cpuset.New(0), + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + cpuRequest := strconv.Itoa(cpuCount) + podLevelResources := &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(cpuRequest), + v1.ResourceMemory: resource.MustParse("300Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse(cpuRequest), + v1.ResourceMemory: resource.MustParse("300Mi"), + }, + } + + guaranteedContainerName := "gu-container" + containers := []ctnAttribute{ + { + ctnName: guaranteedContainerName, cpuRequest: "1", cpuLimit: "1", ctnCommand: "echo 'container done' && sleep 2", + }, + } + + pod1 := makeCPUManagerPod("gu-pod-sequential-1", containers) + pod1.Spec.Resources = podLevelResources + pod1.Spec.RestartPolicy = v1.RestartPolicyNever + + ginkgo.By("creating the first test pod") + pod1 = createPodSync(ctx, pod1) + + ginkgo.By("waiting for the first pod to succeed") + err := e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod1.Name, f.Namespace.Name) + framework.ExpectNoError(err, "first pod failed to complete") + + pod2 := makeCPUManagerPod("gu-pod-sequential-2", containers) + pod2.Spec.Resources = podLevelResources + pod2.Spec.RestartPolicy = v1.RestartPolicyNever + + ginkgo.By("creating the second test pod") + pod2 = createPodSync(ctx, pod2) + + ginkgo.By("waiting for the second pod to succeed") + err = e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod2.Name, f.Namespace.Name) + framework.ExpectNoError(err, "second pod failed to complete") + }) + + // This test verifies that the pod shared pool CPU set is not removed until all containers + // in the pod have finished. It creates a pod with two containers sharing the pool, + // allows one to terminate, and then ensures that a new pod does not overlap with the + // remaining running container. + ginkgo.It("should not release pod shared pool cpus while at least one container is using them", ginkgo.Label("pod-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 3) + + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(currentCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + // Setup Pod A with one short-lived and one long-lived container. + podA := makeCPUManagerPod("pod-a-shared-pool", []ctnAttribute{ + { + ctnName: "short-container", + ctnCommand: "sleep 5", + }, + { + ctnName: "long-container", + ctnCommand: "sleep 3600", + }, + }) + podA.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("200Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("200Mi"), + }, + } + podA.Spec.RestartPolicy = v1.RestartPolicyNever + + ginkgo.By("creating Pod A with shared containers") + podA = createPodSync(ctx, podA) + + ginkgo.By("waiting for long-container to be running") + framework.ExpectNoError(e2epod.WaitForContainerRunning(ctx, f.ClientSet, podA.Namespace, podA.Name, "long-container", 2*time.Minute)) + + ginkgo.By("waiting for short-container to terminate") + framework.ExpectNoError(e2epod.WaitForContainerTerminated(ctx, f.ClientSet, podA.Namespace, podA.Name, "short-container", 2*time.Minute)) + + // Even when a non-guaranteed container from Pod A finished early, Pod B should not have access to any pod shared pool resources. + podB := makeCPUManagerPod("pod-b-attacker", []ctnAttribute{ + { + ctnName: "attacker-container", + cpuRequest: "1", + cpuLimit: "1", + }, + }) + podB.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + + ginkgo.By("creating Pod B") + podB = createPodSync(ctx, podB) + + ginkgo.By("verifying Pod B does NOT overlap with Pod A's long-running container") + cpusA, err := getContainerAllowedCPUs(podA, "long-container", false) + framework.ExpectNoError(err) + + // Verify Pod A's shared pool size remains correct + gomega.Expect(cpusA.Size()).To(gomega.Equal(2), "Pod A shared pool size should remain 2") + + cpusB, err := getContainerAllowedCPUs(podB, "attacker-container", false) + framework.ExpectNoError(err) + + framework.Logf("Pod A (long) CPUs: %v", cpusA) + framework.Logf("Pod B (attacker) CPUs: %v", cpusB) + + gomega.Expect(cpusA.Intersection(cpusB).IsEmpty()).To(gomega.BeTrueBecause("Pod B CPUs %v overlapped with Pod A CPUs %v", cpusB, cpusA)) + }) + + // This test verifies that resource accounting is performed at the pod level. + // It creates two pods, each requesting a dynamic amount of CPUs at the pod level + // but containing three non-guaranteed containers. If accounting were incorrect + // (e.g., summing container resources), the pods might be rejected. + // The test ensures both pods are admitted and run concurrently. + ginkgo.It("should admit multiple pods with pod-level resources and many non-guaranteed containers", ginkgo.Label("pod-scope"), func(ctx context.Context) { + node := getLocalNode(ctx, f) + nodeCPUDetails := cpuDetailsFromNode(node) + // We need at least 2 CPUs to run two pods concurrently. + if nodeCPUDetails.Allocatable < 2 { + e2eskipper.Skipf("Skipping test: requires at least 2 allocatable CPUs, but has %d", int(nodeCPUDetails.Allocatable)) + } + + // Each pod will request half of the allocatable CPUs. + cpuCountPerPod := int(nodeCPUDetails.Allocatable / 2) + framework.Logf("Node has %d allocatable CPUs. Requesting %d for each of the 2 pods.", int(nodeCPUDetails.Allocatable), cpuCountPerPod) + skipIfAllocatableCPUsLessThan(node, cpuCountPerPod*2) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + reservedSystemCPUs: cpuset.New(0), + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + podRequest := strconv.Itoa(cpuCountPerPod) + podMemRequest := "400Mi" + + makeTestPod := func(name string) *v1.Pod { + pod := makeCPUManagerPod(name, []ctnAttribute{ + {ctnName: "ngu-container-1", ctnCommand: "sleep 1d"}, + {ctnName: "ngu-container-2", ctnCommand: "sleep 1d"}, + {ctnName: "ngu-container-3", ctnCommand: "sleep 1d"}, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse(podRequest), v1.ResourceMemory: resource.MustParse(podMemRequest)}, + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse(podRequest), v1.ResourceMemory: resource.MustParse(podMemRequest)}, + } + return pod + } + + pod1 := makeTestPod("concurrent-pod-1") + pod2 := makeTestPod("concurrent-pod-2") + + ginkgo.By("creating two concurrent pods") + pod1 = createPodSync(ctx, pod1) + pod2 = createPodSync(ctx, pod2) + + ginkgo.By("waiting for both pods to be running") + framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod1)) + framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod2)) + + ginkgo.By("verifying CPU allocation for both pods") + gomega.Expect(pod1).To(HavePodExclusiveCPUs(cpuCountPerPod)) + gomega.Expect(pod2).To(HavePodExclusiveCPUs(cpuCountPerPod)) + + ginkgo.By("verifying CPU sets do not overlap") + + // It's sufficient to check one container from each pod, as all containers in a pod share the same pod-level cpuset. + cpus1, err := getContainerAllowedCPUs(pod1, "ngu-container-1", false) + framework.ExpectNoError(err) + cpus2, err := getContainerAllowedCPUs(pod2, "ngu-container-1", false) + framework.ExpectNoError(err) + gomega.Expect(cpus1.Intersection(cpus2).IsEmpty()).To(gomega.BeTrueBecause("CPU sets of the two pods should not overlap")) + }) + + ginkgo.It("should not maintain CPU quota for a pod with pod-level resources and guaranteed containers in Pod scope, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + reservedSystemCPUs: cpuset.CPUSet{}, + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + disableCPUQuotaWithExclusiveCPUs: true, + })) + + pod := makeCPUManagerPod("gu-pod-level-resources-quota", []ctnAttribute{ + { + ctnName: "gu-container", + cpuRequest: "1", + cpuLimit: "1", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the CPU quota is disabled") + gomega.Expect(pod).To(HaveSandboxQuota("max")) + gomega.Expect(pod).To(HaveContainerQuota("gu-container", "max")) + }) + + ginkgo.It("should maintain CPU quota for a pod with pod-level resources and non-guaranteed containers in Pod scope, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + reservedSystemCPUs: cpuset.CPUSet{}, + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + disableCPUQuotaWithExclusiveCPUs: true, + })) + + pod := makeCPUManagerPod("gu-pod-level-resources-quota", []ctnAttribute{ + { + ctnName: "ngu-container", + cpuRequest: "0", + cpuLimit: "1", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the CPU quota is enabled") + gomega.Expect(pod).To(HaveSandboxQuota("100000")) + gomega.Expect(pod).To(HaveContainerQuota("ngu-container", "100000")) + }) + }) + + ginkgo.Context("when the topology manager scope is 'container'", func() { + ginkgo.It("should allocate exclusive CPUs to a guaranteed pod with pod-level resources and guaranteed container, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 1) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + pod := makeCPUManagerPod("gu-pod-level-gu-ctn-ctn-scope", []ctnAttribute{ + { + ctnName: "gu-container", + cpuRequest: "1", + cpuLimit: "1", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the expected cpuset was assigned") + + gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", 1)) + gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs)) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs)) + }) + + ginkgo.It("should not allocate exclusive CPUs to a guaranteed pod with pod-level resources and non-guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 1) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + pod := makeCPUManagerPod("gu-pod-level-ngu-ctn-ctn-scope", []ctnAttribute{ + { + ctnName: "ngu-container", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the expected cpuset was assigned") + + gomega.Expect(pod).To(HaveContainerCPUsEqualTo("ngu-container", onlineCPUs)) + }) + + ginkgo.It("should allocate exclusive CPUs to a guaranteed pod with pod-level resources and mix of guaranteed and non-guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 1) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + pod := makeCPUManagerPod("gu-pod-level-mix-ctn-ctn-scope", []ctnAttribute{ + { + ctnName: "gu-container", + cpuRequest: "1", + cpuLimit: "1", + }, + { + ctnName: "ngu-container", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("300Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("300Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the expected cpuset was assigned") + + gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", 1)) + gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs)) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs)) + + exclusiveCPUs, err := getContainerAllowedCPUs(pod, "gu-container", false) + framework.ExpectNoError(err) + expectedSharedCPUs := onlineCPUs.Difference(exclusiveCPUs) + gomega.Expect(pod).To(HaveContainerCPUsEqualTo("ngu-container", expectedSharedCPUs)) + }) + + ginkgo.It("should allocate exclusive CPUs to a guaranteed pod with pod-level resources and mix of guaranteed and non-guaranteed standard and init containers, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 2) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + pod := makeCPUManagerPod("gu-pod-level-mix-init-ctn-ctn-scope", []ctnAttribute{ + { + ctnName: "gu-container", + cpuRequest: "1", + cpuLimit: "1", + }, + { + ctnName: "ngu-container", + }, + }) + pod.Spec.InitContainers = []v1.Container{ + { + Name: "gu-init-container", + Image: busyboxImage, + Command: []string{"sh", "-c", "echo hello"}, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + }, + }, + } + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("200Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("200Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the expected cpuset was assigned") + + gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container", 1)) + gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container", onlineCPUs)) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container", reservedCPUs)) + + exclusiveCPUs, err := getContainerAllowedCPUs(pod, "gu-container", false) + framework.ExpectNoError(err) + expectedSharedCPUs := onlineCPUs.Difference(exclusiveCPUs) + gomega.Expect(pod).To(HaveContainerCPUsEqualTo("ngu-container", expectedSharedCPUs)) + }) + + ginkgo.It("should not allocate exclusive CPUs to a non-guaranteed pod with pod-level resources and guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 2) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + pod := makeCPUManagerPod("ngu-pod-level-gu-ctn-ctn-scope", []ctnAttribute{ + { + ctnName: "gu-container", + cpuRequest: "1", + cpuLimit: "1", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the expected cpuset was assigned") + + gomega.Expect(pod).To(HaveContainerCPUsEqualTo("gu-container", onlineCPUs)) + }) + + ginkgo.It("should not allocate exclusive CPUs to a non-guaranteed pod with pod-level resources and non-guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 1) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + pod := makeCPUManagerPod("ngu-pod-level-ngu-ctn-ctn-scope", []ctnAttribute{ + { + ctnName: "ngu-container", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the expected cpuset was assigned") + + gomega.Expect(pod).To(HaveContainerCPUsEqualTo("ngu-container", onlineCPUs)) + }) + + ginkgo.It("should not reject a pod that would result in an empty pod shared pool, no pod shared pool in container scope", ginkgo.Label("container-scope"), func(ctx context.Context) { + skipIfAllocatableCPUsLessThan(getLocalNode(ctx, f), 2) + + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + pod := makeCPUManagerPod("gu-pod-level-empty-shared-ctn-scope", []ctnAttribute{ + { + ctnName: "gu-container-1", + cpuRequest: "1", + cpuLimit: "1", + }, + { + ctnName: "gu-container-2", + cpuRequest: "1", + cpuLimit: "1", + }, + { + ctnName: "ngu-container", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("300Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2"), + v1.ResourceMemory: resource.MustParse("300Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the expected cpuset was assigned") + + gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-1", 1)) + gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-1", onlineCPUs)) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-1", reservedCPUs)) + + gomega.Expect(pod).To(HaveContainerCPUsCount("gu-container-2", 1)) + gomega.Expect(pod).To(HaveContainerCPUsASubsetOf("gu-container-2", onlineCPUs)) + gomega.Expect(pod).ToNot(HaveContainerCPUsOverlapWith("gu-container-2", reservedCPUs)) + + exclusiveCPUs1, err := getContainerAllowedCPUs(pod, "gu-container-1", false) + framework.ExpectNoError(err) + exclusiveCPUs2, err := getContainerAllowedCPUs(pod, "gu-container-2", false) + framework.ExpectNoError(err) + exclusiveCPUs := exclusiveCPUs1.Union(exclusiveCPUs2) + expectedSharedCPUs := onlineCPUs.Difference(exclusiveCPUs) + gomega.Expect(pod).To(HaveContainerCPUsEqualTo("ngu-container", expectedSharedCPUs)) + }) + + ginkgo.It("should not maintain CPU quota for a pod with pod-level resources and guaranteed containers in Container scope, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + reservedSystemCPUs: cpuset.CPUSet{}, + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + disableCPUQuotaWithExclusiveCPUs: true, + })) + + pod := makeCPUManagerPod("gu-pod-level-resources-quota", []ctnAttribute{ + { + ctnName: "gu-container", + cpuRequest: "1", + cpuLimit: "1", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the CPU quota is disabled") + gomega.Expect(pod).To(HaveSandboxQuota("max")) + gomega.Expect(pod).To(HaveContainerQuota("gu-container", "max")) + }) + + ginkgo.It("should maintain CPU quota for a pod with pod-level resources and non-guaranteed containers in Container scope, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + reservedSystemCPUs: cpuset.CPUSet{}, + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + disableCPUQuotaWithExclusiveCPUs: true, + })) + + pod := makeCPUManagerPod("gu-pod-level-resources-quota", []ctnAttribute{ + { + ctnName: "ngu-container", + cpuRequest: "0", + cpuLimit: "1", + }, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1"), + v1.ResourceMemory: resource.MustParse("100Mi"), + }, + } + ginkgo.By("creating the test pod") + pod = createPodSync(ctx, pod) + + ginkgo.By("checking if the CPU quota is enabled") + gomega.Expect(pod).To(HaveSandboxQuota("100000")) + gomega.Expect(pod).To(HaveContainerQuota("ngu-container", "100000")) + }) + + ginkgo.It("should run on the shared CPU pool for a pod with pod-level resources when PodLevelResourceManagers is disabled and no reserved system CPUs", func(ctx context.Context) { + updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ + policyName: string(cpumanager.PolicyStatic), + reservedSystemCPUs: cpuset.CPUSet{}, + enablePodLevelResources: true, + enablePodLevelResourceManagers: false, })) pod := makeCPUManagerPod("gu-pod-level-resources", []ctnAttribute{ @@ -2057,20 +3158,18 @@ var _ = SIGDescribe("CPU Manager Incompatibility Pod Level Resources", ginkgo.Or } ginkgo.By("creating the test pod") - pod = e2epod.NewPodClient(f).CreateSync(ctx, pod) - podMap[string(pod.UID)] = pod + pod = createPodSync(ctx, pod) ginkgo.By("checking if the expected cpuset was assigned") gomega.Expect(pod).To(HaveContainerCPUsEqualTo("gu-container", onlineCPUs)) }) - ginkgo.It("should let the container access all the online CPUs when using a reserved CPUs set", func(ctx context.Context) { - reservedCPUs = cpuset.New(0) - + ginkgo.It("should run on the shared CPU pool for a pod with pod-level resources when PodLevelResourceManagers is disabled and with reserved system CPUs", func(ctx context.Context) { updateKubeletConfigIfNeeded(ctx, f, configureCPUManagerInKubelet(oldCfg, &cpuManagerKubeletArguments{ - policyName: string(cpumanager.PolicyStatic), - reservedSystemCPUs: reservedCPUs, // Not really needed for the tests but helps to make a more precise check - enablePodLevelResources: true, + policyName: string(cpumanager.PolicyStatic), + reservedSystemCPUs: cpuset.New(0), // Not really needed for the tests but helps to make a more precise check + enablePodLevelResources: true, + enablePodLevelResourceManagers: false, })) pod := makeCPUManagerPod("gu-pod-level-resources", []ctnAttribute{ @@ -2092,8 +3191,7 @@ var _ = SIGDescribe("CPU Manager Incompatibility Pod Level Resources", ginkgo.Or } ginkgo.By("creating the test pod") - pod = e2epod.NewPodClient(f).CreateSync(ctx, pod) - podMap[string(pod.UID)] = pod + pod = createPodSync(ctx, pod) ginkgo.By("checking if the expected cpuset was assigned") gomega.Expect(pod).To(HaveContainerCPUsEqualTo("gu-container", onlineCPUs)) @@ -2174,6 +3272,21 @@ func HaveContainerCPUsOverlapWith(ctnName string, ref cpuset.CPUSet) types.Gomeg }).WithTemplate("Pod {{.Actual.Namespace}}/{{.Actual.Name}} UID {{.Actual.UID}} has allowed CPUs <{{.Data.CurrentCPUs}}> overlapping with expected CPUs <{{.Data.ExpectedCPUs}}> for container {{.Data.Name}}", md) } +func HaveContainerCPUsOverlap(containerName1, containerName2 string) types.GomegaMatcher { + return gcustom.MakeMatcher(func(pod *v1.Pod) (bool, error) { + cpus1, err := getContainerAllowedCPUs(pod, containerName1, false) + if err != nil { + return false, err + } + cpus2, err := getContainerAllowedCPUs(pod, containerName2, false) + if err != nil { + return false, err + } + + return !cpus1.Intersection(cpus2).IsEmpty(), nil + }) +} + func HaveContainerCPUsASubsetOf(ctnName string, ref cpuset.CPUSet) types.GomegaMatcher { md := &msgData{ Name: ctnName, @@ -2371,6 +3484,36 @@ func HaveContainerCPUsShareUncoreCacheWith(ctnName string, ref cpuset.CPUSet) ty ) } +// HavePodExclusiveCPUs performs a pod-level check for CPU allocation. Verifies +// the total number of unique CPUs assigned across all containers in the pod equals +// the expected count. +func HavePodExclusiveCPUs(expectedTotalCPUs int) types.GomegaMatcher { + return gcustom.MakeMatcher(func(pod *v1.Pod) (bool, error) { + allCPUSets := make(map[string]cpuset.CPUSet) + + // Classify containers and fetch their CPU sets + for _, container := range pod.Spec.Containers { + cpus, err := getContainerAllowedCPUs(pod, container.Name, false) + if err != nil { + return false, fmt.Errorf("failed to get CPUs for container %s: %w", container.Name, err) + } + + allCPUSets[container.Name] = cpus + } + + // Check total CPU footprint of the pod + totalPodCPUs := cpuset.New() + for _, cpus := range allCPUSets { + totalPodCPUs = totalPodCPUs.Union(cpus) + } + if totalPodCPUs.Size() != expectedTotalCPUs { + return false, fmt.Errorf("expected pod to have a total of %d exclusive CPUs, but it has %d (%s)", expectedTotalCPUs, totalPodCPUs.Size(), totalPodCPUs.String()) + } + + return true, nil + }).WithMessage("to have exclusive CPUs assigned correctly at the pod level") +} + // Custom matcher for checking packed CPUs. func BePackedCPUs() types.GomegaMatcher { return gcustom.MakeMatcher(func(allocatedCPUs cpuset.CPUSet) (bool, error) { @@ -2732,19 +3875,33 @@ type ctnAttribute struct { func makeCPUManagerPod(podName string, ctnAttributes []ctnAttribute) *v1.Pod { var containers []v1.Container for _, ctnAttr := range ctnAttributes { - cpusetCmd := "grep Cpus_allowed_list /proc/self/status | cut -f2 && sleep 1d" + var cpusetCmd string + if ctnAttr.ctnCommand != "" { + cpusetCmd = ctnAttr.ctnCommand + } else { + cpusetCmd = "grep Cpus_allowed_list /proc/self/status | cut -f2 && sleep 1d" + } + + requests := v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("100Mi"), + } + if ctnAttr.cpuRequest != "" { + requests[v1.ResourceCPU] = resource.MustParse(ctnAttr.cpuRequest) + } + + limits := v1.ResourceList{ + v1.ResourceMemory: resource.MustParse("100Mi"), + } + if ctnAttr.cpuLimit != "" { + limits[v1.ResourceCPU] = resource.MustParse(ctnAttr.cpuLimit) + } + ctn := v1.Container{ Name: ctnAttr.ctnName, Image: busyboxImage, Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse(ctnAttr.cpuRequest), - v1.ResourceMemory: resource.MustParse("100Mi"), - }, - Limits: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse(ctnAttr.cpuLimit), - v1.ResourceMemory: resource.MustParse("100Mi"), - }, + Requests: requests, + Limits: limits, }, Command: []string{"sh", "-c", cpusetCmd}, VolumeMounts: []v1.VolumeMount{ @@ -2855,10 +4012,13 @@ func makeCPUManagerInitContainersPod(podName string, ctnAttributes []ctnAttribut type cpuManagerKubeletArguments struct { policyName string + topologyManagerPolicy string + topologyManagerScope string enableCPUManagerOptions bool disableCPUQuotaWithExclusiveCPUs bool enablePodLevelResources bool customCPUCFSQuotaPeriod time.Duration + enablePodLevelResourceManagers bool reservedSystemCPUs cpuset.CPUSet options map[string]string } @@ -2873,6 +4033,7 @@ func configureCPUManagerInKubelet(oldCfg *kubeletconfig.KubeletConfiguration, ku newCfg.FeatureGates["CPUManagerPolicyAlphaOptions"] = kubeletArguments.enableCPUManagerOptions newCfg.FeatureGates["DisableCPUQuotaWithExclusiveCPUs"] = kubeletArguments.disableCPUQuotaWithExclusiveCPUs newCfg.FeatureGates["PodLevelResources"] = kubeletArguments.enablePodLevelResources + newCfg.FeatureGates["PodLevelResourceManagers"] = kubeletArguments.enablePodLevelResourceManagers if kubeletArguments.customCPUCFSQuotaPeriod != 0 { newCfg.FeatureGates["CustomCPUCFSQuotaPeriod"] = true @@ -2882,6 +4043,8 @@ func configureCPUManagerInKubelet(oldCfg *kubeletconfig.KubeletConfiguration, ku } newCfg.CPUManagerPolicy = kubeletArguments.policyName + newCfg.TopologyManagerPolicy = kubeletArguments.topologyManagerPolicy + newCfg.TopologyManagerScope = kubeletArguments.topologyManagerScope newCfg.CPUManagerReconcilePeriod = metav1.Duration{Duration: 1 * time.Second} if kubeletArguments.options != nil { @@ -2907,3 +4070,17 @@ func configureCPUManagerInKubelet(oldCfg *kubeletconfig.KubeletConfiguration, ku return newCfg } + +func checkAllocatableCPUs(node *v1.Node, val int, reservedCPUs cpuset.CPUSet, onlineCPUs cpuset.CPUSet, smtLevel int) { + ginkgo.GinkgoHelper() + cpuReq := int64(val + reservedCPUs.Size()) // reserved CPUs are not usable, need to account them + // the framework is initialized using an injected BeforeEach node, so the + // earliest we can do is to initialize the other objects here + nodeCPUDetails := cpuDetailsFromNode(node) + + msg := fmt.Sprintf("%v full CPUs (detected=%v requested=%v reserved=%v online=%v smt=%v)", cpuReq, nodeCPUDetails.Allocatable, val, reservedCPUs.Size(), onlineCPUs.Size(), smtLevel) + ginkgo.By("Checking if allocatable: " + msg) + if nodeCPUDetails.Allocatable < cpuReq { + e2eskipper.Skipf("Skipping CPU Manager test: not allocatable %s", msg) + } +} diff --git a/test/e2e_node/memory_manager_test.go b/test/e2e_node/memory_manager_test.go index c28f5d4eb4a..ee624d2e153 100644 --- a/test/e2e_node/memory_manager_test.go +++ b/test/e2e_node/memory_manager_test.go @@ -31,11 +31,13 @@ import ( v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" kubeletpodresourcesv1 "k8s.io/kubelet/pkg/apis/podresources/v1" "k8s.io/kubernetes/pkg/features" kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config" "k8s.io/kubernetes/pkg/kubelet/apis/podresources" "k8s.io/kubernetes/pkg/kubelet/cm/memorymanager/state" + "k8s.io/kubernetes/pkg/kubelet/cm/topologymanager" "k8s.io/kubernetes/pkg/kubelet/util" "k8s.io/kubernetes/test/e2e/feature" "k8s.io/kubernetes/test/e2e/framework" @@ -69,16 +71,24 @@ func makeMemoryManagerContainers(ctnCmd string, ctnAttributes []memoryManagerCtn hugepagesMount := false var containers []v1.Container for _, ctnAttr := range ctnAttributes { + res := v1.ResourceRequirements{ + Limits: v1.ResourceList{}, + Requests: v1.ResourceList{}, + } + if ctnAttr.cpus != "" { + res.Limits[v1.ResourceCPU] = resource.MustParse(ctnAttr.cpus) + res.Requests[v1.ResourceCPU] = resource.MustParse(ctnAttr.cpus) + } + if ctnAttr.memory != "" { + res.Limits[v1.ResourceMemory] = resource.MustParse(ctnAttr.memory) + res.Requests[v1.ResourceMemory] = resource.MustParse(ctnAttr.memory) + } + ctn := v1.Container{ - Name: ctnAttr.ctnName, - Image: busyboxImage, - Resources: v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse(ctnAttr.cpus), - v1.ResourceMemory: resource.MustParse(ctnAttr.memory), - }, - }, - Command: []string{"sh", "-c", ctnCmd}, + Name: ctnAttr.ctnName, + Image: busyboxImage, + Resources: res, + Command: []string{"sh", "-c", ctnCmd}, } if ctnAttr.hugepages2Mi != "" { hugepagesMount = true @@ -240,6 +250,7 @@ func getAllNUMANodes() []int { } func verifyMemoryPinning(f *framework.Framework, ctx context.Context, pod *v1.Pod, numaNodeIDs []int) { + ginkgo.GinkgoHelper() ginkgo.By("Verifying the NUMA pinning") output, err := e2epod.GetPodLogs(ctx, f.ClientSet, f.Namespace.Name, pod.Name, pod.Spec.Containers[0].Name) @@ -251,6 +262,74 @@ func verifyMemoryPinning(f *framework.Framework, ctx context.Context, pod *v1.Po gomega.Expect(numaNodeIDs).To(gomega.Equal(currentNUMANodeIDs.List())) } +func verifyMemoryManagerAllocations(ctx context.Context, pod *v1.Pod, expectedGuaranteedContainers []string, expectedSharedMemorySize int64) { + ginkgo.GinkgoHelper() + ginkgo.By("Verifying memory manager allocations via pod resource API") + endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket) + framework.ExpectNoError(err) + + cli, conn, err := podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize) + framework.ExpectNoError(err) + defer conn.Close() //nolint:errcheck + + gomega.Eventually(ctx, func(ctx context.Context) error { + _, err := cli.List(ctx, &kubeletpodresourcesv1.ListPodResourcesRequest{}) + return err + }).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should(gomega.Succeed()) + resp, err := cli.List(ctx, &kubeletpodresourcesv1.ListPodResourcesRequest{}) + framework.ExpectNoError(err) + + expectedGuaranteedSet := sets.NewString(expectedGuaranteedContainers...) + + for _, podResource := range resp.PodResources { + if podResource.Name != pod.Name { + continue + } + + for _, containerResource := range podResource.Containers { + // find the container in the pod spec + var containerSpec *v1.Container + for i := range pod.Spec.Containers { + if pod.Spec.Containers[i].Name == containerResource.Name { + containerSpec = &pod.Spec.Containers[i] + break + } + } + if containerSpec == nil { + // could be an init container + for i := range pod.Spec.InitContainers { + if pod.Spec.InitContainers[i].Name == containerResource.Name { + containerSpec = &pod.Spec.InitContainers[i] + break + } + } + } + gomega.Expect(containerSpec).ToNot(gomega.BeNil(), "container spec for %s not found", containerResource.Name) + + if expectedGuaranteedSet.Has(containerResource.Name) { + gomega.Expect(containerResource.Memory).ToNot(gomega.BeEmpty(), "expected memory resources for container %s", containerResource.Name) + for _, mem := range containerResource.Memory { + q := containerSpec.Resources.Limits[v1.ResourceName(mem.MemoryType)] + val, ok := q.AsInt64() + gomega.Expect(ok).To(gomega.BeTrueBecause("cannot convert value to integer")) + gomega.Expect(val).To(gomega.BeEquivalentTo(mem.Size)) + } + } else { + if expectedSharedMemorySize > 0 { + gomega.Expect(containerResource.Memory).ToNot(gomega.BeEmpty(), "expected memory resources for non-guaranteed container %s", containerResource.Name) + var totalAllocated uint64 + for _, mem := range containerResource.Memory { + totalAllocated += mem.Size + } + gomega.Expect(totalAllocated).To(gomega.BeEquivalentTo(expectedSharedMemorySize), "container %s memory should match shared pool size", containerResource.Name) + } else { + gomega.Expect(containerResource.Memory).To(gomega.BeEmpty(), "expected no memory resources for container %s", containerResource.Name) + } + } + } + } +} + // Serial because the test updates kubelet configuration. var _ = SIGDescribe("Memory Manager", "[LinuxOnly]", framework.WithDisruptive(), framework.WithSerial(), feature.MemoryManager, func() { // TODO: add more complex tests that will include interaction between CPUManager, MemoryManager and TopologyManager @@ -303,7 +382,7 @@ var _ = SIGDescribe("Memory Manager", "[LinuxOnly]", framework.WithDisruptive(), return fmt.Errorf("the actual size %d is different from the expected one %d", size, expectedSize) } return nil - }, time.Minute, framework.Poll).Should(gomega.BeNil()) + }).WithTimeout(time.Minute).WithPolling(framework.Poll).Should(gomega.BeNil()) } ginkgo.BeforeEach(func(ctx context.Context) { @@ -324,7 +403,7 @@ var _ = SIGDescribe("Memory Manager", "[LinuxOnly]", framework.WithDisruptive(), ginkgo.By("Configuring hugepages") gomega.Eventually(ctx, func() error { return configureHugePages(hugepagesSize2M, hugepages2MiCount, ptr.To[int](0)) - }, 30*time.Second, framework.Poll).Should(gomega.BeNil()) + }).WithTimeout(30 * time.Second).WithPolling(framework.Poll).Should(gomega.BeNil()) } }) @@ -357,7 +436,7 @@ var _ = SIGDescribe("Memory Manager", "[LinuxOnly]", framework.WithDisruptive(), gomega.Eventually(ctx, func() error { // configure hugepages on the NUMA node 0 to avoid hugepages split across NUMA nodes return configureHugePages(hugepagesSize2M, 0, ptr.To[int](0)) - }, 90*time.Second, 15*time.Second).ShouldNot(gomega.HaveOccurred(), "failed to release hugepages") + }).WithTimeout(90*time.Second).WithPolling(15*time.Second).ShouldNot(gomega.HaveOccurred(), "failed to release hugepages") } }) @@ -442,9 +521,9 @@ var _ = SIGDescribe("Memory Manager", "[LinuxOnly]", framework.WithDisruptive(), // it no taste to verify NUMA pinning when the node has only one NUMA node if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") return } - verifyMemoryPinning(f, ctx, testPod, []int{0}) }) }) @@ -467,9 +546,9 @@ var _ = SIGDescribe("Memory Manager", "[LinuxOnly]", framework.WithDisruptive(), // it no taste to verify NUMA pinning when the node has only one NUMA node if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") return } - verifyMemoryPinning(f, ctx, testPod, []int{0}) }) }) @@ -501,9 +580,9 @@ var _ = SIGDescribe("Memory Manager", "[LinuxOnly]", framework.WithDisruptive(), // it no taste to verify NUMA pinning when the node has only one NUMA node if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") return } - verifyMemoryPinning(f, ctx, testPod, []int{0}) verifyMemoryPinning(f, ctx, testPod2, []int{0}) }) @@ -623,7 +702,7 @@ var _ = SIGDescribe("Memory Manager", "[LinuxOnly]", framework.WithDisruptive(), } return true - }, time.Minute, 5*time.Second).Should( + }).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should( gomega.BeTrueBecause( "the pod succeeded to start, when it should fail with the admission error", )) @@ -704,26 +783,51 @@ var _ = SIGDescribe("Memory Manager", "[LinuxOnly]", framework.WithDisruptive(), // it no taste to verify NUMA pinning when the node has only one NUMA node if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") return } - verifyMemoryPinning(f, ctx, testPod, allNUMANodes) }) }) }) }) -var _ = SIGDescribe("Memory Manager Incompatibility Pod Level Resources", framework.WithDisruptive(), framework.WithSerial(), feature.MemoryManager, feature.PodLevelResources, framework.WithFeatureGate(features.PodLevelResources), func() { - var ( - allNUMANodes []int - ctnParams, initCtnParams []memoryManagerCtnAttributes - isMultiNUMASupported *bool - testPod *v1.Pod - ) +type memoryManagerKubeletArguments struct { + policyName string + topologyManagerPolicy string + topologyManagerScope string + enablePodLevelResources bool + enablePodLevelResourceManagers bool +} - f := framework.NewDefaultFramework("memory-manager-incompatibility-pod-level-resources-test") +func configureMemoryManagerInKubelet(oldCfg *kubeletconfig.KubeletConfiguration, kubeletArguments *memoryManagerKubeletArguments) *kubeletconfig.KubeletConfiguration { + newCfg := oldCfg.DeepCopy() + if newCfg.FeatureGates == nil { + newCfg.FeatureGates = make(map[string]bool) + } + + newCfg.FeatureGates["PodLevelResources"] = kubeletArguments.enablePodLevelResources + newCfg.FeatureGates["PodLevelResourceManagers"] = kubeletArguments.enablePodLevelResourceManagers + + newCfg.MemoryManagerPolicy = kubeletArguments.policyName + newCfg.TopologyManagerPolicy = kubeletArguments.topologyManagerPolicy + newCfg.TopologyManagerScope = kubeletArguments.topologyManagerScope + + return newCfg +} + +var _ = SIGDescribe("Memory Manager Pod Level Resources", ginkgo.Ordered, ginkgo.ContinueOnFailure, framework.WithDisruptive(), framework.WithSerial(), feature.MemoryManager, feature.PodLevelResources, feature.PodLevelResourceManagers, framework.WithFeatureGate(features.PodLevelResources), framework.WithFeatureGate(features.PodLevelResourceManagers), func() { + f := framework.NewDefaultFramework("memory-manager-pod-level-resources") f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + var ( + allNUMANodes []int + isMultiNUMASupported *bool + oldCfg *kubeletconfig.KubeletConfiguration + podMap map[string]*v1.Pod + createPodSync func(ctx context.Context, pod *v1.Pod) *v1.Pod + ) + memoryQuantity := resource.MustParse("1100Mi") defaultKubeParams := &memoryManagerKubeletParams{ systemReservedMemory: []kubeletconfig.MemoryReservation{ @@ -739,7 +843,11 @@ var _ = SIGDescribe("Memory Manager Incompatibility Pod Level Resources", framew evictionHard: map[string]string{evictionHardMemory: "100Mi"}, } - ginkgo.BeforeEach(func(ctx context.Context) { + ginkgo.BeforeAll(func(ctx context.Context) { + var err error + oldCfg, err = getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + if isMultiNUMASupported == nil { isMultiNUMASupported = ptr.To(isMultiNUMA()) } @@ -749,28 +857,21 @@ var _ = SIGDescribe("Memory Manager Incompatibility Pod Level Resources", framew } }) - // dynamically update the kubelet configuration - ginkgo.JustBeforeEach(func(ctx context.Context) { - if len(ctnParams) > 0 { - testPod = makeMemoryManagerPod(ctnParams[0].ctnName, initCtnParams, ctnParams) - testPod.Spec.Resources = &v1.ResourceRequirements{ - Limits: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100m"), - v1.ResourceMemory: resource.MustParse("128Mi"), - }, - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100m"), - v1.ResourceMemory: resource.MustParse("128Mi"), - }, - } + ginkgo.AfterAll(func(ctx context.Context) { + updateKubeletConfig(ctx, f, oldCfg, true) + }) + + ginkgo.BeforeEach(func(ctx context.Context) { + podMap = make(map[string]*v1.Pod) + createPodSync = func(ctx context.Context, pod *v1.Pod) *v1.Pod { + newPod := e2epod.NewPodClient(f).CreateSync(ctx, pod) + podMap[string(newPod.UID)] = newPod + return newPod } }) - ginkgo.JustAfterEach(func(ctx context.Context) { - // delete the test pod - if testPod != nil && testPod.Name != "" { - e2epod.NewPodClient(f).DeleteSync(ctx, testPod.Name, metav1.DeleteOptions{}, f.Timeouts.PodDelete) - } + ginkgo.AfterEach(func(ctx context.Context) { + deletePodsAsync(ctx, f, podMap) }) ginkgo.Context("with static policy", func() { @@ -781,66 +882,1291 @@ var _ = SIGDescribe("Memory Manager Incompatibility Pod Level Resources", framew if initialConfig.FeatureGates == nil { initialConfig.FeatureGates = make(map[string]bool) } - initialConfig.FeatureGates["PodLevelResources"] = true }) - ginkgo.Context("", func() { - ginkgo.BeforeEach(func() { - // override pod parameters - ctnParams = []memoryManagerCtnAttributes{ + ginkgo.Context("when the topology manager scope is 'pod'", func() { + ginkgo.It("should allocate exclusive memory to a guaranteed pod with pod-level resources and guaranteed container, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + ctnParams := []memoryManagerCtnAttributes{ { - ctnName: "memory-manager-none", + ctnName: "gu-container", cpus: "100m", memory: "128Mi", }, } - }) - - ginkgo.JustAfterEach(func() { - // reset containers attributes - ctnParams = []memoryManagerCtnAttributes{} - initCtnParams = []memoryManagerCtnAttributes{} - }) - - ginkgo.It("should not report any memory data during request to pod resources List when pod has pod level resources", func(ctx context.Context) { - testPod = e2epod.NewPodClient(f).CreateSync(ctx, testPod) - - endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket) - framework.ExpectNoError(err) - - var resp *kubeletpodresourcesv1.ListPodResourcesResponse - gomega.Eventually(ctx, func(ctx context.Context) error { - cli, conn, err := podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize) - if err != nil { - return err - } - defer conn.Close() //nolint:errcheck - resp, err = cli.List(ctx, &kubeletpodresourcesv1.ListPodResourcesRequest{}) - - return err - }, time.Minute, 5*time.Second).Should(gomega.Succeed()) - - for _, podResource := range resp.PodResources { - if podResource.Name != testPod.Name { - continue - } - - for _, containerResource := range podResource.Containers { - gomega.Expect(containerResource.Memory).To(gomega.BeEmpty()) - } + testPod := makeMemoryManagerPod("gu-pod-level-gu-ctn", nil, ctnParams) + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, } - }) - ginkgo.It("should succeed to start the pod when it has pod level resources", func(ctx context.Context) { - testPod = e2epod.NewPodClient(f).CreateSync(ctx, testPod) + ginkgo.By("Running the test pod") + testPod = createPodSync(ctx, testPod) + + verifyMemoryManagerAllocations(ctx, testPod, []string{ + "gu-container", + }, 0) - // it no taste to verify NUMA pinning when the node has only one NUMA node if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") return } + verifyMemoryPinning(f, ctx, testPod, []int{0}) + }) + ginkgo.It("should allocate exclusive memory to a guaranteed pod with pod-level resources and non-guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + testPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "gu-pod-level-ngu-ctn-", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{ + { + Name: "ngu-container", + Image: busyboxImage, + Command: []string{"sh", "-c", "grep Mems_allowed_list /proc/self/status | cut -f2 && sleep 1d"}, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + }, + }, + }, + }, + } + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + + ginkgo.By("Running the test pod") + testPod = createPodSync(ctx, testPod) + + verifyMemoryManagerAllocations(ctx, testPod, nil, 128*1024*1024) + + if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") + return + } + verifyMemoryPinning(f, ctx, testPod, []int{0}) + }) + + ginkgo.It("should allocate exclusive memory to a guaranteed pod with pod-level resources and mix of guaranteed and non-guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + ctnParams := []memoryManagerCtnAttributes{ + { + ctnName: "gu-container", + cpus: "100m", + memory: "64Mi", + }, + } + testPod := makeMemoryManagerPod("gu-pod-level-mix-ctn", nil, ctnParams) + testPod.Spec.Containers = append(testPod.Spec.Containers, v1.Container{ + Name: "ngu-container", + Image: busyboxImage, + Command: []string{"sh", "-c", "grep Mems_allowed_list /proc/self/status | cut -f2 && sleep 1d"}, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + }, + }) + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + + ginkgo.By("Running the test pod") + testPod = createPodSync(ctx, testPod) + + verifyMemoryManagerAllocations(ctx, testPod, []string{"gu-container"}, 64*1024*1024) + + if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") + return + } + verifyMemoryPinning(f, ctx, testPod, []int{0}) + }) + + ginkgo.It("should allocate exclusive memory to a guaranteed pod with pod-level resources and mix of guaranteed and non-guaranteed standard and init containers, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + initCtnParams := []memoryManagerCtnAttributes{ + { + ctnName: "gu-init-container", + cpus: "100m", + memory: "64Mi", + }, + } + ctnParams := []memoryManagerCtnAttributes{ + { + ctnName: "gu-container", + cpus: "100m", + memory: "64Mi", + }, + } + testPod := makeMemoryManagerPod("gu-pod-level-mix-init-ctn", initCtnParams, ctnParams) + testPod.Spec.Containers = append(testPod.Spec.Containers, v1.Container{ + Name: "ngu-container", + Image: busyboxImage, + Command: []string{"sh", "-c", "grep Mems_allowed_list /proc/self/status | cut -f2 && sleep 1d"}, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + }, + }) + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + + ginkgo.By("Running the test pod") + testPod = createPodSync(ctx, testPod) + + verifyMemoryManagerAllocations(ctx, testPod, []string{"gu-init-container", "gu-container"}, 64*1024*1024) + + if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") + return + } + verifyMemoryPinning(f, ctx, testPod, []int{0}) + }) + + ginkgo.It("should not allocate exclusive memory to a non-guaranteed pod with pod-level resources and guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + ctnParams := []memoryManagerCtnAttributes{ + { + ctnName: "gu-container", + cpus: "100m", + memory: "64Mi", + }, + } + testPod := makeMemoryManagerPod("ngu-pod-level-gu-ctn", nil, ctnParams) + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("64Mi"), + }, + } + + ginkgo.By("Running the test pod") + testPod = createPodSync(ctx, testPod) + + verifyMemoryManagerAllocations(ctx, testPod, nil, 0) + + if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") + return + } verifyMemoryPinning(f, ctx, testPod, allNUMANodes) }) + + ginkgo.It("should not allocate exclusive memory to a non-guaranteed pod with pod-level resources and non-guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + testPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "ngu-pod-level-ngu-ctn-", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{ + { + Name: "ngu-container", + Image: busyboxImage, + Command: []string{"sh", "-c", "grep Mems_allowed_list /proc/self/status | cut -f2 && sleep 1d"}, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + }, + }, + }, + }, + } + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("64Mi"), + }, + } + + ginkgo.By("Running the test pod") + testPod = createPodSync(ctx, testPod) + + verifyMemoryManagerAllocations(ctx, testPod, nil, 0) + + if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") + return + } + verifyMemoryPinning(f, ctx, testPod, allNUMANodes) + }) + + ginkgo.It("should reject a pod that would result in an empty pod shared pool, topologymanager.PolicyBestEffort, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyBestEffort, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + ctnParams := []memoryManagerCtnAttributes{ + { + ctnName: "gu-container-1", + cpus: "100m", + memory: "64Mi", + }, + { + ctnName: "gu-container-2", + cpus: "100m", + memory: "64Mi", + }, + } + testPod := makeMemoryManagerPod("gu-pod-level-empty-shared", nil, ctnParams) + testPod.Spec.Containers = append(testPod.Spec.Containers, v1.Container{ + Name: "ngu-container", + Image: busyboxImage, + Command: []string{"sh", "-c", "sleep 1d"}, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + }, + }) + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + + ginkgo.By("Creating the pod") + testPod = e2epod.NewPodClient(f).Create(ctx, testPod) + podMap[string(testPod.UID)] = testPod + + ginkgo.By("Checking that pod failed to start because of admission error") + gomega.Eventually(ctx, func(g gomega.Gomega) { + tmpPod, err := e2epod.NewPodClient(f).Get(ctx, testPod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + g.Expect(tmpPod.Status.Phase).To(gomega.Equal(v1.PodFailed)) + g.Expect(tmpPod.Status.Message).To(gomega.ContainSubstring("sum of exclusive container memory requests equals pod budget")) + }).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should(gomega.Succeed()) + }) + + ginkgo.It("should reject a pod that would result in an empty pod shared pool, topologymanager.PolicyRestricted, PodLevelResourceManagers enabled", ginkgo.Label("pod-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + ctnParams := []memoryManagerCtnAttributes{ + { + ctnName: "gu-container-1", + cpus: "100m", + memory: "64Mi", + }, + { + ctnName: "gu-container-2", + cpus: "100m", + memory: "64Mi", + }, + } + testPod := makeMemoryManagerPod("gu-pod-level-empty-shared", nil, ctnParams) + testPod.Spec.Containers = append(testPod.Spec.Containers, v1.Container{ + Name: "ngu-container", + Image: busyboxImage, + Command: []string{"sh", "-c", "sleep 1d"}, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + }, + }) + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + + ginkgo.By("Creating the pod") + testPod = e2epod.NewPodClient(f).Create(ctx, testPod) + podMap[string(testPod.UID)] = testPod + + ginkgo.By("Checking that pod failed to start because of admission error") + gomega.Eventually(ctx, func(g gomega.Gomega) { + tmpPod, err := e2epod.NewPodClient(f).Get(ctx, testPod.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + g.Expect(tmpPod.Status.Phase).To(gomega.Equal(v1.PodFailed)) + g.Expect(tmpPod.Status.Message).To(gomega.ContainSubstring("pod with pod-level resources failed admission under pod-scope topology manager")) + }).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should(gomega.Succeed()) + }) + + // This test demonstrates that memory resources are correctly released and re-allocated. + // It runs two pods sequentially that each request a significant amount of memory from a single NUMA node. + // It verifies that the second pod is allocated the same NUMA node(s) as the first, + // proving that the memory manager freed the resources after the first pod completed. + ginkgo.It("should release and re-allocate memory correctly for sequential guaranteed pods with guaranteed containers and empty shared pool", ginkgo.Label("pod-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + stateData, err := getMemoryManagerState() + framework.ExpectNoError(err) + + // Find the first NUMA node with enough allocatable memory to run the test. + var memRequest string + foundNuma := false + for _, numaState := range stateData.MachineState { + memToRequest := numaState.MemoryMap[v1.ResourceMemory].Allocatable - (1 * 1024 * 1024) // leave 1Mi buffer + memRequest = fmt.Sprintf("%d", memToRequest) + foundNuma = true + break + } + if !foundNuma { + ginkgo.Skip("Skipping test: no NUMA node found with sufficient allocatable memory") + } + framework.Logf("Node has sufficient memory. Requesting %s for pods.", memRequest) + + podLevelResources := &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse(memRequest), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse(memRequest), + }, + } + + // A command that prints the allowed NUMA nodes and then exits. + command := []string{"sh", "-c", "grep Mems_allowed_list /proc/self/status | cut -f2 && sleep 2"} + + makeTestPod := func(name string) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{ + { + Name: "gu-container", + Image: busyboxImage, + Command: command, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse(memRequest), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse(memRequest), + }, + }, + }, + }, + Resources: podLevelResources, + }, + } + } + + pod1 := makeTestPod("gu-pod-sequential-1") + pod1.Spec.RestartPolicy = v1.RestartPolicyNever + + ginkgo.By("creating the first test pod") + pod1 = createPodSync(ctx, pod1) + + ginkgo.By("getting NUMA allocation for the first pod") + logs1, err := e2epod.GetPodLogs(ctx, f.ClientSet, pod1.Namespace, pod1.Name, "gu-container") + framework.ExpectNoError(err, "failed to get logs for the first pod") + numaNodes1, err := cpuset.Parse(strings.TrimSpace(logs1)) + framework.ExpectNoError(err, "failed to parse NUMA nodes from logs for the first pod") + framework.Logf("Pod 1 allocated NUMA nodes: %s", numaNodes1.String()) + + ginkgo.By("waiting for the first pod to succeed") + err = e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod1.Name, f.Namespace.Name) + framework.ExpectNoError(err, "first pod failed to complete") + + pod2 := makeTestPod("gu-pod-sequential-2") + pod2.Spec.RestartPolicy = v1.RestartPolicyNever + + ginkgo.By("creating the second test pod") + pod2 = createPodSync(ctx, pod2) + + ginkgo.By("getting NUMA allocation for the second pod") + logs2, err := e2epod.GetPodLogs(ctx, f.ClientSet, pod2.Namespace, pod2.Name, "gu-container") + framework.ExpectNoError(err, "failed to get logs for the second pod") + numaNodes2, err := cpuset.Parse(strings.TrimSpace(logs2)) + framework.ExpectNoError(err, "failed to parse NUMA nodes from logs for the second pod") + framework.Logf("Pod 2 allocated NUMA nodes: %s", numaNodes2.String()) + + ginkgo.By("verifying the second pod reused the NUMA nodes from the first pod") + gomega.Expect(numaNodes2).To(gomega.Equal(numaNodes1), "The NUMA set for the second pod should be the same as the first pod's.") + }) + + // This test demonstrates that memory is correctly released from the pod shared + // pool, even when no containers are linked to it. This test demonstrates that + // the remove container logic cleans resources properly using the state. + ginkgo.It("should release resources from the pod shared pool when no containers used it", ginkgo.Label("pod-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + stateData, err := getMemoryManagerState() + framework.ExpectNoError(err) + + // Find the first NUMA node with enough allocatable memory to run the test. + var memRequest string + foundNuma := false + for _, numaState := range stateData.MachineState { + memToRequest := numaState.MemoryMap[v1.ResourceMemory].Allocatable - (1 * 1024 * 1024) // leave 1Mi buffer + memRequest = fmt.Sprintf("%d", memToRequest) + foundNuma = true + break + } + if !foundNuma { + ginkgo.Skip("Skipping test: no NUMA node found with sufficient allocatable memory") + } + framework.Logf("Node has sufficient memory. Requesting %s for pods.", memRequest) + + podLevelResources := &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse(memRequest), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse(memRequest), + }, + } + + // A command that prints the allowed NUMA nodes and then exits. + command := []string{"sh", "-c", "grep Mems_allowed_list /proc/self/status | cut -f2 && sleep 2"} + + makeTestPod := func(name string) *v1.Pod { + return &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{ + { + Name: "gu-container", + Image: busyboxImage, + Command: command, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + }, + }, + }, + Resources: podLevelResources, + }, + } + } + pod1 := makeTestPod("gu-pod-sequential-1") + pod1.Spec.RestartPolicy = v1.RestartPolicyNever + + ginkgo.By("creating the first test pod") + pod1 = createPodSync(ctx, pod1) + + ginkgo.By("getting NUMA allocation for the first pod") + logs1, err := e2epod.GetPodLogs(ctx, f.ClientSet, pod1.Namespace, pod1.Name, "gu-container") + framework.ExpectNoError(err, "failed to get logs for the first pod") + numaNodes1, err := cpuset.Parse(strings.TrimSpace(logs1)) + framework.ExpectNoError(err, "failed to parse NUMA nodes from logs for the first pod") + framework.Logf("Pod 1 allocated NUMA nodes: %s", numaNodes1.String()) + + ginkgo.By("waiting for the first pod to succeed") + err = e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod1.Name, f.Namespace.Name) + framework.ExpectNoError(err, "first pod failed to complete") + + pod2 := makeTestPod("gu-pod-sequential-2") + pod2.Spec.RestartPolicy = v1.RestartPolicyNever + + ginkgo.By("creating the second test pod") + pod2 = createPodSync(ctx, pod2) + + ginkgo.By("getting NUMA allocation for the second pod") + logs2, err := e2epod.GetPodLogs(ctx, f.ClientSet, pod2.Namespace, pod2.Name, "gu-container") + framework.ExpectNoError(err, "failed to get logs for the second pod") + numaNodes2, err := cpuset.Parse(strings.TrimSpace(logs2)) + framework.ExpectNoError(err, "failed to parse NUMA nodes from logs for the second pod") + framework.Logf("Pod 2 allocated NUMA nodes: %s", numaNodes2.String()) + + ginkgo.By("waiting for the second pod to succeed") + err = e2epod.WaitForPodSuccessInNamespace(ctx, f.ClientSet, pod2.Name, f.Namespace.Name) + framework.ExpectNoError(err, "second pod failed to complete") + }) + + // This test verifies that the pod shared pool memory is not removed until all containers + // in the pod have finished. It creates a pod with two containers sharing the pool, + // allows one to terminate, and then ensures that the memory remains reserved in the state. + ginkgo.It("should not release pod shared pool memory while at least one container is using it", ginkgo.Label("pod-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + stateData, err := getMemoryManagerState() + framework.ExpectNoError(err) + + // Find a NUMA node with sufficient memory. + var memRequestString string + var memRequestValue int64 + foundNuma := false + for _, numaState := range stateData.MachineState { + allocatable := numaState.MemoryMap[v1.ResourceMemory].Allocatable + if allocatable > 256*1024*1024 { + // Request half the allocatable memory to be safe, but substantial enough to verify reservation. + memRequestValue = int64(allocatable / 2) + memRequestString = fmt.Sprintf("%d", memRequestValue) + foundNuma = true + break + } + } + if !foundNuma { + ginkgo.Skip("Skipping test: no NUMA node found with sufficient allocatable memory") + } + + podA := makeMemoryManagerPod("pod-a-shared-pool", nil, []memoryManagerCtnAttributes{ + { + ctnName: "short-container", + cpus: "50m", + }, + { + ctnName: "long-container", + cpus: "50m", + }, + }) + podA.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse(memRequestString), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse(memRequestString), + }, + } + podA.Spec.Containers[0].Command = []string{"sh", "-c", "sleep 5"} + podA.Spec.RestartPolicy = v1.RestartPolicyNever + + ginkgo.By("creating Pod A with shared containers") + podA = createPodSync(ctx, podA) + + ginkgo.By("waiting for long-container to be running") + framework.ExpectNoError(e2epod.WaitForContainerRunning(ctx, f.ClientSet, podA.Namespace, podA.Name, "long-container", 2*time.Minute)) + + ginkgo.By("waiting for short-container to terminate") + framework.ExpectNoError(e2epod.WaitForContainerTerminated(ctx, f.ClientSet, podA.Namespace, podA.Name, "short-container", 2*time.Minute)) + + // Identify the NUMA node used by Pod A + logs, err := e2epod.GetPodLogs(ctx, f.ClientSet, podA.Namespace, podA.Name, "long-container") + framework.ExpectNoError(err, "failed to get logs for long-container") + numaNodes, err := cpuset.Parse(strings.TrimSpace(logs)) + framework.ExpectNoError(err, "failed to parse NUMA nodes from logs") + framework.Logf("Pod A allocated NUMA nodes: %s", numaNodes.String()) + + ginkgo.By("verifying memory is still reserved in Memory Manager state") + stateAfterExit, err := getMemoryManagerState() + framework.ExpectNoError(err) + + for _, nodeID := range numaNodes.List() { + nodeState := stateAfterExit.MachineState[nodeID] + reserved := nodeState.MemoryMap[v1.ResourceMemory].Reserved + // We verify that Reserved memory is at least what we requested, the pod shared pool should not be affected. + gomega.Expect(int64(reserved)).To(gomega.BeNumerically(">=", memRequestValue), "Reserved memory on node %d should be at least %d", nodeID, memRequestValue) + } + }) + + // This test verifies that resource accounting is performed at the pod level for memory. + // It creates two pods, each requesting half of the node's allocatable memory at the pod level + // but containing three non-guaranteed containers. If accounting were incorrect + // (e.g., summing container resources), the pods might be rejected. + // The test ensures both pods are admitted and run concurrently. + ginkgo.It("should admit multiple pods with pod-level resources and many non-guaranteed containers", ginkgo.Label("pod-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.PodTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + node, err := f.ClientSet.CoreV1().Nodes().Get(ctx, framework.TestContext.NodeName, metav1.GetOptions{}) + framework.ExpectNoError(err) + allocatableMem := node.Status.Allocatable.Memory().Value() + memRequestPerPodVal := allocatableMem / 2 + framework.Logf("Node has %d allocatable memory. Requesting %d for each of the 2 pods.", allocatableMem, memRequestPerPodVal) + + memRequestPerPod := fmt.Sprintf("%d", memRequestPerPodVal) + + makeTestPod := func(name string) *v1.Pod { + pod := makeMemoryManagerPod(name, nil, []memoryManagerCtnAttributes{ + {ctnName: "ngu-container-1", cpus: "50m"}, + {ctnName: "ngu-container-2", cpus: "50m"}, + {ctnName: "ngu-container-3", cpus: "50m"}, + }) + pod.Spec.Resources = &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse(memRequestPerPod), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse(memRequestPerPod), + }, + } + return pod + } + + pod1 := makeTestPod("concurrent-pod-1") + pod2 := makeTestPod("concurrent-pod-2") + + ginkgo.By("creating two concurrent pods") + pod1 = createPodSync(ctx, pod1) + pod2 = createPodSync(ctx, pod2) + + ginkgo.By("verifying NUMA allocation for both pods") + logs1, err := e2epod.GetPodLogs(ctx, f.ClientSet, pod1.Namespace, pod1.Name, "ngu-container-1") + framework.ExpectNoError(err, "failed to get logs for pod1") + numaNodes1, err := cpuset.Parse(strings.TrimSpace(logs1)) + framework.ExpectNoError(err, "failed to parse NUMA nodes for pod1") + + logs2, err := e2epod.GetPodLogs(ctx, f.ClientSet, pod2.Namespace, pod2.Name, "ngu-container-1") + framework.ExpectNoError(err, "failed to get logs for pod2") + numaNodes2, err := cpuset.Parse(strings.TrimSpace(logs2)) + framework.ExpectNoError(err, "failed to parse NUMA nodes for pod2") + + ginkgo.By("verifying NUMA sets do not overlap if on different nodes") + // This is a soft check. If the machine has enough NUMA nodes, they should be different. + // If not, they might be the same, which is also valid. The main point is that both were admitted. + if numaNodes1.String() != numaNodes2.String() { + framework.Logf("Pods are on different NUMA nodes, verifying no overlap.") + gomega.Expect(numaNodes1.Intersection(numaNodes2).IsEmpty()).To(gomega.BeTrueBecause("NUMA sets of the two pods should not overlap")) + } else { + framework.Logf("Pods are on the same NUMA node (%s). This is acceptable.", numaNodes1.String()) + } + }) + }) + + ginkgo.Context("when the topology manager scope is 'container'", func() { + ginkgo.It("should allocate exclusive memory to a guaranteed pod with pod-level resources and guaranteed container, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + ctnParams := []memoryManagerCtnAttributes{ + { + ctnName: "gu-container", + cpus: "100m", + memory: "128Mi", + }, + } + testPod := makeMemoryManagerPod("gu-pod-level-gu-ctn-ctn-scope", nil, ctnParams) + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + + ginkgo.By("Running the test pod") + testPod = createPodSync(ctx, testPod) + + verifyMemoryManagerAllocations(ctx, testPod, []string{"gu-container"}, 0) + + if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") + return + } + verifyMemoryPinning(f, ctx, testPod, []int{0}) + }) + + ginkgo.It("should not allocate exclusive memory to a guaranteed pod with pod-level resources and non-guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + testPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "gu-pod-level-ngu-ctn-ctn-scope-", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{ + { + Name: "ngu-container", + Image: busyboxImage, + Command: []string{"sh", "-c", "grep Mems_allowed_list /proc/self/status | cut -f2 && sleep 1d"}, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + }, + }, + }, + }, + } + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + + ginkgo.By("Running the test pod") + testPod = createPodSync(ctx, testPod) + + verifyMemoryManagerAllocations(ctx, testPod, nil, 0) + + if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") + return + } + verifyMemoryPinning(f, ctx, testPod, allNUMANodes) + }) + + ginkgo.It("should allocate exclusive memory to a guaranteed pod with pod-level resources and mix of guaranteed and non-guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + ctnParams := []memoryManagerCtnAttributes{ + { + ctnName: "gu-container", + cpus: "100m", + memory: "64Mi", + }, + } + testPod := makeMemoryManagerPod("gu-pod-level-mix-ctn-ctn-scope", nil, ctnParams) + testPod.Spec.Containers = append(testPod.Spec.Containers, v1.Container{ + Name: "ngu-container", + Image: busyboxImage, + Command: []string{"sh", "-c", "grep Mems_allowed_list /proc/self/status | cut -f2 && sleep 1d"}, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + }, + }) + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("200m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + + ginkgo.By("Running the test pod") + testPod = createPodSync(ctx, testPod) + + verifyMemoryManagerAllocations(ctx, testPod, []string{"gu-container"}, 0) + + if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") + return + } + verifyMemoryPinning(f, ctx, testPod, []int{0}) + }) + + ginkgo.It("should allocate exclusive memory to a guaranteed pod with pod-level resources and mix of guaranteed and non-guaranteed standard and init containers, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + initCtnParams := []memoryManagerCtnAttributes{ + { + ctnName: "gu-init-container", + cpus: "100m", + memory: "64Mi", + }, + } + ctnParams := []memoryManagerCtnAttributes{ + { + ctnName: "gu-container", + cpus: "100m", + memory: "64Mi", + }, + } + testPod := makeMemoryManagerPod("gu-pod-level-mix-init-ctn-ctn-scope", initCtnParams, ctnParams) + testPod.Spec.Containers = append(testPod.Spec.Containers, v1.Container{ + Name: "ngu-container", + Image: busyboxImage, + Command: []string{"sh", "-c", "grep Mems_allowed_list /proc/self/status | cut -f2 && sleep 1d"}, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + }, + }) + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + + ginkgo.By("Running the test pod") + testPod = createPodSync(ctx, testPod) + + verifyMemoryManagerAllocations(ctx, testPod, []string{"gu-init-container", "gu-container"}, 0) + + if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") + return + } + verifyMemoryPinning(f, ctx, testPod, []int{0}) + }) + + ginkgo.It("should not allocate exclusive memory to a non-guaranteed pod with pod-level resources and guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + ctnParams := []memoryManagerCtnAttributes{ + { + ctnName: "gu-container", + cpus: "100m", + memory: "64Mi", + }, + } + testPod := makeMemoryManagerPod("ngu-pod-level-gu-ctn-ctn-scope", nil, ctnParams) + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("64Mi"), + }, + } + + ginkgo.By("Running the test pod") + testPod = createPodSync(ctx, testPod) + + if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") + return + } + verifyMemoryPinning(f, ctx, testPod, allNUMANodes) + }) + + ginkgo.It("should not allocate exclusive memory to a non-guaranteed pod with pod-level resources and non-guaranteed containers, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + testPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "ngu-pod-level-ngu-ctn-ctn-scope-", + }, + Spec: v1.PodSpec{ + RestartPolicy: v1.RestartPolicyNever, + Containers: []v1.Container{ + { + Name: "ngu-container", + Image: busyboxImage, + Command: []string{"sh", "-c", "grep Mems_allowed_list /proc/self/status | cut -f2 && sleep 1d"}, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + }, + }, + }, + }, + } + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("64Mi"), + }, + } + + ginkgo.By("Running the test pod") + testPod = createPodSync(ctx, testPod) + + verifyMemoryManagerAllocations(ctx, testPod, nil, 0) + + if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") + return + } + verifyMemoryPinning(f, ctx, testPod, allNUMANodes) + }) + + ginkgo.It("should not reject a pod that would result in an empty pod shared pool, no pod shared pool in container scope, PodLevelResourceManagers enabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: true, + })) + + ctnParams := []memoryManagerCtnAttributes{ + { + ctnName: "gu-container-1", + cpus: "100m", + memory: "64Mi", + }, + { + ctnName: "gu-container-2", + cpus: "100m", + memory: "64Mi", + }, + } + testPod := makeMemoryManagerPod("gu-pod-level-empty-shared-ctn-scope", nil, ctnParams) + testPod.Spec.Containers = append(testPod.Spec.Containers, v1.Container{ + Name: "ngu-container", + Image: busyboxImage, + Command: []string{"sh", "-c", "sleep 1d"}, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + }, + }, + }) + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("300m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + + ginkgo.By("Running the test pod") + testPod = createPodSync(ctx, testPod) + + if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") + return + } + verifyMemoryPinning(f, ctx, testPod, []int{0}) + }) + }) + + ginkgo.It("should not report any memory data during request and run on the shared memory pool for a pod with pod-level resources when PodLevelResourceManagers is disabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: false, + })) + + ctnParams := []memoryManagerCtnAttributes{ + { + ctnName: "memory-manager-none", + cpus: "100m", + memory: "128Mi", + }, + } + testPod := makeMemoryManagerPod(ctnParams[0].ctnName, nil, ctnParams) + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + testPod = createPodSync(ctx, testPod) + + endpoint, err := util.LocalEndpoint(defaultPodResourcesPath, podresources.Socket) + framework.ExpectNoError(err) + + var resp *kubeletpodresourcesv1.ListPodResourcesResponse + gomega.Eventually(ctx, func(ctx context.Context) error { + cli, conn, err := podresources.GetV1Client(endpoint, defaultPodResourcesTimeout, defaultPodResourcesMaxSize) + if err != nil { + return err + } + defer conn.Close() //nolint:errcheck + resp, err = cli.List(ctx, &kubeletpodresourcesv1.ListPodResourcesRequest{}) + + return err + }).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should(gomega.Succeed()) + for _, podResource := range resp.PodResources { + if podResource.Name != testPod.Name { + continue + } + + for _, containerResource := range podResource.Containers { + gomega.Expect(containerResource.Memory).To(gomega.BeEmpty()) + } + } + }) + + ginkgo.It("should run on the shared memory pool for a pod with pod-level resources when PodLevelResourceManagers is disabled", ginkgo.Label("container-scope"), func(ctx context.Context) { + currentCfg, err := getCurrentKubeletConfig(ctx) + framework.ExpectNoError(err) + updateKubeletConfigIfNeeded(ctx, f, configureMemoryManagerInKubelet(currentCfg, &memoryManagerKubeletArguments{ + policyName: string(staticPolicy), + topologyManagerPolicy: topologymanager.PolicyRestricted, + topologyManagerScope: topologymanager.ContainerTopologyScope, + enablePodLevelResources: true, + enablePodLevelResourceManagers: false, + })) + + ctnParams := []memoryManagerCtnAttributes{ + { + ctnName: "memory-manager-none", + cpus: "100m", + memory: "128Mi", + }, + } + testPod := makeMemoryManagerPod(ctnParams[0].ctnName, nil, ctnParams) + testPod.Spec.Resources = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + testPod = createPodSync(ctx, testPod) + + if !*isMultiNUMASupported { + framework.Logf("Skipping memory pinning verification on single-NUMA machine") + return + } + verifyMemoryPinning(f, ctx, testPod, allNUMANodes) }) }) })