This commit is contained in:
Sohan Kunkerkar 2026-07-08 01:02:08 -04:00 committed by GitHub
commit ad2ac9eceb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 320 additions and 8 deletions

View file

@ -940,6 +940,7 @@ func run(ctx context.Context, s *options.KubeletServer, kubeDeps *kubelet.Depend
MemoryManagerPolicy: s.MemoryManagerPolicy,
MemoryManagerReservedMemory: s.ReservedMemory,
MemoryReservationPolicy: s.MemoryReservationPolicy,
MemoryThrottlingFactor: s.MemoryThrottlingFactor,
PodPidsLimit: s.PodPidsLimit,
EnforceCPULimits: s.CPUCFSQuota,
CPUCFSQuotaPeriod: s.CPUCFSQuotaPeriod.Duration,

View file

@ -194,6 +194,7 @@ type NodeConfig struct {
MemoryManagerPolicy string
MemoryManagerReservedMemory []kubeletconfig.MemoryReservation
MemoryReservationPolicy kubeletconfig.MemoryReservationPolicy
MemoryThrottlingFactor *float64
PodPidsLimit int64
EnforceCPULimits bool
CPUCFSQuotaPeriod time.Duration

View file

@ -410,6 +410,7 @@ func (cm *containerManagerImpl) NewPodContainerManager() PodContainerManager {
cpuCFSQuotaPeriod: uint64(cm.CPUCFSQuotaPeriod / time.Microsecond),
podContainerManager: cm,
memoryReservationPolicy: cm.MemoryReservationPolicy,
memoryThrottlingFactor: cm.MemoryThrottlingFactor,
}
}
return &podContainerManagerNoop{

View file

@ -38,6 +38,60 @@ import (
"k8s.io/kubernetes/pkg/kubelet/cm/util"
)
// ApplyPodLevelMemoryHigh sets memory.high on the pod cgroup using the KEP-2570 formula.
// Skips pods where not all containers declare memory limits (partial sums are unreliable)
// unless PodLevelResources provides a pod-level memory limit.
func ApplyPodLevelMemoryHigh(pod *v1.Pod, rc *ResourceConfig, throttlingFactor float64) {
podLevelResourcesEnabled := utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodLevelResources)
reqs := resourcehelper.PodRequests(pod, resourcehelper.PodResourcesOptions{
SkipPodLevelResources: !podLevelResourcesEnabled,
})
memoryLimitsDeclared := true
limits := resourcehelper.PodLimits(pod, resourcehelper.PodResourcesOptions{
SkipPodLevelResources: !podLevelResourcesEnabled,
ContainerFn: func(res v1.ResourceList, _ resourcehelper.ContainerType) {
if res.Memory().IsZero() {
memoryLimitsDeclared = false
}
},
})
if podLevelResourcesEnabled && resourcehelper.IsPodLevelResourcesSet(pod) &&
!pod.Spec.Resources.Limits.Memory().IsZero() {
memoryLimitsDeclared = true
}
if !memoryLimitsDeclared {
return
}
memoryRequest := int64(0)
memoryLimit := int64(0)
if req, found := reqs[v1.ResourceMemory]; found {
memoryRequest = req.Value()
}
if lim, found := limits[v1.ResourceMemory]; found {
memoryLimit = lim.Value()
}
if val := memoryHighForPod(memoryRequest, memoryLimit, throttlingFactor); val != "" {
if rc.Unified == nil {
rc.Unified = map[string]string{}
}
rc.Unified[Cgroup2MemoryHigh] = val
}
}
func memoryHighForPod(memoryRequest, memoryLimit int64, throttlingFactor float64) string {
if (memoryRequest == memoryLimit && memoryRequest != 0) || memoryLimit == 0 {
return ""
}
pageSize := int64(os.Getpagesize())
memoryHigh := int64(math.Floor(
float64(memoryRequest)+
(float64(memoryLimit)-float64(memoryRequest))*throttlingFactor)/float64(pageSize)) * pageSize
if memoryHigh > 0 && memoryHigh > memoryRequest {
return strconv.FormatInt(memoryHigh, 10)
}
return ""
}
const (
// These limits are defined in the kernel:
// https://github.com/torvalds/linux/blob/0bddd227f3dc55975e2b8dfa7fc6f959b062a2c7/kernel/sched/sched.h#L427-L428

View file

@ -857,3 +857,44 @@ func TestResourceConfigForPodWithEnforceMemoryQoS(t *testing.T) {
}
}
}
func TestApplyPodLevelMemoryHigh(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.PodLevelResources, true)
t.Run("cpu-only pod-level resources does not set memory.high", func(t *testing.T) {
pod := &v1.Pod{
Spec: v1.PodSpec{
Resources: &v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceCPU: resource.MustParse("2"),
},
},
Containers: []v1.Container{
{Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", ""))},
},
},
}
rc := &ResourceConfig{}
ApplyPodLevelMemoryHigh(pod, rc, 0.9)
if rc.Unified != nil {
t.Errorf("expected no Unified map, got %v", rc.Unified)
}
})
t.Run("mixed declared/undeclared memory limits does not set memory.high", func(t *testing.T) {
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.PodLevelResources, false)
pod := &v1.Pod{
Spec: v1.PodSpec{
Containers: []v1.Container{
{Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("200m", "200Mi"))},
{Resources: getResourceRequirements(getResourceList("100m", "100Mi"), getResourceList("", ""))},
},
},
}
rc := &ResourceConfig{}
ApplyPodLevelMemoryHigh(pod, rc, 0.9)
if rc.Unified != nil {
t.Errorf("expected no Unified map for mixed limits, got %v", rc.Unified)
}
})
}

View file

@ -60,6 +60,8 @@ type podContainerManagerImpl struct {
podContainerManager ContainerManager
// memoryReservationPolicy controls memory reservation protection behavior
memoryReservationPolicy kubeletconfig.MemoryReservationPolicy
// memoryThrottlingFactor is used to compute pod-level memory.high
memoryThrottlingFactor *float64
}
// Make sure that podContainerManagerImpl implements the PodContainerManager interface
@ -98,6 +100,7 @@ func (m *podContainerManagerImpl) EnsureExists(logger klog.Logger, pod *v1.Pod)
containerConfig.ResourceParameters.PidsLimit = &m.podPidsLimit
}
if enforceMemoryQoS {
m.applyPodLevelMemoryHigh(pod, containerConfig.ResourceParameters)
logger.V(4).Info("MemoryQoS config for pod", "pod", klog.KObj(pod), "unified", containerConfig.ResourceParameters.Unified)
}
if err := m.cgroupManager.Create(logger, containerConfig); err != nil {
@ -108,6 +111,12 @@ func (m *podContainerManagerImpl) EnsureExists(logger klog.Logger, pod *v1.Pod)
return nil
}
func (m *podContainerManagerImpl) applyPodLevelMemoryHigh(pod *v1.Pod, rc *ResourceConfig) {
if m.memoryThrottlingFactor != nil {
ApplyPodLevelMemoryHigh(pod, rc, *m.memoryThrottlingFactor)
}
}
// GetPodContainerName returns the CgroupName identifier, and its literal cgroupfs form on the host.
func (m *podContainerManagerImpl) GetPodContainerName(pod *v1.Pod) (CgroupName, string) {
podQOS := v1qos.GetPodQOS(pod)

View file

@ -154,7 +154,7 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerResources(ctx context.
if enforceMemoryQoS {
unified := map[string]string{}
memoryRequest := container.Resources.Requests.Memory().Value()
memoryLimit := container.Resources.Limits.Memory().Value()
memoryLimitValue := container.Resources.Limits.Memory().Value()
if memoryRequest != 0 && m.memoryReservationPolicy == kubeletconfiginternal.TieredReservationMemoryReservationPolicy {
// Guaranteed pods get memory.min (hard protection).
// Burstable pods get memory.low (soft protection).
@ -168,19 +168,24 @@ func (m *kubeGenericRuntimeManager) generateLinuxContainerResources(ctx context.
unified[cm.Cgroup2MemoryLow] = "0"
}
// Skip memory.high only for equal, positive memory request/limit (container guaranteed memory).
// For Burstable pods, memory.high uses the memory limit, for BestEffort pods (request=limit=0), node allocatable is used.
if memoryRequest != memoryLimit || memoryRequest == 0 {
// When PodLevelResources is active and the container has no memory limit,
// skip container-level memory.high — the pod cgroup's memory.high handles
// throttling hierarchically (kernel walks ancestors in try_charge_memcg).
skipContainerMemoryHigh := memoryLimitValue == 0 &&
utilfeature.DefaultFeatureGate.Enabled(kubefeatures.PodLevelResources) &&
resourcehelper.IsPodLevelResourcesSet(pod) &&
!pod.Spec.Resources.Limits.Memory().IsZero()
if !skipContainerMemoryHigh && (memoryRequest != memoryLimitValue || memoryRequest == 0) {
// The formula for memory.high for container cgroup is modified in Alpha stage of the feature in K8s v1.27.
// It will be set based on formula:
// `memory.high=floor[(requests.memory + memory throttling factor * (limits.memory or node allocatable memory - requests.memory))/pageSize] * pageSize`
// where default value of memory throttling factor is set to 0.9
// More info: https://git.k8s.io/enhancements/keps/sig-node/2570-memory-qos
memoryHigh := int64(0)
if memoryLimit != 0 {
if memoryLimitValue != 0 {
memoryHigh = int64(math.Floor(
float64(memoryRequest)+
(float64(memoryLimit)-float64(memoryRequest))*float64(m.memoryThrottlingFactor))/float64(defaultPageSize)) * defaultPageSize
(float64(memoryLimitValue)-float64(memoryRequest))*float64(m.memoryThrottlingFactor))/float64(defaultPageSize)) * defaultPageSize
} else {
allocatable := m.getNodeAllocatable()
allocatableMemory, ok := allocatable[v1.ResourceMemory]
@ -434,6 +439,18 @@ func toKubeContainerResources(statusResources *runtimeapi.ContainerResources) *k
// the cgroup version would solely depend on the environment running the test.
var isCgroup2UnifiedMode = libcontainercgroups.IsCgroup2UnifiedMode
// isMemoryQoSEnforced reports whether MemoryQoS is enabled on cgroup v2.
func (m *kubeGenericRuntimeManager) isMemoryQoSEnforced() bool {
return utilfeature.DefaultFeatureGate.Enabled(kubefeatures.MemoryQoS) && isCgroup2UnifiedMode()
}
// applyPodLevelMemoryHigh sets pod-level memory.high; kernel enforces hierarchically.
func (m *kubeGenericRuntimeManager) applyPodLevelMemoryHigh(pod *v1.Pod, rc *cm.ResourceConfig) {
if m.memoryThrottlingFactor != 0 {
cm.ApplyPodLevelMemoryHigh(pod, rc, m.memoryThrottlingFactor)
}
}
// checkSwapControllerAvailability checks if swap controller is available.
// It returns true if the swap controller is available, false otherwise.
func checkSwapControllerAvailability(ctx context.Context) bool {

View file

@ -2170,3 +2170,100 @@ func setCgroupVersionDuringTest(version CgroupVersion) {
return version == cgroupV2
}
}
func TestContainerMemoryHighSkippedWithPodLevelResources(t *testing.T) {
tCtx := ktesting.Init(t)
_, _, m, err := createTestRuntimeManager(tCtx)
require.NoError(t, err)
setCgroupVersionDuringTest(cgroupV2)
m.memoryThrottlingFactor = 0.9
m.memoryReservationPolicy = kubeletconfiginternal.NoneMemoryReservationPolicy
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MemoryQoS, true)
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLevelResources, true)
containerRequest := resource.MustParse("256Mi")
containerLimit := resource.MustParse("512Mi")
podLimitMemory := resource.MustParse("1Gi")
pod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{
UID: "12345678",
Name: "pod-level-test",
Namespace: "test",
},
Spec: v1.PodSpec{
Resources: &v1.ResourceRequirements{
Limits: v1.ResourceList{
v1.ResourceMemory: podLimitMemory,
},
},
Containers: []v1.Container{
{
Name: "c1-with-limit",
Image: "busybox",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: containerRequest,
v1.ResourceCPU: resource.MustParse("100m"),
},
Limits: v1.ResourceList{
v1.ResourceMemory: containerLimit,
v1.ResourceCPU: resource.MustParse("200m"),
},
},
},
{
Name: "c2-no-limit",
Image: "busybox",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: containerRequest,
v1.ResourceCPU: resource.MustParse("100m"),
},
},
},
},
},
}
t.Run("container with own limit gets per-container memory.high", func(t *testing.T) {
lcr := m.generateLinuxContainerResources(tCtx, pod, &pod.Spec.Containers[0], true)
pageSize := int64(os.Getpagesize())
expectedHigh := int64(math.Floor(
float64(containerRequest.Value())+
(float64(containerLimit.Value())-float64(containerRequest.Value()))*0.9)/float64(pageSize)) * pageSize
actualHigh, ok := lcr.Unified[cm.Cgroup2MemoryHigh]
assert.True(t, ok, "memory.high should be set for container with own limit")
assert.Equal(t, strconv.FormatInt(expectedHigh, 10), actualHigh)
})
t.Run("container without limit skips memory.high (pod-level handles it)", func(t *testing.T) {
lcr := m.generateLinuxContainerResources(tCtx, pod, &pod.Spec.Containers[1], true)
_, ok := lcr.Unified[cm.Cgroup2MemoryHigh]
assert.False(t, ok, "memory.high should NOT be set on container without own limit when PodLevelResources is active")
})
t.Run("cpu-only pod-level resources does not skip container memory.high", func(t *testing.T) {
cpuOnlyPod := &v1.Pod{
ObjectMeta: metav1.ObjectMeta{UID: "cpu-only", Name: "cpu-only-test", Namespace: "test"},
Spec: v1.PodSpec{
Resources: &v1.ResourceRequirements{
Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("2")},
},
Containers: []v1.Container{{
Name: "c1", Image: "busybox",
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{
v1.ResourceMemory: containerRequest,
v1.ResourceCPU: resource.MustParse("100m"),
},
},
}},
},
}
lcr := m.generateLinuxContainerResources(tCtx, cpuOnlyPod, &cpuOnlyPod.Spec.Containers[0], true)
_, ok := lcr.Unified[cm.Cgroup2MemoryHigh]
assert.True(t, ok, "memory.high should be set via node-allocatable fallback when pod has CPU-only resources")
})
}

View file

@ -59,3 +59,10 @@ func (m *kubeGenericRuntimeManager) GetContainerSwapBehavior(pod *v1.Pod, contai
func initSwapControllerAvailabilityCheck(ctx context.Context) func() bool {
return func() bool { return false }
}
func (m *kubeGenericRuntimeManager) isMemoryQoSEnforced() bool {
return false
}
func (m *kubeGenericRuntimeManager) applyPodLevelMemoryHigh(_ *v1.Pod, _ *cm.ResourceConfig) {
}

View file

@ -195,3 +195,10 @@ func (m *kubeGenericRuntimeManager) GetContainerSwapBehavior(pod *v1.Pod, contai
func initSwapControllerAvailabilityCheck(ctx context.Context) func() bool {
return func() bool { return false }
}
func (m *kubeGenericRuntimeManager) isMemoryQoSEnforced() bool {
return false
}
func (m *kubeGenericRuntimeManager) applyPodLevelMemoryHigh(_ *v1.Pod, _ *cm.ResourceConfig) {
}

View file

@ -827,18 +827,20 @@ func (m *kubeGenericRuntimeManager) doPodResizeAction(ctx context.Context, pod *
resizeResult := kubecontainer.NewSyncResult(kubecontainer.ResizePodInPlace, format.Pod(pod))
pcm := m.containerManager.NewPodContainerManager()
//TODO(vinaykul,InPlacePodVerticalScaling): Figure out best way to get enforceMemoryQoS value (parameter #4 below) in platform-agnostic way
enforceCPULimits := m.cpuCFSQuota
if utilfeature.DefaultFeatureGate.Enabled(features.DisableCPUQuotaWithExclusiveCPUs) && m.containerManager.PodHasExclusiveCPUs(logger, pod) {
enforceCPULimits = false
logger.V(2).Info("Disabled CFS quota", "pod", klog.KObj(pod))
}
podResources := cm.ResourceConfigForPod(pod, enforceCPULimits, uint64((m.cpuCFSQuotaPeriod.Duration)/time.Microsecond), false, kubeletconfiginternal.NoneMemoryReservationPolicy)
podResources := cm.ResourceConfigForPod(pod, enforceCPULimits, uint64((m.cpuCFSQuotaPeriod.Duration)/time.Microsecond), m.isMemoryQoSEnforced(), m.memoryReservationPolicy)
if podResources == nil {
logger.Error(nil, "Unable to get resource configuration", "pod", klog.KObj(pod))
resizeResult.Fail(kubecontainer.ErrResizePodInPlace, fmt.Sprintf("unable to get resource configuration processing resize for pod %q", format.Pod(pod)))
return resizeResult
}
if m.isMemoryQoSEnforced() {
m.applyPodLevelMemoryHigh(pod, podResources)
}
currentPodMemoryConfig, err := pcm.GetPodCgroupConfig(pod, v1.ResourceMemory)
if err != nil {
logger.Error(err, "Unable to get pod cgroup memory config", "pod", klog.KObj(pod))
@ -932,6 +934,7 @@ func (m *kubeGenericRuntimeManager) doPodResizeAction(ctx context.Context, pod *
return nil
}
resizedResources.Memory = podResources.Memory
resizedResources.Unified = podResources.Unified
}
// Notify the runtime first. If this fails, the runtime has rejected the resize.

View file

@ -1191,4 +1191,78 @@ var _ = SIGDescribe("MemoryQoS", framework.WithSerial(), func() {
"container-level memory.low persists after rollback (CRI limitation)")
})
})
f.Describe("IPPR resize with memory protection", func() {
ginkgo.AfterEach(func(ctx context.Context) { restoreConfig(ctx) })
ginkgo.It("should update pod-level memory.high and memory.low after Burstable pod resize", func(ctx context.Context) {
throttlingFactor := 0.9
configureMemoryQoSWithPolicy(ctx, throttlingFactor, kubeletconfig.TieredReservationMemoryReservationPolicy)
initialRequest := resource.MustParse("128Mi")
initialLimit := resource.MustParse("256Mi")
pod := memqosMakePod("memqos-resize-protection", f.Namespace.Name,
v1.ResourceList{
v1.ResourceMemory: initialRequest,
v1.ResourceCPU: resource.MustParse("50m"),
},
v1.ResourceList{
v1.ResourceMemory: initialLimit,
v1.ResourceCPU: resource.MustParse("100m"),
},
)
pod.Spec.Containers[0].ResizePolicy = []v1.ContainerResizePolicy{
{ResourceName: v1.ResourceMemory, RestartPolicy: v1.NotRequired},
{ResourceName: v1.ResourceCPU, RestartPolicy: v1.NotRequired},
}
pod = e2epod.NewPodClient(f).CreateSync(ctx, pod)
podCgroupPath := memqosGetPodCgroupPath(pod, cgroupDriver)
gomega.Expect(podCgroupPath).NotTo(gomega.BeEmpty())
expectedMemoryHigh := func(req, lim int64) int64 {
pageSize := int64(os.Getpagesize())
return int64(math.Floor(float64(req)+(float64(lim)-float64(req))*throttlingFactor)/float64(pageSize)) * pageSize
}
ginkgo.By("Verifying initial memory.low and memory.high")
podMemLow, err := memqosReadCgroupInt64(podCgroupPath, cgroupMemoryLow)
framework.ExpectNoError(err)
gomega.Expect(podMemLow).To(gomega.Equal(initialRequest.Value()),
"pod memory.low should equal initial request")
podMemHigh, err := memqosReadCgroupInt64(podCgroupPath, cgroupMemoryHigh)
framework.ExpectNoError(err)
gomega.Expect(podMemHigh).To(gomega.Equal(expectedMemoryHigh(initialRequest.Value(), initialLimit.Value())),
"pod memory.high should match KEP-2570 formula")
ginkgo.By("Resizing memory request and limit via IPPR")
pod, err = f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{})
framework.ExpectNoError(err)
newRequest := resource.MustParse("192Mi")
newLimit := resource.MustParse("384Mi")
pod.Spec.Containers[0].Resources.Requests[v1.ResourceMemory] = newRequest
pod.Spec.Containers[0].Resources.Limits[v1.ResourceMemory] = newLimit
_, err = f.ClientSet.CoreV1().Pods(pod.Namespace).UpdateResize(ctx, pod.Name, pod, metav1.UpdateOptions{})
framework.ExpectNoError(err)
ginkgo.By("Verifying pod-level memory.high updated after resize")
newExpectedHigh := expectedMemoryHigh(newRequest.Value(), newLimit.Value())
gomega.Eventually(ctx, func() int64 {
val, _ := memqosReadCgroupInt64(podCgroupPath, cgroupMemoryHigh)
return val
}).WithTimeout(2*time.Minute).WithPolling(5*time.Second).Should(
gomega.Equal(newExpectedHigh),
"pod memory.high should update after IPPR resize")
ginkgo.By("Verifying pod-level memory.low updated after resize")
gomega.Eventually(ctx, func() int64 {
val, _ := memqosReadCgroupInt64(podCgroupPath, cgroupMemoryLow)
return val
}).WithTimeout(2*time.Minute).WithPolling(5*time.Second).Should(
gomega.Equal(newRequest.Value()),
"pod memory.low should update to new request after IPPR resize")
})
})
})