From 06c47136fefbce67d609f9f242e5b71bb1b5ce38 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Fri, 26 Sep 2025 11:38:33 -0400 Subject: [PATCH 1/3] implement KEP-5573 by not starting kubelet on cgroup v1 by default --- pkg/generated/openapi/zz_generated.openapi.go | 2 +- pkg/kubelet/apis/config/v1beta1/defaults.go | 2 +- pkg/kubelet/apis/config/v1beta1/defaults_test.go | 14 +++++++------- .../apis/config/validation/validation_linux.go | 2 +- pkg/kubelet/cm/container_manager.go | 2 +- pkg/kubelet/kubelet_linux.go | 4 ++-- staging/src/k8s.io/kubelet/config/v1beta1/types.go | 8 ++++---- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index ccc4bc0a153..43f57d0ef8d 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -71925,7 +71925,7 @@ func schema_k8sio_kubelet_config_v1beta1_KubeletConfiguration(ref common.Referen }, "failCgroupV1": { SchemaProps: spec.SchemaProps{ - Description: "FailCgroupV1 prevents the kubelet from starting on hosts that use cgroup v1. By default, this is set to 'false', meaning the kubelet is allowed to start on cgroup v1 hosts unless this option is explicitly enabled. Default: false", + Description: "FailCgroupV1 prevents the kubelet from starting on hosts that use cgroup v1. By default, this is set to 'true', meaning the kubelet will not start on cgroup v1 hosts unless this option is explicitly disabled. Default: true", Type: []string{"boolean"}, Format: "", }, diff --git a/pkg/kubelet/apis/config/v1beta1/defaults.go b/pkg/kubelet/apis/config/v1beta1/defaults.go index 04d71d00bd7..ba5e7aeb600 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults.go @@ -290,7 +290,7 @@ func SetDefaults_KubeletConfiguration(obj *kubeletconfigv1beta1.KubeletConfigura obj.SeccompDefault = ptr.To(false) } if obj.FailCgroupV1 == nil { - obj.FailCgroupV1 = ptr.To(false) + obj.FailCgroupV1 = ptr.To(true) } if obj.MemoryThrottlingFactor == nil { obj.MemoryThrottlingFactor = ptr.To(DefaultMemoryThrottlingFactor) diff --git a/pkg/kubelet/apis/config/v1beta1/defaults_test.go b/pkg/kubelet/apis/config/v1beta1/defaults_test.go index 9c79a088cc0..d6cedf9a1af 100644 --- a/pkg/kubelet/apis/config/v1beta1/defaults_test.go +++ b/pkg/kubelet/apis/config/v1beta1/defaults_test.go @@ -126,7 +126,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: ptr.To(true), EnableDebugFlagsHandler: ptr.To(true), SeccompDefault: ptr.To(false), - FailCgroupV1: ptr.To(false), + FailCgroupV1: ptr.To(true), MemoryThrottlingFactor: ptr.To(DefaultMemoryThrottlingFactor), RegisterNode: ptr.To(true), LocalStorageCapacityIsolation: ptr.To(true), @@ -261,7 +261,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: ptr.To(false), EnableDebugFlagsHandler: ptr.To(false), SeccompDefault: ptr.To(false), - FailCgroupV1: ptr.To(false), + FailCgroupV1: ptr.To(true), MemoryThrottlingFactor: ptr.To[float64](0), RegisterNode: ptr.To(false), LocalStorageCapacityIsolation: ptr.To(false), @@ -366,7 +366,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: ptr.To(false), EnableDebugFlagsHandler: ptr.To(false), SeccompDefault: ptr.To(false), - FailCgroupV1: ptr.To(false), + FailCgroupV1: ptr.To(true), MemoryThrottlingFactor: ptr.To[float64](0), RegisterNode: ptr.To(false), LocalStorageCapacityIsolation: ptr.To(false), @@ -779,7 +779,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: ptr.To(true), EnableDebugFlagsHandler: ptr.To(true), SeccompDefault: ptr.To(false), - FailCgroupV1: ptr.To(false), + FailCgroupV1: ptr.To(true), MemoryThrottlingFactor: ptr.To(DefaultMemoryThrottlingFactor), RegisterNode: ptr.To(true), LocalStorageCapacityIsolation: ptr.To(true), @@ -875,7 +875,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: ptr.To(true), EnableDebugFlagsHandler: ptr.To(true), SeccompDefault: ptr.To(false), - FailCgroupV1: ptr.To(false), + FailCgroupV1: ptr.To(true), MemoryThrottlingFactor: ptr.To(DefaultMemoryThrottlingFactor), RegisterNode: ptr.To(true), LocalStorageCapacityIsolation: ptr.To(true), @@ -971,7 +971,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: ptr.To(true), EnableDebugFlagsHandler: ptr.To(true), SeccompDefault: ptr.To(false), - FailCgroupV1: ptr.To(false), + FailCgroupV1: ptr.To(true), MemoryThrottlingFactor: ptr.To(DefaultMemoryThrottlingFactor), RegisterNode: ptr.To(true), LocalStorageCapacityIsolation: ptr.To(true), @@ -1067,7 +1067,7 @@ func TestSetDefaultsKubeletConfiguration(t *testing.T) { EnableProfilingHandler: ptr.To(true), EnableDebugFlagsHandler: ptr.To(true), SeccompDefault: ptr.To(false), - FailCgroupV1: ptr.To(false), + FailCgroupV1: ptr.To(true), MemoryThrottlingFactor: ptr.To(DefaultMemoryThrottlingFactor), RegisterNode: ptr.To(true), LocalStorageCapacityIsolation: ptr.To(true), diff --git a/pkg/kubelet/apis/config/validation/validation_linux.go b/pkg/kubelet/apis/config/validation/validation_linux.go index dca9c69b5a3..76c976f6c00 100644 --- a/pkg/kubelet/apis/config/validation/validation_linux.go +++ b/pkg/kubelet/apis/config/validation/validation_linux.go @@ -34,7 +34,7 @@ const userNsUnitLength = 65536 func validateKubeletOSConfiguration(kc *kubeletconfig.KubeletConfiguration) error { isCgroup1 := !libcontainercgroups.IsCgroup2UnifiedMode() if kc.FailCgroupV1 && isCgroup1 { - return fmt.Errorf("kubelet is configured to not run on a host using cgroup v1. cgroup v1 support is in maintenance mode") + return fmt.Errorf("kubelet is configured to not run on a host using cgroup v1. cgroup v1 support is unsupported and will be removed in a future release") } if isCgroup1 && kc.SingleProcessOOMKill != nil && !ptr.Deref(kc.SingleProcessOOMKill, true) { diff --git a/pkg/kubelet/cm/container_manager.go b/pkg/kubelet/cm/container_manager.go index 0f4a2080753..de05b1a3d9d 100644 --- a/pkg/kubelet/cm/container_manager.go +++ b/pkg/kubelet/cm/container_manager.go @@ -49,7 +49,7 @@ import ( const ( // Warning message for the users still using cgroup v1 - CgroupV1MaintenanceModeWarning = "cgroup v1 support is in maintenance mode, please migrate to cgroup v2" + CgroupV1UnsupportedWarning = "cgroup v1 support is unsupported and will be removed in a future release. Please migrate to cgroup v2. More information at https://git.k8s.io/enhancements/keps/sig-node/5573-cgroup-v1-unsupported" // Warning message for the users using cgroup v2 on kernel doesn't support root `cpu.stat`. // `cpu.stat` was added to root cgroup in kernel 5.8. diff --git a/pkg/kubelet/kubelet_linux.go b/pkg/kubelet/kubelet_linux.go index c1f82b1af69..44c92461144 100644 --- a/pkg/kubelet/kubelet_linux.go +++ b/pkg/kubelet/kubelet_linux.go @@ -36,8 +36,8 @@ func (kl *Kubelet) cgroupVersionCheck() error { metrics.CgroupVersion.Set(float64(cgroupVersion)) switch cgroupVersion { case 1: - kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.CgroupV1, cm.CgroupV1MaintenanceModeWarning) - return errors.New(cm.CgroupV1MaintenanceModeWarning) + kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.CgroupV1, cm.CgroupV1UnsupportedWarning) + return errors.New(cm.CgroupV1UnsupportedWarning) case 2: cpustat := filepath.Join(util.CgroupRoot, "cpu.stat") if _, err := os.Stat(cpustat); os.IsNotExist(err) { diff --git a/staging/src/k8s.io/kubelet/config/v1beta1/types.go b/staging/src/k8s.io/kubelet/config/v1beta1/types.go index 610737c87ef..1794d8eb537 100644 --- a/staging/src/k8s.io/kubelet/config/v1beta1/types.go +++ b/staging/src/k8s.io/kubelet/config/v1beta1/types.go @@ -915,10 +915,10 @@ type KubeletConfiguration struct { ImageServiceEndpoint string `json:"imageServiceEndpoint,omitempty"` // FailCgroupV1 prevents the kubelet from starting on hosts - // that use cgroup v1. By default, this is set to 'false', meaning - // the kubelet is allowed to start on cgroup v1 hosts unless this - // option is explicitly enabled. - // Default: false + // that use cgroup v1. By default, this is set to 'true', meaning + // the kubelet will not start on cgroup v1 hosts unless this + // option is explicitly disabled. + // Default: true // +optional FailCgroupV1 *bool `json:"failCgroupV1,omitempty"` From 7e27b9c241fa2d369e2585890f07905ea2943c90 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Sun, 28 Sep 2025 13:12:59 -0400 Subject: [PATCH 2/3] change default for failCgroupv1 --- .../scheme/testdata/KubeletConfiguration/after/v1beta1.yaml | 2 +- .../KubeletConfiguration/roundtrip/default/v1beta1.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml index 1acdd87f50f..515a995b572 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/after/v1beta1.yaml @@ -38,7 +38,7 @@ enforceNodeAllocatable: eventBurst: 100 eventRecordQPS: 50 evictionPressureTransitionPeriod: 5m0s -failCgroupV1: false +failCgroupV1: true failSwapOn: true fileCheckFrequency: 20s hairpinMode: promiscuous-bridge diff --git a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml index 33c9777a94a..8d065e6447c 100644 --- a/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml +++ b/pkg/kubelet/apis/config/scheme/testdata/KubeletConfiguration/roundtrip/default/v1beta1.yaml @@ -38,7 +38,7 @@ enforceNodeAllocatable: eventBurst: 10 eventRecordQPS: 5 evictionPressureTransitionPeriod: 5m0s -failCgroupV1: false +failCgroupV1: true failSwapOn: true fileCheckFrequency: 20s hairpinMode: promiscuous-bridge From f27346db6c5d872ef4a95f45ef6afce6aa05fff6 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Fri, 10 Oct 2025 10:00:08 -0400 Subject: [PATCH 3/3] update warning to reflect kep changes --- pkg/kubelet/cm/container_manager.go | 2 +- pkg/kubelet/kubelet_linux.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/kubelet/cm/container_manager.go b/pkg/kubelet/cm/container_manager.go index de05b1a3d9d..69704223e23 100644 --- a/pkg/kubelet/cm/container_manager.go +++ b/pkg/kubelet/cm/container_manager.go @@ -49,7 +49,7 @@ import ( const ( // Warning message for the users still using cgroup v1 - CgroupV1UnsupportedWarning = "cgroup v1 support is unsupported and will be removed in a future release. Please migrate to cgroup v2. More information at https://git.k8s.io/enhancements/keps/sig-node/5573-cgroup-v1-unsupported" + CgroupV1DeprecatedWarning = "cgroup v1 detected. cgroup v1 support is deprecated and will be removed in a future release. Please migrate to cgroup v2. More information at https://git.k8s.io/enhancements/keps/sig-node/5573-remove-cgroup-v1" // Warning message for the users using cgroup v2 on kernel doesn't support root `cpu.stat`. // `cpu.stat` was added to root cgroup in kernel 5.8. diff --git a/pkg/kubelet/kubelet_linux.go b/pkg/kubelet/kubelet_linux.go index 44c92461144..6dd637a6152 100644 --- a/pkg/kubelet/kubelet_linux.go +++ b/pkg/kubelet/kubelet_linux.go @@ -36,8 +36,8 @@ func (kl *Kubelet) cgroupVersionCheck() error { metrics.CgroupVersion.Set(float64(cgroupVersion)) switch cgroupVersion { case 1: - kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.CgroupV1, cm.CgroupV1UnsupportedWarning) - return errors.New(cm.CgroupV1UnsupportedWarning) + kl.recorder.Eventf(kl.nodeRef, v1.EventTypeWarning, events.CgroupV1, cm.CgroupV1DeprecatedWarning) + return errors.New(cm.CgroupV1DeprecatedWarning) case 2: cpustat := filepath.Join(util.CgroupRoot, "cpu.stat") if _, err := os.Stat(cpustat); os.IsNotExist(err) {