From e237faffd1a7fd2d6f424a402b64b4cef775ae5e Mon Sep 17 00:00:00 2001 From: YeRongyu Date: Thu, 30 Apr 2026 11:30:15 +0800 Subject: [PATCH 1/2] apiserver: add feature gate to skip consistent list storage fallback --- pkg/features/kube_features.go | 6 ++++++ .../k8s.io/apiserver/pkg/features/kube_features.go | 11 +++++++++++ .../apiserver/pkg/storage/cacher/delegator.go | 13 ++++++++++--- .../reference/feature_list.md | 1 + .../reference/versioned_feature_list.yaml | 6 ++++++ 5 files changed, 34 insertions(+), 3 deletions(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 929d2b599fe..426124cf16e 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -2195,6 +2195,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, }, + genericfeatures.ConsistentListFromCacheSkipStorageFallback: { + {Version: version.MustParse("1.37"), Default: false, PreRelease: featuregate.Alpha}, + }, + genericfeatures.ConstrainedImpersonation: { {Version: version.MustParse("1.35"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.36"), Default: true, PreRelease: featuregate.Beta}, @@ -2737,6 +2741,8 @@ var defaultKubernetesFeatureGateDependencies = map[featuregate.Feature][]feature genericfeatures.ConsistentListFromCache: {}, + genericfeatures.ConsistentListFromCacheSkipStorageFallback: {}, + genericfeatures.ConstrainedImpersonation: {}, genericfeatures.DeclarativeValidation: {}, diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index 0c54a20c3d4..cce4fc657eb 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -109,6 +109,13 @@ const ( // Allow the API server to serve consistent lists from cache ConsistentListFromCache featuregate.Feature = "ConsistentListFromCache" + // owner: @yedou37 + // + // Skip falling back to storage when a consistent LIST from cache cannot be served, + // and return a retryable response instead. + // See https://github.com/kubernetes/kubernetes/issues/138494. + ConsistentListFromCacheSkipStorageFallback featuregate.Feature = "ConsistentListFromCacheSkipStorageFallback" + // owner: @enj @qiujian16 // kep: https://kep.k8s.io/5284 // @@ -391,6 +398,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, }, + ConsistentListFromCacheSkipStorageFallback: { + {Version: version.MustParse("1.37"), Default: false, PreRelease: featuregate.Alpha}, + }, + ConstrainedImpersonation: { {Version: version.MustParse("1.35"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.36"), Default: true, PreRelease: featuregate.Beta}, diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/delegator.go b/staging/src/k8s.io/apiserver/pkg/storage/cacher/delegator.go index 4e1fabfaba8..f60a4de89ea 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/delegator.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher/delegator.go @@ -217,10 +217,17 @@ func (c *CacheDelegator) GetList(ctx context.Context, key string, opts storage.L } if result.ConsistentRead { // IsTooLargeResourceVersion occurs when the requested RV is higher than cache's current RV - // and cache hasn't caught up within the timeout period. Fall back to etcd. + // and cache hasn't caught up within the timeout period. By default we fall back + // to etcd, but the ConsistentListFromCacheSkipStorageFallback feature gate turns + // this path into a retryable 429 to avoid amplifying storage load in degraded + // scenarios. See https://github.com/kubernetes/kubernetes/issues/138494. if storage.IsTooLargeResourceVersion(err) { - fallback = "true" - err = c.storage.GetList(ctx, key, opts, listObj) + if utilfeature.DefaultFeatureGate.Enabled(features.ConsistentListFromCacheSkipStorageFallback) { + err = errors.NewTooManyRequests(err.Error(), resourceVersionTooHighRetrySeconds) + } else { + fallback = "true" + err = c.storage.GetList(ctx, key, opts, listObj) + } } if err != nil { success = "false" diff --git a/test/compatibility_lifecycle/reference/feature_list.md b/test/compatibility_lifecycle/reference/feature_list.md index fb81ac8e063..9c7f0e32a2e 100644 --- a/test/compatibility_lifecycle/reference/feature_list.md +++ b/test/compatibility_lifecycle/reference/feature_list.md @@ -45,6 +45,7 @@ | ComponentStatusz | :ballot_box_with_check: 1.36+ | | 1.32–1.35 | 1.36– | | | | [code](https://cs.k8s.io/?q=%5CbComponentStatusz%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbComponentStatusz%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ConcurrentWatchObjectDecode | | | | 1.31– | | | | [code](https://cs.k8s.io/?q=%5CbConcurrentWatchObjectDecode%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbConcurrentWatchObjectDecode%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ConsistentListFromCache | :ballot_box_with_check: 1.31+ | :closed_lock_with_key: 1.34+ | 1.28–1.30 | 1.31–1.33 | 1.34– | | | [code](https://cs.k8s.io/?q=%5CbConsistentListFromCache%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbConsistentListFromCache%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | +| ConsistentListFromCacheSkipStorageFallback | | | 1.37– | | | | | [code](https://cs.k8s.io/?q=%5CbConsistentListFromCacheSkipStorageFallback%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbConsistentListFromCacheSkipStorageFallback%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ConstrainedImpersonation | :ballot_box_with_check: 1.36+ | | 1.35 | 1.36– | | | | [code](https://cs.k8s.io/?q=%5CbConstrainedImpersonation%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbConstrainedImpersonation%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ContainerCheckpoint | :ballot_box_with_check: 1.30+ | | 1.25–1.29 | 1.30– | | | | [code](https://cs.k8s.io/?q=%5CbContainerCheckpoint%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbContainerCheckpoint%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ContainerRestartRules | :ballot_box_with_check: 1.35+ | | 1.34 | 1.35– | | | | [code](https://cs.k8s.io/?q=%5CbContainerRestartRules%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbContainerRestartRules%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | diff --git a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml index 42a2ed523f3..fc58dd6c79a 100644 --- a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml +++ b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml @@ -263,6 +263,12 @@ lockToDefault: true preRelease: GA version: "1.34" +- name: ConsistentListFromCacheSkipStorageFallback + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.37" - name: ConstrainedImpersonation versionedSpecs: - default: false From 9a07228d25e476bc4414d7833442cba5dba118d6 Mon Sep 17 00:00:00 2001 From: YeRongyu Date: Thu, 30 Apr 2026 16:49:47 +0800 Subject: [PATCH 2/2] apiserver: add feature gate to skip consistent list storage fallback --- pkg/features/kube_features.go | 4 ++-- .../apiserver/pkg/features/kube_features.go | 6 ++--- .../storage/cacher/cacher_whitebox_test.go | 24 +++++++++++++++++++ .../apiserver/pkg/storage/cacher/delegator.go | 8 +++---- .../reference/feature_list.md | 2 +- .../reference/versioned_feature_list.yaml | 2 +- 6 files changed, 34 insertions(+), 12 deletions(-) diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 426124cf16e..b7ee44cbee2 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -2195,7 +2195,7 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, }, - genericfeatures.ConsistentListFromCacheSkipStorageFallback: { + genericfeatures.ConsistentListFromCacheSkipTimeoutFallback: { {Version: version.MustParse("1.37"), Default: false, PreRelease: featuregate.Alpha}, }, @@ -2741,7 +2741,7 @@ var defaultKubernetesFeatureGateDependencies = map[featuregate.Feature][]feature genericfeatures.ConsistentListFromCache: {}, - genericfeatures.ConsistentListFromCacheSkipStorageFallback: {}, + genericfeatures.ConsistentListFromCacheSkipTimeoutFallback: {genericfeatures.ConsistentListFromCache}, genericfeatures.ConstrainedImpersonation: {}, diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index cce4fc657eb..a4e9faecb64 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -111,10 +111,10 @@ const ( // owner: @yedou37 // - // Skip falling back to storage when a consistent LIST from cache cannot be served, + // Skip the timeout fallback to storage when a consistent LIST from cache cannot be served, // and return a retryable response instead. // See https://github.com/kubernetes/kubernetes/issues/138494. - ConsistentListFromCacheSkipStorageFallback featuregate.Feature = "ConsistentListFromCacheSkipStorageFallback" + ConsistentListFromCacheSkipTimeoutFallback featuregate.Feature = "ConsistentListFromCacheSkipTimeoutFallback" // owner: @enj @qiujian16 // kep: https://kep.k8s.io/5284 @@ -398,7 +398,7 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, }, - ConsistentListFromCacheSkipStorageFallback: { + ConsistentListFromCacheSkipTimeoutFallback: { {Version: version.MustParse("1.37"), Default: false, PreRelease: featuregate.Alpha}, }, diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher_whitebox_test.go b/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher_whitebox_test.go index 8005bf1ae70..01a75da4116 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher_whitebox_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher_whitebox_test.go @@ -373,11 +373,13 @@ func TestConsistentReadFallback(t *testing.T) { tcs := []struct { name string consistentReadsEnabled bool + skipStorageFallback bool watchCacheRV string storageRV string fallbackError bool expectError bool + expectTooManyRequests bool expectRV string expectBlock bool expectRequestsToStorage int @@ -423,6 +425,22 @@ apiserver_watch_cache_consistent_read_total{fallback="true", group="", resource= # HELP apiserver_watch_cache_consistent_read_total [ALPHA] Counter for consistent reads from cache. # TYPE apiserver_watch_cache_consistent_read_total counter apiserver_watch_cache_consistent_read_total{fallback="true", group="", resource="pods", success="false"} 1 +`, + }, + { + name: "Skip Storage Fallback", + consistentReadsEnabled: true, + skipStorageFallback: true, + watchCacheRV: "2", + storageRV: "42", + expectError: true, + expectTooManyRequests: true, + expectBlock: true, + expectRequestsToStorage: 1, + expectMetric: ` +# HELP apiserver_watch_cache_consistent_read_total [ALPHA] Counter for consistent reads from cache. +# TYPE apiserver_watch_cache_consistent_read_total counter +apiserver_watch_cache_consistent_read_total{fallback="skipped", group="", resource="pods", success="false"} 1 `, }, { @@ -442,6 +460,9 @@ apiserver_watch_cache_consistent_read_total{fallback="true", group="", resource= featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.33")) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, false) } + if tc.skipStorageFallback { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCacheSkipTimeoutFallback, true) + } registry := k8smetrics.NewKubeRegistry() metrics.ConsistentReadTotal.Reset() @@ -518,6 +539,9 @@ apiserver_watch_cache_consistent_read_total{fallback="true", group="", resource= if (err != nil) != tc.expectError { t.Fatalf("Unexpected error err: %v", err) } + if tc.expectTooManyRequests && !apierrors.IsTooManyRequests(err) { + t.Fatalf("Unexpected error, got: %v, want TooManyRequests", err) + } if result.ResourceVersion != tc.expectRV { t.Fatalf("Unexpected List response RV, got: %q, want: %q", result.ResourceVersion, tc.expectRV) } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/delegator.go b/staging/src/k8s.io/apiserver/pkg/storage/cacher/delegator.go index f60a4de89ea..4ce39140553 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/delegator.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher/delegator.go @@ -217,12 +217,10 @@ func (c *CacheDelegator) GetList(ctx context.Context, key string, opts storage.L } if result.ConsistentRead { // IsTooLargeResourceVersion occurs when the requested RV is higher than cache's current RV - // and cache hasn't caught up within the timeout period. By default we fall back - // to etcd, but the ConsistentListFromCacheSkipStorageFallback feature gate turns - // this path into a retryable 429 to avoid amplifying storage load in degraded - // scenarios. See https://github.com/kubernetes/kubernetes/issues/138494. + // and cache hasn't caught up within the timeout period. if storage.IsTooLargeResourceVersion(err) { - if utilfeature.DefaultFeatureGate.Enabled(features.ConsistentListFromCacheSkipStorageFallback) { + if utilfeature.DefaultFeatureGate.Enabled(features.ConsistentListFromCacheSkipTimeoutFallback) { + fallback = "skipped" err = errors.NewTooManyRequests(err.Error(), resourceVersionTooHighRetrySeconds) } else { fallback = "true" diff --git a/test/compatibility_lifecycle/reference/feature_list.md b/test/compatibility_lifecycle/reference/feature_list.md index 9c7f0e32a2e..42b07eff796 100644 --- a/test/compatibility_lifecycle/reference/feature_list.md +++ b/test/compatibility_lifecycle/reference/feature_list.md @@ -45,7 +45,7 @@ | ComponentStatusz | :ballot_box_with_check: 1.36+ | | 1.32–1.35 | 1.36– | | | | [code](https://cs.k8s.io/?q=%5CbComponentStatusz%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbComponentStatusz%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ConcurrentWatchObjectDecode | | | | 1.31– | | | | [code](https://cs.k8s.io/?q=%5CbConcurrentWatchObjectDecode%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbConcurrentWatchObjectDecode%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ConsistentListFromCache | :ballot_box_with_check: 1.31+ | :closed_lock_with_key: 1.34+ | 1.28–1.30 | 1.31–1.33 | 1.34– | | | [code](https://cs.k8s.io/?q=%5CbConsistentListFromCache%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbConsistentListFromCache%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | -| ConsistentListFromCacheSkipStorageFallback | | | 1.37– | | | | | [code](https://cs.k8s.io/?q=%5CbConsistentListFromCacheSkipStorageFallback%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbConsistentListFromCacheSkipStorageFallback%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | +| ConsistentListFromCacheSkipTimeoutFallback | | | 1.37– | | | | ConsistentListFromCache | [code](https://cs.k8s.io/?q=%5CbConsistentListFromCacheSkipTimeoutFallback%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbConsistentListFromCacheSkipTimeoutFallback%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ConstrainedImpersonation | :ballot_box_with_check: 1.36+ | | 1.35 | 1.36– | | | | [code](https://cs.k8s.io/?q=%5CbConstrainedImpersonation%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbConstrainedImpersonation%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ContainerCheckpoint | :ballot_box_with_check: 1.30+ | | 1.25–1.29 | 1.30– | | | | [code](https://cs.k8s.io/?q=%5CbContainerCheckpoint%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbContainerCheckpoint%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ContainerRestartRules | :ballot_box_with_check: 1.35+ | | 1.34 | 1.35– | | | | [code](https://cs.k8s.io/?q=%5CbContainerRestartRules%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbContainerRestartRules%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | diff --git a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml index fc58dd6c79a..221e568d57d 100644 --- a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml +++ b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml @@ -263,7 +263,7 @@ lockToDefault: true preRelease: GA version: "1.34" -- name: ConsistentListFromCacheSkipStorageFallback +- name: ConsistentListFromCacheSkipTimeoutFallback versionedSpecs: - default: false lockToDefault: false