diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index b7f75919245..284391dbc03 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -2176,6 +2176,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, }, + genericfeatures.ConsistentListFromCacheSkipTimeoutFallback: { + {Version: version.MustParse("1.37"), Default: false, PreRelease: featuregate.Alpha}, + }, + genericfeatures.ConstrainedImpersonation: { {Version: version.MustParse("1.35"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.36"), Default: true, PreRelease: featuregate.Beta}, @@ -2714,6 +2718,8 @@ var defaultKubernetesFeatureGateDependencies = map[featuregate.Feature][]feature genericfeatures.ConsistentListFromCache: {}, + genericfeatures.ConsistentListFromCacheSkipTimeoutFallback: {genericfeatures.ConsistentListFromCache}, + genericfeatures.ConstrainedImpersonation: {}, genericfeatures.DeclarativeValidation: {}, diff --git a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go index 9f7b42c01cb..ae5c89ad57d 100644 --- a/staging/src/k8s.io/apiserver/pkg/features/kube_features.go +++ b/staging/src/k8s.io/apiserver/pkg/features/kube_features.go @@ -103,6 +103,13 @@ const ( // Allow the API server to serve consistent lists from cache ConsistentListFromCache featuregate.Feature = "ConsistentListFromCache" + // owner: @yedou37 + // + // Skip the timeout fallback to storage when a consistent LIST from cache cannot be served, + // and return a retryable response instead. + // See https://github.com/kubernetes/kubernetes/issues/138494. + ConsistentListFromCacheSkipTimeoutFallback featuregate.Feature = "ConsistentListFromCacheSkipTimeoutFallback" + // owner: @enj @qiujian16 // kep: https://kep.k8s.io/5284 // @@ -379,6 +386,10 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate {Version: version.MustParse("1.34"), Default: true, PreRelease: featuregate.GA, LockToDefault: true}, }, + ConsistentListFromCacheSkipTimeoutFallback: { + {Version: version.MustParse("1.37"), Default: false, PreRelease: featuregate.Alpha}, + }, + ConstrainedImpersonation: { {Version: version.MustParse("1.35"), Default: false, PreRelease: featuregate.Alpha}, {Version: version.MustParse("1.36"), Default: true, PreRelease: featuregate.Beta}, diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher_whitebox_test.go b/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher_whitebox_test.go index 8005bf1ae70..01a75da4116 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher_whitebox_test.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher/cacher_whitebox_test.go @@ -373,11 +373,13 @@ func TestConsistentReadFallback(t *testing.T) { tcs := []struct { name string consistentReadsEnabled bool + skipStorageFallback bool watchCacheRV string storageRV string fallbackError bool expectError bool + expectTooManyRequests bool expectRV string expectBlock bool expectRequestsToStorage int @@ -423,6 +425,22 @@ apiserver_watch_cache_consistent_read_total{fallback="true", group="", resource= # HELP apiserver_watch_cache_consistent_read_total [ALPHA] Counter for consistent reads from cache. # TYPE apiserver_watch_cache_consistent_read_total counter apiserver_watch_cache_consistent_read_total{fallback="true", group="", resource="pods", success="false"} 1 +`, + }, + { + name: "Skip Storage Fallback", + consistentReadsEnabled: true, + skipStorageFallback: true, + watchCacheRV: "2", + storageRV: "42", + expectError: true, + expectTooManyRequests: true, + expectBlock: true, + expectRequestsToStorage: 1, + expectMetric: ` +# HELP apiserver_watch_cache_consistent_read_total [ALPHA] Counter for consistent reads from cache. +# TYPE apiserver_watch_cache_consistent_read_total counter +apiserver_watch_cache_consistent_read_total{fallback="skipped", group="", resource="pods", success="false"} 1 `, }, { @@ -442,6 +460,9 @@ apiserver_watch_cache_consistent_read_total{fallback="true", group="", resource= featuregatetesting.SetFeatureGateEmulationVersionDuringTest(t, utilfeature.DefaultFeatureGate, version.MustParse("1.33")) featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCache, false) } + if tc.skipStorageFallback { + featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.ConsistentListFromCacheSkipTimeoutFallback, true) + } registry := k8smetrics.NewKubeRegistry() metrics.ConsistentReadTotal.Reset() @@ -518,6 +539,9 @@ apiserver_watch_cache_consistent_read_total{fallback="true", group="", resource= if (err != nil) != tc.expectError { t.Fatalf("Unexpected error err: %v", err) } + if tc.expectTooManyRequests && !apierrors.IsTooManyRequests(err) { + t.Fatalf("Unexpected error, got: %v, want TooManyRequests", err) + } if result.ResourceVersion != tc.expectRV { t.Fatalf("Unexpected List response RV, got: %q, want: %q", result.ResourceVersion, tc.expectRV) } diff --git a/staging/src/k8s.io/apiserver/pkg/storage/cacher/delegator.go b/staging/src/k8s.io/apiserver/pkg/storage/cacher/delegator.go index 4e1fabfaba8..4ce39140553 100644 --- a/staging/src/k8s.io/apiserver/pkg/storage/cacher/delegator.go +++ b/staging/src/k8s.io/apiserver/pkg/storage/cacher/delegator.go @@ -217,10 +217,15 @@ func (c *CacheDelegator) GetList(ctx context.Context, key string, opts storage.L } if result.ConsistentRead { // IsTooLargeResourceVersion occurs when the requested RV is higher than cache's current RV - // and cache hasn't caught up within the timeout period. Fall back to etcd. + // and cache hasn't caught up within the timeout period. if storage.IsTooLargeResourceVersion(err) { - fallback = "true" - err = c.storage.GetList(ctx, key, opts, listObj) + if utilfeature.DefaultFeatureGate.Enabled(features.ConsistentListFromCacheSkipTimeoutFallback) { + fallback = "skipped" + err = errors.NewTooManyRequests(err.Error(), resourceVersionTooHighRetrySeconds) + } else { + fallback = "true" + err = c.storage.GetList(ctx, key, opts, listObj) + } } if err != nil { success = "false" diff --git a/test/compatibility_lifecycle/reference/feature_list.md b/test/compatibility_lifecycle/reference/feature_list.md index c64a173ec41..ca22798035d 100644 --- a/test/compatibility_lifecycle/reference/feature_list.md +++ b/test/compatibility_lifecycle/reference/feature_list.md @@ -43,6 +43,7 @@ | ComponentStatusz | :ballot_box_with_check: 1.36+ | | 1.32–1.35 | 1.36– | | | | [code](https://cs.k8s.io/?q=%5CbComponentStatusz%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbComponentStatusz%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ConcurrentWatchObjectDecode | | | | 1.31– | | | | [code](https://cs.k8s.io/?q=%5CbConcurrentWatchObjectDecode%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbConcurrentWatchObjectDecode%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ConsistentListFromCache | :ballot_box_with_check: 1.31+ | :closed_lock_with_key: 1.34+ | 1.28–1.30 | 1.31–1.33 | 1.34– | | | [code](https://cs.k8s.io/?q=%5CbConsistentListFromCache%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbConsistentListFromCache%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | +| ConsistentListFromCacheSkipTimeoutFallback | | | 1.37– | | | | ConsistentListFromCache | [code](https://cs.k8s.io/?q=%5CbConsistentListFromCacheSkipTimeoutFallback%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbConsistentListFromCacheSkipTimeoutFallback%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ConstrainedImpersonation | :ballot_box_with_check: 1.36+ | | 1.35 | 1.36– | | | | [code](https://cs.k8s.io/?q=%5CbConstrainedImpersonation%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbConstrainedImpersonation%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ContainerCheckpoint | :ballot_box_with_check: 1.30+ | | 1.25–1.29 | 1.30– | | | | [code](https://cs.k8s.io/?q=%5CbContainerCheckpoint%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbContainerCheckpoint%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | | ContainerRestartRules | :ballot_box_with_check: 1.35+ | | 1.34 | 1.35– | | | | [code](https://cs.k8s.io/?q=%5CbContainerRestartRules%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/kubernetes) [KEPs](https://cs.k8s.io/?q=%5CbContainerRestartRules%5Cb&i=nope&files=&excludeFiles=CHANGELOG&repos=kubernetes/enhancements) | diff --git a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml index a5d6cdf4e18..e300711db10 100644 --- a/test/compatibility_lifecycle/reference/versioned_feature_list.yaml +++ b/test/compatibility_lifecycle/reference/versioned_feature_list.yaml @@ -235,6 +235,12 @@ lockToDefault: true preRelease: GA version: "1.34" +- name: ConsistentListFromCacheSkipTimeoutFallback + versionedSpecs: + - default: false + lockToDefault: false + preRelease: Alpha + version: "1.37" - name: ConstrainedImpersonation versionedSpecs: - default: false