upstream/kubernetes
1
0
Fork 0
mirror of https://github.com/kubernetes/kubernetes.git synced 2026-06-10 17:35:44 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #135017 from liggitt/stateful-set-noop-rollout

Browse source 
Fix spurious statefulset rollout from 1.33 → 1.34
This commit is contained in:
Kubernetes Prow Robot 2025-11-03 19:58:11 -08:00 committed by GitHub
commit 48c56e04e0
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 479 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
pkg/controller/history/controller_history.go
View file

@ -111,27 +111,9 @@ func SortControllerRevisions(revisions []*apps.ControllerRevision) {
// EqualRevision returns true if lhs and rhs are either both nil, or both point to non-nil ControllerRevisions that
// contain semantically equivalent data. Otherwise this method returns false.
func EqualRevision(lhs *apps.ControllerRevision, rhs *apps.ControllerRevision) bool {
var lhsHash, rhsHash *uint32
if lhs == nil || rhs == nil {
return lhs == rhs
}
if hs, found := lhs.Labels[ControllerRevisionHashLabel]; found {
hash, err := strconv.ParseInt(hs, 10, 32)
if err == nil {
lhsHash = new(uint32)
*lhsHash = uint32(hash)
}
}
if hs, found := rhs.Labels[ControllerRevisionHashLabel]; found {
hash, err := strconv.ParseInt(hs, 10, 32)
if err == nil {
rhsHash = new(uint32)
*rhsHash = uint32(hash)
}
}
if lhsHash != nil && rhsHash != nil && *lhsHash != *rhsHash {
return false
}
return bytes.Equal(lhs.Data.Raw, rhs.Data.Raw) && apiequality.Semantic.DeepEqual(lhs.Data.Object, rhs.Data.Object)
}

149
pkg/controller/statefulset/stateful_set_compatibility_test.go Normal file
View file

@ -0,0 +1,149 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package statefulset
import (
"os"
"reflect"
"testing"
"github.com/google/go-cmp/cmp"
"sigs.k8s.io/json"
appsv1 "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
"k8s.io/kubernetes/pkg/api/legacyscheme"
)
func TestStatefulSetCompatibility(t *testing.T) {
set133 := &appsv1.StatefulSet{}
set134 := &appsv1.StatefulSet{}
rev133 := &appsv1.ControllerRevision{}
rev134 := &appsv1.ControllerRevision{}
load(t, "compatibility_set_1.33.0.json", set133)
load(t, "compatibility_set_1.34.0.json", set134)
load(t, "compatibility_revision_1.33.0.json", rev133)
load(t, "compatibility_revision_1.34.0.json", rev134)
testcases := []struct {
name string
set *appsv1.StatefulSet
revisions []*appsv1.ControllerRevision
}{
{
name: "1.33 set, 1.33 rev",
set: set133.DeepCopy(),
revisions: []*appsv1.ControllerRevision{rev133.DeepCopy()},
},
{
name: "1.34 set, 1.34 rev",
set: set134.DeepCopy(),
revisions: []*appsv1.ControllerRevision{rev134.DeepCopy()},
},
{
name: "1.34 set, 1.33+1.34 rev",
set: set134.DeepCopy(),
revisions: []*appsv1.ControllerRevision{rev133.DeepCopy(), rev134.DeepCopy()},
},
}
for _, tc := range testcases {
t.Run(tc.name, func(t *testing.T) {
latestRev := tc.revisions[len(tc.revisions)-1]
client := fake.NewClientset(tc.set)
_, _, ssc := setupController(client)
currentRev, updateRev, _, err := ssc.(*defaultStatefulSetControl).getStatefulSetRevisions(tc.set, tc.revisions)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(currentRev, latestRev) {
t.Fatalf("expected no change from latestRev, got %s", cmp.Diff(latestRev, currentRev))
}
if !reflect.DeepEqual(updateRev, latestRev) {
t.Fatalf("expected no change from latestRev, got %s", cmp.Diff(latestRev, updateRev))
}
})
}
}
func BenchmarkStatefulSetCompatibility(b *testing.B) {
set133 := &appsv1.StatefulSet{}
set134 := &appsv1.StatefulSet{}
rev133 := &appsv1.ControllerRevision{}
rev134 := &appsv1.ControllerRevision{}
load(b, "compatibility_set_1.33.0.json", set133)
load(b, "compatibility_set_1.34.0.json", set134)
load(b, "compatibility_revision_1.33.0.json", rev133)
load(b, "compatibility_revision_1.34.0.json", rev134)
testcases := []struct {
name string
set *appsv1.StatefulSet
revisions []*appsv1.ControllerRevision
}{
{
name: "1.33 set, 1.33 rev",
set: set133.DeepCopy(),
revisions: []*appsv1.ControllerRevision{rev133.DeepCopy()},
},
{
name: "1.34 set, 1.34 rev",
set: set134.DeepCopy(),
revisions: []*appsv1.ControllerRevision{rev134.DeepCopy()},
},
{
name: "1.34 set, 1.33+1.34 rev",
set: set134.DeepCopy(),
revisions: []*appsv1.ControllerRevision{rev133.DeepCopy(), rev134.DeepCopy()},
},
}
for _, tc := range testcases {
b.Run(tc.name, func(b *testing.B) {
latestRev := tc.revisions[len(tc.revisions)-1]
client := fake.NewClientset(tc.set)
_, _, ssc := setupController(client)
for i := 0; i < b.N; i++ {
currentRev, updateRev, _, err := ssc.(*defaultStatefulSetControl).getStatefulSetRevisions(tc.set, tc.revisions)
if err != nil {
b.Fatal(err)
}
if !reflect.DeepEqual(currentRev, latestRev) {
b.Fatalf("expected no change from latestRev, got %s", cmp.Diff(latestRev, currentRev))
}
if !reflect.DeepEqual(updateRev, latestRev) {
b.Fatalf("expected no change from latestRev, got %s", cmp.Diff(latestRev, updateRev))
}
}
})
}
}
func load(t testing.TB, filename string, object runtime.Object) {
data, err := os.ReadFile("testdata/" + filename)
if err != nil {
t.Fatal(err)
}
if strictErrs, err := json.UnmarshalStrict(data, object); err != nil {
t.Fatal(err)
} else if len(strictErrs) > 0 {
t.Fatal(strictErrs)
}
// apply defaulting just as if it was read from etcd
legacyscheme.Scheme.Default(object)
}

53
pkg/controller/statefulset/stateful_set_control.go
View file

@ -22,13 +22,16 @@ import (
"sync"
"k8s.io/klog/v2"
"k8s.io/utils/lru"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/controller/history"
"k8s.io/kubernetes/pkg/controller/statefulset/metrics"
"k8s.io/kubernetes/pkg/features"
@ -63,13 +66,15 @@ func NewDefaultStatefulSetControl(
podControl *StatefulPodControl,
statusUpdater StatefulSetStatusUpdaterInterface,
controllerHistory history.Interface) StatefulSetControlInterface {
return &defaultStatefulSetControl{podControl, statusUpdater, controllerHistory}
return &defaultStatefulSetControl{podControl, statusUpdater, controllerHistory, lru.New(maxRevisionEqualityCacheEntries)}
}
type defaultStatefulSetControl struct {
podControl *StatefulPodControl
statusUpdater StatefulSetStatusUpdaterInterface
controllerHistory history.Interface
revisionEqualityCache *lru.Cache
}
// UpdateStatefulSet executes the core logic loop for a stateful set, applying the predictable and
@ -209,6 +214,49 @@ func (ssc *defaultStatefulSetControl) truncateHistory(
return nil
}
// maxRevisionEqualityCacheEntries is the size of the memory cache for equal set/controllerrevisions.
// Allowing up to 10,000 entries takes ~1MB. Each entry consumes up to ~111 bytes:
// - 40 bytes for the cache key (revisionEqualityKey{})
// - 16 for the cache value (interface{} --> struct{}{})
// - 36 bytes for the setUID string
// - 19 bytes for the revisionResourceVersion string
const maxRevisionEqualityCacheEntries = 10_000
// revisionEqualityKey is the cache key for remembering a particular revision RV
// is equal to the revision that results from a particular set UID at a particular set generation.
type revisionEqualityKey struct {
setUID types.UID
setGeneration int64
revisionResourceVersion string
}
// setMatchesLatestExistingRevision returns true if the set/proposedRevision already matches what would be produced from restoring latestExistingRevision.
func setMatchesLatestExistingRevision(set *apps.StatefulSet, proposedRevision *apps.ControllerRevision, latestExistingRevision *apps.ControllerRevision, memory *lru.Cache) bool {
if !utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetSemanticRevisionComparison) {
return false
}
equalityCacheKey := revisionEqualityKey{setUID: set.UID, setGeneration: set.Generation, revisionResourceVersion: latestExistingRevision.ResourceVersion}
if _, ok := memory.Get(equalityCacheKey); ok {
return true
}
// see if reverting to the latest existing revision would produce the same thing as proposedRevision
latestSet, err := ApplyRevision(set, latestExistingRevision)
if err != nil {
return false
}
legacyscheme.Scheme.Default(latestSet)
reconstructedLatestRevision, err := newRevision(latestSet, -1, nil)
if err != nil {
return false
}
// if they match, cache this combination of set(uid,generation)+revision(resourceVersion) to minimize expensive comparisons in steady state
if history.EqualRevision(proposedRevision, reconstructedLatestRevision) {
memory.Add(equalityCacheKey, struct{}{})
return true
}
return false
}
// getStatefulSetRevisions returns the current and update ControllerRevisions for set. It also
// returns a collision count that records the number of name collisions set saw when creating
// new ControllerRevisions. This count is incremented on every name collision and is used in
@ -252,6 +300,9 @@ func (ssc *defaultStatefulSetControl) getStatefulSetRevisions(
if err != nil {
return nil, nil, collisionCount, err
}
} else if revisionCount > 0 && setMatchesLatestExistingRevision(set, updateRevision, revisions[revisionCount-1], ssc.revisionEqualityCache) {
// the update revision has not changed
updateRevision = revisions[revisionCount-1]
} else {
//if there is no equivalent revision we create a new one
updateRevision, err = ssc.controllerHistory.CreateControllerRevision(set, updateRevision, &collisionCount)

3
pkg/controller/statefulset/stateful_set_control_test.go
View file

@ -31,6 +31,7 @@ import (
"testing"
"time"
"k8s.io/utils/lru"
"k8s.io/utils/ptr"
apps "k8s.io/api/apps/v1"
@ -855,7 +856,7 @@ func TestStatefulSetControl_getSetRevisions(t *testing.T) {
informerFactory := informers.NewSharedInformerFactory(client, controller.NoResyncPeriodFunc())
spc := NewStatefulPodControlFromManager(newFakeObjectManager(informerFactory), &noopRecorder{})
ssu := newFakeStatefulSetStatusUpdater(informerFactory.Apps().V1().StatefulSets())
ssc := defaultStatefulSetControl{spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions())}
ssc := defaultStatefulSetControl{spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()), lru.New(maxRevisionEqualityCacheEntries)}
stop := make(chan struct{})
defer close(stop)

25
pkg/controller/statefulset/testdata/compatibility_revision_1.33.0.json vendored Normal file
View file

@ -0,0 +1,25 @@
{
"apiVersion":"apps/v1",
"kind":"ControllerRevision",
"metadata":{
"creationTimestamp":"2025-10-31T18:19:02Z",
"labels":{
"app":"foo",
"controller.kubernetes.io/hash":"c77f6d978"
},
"name":"test-c77f6d978",
"namespace":"default",
"ownerReferences":[{
"apiVersion":"apps/v1",
"blockOwnerDeletion":true,
"controller":true,
"kind":"StatefulSet",
"name":"test",
"uid":"ec335e25-1045-4216-8634-50cfbe05f3d6"
}],
"resourceVersion":"2209",
"uid":"af6e1945-ed14-4d1a-b420-813aa683a0fd"
},
"data":{"spec":{"template":{"$patch":"replace","metadata":{"annotations":{"test":"value"},"creationTimestamp":null,"labels":{"app":"foo"}},"spec":{"containers":[{"image":"test","imagePullPolicy":"Always","name":"test","resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File"}],"dnsPolicy":"ClusterFirst","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"terminationGracePeriodSeconds":30}}}},
"revision":1
}

25
pkg/controller/statefulset/testdata/compatibility_revision_1.34.0.json vendored Normal file
View file

@ -0,0 +1,25 @@
{
"apiVersion":"apps/v1",
"kind":"ControllerRevision",
"metadata":{
"creationTimestamp":"2025-11-03T19:46:23Z",
"labels":{
"app":"foo",
"controller.kubernetes.io/hash":"776999688b"
},
"name":"test-776999688b",
"namespace":"default",
"ownerReferences":[{
"apiVersion":"apps/v1",
"blockOwnerDeletion":true,
"controller":true,
"kind":"StatefulSet",
"name":"test",
"uid":"ec335e25-1045-4216-8634-50cfbe05f3d6"
}],
"resourceVersion":"16318",
"uid":"47df387b-5f17-40b6-9964-4c43cf6ad5d1"
},
"data":{"spec":{"template":{"$patch":"replace","metadata":{"annotations":{"test":"value"},"labels":{"app":"foo"}},"spec":{"containers":[{"image":"test","imagePullPolicy":"Always","name":"test","resources":{},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File"}],"dnsPolicy":"ClusterFirst","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"terminationGracePeriodSeconds":30}}}},
"revision":2
}

104
pkg/controller/statefulset/testdata/compatibility_set_1.33.0.json vendored Normal file
View file

@ -0,0 +1,104 @@
{
"apiVersion": "apps/v1",
"kind": "StatefulSet",
"metadata": {
"creationTimestamp": "2025-10-31T18:19:02Z",
"generation": 1,
"labels": {
"sslabel": "value"
},
"name": "test",
"namespace": "default",
"resourceVersion": "2219",
"uid": "ec335e25-1045-4216-8634-50cfbe05f3d6"
},
"spec": {
"persistentVolumeClaimRetentionPolicy": {
"whenDeleted": "Retain",
"whenScaled": "Retain"
},
"podManagementPolicy": "OrderedReady",
"replicas": 1,
"revisionHistoryLimit": 10,
"selector": {
"matchLabels": {
"app": "foo"
}
},
"serviceName": "",
"template": {
"metadata": {
"annotations": {
"test": "value"
},
"creationTimestamp": null,
"labels": {
"app": "foo"
}
},
"spec": {
"containers": [
{
"image": "test",
"imagePullPolicy": "Always",
"name": "test",
"resources": {},
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File"
}
],
"dnsPolicy": "ClusterFirst",
"restartPolicy": "Always",
"schedulerName": "default-scheduler",
"securityContext": {},
"terminationGracePeriodSeconds": 30
}
},
"updateStrategy": {
"rollingUpdate": {
"partition": 0
},
"type": "RollingUpdate"
},
"volumeClaimTemplates": [
{
"apiVersion": "v1",
"kind": "PersistentVolumeClaim",
"metadata": {
"annotations": {
"key": "value"
},
"creationTimestamp": null,
"labels": {
"key": "value"
},
"name": "test"
},
"spec": {
"accessModes": [
"ReadWriteOnce"
],
"resources": {
"requests": {
"storage": "1Gi"
}
},
"volumeMode": "Filesystem"
},
"status": {
"phase": "Pending"
}
}
]
},
"status": {
"availableReplicas": 1,
"collisionCount": 0,
"currentReplicas": 1,
"currentRevision": "test-c77f6d978",
"observedGeneration": 1,
"replicas": 1,
"updateRevision": "test-c77f6d978",
"updatedReplicas": 1
}
}

102
pkg/controller/statefulset/testdata/compatibility_set_1.34.0.json vendored Normal file
View file

@ -0,0 +1,102 @@
{
"apiVersion": "apps/v1",
"kind": "StatefulSet",
"metadata": {
"creationTimestamp": "2025-10-31T18:19:02Z",
"generation": 1,
"labels": {
"sslabel": "value"
},
"name": "test",
"namespace": "default",
"resourceVersion": "16319",
"uid": "ec335e25-1045-4216-8634-50cfbe05f3d6"
},
"spec": {
"persistentVolumeClaimRetentionPolicy": {
"whenDeleted": "Retain",
"whenScaled": "Retain"
},
"podManagementPolicy": "OrderedReady",
"replicas": 1,
"revisionHistoryLimit": 10,
"selector": {
"matchLabels": {
"app": "foo"
}
},
"serviceName": "",
"template": {
"metadata": {
"annotations": {
"test": "value"
},
"labels": {
"app": "foo"
}
},
"spec": {
"containers": [
{
"image": "test",
"imagePullPolicy": "Always",
"name": "test",
"resources": {},
"terminationMessagePath": "/dev/termination-log",
"terminationMessagePolicy": "File"
}
],
"dnsPolicy": "ClusterFirst",
"restartPolicy": "Always",
"schedulerName": "default-scheduler",
"securityContext": {},
"terminationGracePeriodSeconds": 30
}
},
"updateStrategy": {
"rollingUpdate": {
"partition": 0
},
"type": "RollingUpdate"
},
"volumeClaimTemplates": [
{
"apiVersion": "v1",
"kind": "PersistentVolumeClaim",
"metadata": {
"annotations": {
"key": "value"
},
"labels": {
"key": "value"
},
"name": "test"
},
"spec": {
"accessModes": [
"ReadWriteOnce"
],
"resources": {
"requests": {
"storage": "1Gi"
}
},
"volumeMode": "Filesystem"
},
"status": {
"phase": "Pending"
}
}
]
},
"status": {
"availableReplicas": 1,
"collisionCount": 0,
"currentReplicas": 1,
"currentRevision": "test-776999688b",
"observedGeneration": 1,
"replicas": 1,
"updateRevision": "test-776999688b",
"updatedReplicas": 1
}
}

14
pkg/features/kube_features.go
View file

@ -893,6 +893,12 @@ const (
// pod's lifecycle and will not block pod termination.
SidecarContainers featuregate.Feature = "SidecarContainers"
// owner: @liggitt
//
// Mitigates spurious statefulset rollouts due to controller revision comparison mismatches
// which are not semantically significant (e.g. serialization differences or missing defaulted fields).
StatefulSetSemanticRevisionComparison = "StatefulSetSemanticRevisionComparison"
// owner: @cupnes
// kep: https://kep.k8s.io/4049
//
@ -1683,6 +1689,12 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
{Version: version.MustParse("1.33"), Default: true, LockToDefault: true, PreRelease: featuregate.GA}, // GA in 1.33 remove in 1.36
},
StatefulSetSemanticRevisionComparison: {
// This is a mitigation for a 1.34 regression due to serialization differences that cannot be feature-gated,
// so this mitigation should not auto-disable even if emulating versions prior to 1.34 with --emulation-version.
{Version: version.MustParse("1.0"), Default: true, PreRelease: featuregate.Beta},
},
StorageCapacityScoring: {
{Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Alpha},
},
@ -2266,6 +2278,8 @@ var defaultKubernetesFeatureGateDependencies = map[featuregate.Feature][]feature
SidecarContainers: {},
StatefulSetSemanticRevisionComparison: {},
StorageCapacityScoring: {},
StorageNamespaceIndex: {},

6
test/compatibility_lifecycle/reference/versioned_feature_list.yaml
View file

@ -1611,6 +1611,12 @@
lockToDefault: false
preRelease: Beta
version: "1.34"
- name: StatefulSetSemanticRevisionComparison
versionedSpecs:
- default: true
lockToDefault: false
preRelease: Beta
version: "1.0"
- name: StorageCapacityScoring
versionedSpecs:
- default: false