Add a preshutdownhook to cleanup apisever identity lease

pkg/controlplane/apiserver/server.go

@@ -262,26 +262,31 @@ func (c completedConfig) New(name string, delegationTarget genericapiserver.Dele
 	})
 
 	if utilfeature.DefaultFeatureGate.Enabled(apiserverfeatures.APIServerIdentity) {
-		s.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-identity-lease-controller", func(hookContext genericapiserver.PostStartHookContext) error {
-			leaseName := s.GenericAPIServer.APIServerID
-			holderIdentity := s.GenericAPIServer.APIServerID + "_" + string(uuid.NewUUID())
+		leaseName := s.GenericAPIServer.APIServerID
+		holderIdentity := s.GenericAPIServer.APIServerID + "_" + string(uuid.NewUUID())
+		peeraddress := getPeerAddress(c.Extra.PeerAdvertiseAddress, c.Generic.PublicAddress, publicServicePort)
 
-			peeraddress := getPeerAddress(c.Extra.PeerAdvertiseAddress, c.Generic.PublicAddress, publicServicePort)
-			// must replace ':,[]' in [ip:port] to be able to store this as a valid label value
-			controller := lease.NewController(
-				clock.RealClock{},
-				client,
-				holderIdentity,
-				int32(IdentityLeaseDurationSeconds),
-				nil,
-				IdentityLeaseRenewIntervalPeriod,
-				leaseName,
-				metav1.NamespaceSystem,
-				// TODO: receive identity label value as a parameter when post start hook is moved to generic apiserver.
-				labelAPIServerHeartbeatFunc(name, peeraddress))
-			go controller.Run(hookContext)
+		// must replace ':,[]' in [ip:port] to be able to store this as a valid label value
+		controller := lease.NewController(
+			clock.RealClock{},
+			client,
+			holderIdentity,
+			int32(IdentityLeaseDurationSeconds),
+			nil,
+			IdentityLeaseRenewIntervalPeriod,
+			leaseName,
+			metav1.NamespaceSystem,
+			// TODO: receive identity label value as a parameter when post start hook is moved to generic apiserver.
+			labelAPIServerHeartbeatFunc(name, peeraddress),
+		)
+		s.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-identity-lease-controller", func(hookContext genericapiserver.PostStartHookContext) error {
+			return controller.Start(hookContext.Done())
+		})
+		s.GenericAPIServer.AddPreShutdownHookOrDie("stop-kube-apiserver-identity-lease-controller", func() error {
+			controller.Stop()
 			return nil
 		})
+
 		// TODO: move this into generic apiserver and make the lease identity value configurable
 		s.GenericAPIServer.AddPostStartHookOrDie("start-kube-apiserver-identity-lease-garbage-collector", func(hookContext genericapiserver.PostStartHookContext) error {
 			go apiserverleasegc.NewAPIServerLeaseGC(

staging/src/k8s.io/component-helpers/apimachinery/lease/controller.go

@@ -40,11 +40,15 @@ const (
 	maxUpdateRetries = 5
 	// maxBackoff is the maximum sleep time during backoff (e.g. in backoffEnsureLease)
 	maxBackoff = 7 * time.Second
+	// shutdownTimeout is the timeout for deleting the lease on shutdown.
+	shutdownTimeout = 10 * time.Second
 )
 
 // Controller manages creating and renewing the lease for this component (kube-apiserver, kubelet, etc.)
 type Controller interface {
 	Run(ctx context.Context)
+	Start(stopCh <-chan struct{}) error
+	Stop()
 }
 
 // ProcessLeaseFunc processes the given lease in-place
@@ -90,6 +94,43 @@ func NewController(clock clock.Clock, client clientset.Interface, holderIdentity
 	}
 }
 
+// Start performs setup actions, and then starts the lease controller.
+func (c *controller) Start(stopCh <-chan struct{}) error {
+	if c.leaseClient == nil {
+		klog.Info("lease controller has nil lease client, will not claim or renew leases")
+		return nil
+	}
+
+	// We delete the lease on startup to ensure that if a new apiserver starts up,
+	// it removes any stale lease from a previous instance. This is important to
+	// prevent other components from discovering and communicating with a dead peer.
+	ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
+	defer cancel()
+	klog.FromContext(ctx).Info("Deleting old lease on startup", "lease", klog.KRef(c.leaseNamespace, c.leaseName))
+	if err := c.leaseClient.Delete(ctx, c.leaseName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
+		// The lease controller will eventually take over and update the lease.
+		klog.FromContext(ctx).Error(err, "error deleting old lease", "lease", klog.KRef(c.leaseNamespace, c.leaseName))
+	}
+
+	go c.Run(wait.ContextForChannel(stopCh))
+	return nil
+}
+
+// Stop gracefully shuts down the controller.
+func (c *controller) Stop() {
+	// if leaseClient is nil, we can't clean up the lease
+	if c.leaseClient == nil {
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
+	defer cancel()
+	klog.FromContext(ctx).Info("Cleaning up lease on exit", "lease", klog.KRef(c.leaseNamespace, c.leaseName))
+	if err := c.leaseClient.Delete(ctx, c.leaseName, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) {
+		klog.FromContext(ctx).Error(err, "Unable to delete lease", "lease", klog.KRef(c.leaseNamespace, c.leaseName))
+	}
+}
+
 // Run runs the controller
 func (c *controller) Run(ctx context.Context) {
 	if c.leaseClient == nil {