diff --git a/pkg/kubelet/kubelet_pods.go b/pkg/kubelet/kubelet_pods.go index 42a58b0dfdc..05b4c1dc3d3 100644 --- a/pkg/kubelet/kubelet_pods.go +++ b/pkg/kubelet/kubelet_pods.go @@ -60,6 +60,7 @@ import ( kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/envvars" "k8s.io/kubernetes/pkg/kubelet/images" + "k8s.io/kubernetes/pkg/kubelet/kuberuntime" "k8s.io/kubernetes/pkg/kubelet/metrics" "k8s.io/kubernetes/pkg/kubelet/status" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" @@ -2620,6 +2621,24 @@ func (kl *Kubelet) convertToAPIContainerStatuses(pod *v1.Pod, podStatus *kubecon if s != nil && s.State == kubecontainer.ContainerStateExited && s.ExitCode == 0 { continue } + // When the pod is static, init container status might be lost. + // We can assume that the init containers are already completed if the main containers are created. + // When the init container is a sidecar container, it should be waiting (and will be restarted). + isSidecar := container.RestartPolicy != nil && *container.RestartPolicy == v1.ContainerRestartPolicyAlways + if s == nil && + kubetypes.IsStaticPod(pod) && + kuberuntime.HasAnyRegularContainerCreated(pod, podStatus) && + !isSidecar && + statuses[container.Name].State.Waiting != nil { + statuses[container.Name].State = v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ + Reason: "Completed", + Message: "Unable to get init container status from container runtime and pod has been initialized, treat it as exited normally", + ExitCode: 0, + }, + } + continue + } } // If a container should be restarted in next syncpod, it is *Waiting*. if !kubecontainer.ShouldContainerBeRestarted(logger, &container, pod, podStatus) { diff --git a/pkg/kubelet/kubelet_pods_test.go b/pkg/kubelet/kubelet_pods_test.go index 7678b70c715..6d4c209fcec 100644 --- a/pkg/kubelet/kubelet_pods_test.go +++ b/pkg/kubelet/kubelet_pods_test.go @@ -3796,6 +3796,29 @@ func TestConvertToAPIContainerStatuses(t *testing.T) { }, RestartPolicy: v1.RestartPolicyAlways, } + desiredStateWithInitContainer := v1.PodSpec{ + NodeName: "machine", + InitContainers: []v1.Container{ + {Name: "init-1"}, + }, + Containers: []v1.Container{ + {Name: "containerA"}, + }, + RestartPolicy: v1.RestartPolicyAlways, + } + desiredStateWithSidecarContainer := v1.PodSpec{ + NodeName: "machine", + InitContainers: []v1.Container{ + { + Name: "sidecar-1", + RestartPolicy: ptr.To(v1.ContainerRestartPolicyAlways), + }, + }, + Containers: []v1.Container{ + {Name: "containerA"}, + }, + RestartPolicy: v1.RestartPolicyAlways, + } now := metav1.Now() tests := []struct { @@ -4057,6 +4080,87 @@ func TestConvertToAPIContainerStatuses(t *testing.T) { withRestartCount(waitingStateWithRestartingAllContainers("containerB"), 1), }, }, + { + name: "Unable to get init container status from container runtime and pod has been initialized, treat it as exited normally", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + Annotations: map[string]string{ + kubetypes.ConfigSourceAnnotationKey: "file", // static pod + }, + }, + Spec: desiredStateWithInitContainer, + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{}, + }, + }, + currentStatus: &kubecontainer.PodStatus{ + ContainerStatuses: []*kubecontainer.Status{ + { + ID: kubecontainer.ContainerID{ID: "foo"}, + Name: "containerA", + StartedAt: time.Unix(1, 0).UTC(), + State: kubecontainer.ContainerStateRunning, + }, + }, + }, + previousStatus: []v1.ContainerStatus{}, + containers: desiredStateWithInitContainer.InitContainers, + expected: []v1.ContainerStatus{ + { + Name: "init-1", + State: v1.ContainerState{ + Terminated: &v1.ContainerStateTerminated{ + Reason: "Completed", + Message: "Unable to get init container status from container runtime and pod has been initialized, treat it as exited normally", + ExitCode: 0, + }, + }, + }, + }, + hasInitContainers: true, + isInitContainer: true, + }, + { + name: "Unable to get sidecar container status from container runtime and pod has been initialized, sidecar container should be waiting", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-pod", + Annotations: map[string]string{ + kubetypes.ConfigSourceAnnotationKey: "file", // static pod + }, + }, + Spec: desiredStateWithSidecarContainer, + Status: v1.PodStatus{ + ContainerStatuses: []v1.ContainerStatus{}, + }, + }, + currentStatus: &kubecontainer.PodStatus{ + ContainerStatuses: []*kubecontainer.Status{ + { + ID: kubecontainer.ContainerID{ID: "foo"}, + Name: "containerA", + StartedAt: time.Unix(1, 0).UTC(), + State: kubecontainer.ContainerStateRunning, + }, + }, + }, + previousStatus: []v1.ContainerStatus{}, + containers: desiredStateWithSidecarContainer.InitContainers, + expected: []v1.ContainerStatus{ + { + Name: "sidecar-1", + State: v1.ContainerState{ + Waiting: &v1.ContainerStateWaiting{ + Reason: "PodInitializing", + Message: "", + }, + }, + }, + }, + hasInitContainers: true, + isInitContainer: true, + }, } featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ features.ContainerRestartRules: true, diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go index 8ef687b5309..e1bc40ea626 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go @@ -1009,9 +1009,9 @@ func (m *kubeGenericRuntimeManager) purgeInitContainers(ctx context.Context, pod } } -// hasAnyRegularContainerCreated returns true if any regular container has been +// HasAnyRegularContainerCreated returns true if any regular container has been // created, which indicates all init containers have been initialized. -func hasAnyRegularContainerCreated(pod *v1.Pod, podStatus *kubecontainer.PodStatus) bool { +func HasAnyRegularContainerCreated(pod *v1.Pod, podStatus *kubecontainer.PodStatus) bool { for _, container := range pod.Spec.Containers { status := podStatus.FindContainerStatusByName(container.Name) if status == nil { diff --git a/pkg/kubelet/kuberuntime/kuberuntime_manager.go b/pkg/kubelet/kuberuntime/kuberuntime_manager.go index 818ca633a1a..90a861b2037 100644 --- a/pkg/kubelet/kuberuntime/kuberuntime_manager.go +++ b/pkg/kubelet/kuberuntime/kuberuntime_manager.go @@ -1188,7 +1188,7 @@ func (m *kubeGenericRuntimeManager) computePodActions(ctx context.Context, pod * // If there is any regular container, it means all init containers have // been initialized. - hasInitialized = hasAnyRegularContainerCreated(pod, podStatus) + hasInitialized = HasAnyRegularContainerCreated(pod, podStatus) if hasInitialized { changes.CreateSandbox = false diff --git a/test/e2e_node/static_pod_test.go b/test/e2e_node/static_pod_test.go new file mode 100644 index 00000000000..8f201e42d81 --- /dev/null +++ b/test/e2e_node/static_pod_test.go @@ -0,0 +1,222 @@ +/* +Copyright 2025 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2enode + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/cli-runtime/pkg/printers" + "k8s.io/kubernetes/test/e2e/framework" + imageutils "k8s.io/kubernetes/test/utils/image" + admissionapi "k8s.io/pod-security-admission/api" + "k8s.io/utils/ptr" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +var _ = SIGDescribe("StaticPod", framework.WithSerial(), func() { + f := framework.NewDefaultFramework("static-pod") + f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged + f.Context("when the static pod has init container", func() { + f.It("should be ready after init container is removed and kubelet restarts", f.WithNodeConformance(), func(ctx context.Context) { + ginkgo.By("create static pod") + staticPod := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "static", + Namespace: f.Namespace.Name, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "main", + Image: imageutils.GetE2EImage(imageutils.Pause), + }, + }, + InitContainers: []v1.Container{ + { + Name: "init", + Image: imageutils.GetE2EImage(imageutils.BusyBox), + Command: []string{"ls"}, + }, + }, + }, + } + staticPodPath, err := createStaticPodFromPod(kubeletCfg.StaticPodPath, staticPod) + framework.ExpectNoError(err) + ginkgo.DeferCleanup(func() { + ginkgo.By("delete static pod") + err = os.Remove(staticPodPath) + framework.ExpectNoError(err) + }) + + var initCtrID string + var startTime *metav1.Time + mirrorPodName := fmt.Sprintf("%s-%s", staticPod.Name, framework.TestContext.NodeName) + ginkgo.By("wait for the mirror pod to be updated") + gomega.Eventually(ctx, func(g gomega.Gomega) { + pod, err := f.ClientSet.CoreV1().Pods(staticPod.Namespace).Get(ctx, mirrorPodName, metav1.GetOptions{}) + g.Expect(err).Should(gomega.Succeed()) + g.Expect(pod.Status.InitContainerStatuses).To(gomega.HaveLen(1)) + cstatus := pod.Status.InitContainerStatuses[0] + // Wait until the init container is terminated. + g.Expect(cstatus.State.Terminated).NotTo(gomega.BeNil()) + g.Expect(cstatus.State.Terminated.ContainerID).NotTo(gomega.BeEmpty()) + initCtrID = cstatus.ContainerID + startTime = pod.Status.StartTime + }, 2*time.Minute, 5*time.Second).Should(gomega.Succeed()) + + ginkgo.By("remove init container") + removeInitContainer(ctx, initCtrID) + + ginkgo.By("restart kubelet") + startKubelet := mustStopKubelet(ctx, f) + startKubelet(ctx) + gomega.Eventually(ctx, func() bool { + return kubeletHealthCheck(kubeletHealthCheckURL) + }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrueBecause("kubelet should be started")) + + ginkgo.By("wait for the mirror pod to be updated") + gomega.Eventually(ctx, func(g gomega.Gomega) { + pod, err := f.ClientSet.CoreV1().Pods(staticPod.Namespace).Get(ctx, mirrorPodName, metav1.GetOptions{}) + g.Expect(pod.Status.StartTime).NotTo(gomega.Equal(startTime)) + g.Expect(err).Should(gomega.Succeed()) + g.Expect(pod.Status.InitContainerStatuses).To(gomega.HaveLen(1)) + cstatus := pod.Status.InitContainerStatuses[0] + // Init container should be completed. + g.Expect(cstatus.State.Terminated).NotTo(gomega.BeNil()) + g.Expect(cstatus.State.Terminated.Reason).To(gomega.Equal("Completed")) + g.Expect(cstatus.State.Terminated.ExitCode).To(gomega.BeZero()) + }, 2*time.Minute, 5*time.Second).Should(gomega.Succeed()) + }) + }) + f.Context("when the static pod has sidecar container", func() { + f.It("should be ready after sidecar container is removed and kubelet restarts", f.WithNodeConformance(), func(ctx context.Context) { + ginkgo.By("create static pod") + staticPod := &v1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "static", + Namespace: f.Namespace.Name, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "main", + Image: imageutils.GetE2EImage(imageutils.Pause), + }, + }, + InitContainers: []v1.Container{ + { + Name: "init", + Image: imageutils.GetE2EImage(imageutils.Pause), + RestartPolicy: ptr.To(v1.ContainerRestartPolicyAlways), + }, + }, + }, + } + staticPodPath, err := createStaticPodFromPod(kubeletCfg.StaticPodPath, staticPod) + framework.ExpectNoError(err) + ginkgo.DeferCleanup(func() { + ginkgo.By("delete static pod") + err = os.Remove(staticPodPath) + framework.ExpectNoError(err) + }) + + var sidecarCtrID string + var startTime *metav1.Time + mirrorPodName := fmt.Sprintf("%s-%s", staticPod.Name, framework.TestContext.NodeName) + ginkgo.By("wait for the mirror pod to be updated") + gomega.Eventually(ctx, func(g gomega.Gomega) { + pod, err := f.ClientSet.CoreV1().Pods(staticPod.Namespace).Get(ctx, mirrorPodName, metav1.GetOptions{}) + g.Expect(err).Should(gomega.Succeed()) + g.Expect(pod.Status.InitContainerStatuses).To(gomega.HaveLen(1)) + cstatus := pod.Status.InitContainerStatuses[0] + // Wait until the sidecar container starts running. + g.Expect(cstatus.State.Running).NotTo(gomega.BeNil()) + sidecarCtrID = cstatus.ContainerID + startTime = pod.Status.StartTime + }, 2*time.Minute, 5*time.Second).Should(gomega.Succeed()) + + // Stop kubelet first not to restart the sidecar container. + ginkgo.By("stop kubelet") + startKubelet := mustStopKubelet(ctx, f) + + ginkgo.By("remove sidecar container") + removeInitContainer(ctx, sidecarCtrID) + + ginkgo.By("start kubelet") + startKubelet(ctx) + gomega.Eventually(ctx, func() bool { + return kubeletHealthCheck(kubeletHealthCheckURL) + }, f.Timeouts.PodStart, f.Timeouts.Poll).Should(gomega.BeTrueBecause("kubelet should be started")) + + ginkgo.By("wait for the mirror pod to be updated") + gomega.Eventually(ctx, func(g gomega.Gomega) { + pod, err := f.ClientSet.CoreV1().Pods(staticPod.Namespace).Get(ctx, mirrorPodName, metav1.GetOptions{}) + g.Expect(pod.Status.StartTime).NotTo(gomega.Equal(startTime)) + g.Expect(err).Should(gomega.Succeed()) + g.Expect(pod.Status.InitContainerStatuses).To(gomega.HaveLen(1)) + cstatus := pod.Status.InitContainerStatuses[0] + // Sidecar container should be restarted and running. + g.Expect(cstatus.State.Running).NotTo(gomega.BeNil()) + }, 2*time.Minute, 5*time.Second).Should(gomega.Succeed()) + }) + }) +}) + +func createStaticPodFromPod(dir string, pod *v1.Pod) (string, error) { + name := pod.Name + namespace := pod.Namespace + file := staticPodPath(dir, name, namespace) + + f, err := os.OpenFile(file, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0666) + if err != nil { + return "", err + } + defer func() { + _ = f.Close() + }() + + y := printers.YAMLPrinter{} + return file, y.PrintObj(pod, f) +} + +func removeInitContainer(ctx context.Context, ctrID string) { + cricli, _, err := getCRIClient() + framework.ExpectNoError(err) + splitID := strings.Split(ctrID, "://") + gomega.Expect(splitID).To(gomega.HaveLen(2)) + ctrID = splitID[1] + // Make sure the container is stopped before removing it. This may fail. + _ = cricli.StopContainer(ctx, ctrID, 0) + err = cricli.RemoveContainer(ctx, ctrID) + framework.ExpectNoError(err) +}