diff --git a/pkg/controller/node/controller_utils.go b/pkg/controller/node/controller_utils.go index f31ccf59efe..b48e21214ab 100644 --- a/pkg/controller/node/controller_utils.go +++ b/pkg/controller/node/controller_utils.go @@ -21,6 +21,7 @@ import ( "strings" "k8s.io/kubernetes/pkg/api" + "k8s.io/kubernetes/pkg/api/errors" "k8s.io/kubernetes/pkg/client/cache" clientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" "k8s.io/kubernetes/pkg/client/record" @@ -28,6 +29,8 @@ import ( "k8s.io/kubernetes/pkg/fields" "k8s.io/kubernetes/pkg/kubelet/util/format" "k8s.io/kubernetes/pkg/types" + utilerrors "k8s.io/kubernetes/pkg/util/errors" + "k8s.io/kubernetes/pkg/util/node" utilruntime "k8s.io/kubernetes/pkg/util/runtime" "k8s.io/kubernetes/pkg/version" @@ -46,6 +49,8 @@ func deletePods(kubeClient clientset.Interface, recorder record.EventRecorder, n selector := fields.OneTermEqualSelector(api.PodHostField, nodeName) options := api.ListOptions{FieldSelector: selector} pods, err := kubeClient.Core().Pods(api.NamespaceAll).List(options) + var updateErrList []error + if err != nil { return remaining, err } @@ -59,6 +64,15 @@ func deletePods(kubeClient clientset.Interface, recorder record.EventRecorder, n if pod.Spec.NodeName != nodeName { continue } + + // Set reason and message in the pod object. + if _, err = setPodTerminationReason(kubeClient, &pod, nodeName); err != nil { + if errors.IsConflict(err) { + updateErrList = append(updateErrList, + fmt.Errorf("update status failed for pod %q: %v", format.Pod(&pod), err)) + continue + } + } // if the pod has already been marked for deletion, we still return true that there are remaining pods. if pod.DeletionGracePeriodSeconds != nil { remaining = true @@ -77,9 +91,31 @@ func deletePods(kubeClient clientset.Interface, recorder record.EventRecorder, n } remaining = true } + + if len(updateErrList) > 0 { + return false, utilerrors.NewAggregate(updateErrList) + } return remaining, nil } +// setPodTerminationReason attempts to set a reason and message in the pod status, updates it in the apiserver, +// and returns an error if it encounters one. +func setPodTerminationReason(kubeClient clientset.Interface, pod *api.Pod, nodeName string) (*api.Pod, error) { + if pod.Status.Reason == node.NodeUnreachablePodReason { + return pod, nil + } + + pod.Status.Reason = node.NodeUnreachablePodReason + pod.Status.Message = fmt.Sprintf(node.NodeUnreachablePodMessage, nodeName, pod.Name) + + var updatedPod *api.Pod + var err error + if updatedPod, err = kubeClient.Core().Pods(pod.Namespace).UpdateStatus(pod); err != nil { + return nil, err + } + return updatedPod, nil +} + func forcefullyDeletePod(c clientset.Interface, pod *api.Pod) error { var zero int64 err := c.Core().Pods(pod.Namespace).Delete(pod.Name, &api.DeleteOptions{GracePeriodSeconds: &zero}) diff --git a/pkg/kubectl/resource_printer.go b/pkg/kubectl/resource_printer.go index 4bd5749ecb7..afb40ab31c6 100644 --- a/pkg/kubectl/resource_printer.go +++ b/pkg/kubectl/resource_printer.go @@ -30,8 +30,6 @@ import ( "text/template" "time" - "github.com/ghodss/yaml" - "github.com/golang/glog" "k8s.io/kubernetes/federation/apis/federation" "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/api/events" @@ -49,7 +47,11 @@ import ( "k8s.io/kubernetes/pkg/runtime" utilerrors "k8s.io/kubernetes/pkg/util/errors" "k8s.io/kubernetes/pkg/util/jsonpath" + "k8s.io/kubernetes/pkg/util/node" "k8s.io/kubernetes/pkg/util/sets" + + "github.com/ghodss/yaml" + "github.com/golang/glog" ) const ( @@ -731,7 +733,10 @@ func printPodBase(pod *api.Pod, w io.Writer, options PrintOptions) error { } } } - if pod.DeletionTimestamp != nil { + + if pod.DeletionTimestamp != nil && pod.Status.Reason == node.NodeUnreachablePodReason { + reason = "Unknown" + } else if pod.DeletionTimestamp != nil { reason = "Terminating" } diff --git a/pkg/util/node/node.go b/pkg/util/node/node.go index 0bb4fb574ef..2357a2dde41 100644 --- a/pkg/util/node/node.go +++ b/pkg/util/node/node.go @@ -31,6 +31,13 @@ import ( "k8s.io/kubernetes/pkg/types" ) +const ( + // The reason and message set on a pod when its state cannot be confirmed as kubelet is unresponsive + // on the node it is (was) running. + NodeUnreachablePodReason = "NodeLost" + NodeUnreachablePodMessage = "Node %v which was running pod %v is unresponsive" +) + func GetHostname(hostnameOverride string) string { var hostname string = hostnameOverride if hostname == "" {