diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 10554d76e..370cebcf4 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -760,11 +760,11 @@ }, { "ImportPath": "k8s.io/api", - "Rev": "73d7eb3bb026" + "Rev": "03aa42fe49ac" }, { "ImportPath": "k8s.io/apimachinery", - "Rev": "ba0f2f062330" + "Rev": "6084ad015ddd" }, { "ImportPath": "k8s.io/cli-runtime", @@ -772,11 +772,11 @@ }, { "ImportPath": "k8s.io/client-go", - "Rev": "b643ec487eb7" + "Rev": "ba26c4506d39" }, { "ImportPath": "k8s.io/code-generator", - "Rev": "5a311e69ffcf" + "Rev": "7dbebec02af3" }, { "ImportPath": "k8s.io/component-base", diff --git a/go.mod b/go.mod index a1ed3a624..b043ea209 100644 --- a/go.mod +++ b/go.mod @@ -34,10 +34,10 @@ require ( github.com/xlab/handysort v0.0.0-20150421192137-fb3537ed64a1 // indirect golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 gopkg.in/yaml.v2 v2.2.8 - k8s.io/api v0.0.0-20200902051604-73d7eb3bb026 - k8s.io/apimachinery v0.0.0-20200902131538-ba0f2f062330 + k8s.io/api v0.0.0-20200902131916-03aa42fe49ac + k8s.io/apimachinery v0.0.0-20200902171538-6084ad015ddd k8s.io/cli-runtime v0.0.0-20200902135640-4d285f01f90f - k8s.io/client-go v0.0.0-20200902132332-b643ec487eb7 + k8s.io/client-go v0.0.0-20200903011833-ba26c4506d39 k8s.io/component-base v0.0.0-20200902133139-1d2c8187d823 k8s.io/klog/v2 v2.2.0 k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 @@ -49,11 +49,11 @@ require ( ) replace ( - k8s.io/api => k8s.io/api v0.0.0-20200902051604-73d7eb3bb026 - k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20200902131538-ba0f2f062330 + k8s.io/api => k8s.io/api v0.0.0-20200902131916-03aa42fe49ac + k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20200902171538-6084ad015ddd k8s.io/cli-runtime => k8s.io/cli-runtime v0.0.0-20200902135640-4d285f01f90f - k8s.io/client-go => k8s.io/client-go v0.0.0-20200902132332-b643ec487eb7 - k8s.io/code-generator => k8s.io/code-generator v0.0.0-20200813011144-5a311e69ffcf + k8s.io/client-go => k8s.io/client-go v0.0.0-20200903011833-ba26c4506d39 + k8s.io/code-generator => k8s.io/code-generator v0.0.0-20200902211251-7dbebec02af3 k8s.io/component-base => k8s.io/component-base v0.0.0-20200902133139-1d2c8187d823 k8s.io/metrics => k8s.io/metrics v0.0.0-20200902135418-05baf84b7531 ) diff --git a/go.sum b/go.sum index de498cf3b..4c3c6d6b1 100644 --- a/go.sum +++ b/go.sum @@ -503,11 +503,11 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -k8s.io/api v0.0.0-20200902051604-73d7eb3bb026/go.mod h1:qMn0Ckwm7+s/YbS3iDBlHErlW/SeOXttpV83LV5iO5M= -k8s.io/apimachinery v0.0.0-20200902131538-ba0f2f062330/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/api v0.0.0-20200902131916-03aa42fe49ac/go.mod h1:RRBubtUUfQknZlqyrIcyOzchPl/ndgo1xAGCzjrKrQU= +k8s.io/apimachinery v0.0.0-20200902171538-6084ad015ddd/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= k8s.io/cli-runtime v0.0.0-20200902135640-4d285f01f90f/go.mod h1:dMX648DBrr+l8yWksbYdwcMzDEM/H0QGyJ8EOKiF9xY= -k8s.io/client-go v0.0.0-20200902132332-b643ec487eb7/go.mod h1:1CDKw+WUI72vWcFo0SpATCUkR78esmyqaIDImRWHmkk= -k8s.io/code-generator v0.0.0-20200813011144-5a311e69ffcf/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= +k8s.io/client-go v0.0.0-20200903011833-ba26c4506d39/go.mod h1:IlEd8lxjDG8cvAB6MHAPJe9pO2IOxBafW1hrpyMFf0U= +k8s.io/code-generator v0.0.0-20200902211251-7dbebec02af3/go.mod h1:moqLn7w0t9cMs4+5CQyxnfA/HV8MF6aAVENF+WZZhgk= k8s.io/component-base v0.0.0-20200902133139-1d2c8187d823/go.mod h1:0gnPCcp/GY+eCzs3ctBHW+6CDRXi6+WyHI39AhX3nHQ= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= diff --git a/pkg/drain/drain.go b/pkg/drain/drain.go index a6657d1a7..30b790a9c 100644 --- a/pkg/drain/drain.go +++ b/pkg/drain/drain.go @@ -254,6 +254,7 @@ func (d *Helper) evictPods(pods []corev1.Pod, policyGroupVersion string, getPodF defer cancel() for _, pod := range pods { go func(pod corev1.Pod, returnCh chan error) { + refreshPod := false for { switch d.DryRunStrategy { case cmdutil.DryRunServer: @@ -268,17 +269,40 @@ func (d *Helper) evictPods(pods []corev1.Pod, policyGroupVersion string, getPodF return default: } - err := d.EvictPod(pod, policyGroupVersion) + + // Create a temporary pod so we don't mutate the pod in the loop. + activePod := pod + if refreshPod { + freshPod, err := getPodFn(pod.Namespace, pod.Name) + // We ignore errors and let eviction sort it out with + // the original pod. + if err == nil { + activePod = *freshPod + } + refreshPod = false + } + + err := d.EvictPod(activePod, policyGroupVersion) if err == nil { break } else if apierrors.IsNotFound(err) { returnCh <- nil return } else if apierrors.IsTooManyRequests(err) { - fmt.Fprintf(d.ErrOut, "error when evicting pods/%q -n %q (will retry after 5s): %v\n", pod.Name, pod.Namespace, err) + fmt.Fprintf(d.ErrOut, "error when evicting pods/%q -n %q (will retry after 5s): %v\n", activePod.Name, activePod.Namespace, err) + time.Sleep(5 * time.Second) + } else if !activePod.ObjectMeta.DeletionTimestamp.IsZero() && apierrors.IsForbidden(err) && apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) { + // an eviction request in a deleting namespace will throw a forbidden error, + // if the pod is already marked deleted, we can ignore this error, an eviction + // request will never succeed, but we will waitForDelete for this pod. + break + } else if apierrors.IsForbidden(err) && apierrors.HasStatusCause(err, corev1.NamespaceTerminatingCause) { + // an eviction request in a deleting namespace will throw a forbidden error, + // if the pod is not marked deleted, we retry until it is. + fmt.Fprintf(d.ErrOut, "error when evicting pod %q (will retry after 5s): %v\n", activePod.Name, err) time.Sleep(5 * time.Second) } else { - returnCh <- fmt.Errorf("error when evicting pods/%q -n %q: %v", pod.Name, pod.Namespace, err) + returnCh <- fmt.Errorf("error when evicting pods/%q -n %q: %v", activePod.Name, activePod.Namespace, err) return } }