From e1ab1debdba5f00c21eeecc306394db69f5772fb Mon Sep 17 00:00:00 2001 From: Lukasz Szaszkiewicz Date: Mon, 2 May 2022 11:57:16 +0200 Subject: [PATCH] hardens integration job tests the job controller used by the tests must wait for the caches to sync since the tests don't check /readyz there is no way the tests can tell it is safe to call the server and requests won't be rejected --- test/integration/job/job_test.go | 40 ++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/test/integration/job/job_test.go b/test/integration/job/job_test.go index a37c3cab22a..05b097e2715 100644 --- a/test/integration/job/job_test.go +++ b/test/integration/job/job_test.go @@ -67,7 +67,7 @@ func TestNonParallelJob(t *testing.T) { closeFn, restConfig, clientSet, ns := setup(t, "simple") defer closeFn() - ctx, cancel := startJobController(restConfig) + ctx, cancel := startJobControllerAndWaitForCaches(restConfig) defer func() { cancel() }() @@ -85,7 +85,7 @@ func TestNonParallelJob(t *testing.T) { // Restarting controller. cancel() - ctx, cancel = startJobController(restConfig) + ctx, cancel = startJobControllerAndWaitForCaches(restConfig) // Failed Pod is replaced. if err := setJobPodsPhase(ctx, clientSet, jobObj, v1.PodFailed, 1); err != nil { @@ -98,7 +98,7 @@ func TestNonParallelJob(t *testing.T) { // Restarting controller. cancel() - ctx, cancel = startJobController(restConfig) + ctx, cancel = startJobControllerAndWaitForCaches(restConfig) // No more Pods are created after the Pod succeeds. if err := setJobPodsPhase(ctx, clientSet, jobObj, v1.PodSucceeded, 1); err != nil { @@ -138,7 +138,7 @@ func TestParallelJob(t *testing.T) { closeFn, restConfig, clientSet, ns := setup(t, "parallel") defer closeFn() - ctx, cancel := startJobController(restConfig) + ctx, cancel := startJobControllerAndWaitForCaches(restConfig) defer cancel() jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, &batchv1.Job{ @@ -226,7 +226,7 @@ func TestParallelJobParallelism(t *testing.T) { closeFn, restConfig, clientSet, ns := setup(t, "parallel") defer closeFn() - ctx, cancel := startJobController(restConfig) + ctx, cancel := startJobControllerAndWaitForCaches(restConfig) defer cancel() jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, &batchv1.Job{ @@ -302,7 +302,7 @@ func TestParallelJobWithCompletions(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, features.JobReadyPods, tc.enableReadyPods)() closeFn, restConfig, clientSet, ns := setup(t, "completions") defer closeFn() - ctx, cancel := startJobController(restConfig) + ctx, cancel := startJobControllerAndWaitForCaches(restConfig) defer cancel() jobObj, err := createJobWithDefaults(ctx, clientSet, ns.Name, &batchv1.Job{ @@ -383,7 +383,7 @@ func TestIndexedJob(t *testing.T) { closeFn, restConfig, clientSet, ns := setup(t, "indexed") defer closeFn() - ctx, cancel := startJobController(restConfig) + ctx, cancel := startJobControllerAndWaitForCaches(restConfig) defer func() { cancel() }() @@ -474,7 +474,7 @@ func TestDisableJobTrackingWithFinalizers(t *testing.T) { closeFn, restConfig, clientSet, ns := setup(t, "simple") defer closeFn() - ctx, cancel := startJobController(restConfig) + ctx, cancel := startJobControllerAndWaitForCaches(restConfig) defer func() { cancel() }() @@ -504,7 +504,7 @@ func TestDisableJobTrackingWithFinalizers(t *testing.T) { } // Restart controller. - ctx, cancel = startJobController(restConfig) + ctx, cancel = startJobControllerAndWaitForCaches(restConfig) // Ensure Job continues to be tracked and finalizers are removed. validateJobPodsStatus(ctx, t, clientSet, jobObj, podsByStatus{ @@ -530,7 +530,7 @@ func TestDisableJobTrackingWithFinalizers(t *testing.T) { } // Restart controller. - ctx, cancel = startJobController(restConfig) + ctx, cancel = startJobControllerAndWaitForCaches(restConfig) // Ensure Job continues to be tracked and finalizers are removed. validateJobPodsStatus(ctx, t, clientSet, jobObj, podsByStatus{ @@ -591,7 +591,7 @@ func TestFinalizersClearedWhenBackoffLimitExceeded(t *testing.T) { closeFn, restConfig, clientSet, ns := setup(t, "simple") defer closeFn() - ctx, cancel := startJobController(restConfig) + ctx, cancel := startJobControllerAndWaitForCaches(restConfig) defer func() { cancel() }() @@ -657,7 +657,7 @@ func TestOrphanPodsFinalizersClearedWithFeatureDisabled(t *testing.T) { closeFn, restConfig, clientSet, ns := setup(t, "simple") defer closeFn() - ctx, cancel := startJobController(restConfig) + ctx, cancel := startJobControllerAndWaitForCaches(restConfig) defer func() { cancel() }() @@ -688,7 +688,7 @@ func TestOrphanPodsFinalizersClearedWithFeatureDisabled(t *testing.T) { } // Restart controller. - ctx, cancel = startJobController(restConfig) + ctx, cancel = startJobControllerAndWaitForCaches(restConfig) if err := wait.Poll(waitInterval, wait.ForeverTestTimeout, func() (done bool, err error) { pods, err := clientSet.CoreV1().Pods(jobObj.Namespace).List(ctx, metav1.ListOptions{}) if err != nil { @@ -752,7 +752,7 @@ func TestSuspendJob(t *testing.T) { closeFn, restConfig, clientSet, ns := setup(t, "suspend") defer closeFn() - ctx, cancel := startJobController(restConfig) + ctx, cancel := startJobControllerAndWaitForCaches(restConfig) defer cancel() events, err := clientSet.EventsV1().Events(ns.Name).Watch(ctx, metav1.ListOptions{}) if err != nil { @@ -805,7 +805,7 @@ func TestSuspendJobControllerRestart(t *testing.T) { closeFn, restConfig, clientSet, ns := setup(t, "suspend") defer closeFn() - ctx, cancel := startJobController(restConfig) + ctx, cancel := startJobControllerAndWaitForCaches(restConfig) defer func() { cancel() }() @@ -847,7 +847,7 @@ func TestNodeSelectorUpdate(t *testing.T) { closeFn, restConfig, clientSet, ns := setup(t, "suspend") defer closeFn() - ctx, cancel := startJobController(restConfig) + ctx, cancel := startJobControllerAndWaitForCaches(restConfig) defer cancel() job, err := createJobWithDefaults(ctx, clientSet, ns.Name, &batchv1.Job{Spec: batchv1.JobSpec{ @@ -1214,11 +1214,17 @@ func setup(t *testing.T, nsBaseName string) (framework.CloseFunc, *restclient.Co return closeFn, &config, clientSet, ns } -func startJobController(restConfig *restclient.Config) (context.Context, context.CancelFunc) { +func startJobControllerAndWaitForCaches(restConfig *restclient.Config) (context.Context, context.CancelFunc) { informerSet := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(restConfig, "job-informers")), 0) jc, ctx, cancel := createJobControllerWithSharedInformers(restConfig, informerSet) informerSet.Start(ctx.Done()) go jc.Run(ctx, 1) + + // since this method starts the controller in a separate goroutine + // and the tests don't check /readyz there is no way + // the tests can tell it is safe to call the server and requests won't be rejected + // thus we wait until caches have synced + informerSet.WaitForCacheSync(ctx.Done()) return ctx, cancel }