From 7c7b1e1018be4a0d380e30f80e194298ab2e14fa Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Wed, 10 Dec 2025 08:18:33 +0100 Subject: [PATCH] DRA e2e: make driver deployment possible in Go unit tests This leverages ktesting as wrapper around Ginkgo and testing.T to make all helper code that is needed to deploy a DRA driver available to Go unit tests and thus integration tests. How to proceed with unifying helper code for integration and E2E testing is open. This is just a minimal first step in that direction. Ideally, such code should be in separate packages where usage of Ginkgo, e2e/framework and gomega.Expect/Eventually/Consistently are forbidden. While at it, the builder gets extended to make cleanup optional. This will be needed for upgrade/downgrade testing with sub-tests. --- test/e2e/dra/dra.go | 406 ++++++++++++++------------ test/e2e/dra/utils/builder.go | 190 ++++++------ test/e2e/dra/utils/deploy.go | 268 ++++++++--------- test/e2e/framework/pod/exec_util.go | 21 +- test/e2e/storage/drivers/csi.go | 2 +- test/e2e/storage/drivers/proxy/io.go | 6 +- test/e2e/storage/utils/create.go | 229 ++++++++------- test/e2e_dra/coredra_test.go | 4 +- test/e2e_dra/gtesting_test.go | 89 ------ test/e2e_dra/upgradedowngrade_test.go | 10 +- 10 files changed, 612 insertions(+), 613 deletions(-) delete mode 100644 test/e2e_dra/gtesting_test.go diff --git a/test/e2e/dra/dra.go b/test/e2e/dra/dra.go index 8d312537328..9e1846a1419 100644 --- a/test/e2e/dra/dra.go +++ b/test/e2e/dra/dra.go @@ -56,6 +56,7 @@ import ( e2edaemonset "k8s.io/kubernetes/test/e2e/framework/daemonset" e2eevents "k8s.io/kubernetes/test/e2e/framework/events" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/utils/ktesting" admissionapi "k8s.io/pod-security-admission/api" "k8s.io/utils/ptr" ) @@ -249,7 +250,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ginkgo.By("waiting for container startup to fail") pod, template := b.PodInline() - b.Create(ctx, pod, template) + b.Create(f.TContext(ctx), pod, template) ginkgo.By("wait for NodePrepareResources call") gomega.Eventually(ctx, func(ctx context.Context) error { @@ -272,7 +273,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ginkgo.It("must not run a pod if a claim is not ready", func(ctx context.Context) { claim := b.ExternalClaim() - b.Create(ctx, claim) + b.Create(f.TContext(ctx), claim) pod := b.PodExternal() // This bypasses scheduling and therefore the pod gets @@ -280,7 +281,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // Because the parameters are missing, the claim // also cannot be allocated later. pod.Spec.NodeName = nodes.NodeNames[0] - b.Create(ctx, pod) + b.Create(f.TContext(ctx), pod) gomega.Consistently(ctx, func(ctx context.Context) error { testPod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) @@ -300,9 +301,9 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { zero := int64(0) pod.Spec.TerminationGracePeriodSeconds = &zero - b.Create(ctx, claim, pod) + b.Create(f.TContext(ctx), claim, pod) - b.TestPod(ctx, f, pod) + b.TestPod(f.TContext(ctx), pod) ginkgo.By(fmt.Sprintf("force delete test pod %s", pod.Name)) err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: &zero}) @@ -321,7 +322,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { for i := range pod.Spec.Containers { pod.Spec.Containers[i].Resources.Claims = nil } - b.Create(ctx, pod, template) + b.Create(f.TContext(ctx), pod, template) framework.ExpectNoError(e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod), "start pod") for host, plugin := range driver.Nodes { gomega.Expect(plugin.GetPreparedResources()).ShouldNot(gomega.BeEmpty(), "claims should be prepared on host %s while pod is running", host) @@ -421,12 +422,12 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { } container1Env = append(container1Env, allContainersEnv...) - b.Create(ctx, claimForAllContainers, claimForContainer0, claimForContainer1, pod) + b.Create(f.TContext(ctx), claimForAllContainers, claimForContainer0, claimForContainer1, pod) err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) framework.ExpectNoError(err, "start pod") - drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[0].Name, true, container0Env...) - drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[1].Name, true, container1Env...) + drautils.TestContainerEnv(f.TContext(ctx), pod, pod.Spec.Containers[0].Name, true, container0Env...) + drautils.TestContainerEnv(f.TContext(ctx), pod, pod.Spec.Containers[1].Name, true, container1Env...) }) // https://github.com/kubernetes/kubernetes/issues/131513 was fixed in master for 1.34 and not backported, @@ -449,8 +450,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { pod := b.PodExternal() node := nodes.NodeNames[0] pod.Spec.NodeSelector = map[string]string{"kubernetes.io/hostname": node} - oldClaim := b.Create(ctx, claim, pod)[0].(*resourceapi.ResourceClaim) - b.TestPod(ctx, f, pod) + oldClaim := b.Create(f.TContext(ctx), claim, pod)[0].(*resourceapi.ResourceClaim) + b.TestPod(f.TContext(ctx), pod) ginkgo.By("Force-delete claim and pod") forceDelete := metav1.DeleteOptions{GracePeriodSeconds: ptr.To(int64(0))} @@ -478,7 +479,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { gomega.Expect(driver.Nodes[node].GetPreparedResources()).Should(gomega.Equal([]testdriverapp.ClaimID{{Name: oldClaim.Name, UID: oldClaim.UID}}), "Old claim should still be prepared.") ginkgo.By("Re-creating the same claim and pod") - newClaim := b.Create(ctx, claim, pod)[0].(*resourceapi.ResourceClaim) + newClaim := b.Create(f.TContext(ctx), claim, pod)[0].(*resourceapi.ResourceClaim) // Keep blocking NodeUnprepareResources for the old pod // until the new pod calls NodePrepareResources and fails. @@ -503,7 +504,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { driver.Fail(unprepareResources, false) - b.TestPod(ctx, f, pod) + b.TestPod(f.TContext(ctx), pod) // The pod must not have started before NodeUnprepareResources was called for the old one, // i.e. what is prepared now must be the new claim. @@ -541,7 +542,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }, } - created := b.Create(ctx, template, daemonSet) + created := b.Create(f.TContext(ctx), template, daemonSet) if !ptr.Deref(created[0].(*resourceapi.ResourceClaimTemplate).Spec.Spec.Devices.Requests[0].Exactly.AdminAccess, false) { framework.Fail("AdminAccess field was cleared. This test depends on the DRAAdminAccess feature.") } @@ -623,16 +624,16 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { pod.Spec.InitContainers[0].Name += "-init" // This must succeed for the pod to start. pod.Spec.InitContainers[0].Command = []string{"sh", "-c", "env | grep user_a=b"} - b.Create(ctx, pod, claim) + b.Create(f.TContext(ctx), pod, claim) - b.TestPod(ctx, f, pod) + b.TestPod(f.TContext(ctx), pod) }) ginkgo.It("removes reservation from claim when pod is done", func(ctx context.Context) { pod := b.PodExternal() claim := b.ExternalClaim() pod.Spec.Containers[0].Command = []string{"true"} - b.Create(ctx, claim, pod) + b.Create(f.TContext(ctx), claim, pod) ginkgo.By("waiting for pod to finish") framework.ExpectNoError(e2epod.WaitForPodNoLongerRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace), "wait for pod to finish") @@ -645,7 +646,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ginkgo.It("deletes generated claims when pod is done", func(ctx context.Context) { pod, template := b.PodInline() pod.Spec.Containers[0].Command = []string{"true"} - b.Create(ctx, template, pod) + b.Create(f.TContext(ctx), template, pod) ginkgo.By("waiting for pod to finish") framework.ExpectNoError(e2epod.WaitForPodNoLongerRunningInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace), "wait for pod to finish") @@ -663,7 +664,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { pod, template := b.PodInline() pod.Spec.Containers[0].Command = []string{"sh", "-c", "sleep 1; exit 1"} pod.Spec.RestartPolicy = v1.RestartPolicyAlways - b.Create(ctx, template, pod) + b.Create(f.TContext(ctx), template, pod) ginkgo.By("waiting for pod to restart twice") gomega.Eventually(ctx, func(ctx context.Context) (*v1.Pod, error) { @@ -675,22 +676,23 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // kubelet tests with individual configurations. f.Context("kubelet", feature.DynamicResourceAllocation, func() { ginkgo.It("runs pod after driver starts", func(ctx context.Context) { - nodes := drautils.NewNodesNow(ctx, f, 1, 4) - driver := drautils.NewDriverInstance(f) - b := drautils.NewBuilderNow(ctx, f, driver) + tCtx := f.TContext(ctx) + nodes := drautils.NewNodesNow(tCtx, 1, 4) + driver := drautils.NewDriverInstance(tCtx) + b := drautils.NewBuilderNow(tCtx, driver) claim := b.ExternalClaim() pod := b.PodExternal() - b.Create(ctx, claim, pod) + b.Create(tCtx, claim, pod) // Cannot run pod, no devices. framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace)) // Set up driver, which makes devices available. - driver.Run(nodes, drautils.DriverResourcesNow(nodes, 1)) + driver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1)) // Now it should run. - b.TestPod(ctx, f, pod) + b.TestPod(tCtx, pod) // We need to clean up explicitly because the normal // cleanup doesn't work (driver shuts down first). @@ -700,38 +702,37 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // Seamless upgrade support was added in Kubernetes 1.33. f.It("rolling update", f.WithLabel("KubeletMinVersion:1.33"), func(ctx context.Context) { - nodes := drautils.NewNodesNow(ctx, f, 1, 1) + tCtx := f.TContext(ctx) + nodes := drautils.NewNodesNow(tCtx, 1, 1) - oldDriver := drautils.NewDriverInstance(f) + oldDriver := drautils.NewDriverInstance(tCtx) oldDriver.InstanceSuffix = "-old" oldDriver.RollingUpdate = true - oldDriver.Run(nodes, drautils.DriverResourcesNow(nodes, 1)) + oldDriver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1)) // We expect one ResourceSlice per node from the driver. getSlices := oldDriver.NewGetSlices() - gomega.Eventually(ctx, getSlices).Should(gomega.HaveField("Items", gomega.HaveLen(len(nodes.NodeNames)))) - initialSlices, err := getSlices(ctx) - framework.ExpectNoError(err) + ktesting.Eventually(tCtx, getSlices).Should(gomega.HaveField("Items", gomega.HaveLen(len(nodes.NodeNames)))) + initialSlices := getSlices(tCtx) // Same driver name, different socket paths because of rolling update. - newDriver := drautils.NewDriverInstance(f) + newDriver := drautils.NewDriverInstance(tCtx) newDriver.InstanceSuffix = "-new" newDriver.RollingUpdate = true - newDriver.Run(nodes, drautils.DriverResourcesNow(nodes, 1)) + newDriver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1)) // Stop old driver instance. - oldDriver.TearDown(ctx) + oldDriver.TearDown(tCtx) // Build behaves the same for both driver instances. - b := drautils.NewBuilderNow(ctx, f, oldDriver) + b := drautils.NewBuilderNow(tCtx, oldDriver) claim := b.ExternalClaim() pod := b.PodExternal() - b.Create(ctx, claim, pod) - b.TestPod(ctx, f, pod) + b.Create(tCtx, claim, pod) + b.TestPod(tCtx, pod) // The exact same slices should still exist. - finalSlices, err := getSlices(ctx) - framework.ExpectNoError(err) + finalSlices := getSlices(tCtx) gomega.Expect(finalSlices.Items).Should(gomega.Equal(initialSlices.Items)) // We need to clean up explicitly because the normal @@ -742,40 +743,39 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // Seamless upgrade support was added in Kubernetes 1.33. f.It("failed update", f.WithLabel("KubeletMinVersion:1.33"), func(ctx context.Context) { - nodes := drautils.NewNodesNow(ctx, f, 1, 1) + tCtx := f.TContext(ctx) + nodes := drautils.NewNodesNow(tCtx, 1, 1) - oldDriver := drautils.NewDriverInstance(f) + oldDriver := drautils.NewDriverInstance(tCtx) oldDriver.InstanceSuffix = "-old" oldDriver.RollingUpdate = true - oldDriver.Run(nodes, drautils.DriverResourcesNow(nodes, 1)) + oldDriver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1)) // We expect one ResourceSlice per node from the driver. getSlices := oldDriver.NewGetSlices() - gomega.Eventually(ctx, getSlices).Should(gomega.HaveField("Items", gomega.HaveLen(len(nodes.NodeNames)))) - initialSlices, err := getSlices(ctx) - framework.ExpectNoError(err) + ktesting.Eventually(tCtx, getSlices).Should(gomega.HaveField("Items", gomega.HaveLen(len(nodes.NodeNames)))) + initialSlices := getSlices(tCtx) // Same driver name, different socket paths because of rolling update. - newDriver := drautils.NewDriverInstance(f) + newDriver := drautils.NewDriverInstance(tCtx) newDriver.InstanceSuffix = "-new" newDriver.RollingUpdate = true newDriver.ExpectResourceSliceRemoval = false - newDriver.Run(nodes, drautils.DriverResourcesNow(nodes, 1)) + newDriver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1)) // Stop new driver instance, simulating the failure of the new instance. // The kubelet should still have the old instance. - newDriver.TearDown(ctx) + newDriver.TearDown(tCtx) // Build behaves the same for both driver instances. - b := drautils.NewBuilderNow(ctx, f, oldDriver) + b := drautils.NewBuilderNow(tCtx, oldDriver) claim := b.ExternalClaim() pod := b.PodExternal() - b.Create(ctx, claim, pod) - b.TestPod(ctx, f, pod) + b.Create(tCtx, claim, pod) + b.TestPod(tCtx, pod) // The exact same slices should still exist. - finalSlices, err := getSlices(ctx) - framework.ExpectNoError(err) + finalSlices := getSlices(tCtx) gomega.Expect(finalSlices.Items).Should(gomega.Equal(initialSlices.Items)) // We need to clean up explicitly because the normal @@ -786,20 +786,18 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // Seamless upgrade support was added in Kubernetes 1.33. f.It("sequential update with pods replacing each other", f.WithLabel("KubeletMinVersion:1.33"), framework.WithSlow(), func(ctx context.Context) { - nodes := drautils.NewNodesNow(ctx, f, 1, 1) + tCtx := f.TContext(ctx) + nodes := drautils.NewNodesNow(tCtx, 1, 1) // Same driver name, same socket path. - oldDriver := drautils.NewDriverInstance(f) + oldDriver := drautils.NewDriverInstance(tCtx) oldDriver.InstanceSuffix = "-old" - oldDriver.Run(nodes, drautils.DriverResourcesNow(nodes, 1)) + oldDriver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1)) // Collect set of resource slices for that driver. - listSlices := framework.ListObjects(f.ClientSet.ResourceV1().ResourceSlices().List, metav1.ListOptions{ - FieldSelector: "spec.driver=" + oldDriver.Name, - }) - gomega.Eventually(ctx, listSlices).Should(gomega.HaveField("Items", gomega.Not(gomega.BeEmpty())), "driver should have published ResourceSlices, got none") - oldSlices, err := listSlices(ctx) - framework.ExpectNoError(err, "list slices published by old driver") + listSlices := oldDriver.NewGetSlices() + ktesting.Eventually(tCtx, listSlices).Should(gomega.HaveField("Items", gomega.Not(gomega.BeEmpty())), "driver should have published ResourceSlices, got none") + oldSlices := listSlices(tCtx) if len(oldSlices.Items) == 0 { framework.Fail("driver should have published ResourceSlices, got none") } @@ -809,18 +807,18 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // its pods when maxSurge is zero. ginkgo.By("reinstall driver") start := time.Now() - oldDriver.TearDown(ctx) - newDriver := drautils.NewDriverInstance(f) + oldDriver.TearDown(tCtx) + newDriver := drautils.NewDriverInstance(tCtx) newDriver.InstanceSuffix = "-new" - newDriver.Run(nodes, drautils.DriverResourcesNow(nodes, 1)) + newDriver.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, drautils.DriverResourcesNow(nodes, 1)) updateDuration := time.Since(start) // Build behaves the same for both driver instances. - b := drautils.NewBuilderNow(ctx, f, oldDriver) + b := drautils.NewBuilderNow(tCtx, oldDriver) claim := b.ExternalClaim() pod := b.PodExternal() - b.Create(ctx, claim, pod) - b.TestPod(ctx, f, pod) + b.Create(tCtx, claim, pod) + b.TestPod(tCtx, pod) // The slices should have survived the update, but only if it happened // quickly enough. If it took too long (= wipingDelay of 30 seconds in pkg/kubelet/cm/dra/manager.go, @@ -828,8 +826,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // the kubelet considered the driver gone and removed them. if updateDuration <= 25*time.Second { framework.Logf("Checking resource slices after downtime of %s.", updateDuration) - newSlices, err := listSlices(ctx) - framework.ExpectNoError(err, "list slices again") + newSlices := listSlices(tCtx) gomega.Expect(newSlices.Items).To(gomega.ConsistOf(oldSlices.Items), "Old slice should have survived a downtime of %s.", updateDuration) } else { framework.Logf("Not checking resource slices, downtime was too long with %s.", updateDuration) @@ -843,8 +840,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // Now shut down for good and wait for the kubelet to react. // This takes time... ginkgo.By("uninstalling driver and waiting for ResourceSlice wiping") - newDriver.TearDown(ctx) - newDriver.IsGone(ctx) + newDriver.TearDown(tCtx) + newDriver.IsGone(tCtx) }) }) @@ -868,12 +865,14 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { expectedEnv = append(expectedEnv, expected...) ginkgo.It("supports claim and class parameters", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod, template := b.PodInline() - b.Create(ctx, pod, template) - b.TestPod(ctx, f, pod, expectedEnv...) + b.Create(tCtx, pod, template) + b.TestPod(tCtx, pod, expectedEnv...) }) ginkgo.It("supports reusing resources", func(ctx context.Context) { + tCtx := f.TContext(ctx) var objects []klog.KMetadata pods := make([]*v1.Pod, numPods) for i := 0; i < numPods; i++ { @@ -882,7 +881,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { objects = append(objects, pod, template) } - b.Create(ctx, objects...) + b.Create(tCtx, objects...) // We don't know the order. All that matters is that all of them get scheduled eventually. var wg sync.WaitGroup @@ -892,7 +891,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { go func() { defer ginkgo.GinkgoRecover() defer wg.Done() - b.TestPod(ctx, f, pod, expectedEnv...) + b.TestPod(tCtx, pod, expectedEnv...) err := f.ClientSet.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{}) framework.ExpectNoError(err, "delete pod") framework.ExpectNoError(e2epod.WaitForPodNotFoundInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace, time.Duration(numPods)*f.Timeouts.PodStartSlow)) @@ -902,6 +901,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) ginkgo.It("supports sharing a claim concurrently", func(ctx context.Context) { + tCtx := f.TContext(ctx) var objects []klog.KMetadata objects = append(objects, b.ExternalClaim()) pods := make([]*v1.Pod, numPods) @@ -911,7 +911,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { objects = append(objects, pod) } - b.Create(ctx, objects...) + b.Create(tCtx, objects...) // We don't know the order. All that matters is that all of them get scheduled eventually. f.Timeouts.PodStartSlow *= time.Duration(numPods) @@ -922,13 +922,14 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { go func() { defer ginkgo.GinkgoRecover() defer wg.Done() - b.TestPod(ctx, f, pod, expectedEnv...) + b.TestPod(tCtx, pod, expectedEnv...) }() } wg.Wait() }) ginkgo.It("retries pod scheduling after creating device class", func(ctx context.Context) { + tCtx := f.TContext(ctx) var objects []klog.KMetadata pod, template := b.PodInline() deviceClassName := template.Spec.Spec.Devices.Requests[0].Exactly.DeviceClassName @@ -937,19 +938,20 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { deviceClassName += "-b" template.Spec.Spec.Devices.Requests[0].Exactly.DeviceClassName = deviceClassName objects = append(objects, template, pod) - b.Create(ctx, objects...) + b.Create(tCtx, objects...) framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace)) class.UID = "" class.ResourceVersion = "" class.Name = deviceClassName - b.Create(ctx, class) + b.Create(tCtx, class) - b.TestPod(ctx, f, pod, expectedEnv...) + b.TestPod(tCtx, pod, expectedEnv...) }) ginkgo.It("retries pod scheduling after updating device class", func(ctx context.Context) { + tCtx := f.TContext(ctx) var objects []klog.KMetadata pod, template := b.PodInline() @@ -968,7 +970,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // Now create the pod. objects = append(objects, template, pod) - b.Create(ctx, objects...) + b.Create(tCtx, objects...) framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, pod.Namespace)) @@ -977,12 +979,13 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { _, err = f.ClientSet.ResourceV1().DeviceClasses().Update(ctx, class, metav1.UpdateOptions{}) framework.ExpectNoError(err) - b.TestPod(ctx, f, pod, expectedEnv...) + b.TestPod(tCtx, pod, expectedEnv...) }) ginkgo.It("runs a pod without a generated resource claim", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod, _ /* template */ := b.PodInline() - created := b.Create(ctx, pod) + created := b.Create(tCtx, pod) pod = created[0].(*v1.Pod) // Normally, this pod would be stuck because the @@ -999,69 +1002,76 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) ginkgo.It("supports simple pod referencing inline resource claim", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod, template := b.PodInline() - b.Create(ctx, pod, template) - b.TestPod(ctx, f, pod) + b.Create(tCtx, pod, template) + b.TestPod(tCtx, pod) }) ginkgo.It("supports inline claim referenced by multiple containers", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod, template := b.PodInlineMultiple() - b.Create(ctx, pod, template) - b.TestPod(ctx, f, pod) + b.Create(tCtx, pod, template) + b.TestPod(tCtx, pod) }) ginkgo.It("supports simple pod referencing external resource claim", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod := b.PodExternal() claim := b.ExternalClaim() - b.Create(ctx, claim, pod) - b.TestPod(ctx, f, pod) + b.Create(tCtx, claim, pod) + b.TestPod(tCtx, pod) }) ginkgo.It("supports external claim referenced by multiple pods", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod1 := b.PodExternal() pod2 := b.PodExternal() pod3 := b.PodExternal() claim := b.ExternalClaim() - b.Create(ctx, claim, pod1, pod2, pod3) + b.Create(tCtx, claim, pod1, pod2, pod3) for _, pod := range []*v1.Pod{pod1, pod2, pod3} { - b.TestPod(ctx, f, pod) + b.TestPod(tCtx, pod) } }) ginkgo.It("supports external claim referenced by multiple containers of multiple pods", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod1 := b.PodExternalMultiple() pod2 := b.PodExternalMultiple() pod3 := b.PodExternalMultiple() claim := b.ExternalClaim() - b.Create(ctx, claim, pod1, pod2, pod3) + b.Create(tCtx, claim, pod1, pod2, pod3) for _, pod := range []*v1.Pod{pod1, pod2, pod3} { - b.TestPod(ctx, f, pod) + b.TestPod(tCtx, pod) } }) ginkgo.It("supports init containers", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod, template := b.PodInline() pod.Spec.InitContainers = []v1.Container{pod.Spec.Containers[0]} pod.Spec.InitContainers[0].Name += "-init" // This must succeed for the pod to start. pod.Spec.InitContainers[0].Command = []string{"sh", "-c", "env | grep user_a=b"} - b.Create(ctx, pod, template) + b.Create(tCtx, pod, template) - b.TestPod(ctx, f, pod) + b.TestPod(tCtx, pod) }) ginkgo.It("must deallocate after use", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod := b.PodExternal() claim := b.ExternalClaim() - b.Create(ctx, claim, pod) + b.Create(tCtx, claim, pod) gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) { return f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{}) }).WithTimeout(f.Timeouts.PodDelete).ShouldNot(gomega.HaveField("Status.Allocation", (*resourceapi.AllocationResult)(nil))) - b.TestPod(ctx, f, pod) + b.TestPod(tCtx, pod) ginkgo.By(fmt.Sprintf("deleting pod %s", klog.KObj(pod))) framework.ExpectNoError(f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{})) @@ -1075,6 +1085,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { if withKubelet { // Serial because the example device plugin can only be deployed with one instance at a time. f.It("supports extended resources together with ResourceClaim", f.WithSerial(), func(ctx context.Context) { + tCtx := f.TContext(ctx) extendedResourceName := deployDevicePlugin(ctx, f, nodes.NodeNames[0:1]) pod := b.PodExternal() @@ -1082,8 +1093,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { pod.Spec.Containers[0].Resources.Requests = resources pod.Spec.Containers[0].Resources.Limits = resources claim := b.ExternalClaim() - b.Create(ctx, claim, pod) - b.TestPod(ctx, f, pod) + b.Create(tCtx, claim, pod) + b.TestPod(tCtx, pod) }) } } @@ -1117,6 +1128,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { b := drautils.NewBuilder(f, driver) ginkgo.It("keeps pod pending because of CEL runtime errors", func(ctx context.Context) { + tCtx := f.TContext(ctx) + // When pod scheduling encounters CEL runtime errors for some nodes, but not all, // it should still not schedule the pod because there is something wrong with it. // Scheduling it would make it harder to detect that there is a problem. @@ -1151,7 +1164,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }, }, ) - b.Create(ctx, pod, template) + b.Create(tCtx, pod, template) framework.ExpectNoError(e2epod.WaitForPodCondition(ctx, f.ClientSet, pod.Namespace, pod.Name, "scheduling failure", f.Timeouts.PodStartShort, func(pod *v1.Pod) (bool, error) { for _, condition := range pod.Status.Conditions { @@ -1176,6 +1189,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { b := drautils.NewBuilder(f, driver) ginkgo.It("uses all resources", func(ctx context.Context) { + tCtx := f.TContext(ctx) var objs []klog.KMetadata var pods []*v1.Pod for i := 0; i < len(nodes.NodeNames); i++ { @@ -1183,7 +1197,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { pods = append(pods, pod) objs = append(objs, pod, template) } - b.Create(ctx, objs...) + b.Create(tCtx, objs...) for _, pod := range pods { if withKubelet { @@ -1254,6 +1268,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { b2.ClassParameters = driver2Params f.It("selects the first subrequest that can be satisfied", func(ctx context.Context) { + tCtx := f.TContext(ctx) name := "external-multiclaim" params := `{"a":"b"}` claim := &resourceapi.ResourceClaim{ @@ -1310,8 +1325,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ResourceClaimName: &externalClaimName, }, } - b1.Create(ctx, claim, pod) - b1.TestPod(ctx, f, pod) + b1.Create(tCtx, claim, pod) + b1.TestPod(tCtx, pod) var allocatedResourceClaim *resourceapi.ResourceClaim gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) { @@ -1326,6 +1341,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) f.It("uses the config for the selected subrequest", func(ctx context.Context) { + tCtx := f.TContext(ctx) name := "external-multiclaim" parentReqParams, parentReqEnv := `{"a":"b"}`, []string{"user_a", "b"} subReq1Params := `{"c":"d"}` @@ -1400,14 +1416,15 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ResourceClaimName: &externalClaimName, }, } - b1.Create(ctx, claim, pod) + b1.Create(tCtx, claim, pod) var expectedEnv []string expectedEnv = append(expectedEnv, parentReqEnv...) expectedEnv = append(expectedEnv, subReq2Env...) - b1.TestPod(ctx, f, pod, expectedEnv...) + b1.TestPod(tCtx, pod, expectedEnv...) }) f.It("chooses the correct subrequest subject to constraints", func(ctx context.Context) { + tCtx := f.TContext(ctx) name := "external-multiclaim" params := `{"a":"b"}` claim := &resourceapi.ResourceClaim{ @@ -1478,8 +1495,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ResourceClaimName: &externalClaimName, }, } - b1.Create(ctx, claim, pod) - b1.TestPod(ctx, f, pod) + b1.Create(tCtx, claim, pod) + b1.TestPod(tCtx, pod) var allocatedResourceClaim *resourceapi.ResourceClaim gomega.Eventually(ctx, func(ctx context.Context) (*resourceapi.ResourceClaim, error) { @@ -1494,6 +1511,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) f.It("filters config correctly for multiple devices", func(ctx context.Context) { + tCtx := f.TContext(ctx) name := "external-multiclaim" req1Params, req1Env := `{"a":"b"}`, []string{"user_a", "b"} req1subReq1Params, _ := `{"c":"d"}`, []string{"user_d", "d"} @@ -1594,7 +1612,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { pod.Spec.Containers[0].Resources.Claims = []v1.ResourceClaim{{Name: name, Request: "request-1"}} pod.Spec.Containers[1].Resources.Claims = []v1.ResourceClaim{{Name: name, Request: "request-2"}} - b1.Create(ctx, claim, pod) + b1.Create(tCtx, claim, pod) err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) framework.ExpectNoError(err, "start pod") @@ -1616,7 +1634,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { req1ExpectedEnv = append(req1ExpectedEnv, req1Env...) req1ExpectedEnv = append(req1ExpectedEnv, req1subReq2Env...) req1ExpectedEnv = append(req1ExpectedEnv, driver1Env...) - drautils.TestContainerEnv(ctx, f, pod, "with-resource-0", true, req1ExpectedEnv...) + drautils.TestContainerEnv(tCtx, pod, "with-resource-0", true, req1ExpectedEnv...) req2ExpectedEnv := []string{ "claim_external_multiclaim_request_2", @@ -1624,7 +1642,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { } req2ExpectedEnv = append(req2ExpectedEnv, req2Env...) req2ExpectedEnv = append(req2ExpectedEnv, driver2Env...) - drautils.TestContainerEnv(ctx, f, pod, "with-resource-1", true, req2ExpectedEnv...) + drautils.TestContainerEnv(tCtx, pod, "with-resource-1", true, req2ExpectedEnv...) }) } @@ -1640,12 +1658,14 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { expectedEnv = append(expectedEnv, expected...) ginkgo.It("supports simple ResourceClaim", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod, template := b.PodInlineWithV1beta1() - b.Create(ctx, pod, template) - b.TestPod(ctx, f, pod, expectedEnv...) + b.Create(tCtx, pod, template) + b.TestPod(tCtx, pod, expectedEnv...) }) f.It("supports requests with alternatives", f.WithFeatureGate(features.DRAPrioritizedList), func(ctx context.Context) { + tCtx := f.TContext(ctx) claimName := "external-multiclaim" parameters, _ := b.ParametersEnv() claim := &resourcev1beta1.ResourceClaim{ @@ -1692,8 +1712,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ResourceClaimName: &claimName, }, } - b.Create(ctx, claim, pod) - b.TestPod(ctx, f, pod, expectedEnv...) + b.Create(tCtx, claim, pod) + b.TestPod(tCtx, pod, expectedEnv...) var allocatedResourceClaim *resourcev1beta1.ResourceClaim gomega.Eventually(ctx, func(ctx context.Context) (*resourcev1beta1.ResourceClaim, error) { @@ -1719,12 +1739,14 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { expectedEnv = append(expectedEnv, expected...) ginkgo.It("supports simple ResourceClaim", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod, template := b.PodInlineWithV1beta2() - b.Create(ctx, pod, template) - b.TestPod(ctx, f, pod, expectedEnv...) + b.Create(tCtx, pod, template) + b.TestPod(tCtx, pod, expectedEnv...) }) f.It("supports requests with alternatives", f.WithFeatureGate(features.DRAPrioritizedList), func(ctx context.Context) { + tCtx := f.TContext(ctx) claimName := "external-multiclaim" parameters, _ := b.ParametersEnv() claim := &resourcev1beta2.ResourceClaim{ @@ -1771,8 +1793,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ResourceClaimName: &claimName, }, } - b.Create(ctx, claim, pod) - b.TestPod(ctx, f, pod, expectedEnv...) + b.Create(tCtx, claim, pod) + b.TestPod(tCtx, pod, expectedEnv...) var allocatedResourceClaim *resourcev1beta2.ResourceClaim gomega.Eventually(ctx, func(ctx context.Context) (*resourcev1beta2.ResourceClaim, error) { @@ -1831,21 +1853,22 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { b := drautils.NewBuilder(f, driver) f.It("must consume and free up counters", func(ctx context.Context) { + tCtx := f.TContext(ctx) // The first pod will use one of the devices. Since both devices are // available, there should be sufficient counters left to allocate // a device. claim := b.ExternalClaim() pod := b.PodExternal() pod.Spec.ResourceClaims[0].ResourceClaimName = &claim.Name - b.Create(ctx, claim, pod) - b.TestPod(ctx, f, pod) + b.Create(tCtx, claim, pod) + b.TestPod(tCtx, pod) // For the second pod, there should not be sufficient counters left, so // it should not succeed. This means the pod should remain in the pending state. claim2 := b.ExternalClaim() pod2 := b.PodExternal() pod2.Spec.ResourceClaims[0].ResourceClaimName = &claim2.Name - b.Create(ctx, claim2, pod2) + b.Create(tCtx, claim2, pod2) gomega.Consistently(ctx, func(ctx context.Context) error { testPod, err := f.ClientSet.CoreV1().Pods(pod2.Namespace).Get(ctx, pod2.Name, metav1.GetOptions{}) @@ -1859,10 +1882,10 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }, 20*time.Second, 200*time.Millisecond).Should(gomega.Succeed()) // Delete the first pod - b.DeletePodAndWaitForNotFound(ctx, pod) + b.DeletePodAndWaitForNotFound(tCtx, pod) // There shoud not be available devices for pod2. - b.TestPod(ctx, f, pod2) + b.TestPod(tCtx, pod2) }) } @@ -1892,6 +1915,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { b := drautils.NewBuilder(f, driver) f.It("must allow multiple allocations and consume capacity", f.WithLabel("KubeletMinVersion:1.34"), func(ctx context.Context) { + tCtx := f.TContext(ctx) // The first pod will use 4Gi of the device. claim := b.ExternalClaim() claim.Spec.Devices.Requests[0].Exactly.Capacity = &resourceapi.CapacityRequirements{ @@ -1901,8 +1925,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { } pod := b.PodExternal() pod.Spec.ResourceClaims[0].ResourceClaimName = &claim.Name - b.Create(ctx, claim, pod) - b.TestPod(ctx, f, pod) + b.Create(tCtx, claim, pod) + b.TestPod(tCtx, pod) // The second pod will be failed to request 8Gi capacity. claim2 := b.ExternalClaim() @@ -1913,7 +1937,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { } pod2 := b.PodExternal() pod2.Spec.ResourceClaims[0].ResourceClaimName = &claim2.Name - b.Create(ctx, claim2, pod2) + b.Create(tCtx, claim2, pod2) // The third pod should be able to use the rest 4Gi of the device. claim3 := b.ExternalClaim() @@ -1924,8 +1948,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { } pod3 := b.PodExternal() pod3.Spec.ResourceClaims[0].ResourceClaimName = &claim3.Name - b.Create(ctx, claim3, pod3) - b.TestPod(ctx, f, pod3) + b.Create(tCtx, claim3, pod3) + b.TestPod(tCtx, pod3) gomega.Consistently(ctx, func(ctx context.Context) error { testPod2, err := f.ClientSet.CoreV1().Pods(pod2.Namespace).Get(ctx, pod2.Name, metav1.GetOptions{}) @@ -1939,11 +1963,11 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }, 20*time.Second, 200*time.Millisecond).Should(gomega.Succeed()) // Delete the first and third pod - b.DeletePodAndWaitForNotFound(ctx, pod) - b.DeletePodAndWaitForNotFound(ctx, pod3) + b.DeletePodAndWaitForNotFound(tCtx, pod) + b.DeletePodAndWaitForNotFound(tCtx, pod3) // There should be available capacity for pod2 now. - b.TestPod(ctx, f, pod2) + b.TestPod(tCtx, pod2) }) } @@ -1975,23 +1999,26 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { b := drautils.NewBuilder(f, driver) f.It("NoSchedule keeps pod pending", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod, template := b.PodInline() - b.Create(ctx, pod, template) + b.Create(tCtx, pod, template) framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name)) }) f.It("NoSchedule can be tolerated", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod, template := b.PodInline() template.Spec.Spec.Devices.Requests[0].Exactly.Tolerations = []resourceapi.DeviceToleration{{ Effect: resourceapi.DeviceTaintEffectNoSchedule, Operator: resourceapi.DeviceTolerationOpExists, // No key: tolerate *all* taints with this effect. }} - b.Create(ctx, pod, template) - b.TestPod(ctx, f, pod) + b.Create(tCtx, pod, template) + b.TestPod(tCtx, pod) }) f.It("DeviceTaintRule evicts pod", f.WithFeatureGate(features.DRADeviceTaintRules), func(ctx context.Context) { + tCtx := f.TContext(ctx) pod, template := b.PodInline() template.Spec.Spec.Devices.Requests[0].Exactly.Tolerations = []resourceapi.DeviceToleration{{ Effect: resourceapi.DeviceTaintEffectNoSchedule, @@ -2000,8 +2027,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }} // Add a finalizer to ensure that we get a chance to test the pod status after eviction (= deletion). pod.Finalizers = []string{"e2e-test/dont-delete-me"} - b.Create(ctx, pod, template) - b.TestPod(ctx, f, pod) + b.Create(tCtx, pod, template) + b.TestPod(tCtx, pod) ginkgo.DeferCleanup(func(ctx context.Context) { gomega.Eventually(ctx, func(ctx context.Context) error { // Unblock shutdown by removing the finalizer. @@ -2037,7 +2064,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }, }, } - createdTaint := b.Create(ctx, taint) + createdTaint := b.Create(tCtx, taint) taint = createdTaint[0].(*resourcealphaapi.DeviceTaintRule) gomega.Expect(*taint).Should(gomega.HaveField("Spec.Taint.TimeAdded.Time", gomega.BeTemporally("~", time.Now(), time.Minute /* allow for some clock drift and delays */))) framework.ExpectNoError(e2epod.WaitForPodTerminatingInNamespaceTimeout(ctx, f.ClientSet, pod.Name, f.Namespace.Name, f.Timeouts.PodStart)) @@ -2063,20 +2090,22 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { b := drautils.NewBuilder(f, driver) f.It("NoExecute keeps pod pending", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod, template := b.PodInline() - b.Create(ctx, pod, template) + b.Create(tCtx, pod, template) framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name)) }) f.It("NoExecute can be tolerated", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod, template := b.PodInline() template.Spec.Spec.Devices.Requests[0].Exactly.Tolerations = []resourceapi.DeviceToleration{{ Effect: resourceapi.DeviceTaintEffectNoExecute, Operator: resourceapi.DeviceTolerationOpExists, // No key: tolerate *all* taints with this effect. }} - b.Create(ctx, pod, template) - b.TestPod(ctx, f, pod) + b.Create(tCtx, pod, template) + b.TestPod(tCtx, pod) // Testing the pod is slow enough that the test fails when the pod gets evicted unexpectedly, // We don't need to a "Consistently" here. @@ -2084,6 +2113,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) extendedResourceTest := func(ctx context.Context, b *drautils.Builder, f *framework.Framework, resourceNames []string, containerEnv []string) { + tCtx := f.TContext(ctx) pod := b.Pod() res := v1.ResourceList{} res2 := v1.ResourceList{} @@ -2111,13 +2141,14 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { pod.Spec.InitContainers[2].Resources.Requests = res pod.Spec.InitContainers[2].Resources.Limits = res - b.Create(ctx, pod) + b.Create(tCtx, pod) err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) framework.ExpectNoError(err, "start pod") - drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[0].Name, false, containerEnv...) + drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...) } runExtendedClaimCleanup := func(ctx context.Context, b *drautils.Builder, shellScript string, waitFn func(context.Context, *v1.Pod) error, cleanupMessage string) { + tCtx := f.TContext(ctx) pod := b.Pod() res := v1.ResourceList{} res[v1.ResourceName(b.ExtendedResourceName(0))] = resource.MustParse("1") @@ -2129,7 +2160,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { pod.Spec.RestartPolicy = v1.RestartPolicyNever ginkgo.By("Creating the pod with extended resource") - b.Create(ctx, pod) + b.Create(tCtx, pod) err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) framework.ExpectNoError(err, "start pod") @@ -2144,7 +2175,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { return false } claimName := updatedPod.Status.ExtendedResourceClaimStatus.ResourceClaimName - extendedResourceClaim, err = b.ClientV1().ResourceClaims(pod.Namespace).Get(ctx, claimName, metav1.GetOptions{}) + extendedResourceClaim, err = b.ClientV1(tCtx).ResourceClaims(pod.Namespace).Get(ctx, claimName, metav1.GetOptions{}) return err == nil && extendedResourceClaim != nil }).WithTimeout(time.Minute).Should(gomega.BeTrueBecause("extended resource claim should be created")) @@ -2154,7 +2185,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ginkgo.By("Verifying extended resource claim is cleaned up") gomega.Eventually(ctx, func(ctx context.Context) bool { - _, err := b.ClientV1().ResourceClaims(pod.Namespace).Get(ctx, extendedResourceClaim.Name, metav1.GetOptions{}) + _, err := b.ClientV1(tCtx).ResourceClaims(pod.Namespace).Get(ctx, extendedResourceClaim.Name, metav1.GetOptions{}) return apierrors.IsNotFound(err) }).WithTimeout(time.Minute).Should(gomega.BeTrueBecause("extended resource claim should be automatically deleted when pod %s", cleanupMessage)) } @@ -2166,6 +2197,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { b.UseExtendedResourceName = true ginkgo.It("must run a pod with extended resource with resource quota", func(ctx context.Context) { + tCtx := f.TContext(ctx) hard := v1.ResourceList{ v1.ResourceName("count/resourceclaims.resource.k8s.io"): resource.MustParse("10"), v1.ResourceName(fmt.Sprintf("requests.%s", b.ExtendedResourceName(0))): resource.MustParse("10"), @@ -2179,7 +2211,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { Hard: hard, }, } - b.Create(ctx, quota) + b.Create(tCtx, quota) pod := b.Pod() res := v1.ResourceList{} @@ -2209,19 +2241,19 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { pod.Spec.InitContainers[2].Resources.Requests = res pod.Spec.InitContainers[2].Resources.Limits = res - b.Create(ctx, pod) + b.Create(tCtx, pod) err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) framework.ExpectNoError(err, "start pod") containerEnv := []string{ "container_3_request_0", "true", "container_3_request_1", "true", } - drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[0].Name, false, containerEnv...) + drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...) claim := b.ExternalClaim() pod2 := b.PodExternal() - b.Create(ctx, claim, pod2) - b.TestPod(ctx, f, pod2) + b.Create(tCtx, claim, pod2) + b.TestPod(tCtx, pod2) // Expect 5 extended and implicit devices / requests resources to be consumed: // @@ -2282,13 +2314,14 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) ginkgo.It("must run a pod with extended resource with one container three resources", func(ctx context.Context) { + tCtx := f.TContext(ctx) var objects []klog.KMetadata for i := range 3 { if i > 0 { objects = append(objects, b.Class(i)) } } - b.Create(ctx, objects...) + b.Create(tCtx, objects...) extendedResourceTest(ctx, b, f, []string{ b.ExtendedResourceName(0), b.ExtendedResourceName(1), @@ -2300,6 +2333,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) }) ginkgo.It("must run a pod with extended resource with three containers one resource each", func(ctx context.Context) { + tCtx := f.TContext(ctx) var objects []klog.KMetadata pod := b.Pod() pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy()) @@ -2319,17 +2353,18 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { } } - b.Create(ctx, objects...) + b.Create(tCtx, objects...) err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) framework.ExpectNoError(err, "start pod") for i := range 3 { containerEnv := []string{ fmt.Sprintf("container_%d_request_0", i), "true", } - drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[i].Name, false, containerEnv...) + drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[i].Name, false, containerEnv...) } }) ginkgo.It("must run a pod with extended resource with three containers multiple resources each", func(ctx context.Context) { + tCtx := f.TContext(ctx) var objects []klog.KMetadata pod := b.Pod() pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy()) @@ -2358,24 +2393,24 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { } objects = append(objects, pod) - b.Create(ctx, objects...) + b.Create(tCtx, objects...) err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) framework.ExpectNoError(err, "start pod") containerEnv := []string{ "container_0_request_0", "true", } - drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[0].Name, false, containerEnv...) + drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...) containerEnv = []string{ "container_1_request_0", "true", "container_1_request_1", "true", } - drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[1].Name, false, containerEnv...) + drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[1].Name, false, containerEnv...) containerEnv = []string{ "container_2_request_0", "true", "container_2_request_1", "true", "container_2_request_2", "true", } - drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[2].Name, false, containerEnv...) + drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[2].Name, false, containerEnv...) }) ginkgo.It("must cleanup extended resource claims when pods complete", func(ctx context.Context) { @@ -2400,6 +2435,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { ) }) f.It("process extended resources after device plugin uninstall", f.WithSerial(), func(ctx context.Context) { + tCtx := f.TContext(ctx) resourceName := b.ExtendedResourceName(drautils.SingletonIndex) extendedResourceName := deployDevicePlugin(ctx, f, nodes.NodeNames[0:1]) gomega.Expect(string(extendedResourceName)).To(gomega.Equal(resourceName)) @@ -2436,7 +2472,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { pod.Spec.Containers[0].Resources.Requests = res pod.Spec.Containers[0].Resources.Limits = res - b.Create(ctx, b.Class(drautils.SingletonIndex), pod) + b.Create(tCtx, b.Class(drautils.SingletonIndex), pod) err = e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) framework.ExpectNoError(err, "start pod") @@ -2445,10 +2481,11 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { containerEnv := []string{ "container_0_request_0", "true", } - drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[0].Name, false, containerEnv...) + drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...) }) ginkgo.It("must prevent overcommitment of extended resources", func(ctx context.Context) { + tCtx := f.TContext(ctx) // Create first pod consuming all available resources pod1 := b.Pod() res := v1.ResourceList{ @@ -2456,7 +2493,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { } pod1.Spec.Containers[0].Resources.Requests = res pod1.Spec.Containers[0].Resources.Limits = res - b.Create(ctx, pod1) + b.Create(tCtx, pod1) err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod1) framework.ExpectNoError(err, "start pod") @@ -2464,21 +2501,21 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { containerEnv := []string{ "container_0_request_0", "true", } - drautils.TestContainerEnv(ctx, f, pod1, pod1.Spec.Containers[0].Name, false, containerEnv...) + drautils.TestContainerEnv(tCtx, pod1, pod1.Spec.Containers[0].Name, false, containerEnv...) // Second pod should remain unschedulable pod2 := b.Pod() pod2.Spec.Containers[0].Resources.Requests = res pod2.Spec.Containers[0].Resources.Limits = res - b.Create(ctx, pod2) + b.Create(tCtx, pod2) framework.ExpectNoError(e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod2.Name, f.Namespace.Name)) // After deleting pod1, pod2 should be schedulable and run successfully - b.DeletePodAndWaitForNotFound(ctx, pod1) + b.DeletePodAndWaitForNotFound(tCtx, pod1) err = e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod2) framework.ExpectNoError(err, "start pod") - drautils.TestContainerEnv(ctx, f, pod2, pod2.Spec.Containers[0].Name, false, containerEnv...) + drautils.TestContainerEnv(tCtx, pod2, pod2.Spec.Containers[0].Name, false, containerEnv...) }) ginkgo.It("must reject pod with invalid extended resource name", func(ctx context.Context) { @@ -2495,6 +2532,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) ginkgo.It("must accurately populate ExtendedResourceClaimStatus", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod := b.Pod() resource0 := b.ExtendedResourceName(0) resource1 := b.ExtendedResourceName(1) @@ -2505,7 +2543,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { pod.Spec.Containers[0].Resources.Requests = res pod.Spec.Containers[0].Resources.Limits = res - b.Create(ctx, b.Class(1), pod) + b.Create(tCtx, b.Class(1), pod) err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) framework.ExpectNoError(err, "start pod") @@ -2514,7 +2552,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { "container_0_request_0", "true", "container_0_request_1", "true", } - drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[0].Name, false, containerEnv...) + drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...) // Verify status updatedPod, err := f.ClientSet.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) @@ -2535,6 +2573,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) f.It("must run a pod with extended resource requesting zero", func(ctx context.Context) { + tCtx := f.TContext(ctx) + pod := b.Pod() res := v1.ResourceList{ v1.ResourceName(b.ExtendedResourceName(0)): resource.MustParse("0"), @@ -2542,7 +2582,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { pod.Spec.Containers[0].Resources.Requests = res pod.Spec.Containers[0].Resources.Limits = res - b.Create(ctx, pod) + b.Create(tCtx, pod) err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) framework.ExpectNoError(err, "start pod") @@ -2553,26 +2593,27 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) f.It("must process extended resource after Device Class creation", func(ctx context.Context) { + tCtx := f.TContext(ctx) resourceName := b.ExtendedResourceName(1) res := v1.ResourceList{v1.ResourceName(resourceName): resource.MustParse("1")} pod := b.Pod() pod.Spec.Containers[0].Resources.Requests = res pod.Spec.Containers[0].Resources.Limits = res - b.Create(ctx, pod) + b.Create(tCtx, pod) err := e2epod.WaitForPodNameUnschedulableInNamespace(ctx, f.ClientSet, pod.Name, f.Namespace.Name) framework.ExpectNoError(err, "pod should be unschedulable before device class creation") ginkgo.By("Creating Device Class after pod creation") - b.Create(ctx, b.Class(1)) + b.Create(tCtx, b.Class(1)) err = e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) framework.ExpectNoError(err, "start pod") ginkgo.By("Check that pod is processed by the DRA driver") containerEnv := []string{"container_0_request_0", "true"} - drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[0].Name, false, containerEnv...) + drautils.TestContainerEnv(tCtx, pod, pod.Spec.Containers[0].Name, false, containerEnv...) }) }) @@ -2588,6 +2629,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { // The test runs two pods, one pod request extended resource backed by DRA, // the other pod requests extended resource by device plugin. f.It("must run pods with extended resource on dra nodes and device plugin nodes", f.WithSerial(), func(ctx context.Context) { + tCtx := f.TContext(ctx) var objects []klog.KMetadata extendedResourceName := deployDevicePlugin(ctx, f, nodes.ExtraNodeNames) // b.ExtendedResourceName(SingletonIndex) must be the same as the returned extendedResourceName. @@ -2607,7 +2649,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { pod2.Spec.Containers[0].Resources.Limits = res objects = append(objects, pod2) - b.Create(ctx, objects...) + b.Create(tCtx, objects...) err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod1) framework.ExpectNoError(err, "start pod1") @@ -2635,7 +2677,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { containerEnv := []string{ "container_0_request_0", "true", } - drautils.TestContainerEnv(ctx, f, draPod, draPod.Spec.Containers[0].Name, false, containerEnv...) + drautils.TestContainerEnv(tCtx, draPod, draPod.Spec.Containers[0].Name, false, containerEnv...) }) }) @@ -2790,13 +2832,14 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) f.It("truncates the name of a generated resource claim", func(ctx context.Context) { + tCtx := f.TContext(ctx) pod, template := b.PodInline() pod.Name = strings.Repeat("p", 63) pod.Spec.ResourceClaims[0].Name = strings.Repeat("c", 63) pod.Spec.Containers[0].Resources.Claims[0].Name = pod.Spec.ResourceClaims[0].Name - b.Create(ctx, template, pod) + b.Create(tCtx, template, pod) - b.TestPod(ctx, f, pod) + b.TestPod(tCtx, pod) }) f.It("supports count/resourceclaims.resource.k8s.io ResourceQuota", func(ctx context.Context) { @@ -2854,12 +2897,13 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }) f.It("must be possible for the driver to update the ResourceClaim.Status.Devices once allocated", f.WithFeatureGate(features.DRAResourceClaimDeviceStatus), func(ctx context.Context) { + tCtx := f.TContext(ctx) pod := b.PodExternal() claim := b.ExternalClaim() - b.Create(ctx, claim, pod) + b.Create(tCtx, claim, pod) // Waits for the ResourceClaim to be allocated and the pod to be scheduled. - b.TestPod(ctx, f, pod) + b.TestPod(tCtx, pod) allocatedResourceClaim, err := f.ClientSet.ResourceV1().ResourceClaims(f.Namespace.Name).Get(ctx, claim.Name, metav1.GetOptions{}) framework.ExpectNoError(err) @@ -2932,12 +2976,13 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { driver.WithKubelet = false f.It("must apply per-node permission checks", func(ctx context.Context) { + tCtx := f.TContext(ctx) // All of the operations use the client set of a kubelet plugin for // a fictional node which both don't exist, so nothing interferes // when we actually manage to create a slice. fictionalNodeName := "dra-fictional-node" gomega.Expect(nodes.NodeNames).NotTo(gomega.ContainElement(fictionalNodeName)) - fictionalNodeClient := driver.ImpersonateKubeletPlugin(&v1.Pod{ + fictionalNodeClient := driver.ImpersonateKubeletPlugin(tCtx, &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: fictionalNodeName + "-dra-plugin", Namespace: f.Namespace.Name, @@ -3089,6 +3134,7 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { b2 := drautils.NewBuilder(f, driver2) ginkgo.It("work", func(ctx context.Context) { + tCtx := f.TContext(ctx) claim1 := b1.ExternalClaim() claim1b := b1.ExternalClaim() claim2 := b2.ExternalClaim() @@ -3103,8 +3149,8 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() { }, ) } - b1.Create(ctx, claim1, claim1b, claim2, claim2b, pod) - b1.TestPod(ctx, f, pod) + b1.Create(tCtx, claim1, claim1b, claim2, claim2b, pod) + b1.TestPod(tCtx, pod) }) } multipleDriversContext := func(prefix string, nodeV1beta1, nodeV1 bool) { diff --git a/test/e2e/dra/utils/builder.go b/test/e2e/dra/utils/builder.go index 147e051e707..b5c2c7f3a5f 100644 --- a/test/e2e/dra/utils/builder.go +++ b/test/e2e/dra/utils/builder.go @@ -42,8 +42,10 @@ import ( draclient "k8s.io/dynamic-resource-allocation/client" "k8s.io/dynamic-resource-allocation/resourceslice" "k8s.io/klog/v2" + "k8s.io/kubernetes/test/e2e/dra/test-driver/app" "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" + "k8s.io/kubernetes/test/utils/ktesting" admissionapi "k8s.io/pod-security-admission/api" "k8s.io/utils/ptr" ) @@ -64,21 +66,22 @@ func (b *Builder) ExtendedResourceName(i int) string { } } -// Builder contains a running counter to make objects unique within thir +// Builder contains a running counter to make objects unique within their // namespace. type Builder struct { - f *framework.Framework + namespace string driver *Driver UseExtendedResourceName bool podCounter int claimCounter int ClassParameters string // JSON + SkipCleanup bool } // ClassName returns the default device class name. func (b *Builder) ClassName() string { - return b.f.UniqueName + b.driver.NameSuffix + "-class" + return b.namespace + b.driver.NameSuffix + "-class" } // SingletonIndex causes Builder.Class and ExtendedResourceName to create a @@ -243,7 +246,7 @@ func (b *Builder) Pod() *v1.Pod { // // It is tempting to use `terminationGraceperiodSeconds: 0`, but that is a very bad // idea because it removes the pod before the kubelet had a chance to react (https://github.com/kubernetes/kubernetes/issues/120671). - pod := e2epod.MakePod(b.f.Namespace.Name, nil, nil, admissionapi.LevelRestricted, "" /* no command = pause */) + pod := e2epod.MakePod(b.namespace, nil, nil, admissionapi.LevelRestricted, "" /* no command = pause */) pod.Labels = make(map[string]string) pod.Spec.RestartPolicy = v1.RestartPolicyNever pod.GenerateName = "" @@ -338,100 +341,108 @@ func (b *Builder) PodExternalMultiple() *v1.Pod { } // Create takes a bunch of objects and calls their Create function. -func (b *Builder) Create(ctx context.Context, objs ...klog.KMetadata) []klog.KMetadata { +func (b *Builder) Create(tCtx ktesting.TContext, objs ...klog.KMetadata) []klog.KMetadata { + tCtx.Helper() + cleanupCtx := tCtx.CleanupCtx + if b.SkipCleanup { + cleanupCtx = func(func(tCtx ktesting.TContext)) {} + } + var createdObjs []klog.KMetadata for _, obj := range objs { - ginkgo.By(fmt.Sprintf("creating %T %s", obj, obj.GetName())) + tCtx.Logf("Creating %T %s", obj, obj.GetName()) var err error var createdObj klog.KMetadata switch obj := obj.(type) { case *resourceapi.DeviceClass: - createdObj, err = b.ClientV1().DeviceClasses().Create(ctx, obj, metav1.CreateOptions{}) - ginkgo.DeferCleanup(func(ctx context.Context) { - err := b.ClientV1().DeviceClasses().Delete(ctx, createdObj.GetName(), metav1.DeleteOptions{}) - framework.ExpectNoError(err, "delete device class") + createdObj, err = b.ClientV1(tCtx).DeviceClasses().Create(tCtx, obj, metav1.CreateOptions{}) + cleanupCtx(func(tCtx ktesting.TContext) { + err := b.ClientV1(tCtx).DeviceClasses().Delete(tCtx, createdObj.GetName(), metav1.DeleteOptions{}) + tCtx.ExpectNoError(err, "delete device class") }) case *v1.Pod: - createdObj, err = b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + createdObj, err = tCtx.Client().CoreV1().Pods(b.namespace).Create(tCtx, obj, metav1.CreateOptions{}) case *v1.ResourceQuota: - createdObj, err = b.f.ClientSet.CoreV1().ResourceQuotas(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + createdObj, err = tCtx.Client().CoreV1().ResourceQuotas(b.namespace).Create(tCtx, obj, metav1.CreateOptions{}) case *v1.ConfigMap: - createdObj, err = b.f.ClientSet.CoreV1().ConfigMaps(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + createdObj, err = tCtx.Client().CoreV1().ConfigMaps(b.namespace).Create(tCtx, obj, metav1.CreateOptions{}) case *resourceapi.ResourceClaim: - createdObj, err = b.ClientV1().ResourceClaims(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + createdObj, err = b.ClientV1(tCtx).ResourceClaims(b.namespace).Create(tCtx, obj, metav1.CreateOptions{}) case *resourcev1beta1.ResourceClaim: - createdObj, err = b.f.ClientSet.ResourceV1beta1().ResourceClaims(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + createdObj, err = tCtx.Client().ResourceV1beta1().ResourceClaims(b.namespace).Create(tCtx, obj, metav1.CreateOptions{}) case *resourcev1beta2.ResourceClaim: - createdObj, err = b.f.ClientSet.ResourceV1beta2().ResourceClaims(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + createdObj, err = tCtx.Client().ResourceV1beta2().ResourceClaims(b.namespace).Create(tCtx, obj, metav1.CreateOptions{}) case *resourceapi.ResourceClaimTemplate: - createdObj, err = b.ClientV1().ResourceClaimTemplates(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + createdObj, err = b.ClientV1(tCtx).ResourceClaimTemplates(b.namespace).Create(tCtx, obj, metav1.CreateOptions{}) case *resourcev1beta1.ResourceClaimTemplate: - createdObj, err = b.f.ClientSet.ResourceV1beta1().ResourceClaimTemplates(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + createdObj, err = tCtx.Client().ResourceV1beta1().ResourceClaimTemplates(b.namespace).Create(tCtx, obj, metav1.CreateOptions{}) case *resourcev1beta2.ResourceClaimTemplate: - createdObj, err = b.f.ClientSet.ResourceV1beta2().ResourceClaimTemplates(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + createdObj, err = tCtx.Client().ResourceV1beta2().ResourceClaimTemplates(b.namespace).Create(tCtx, obj, metav1.CreateOptions{}) case *resourceapi.ResourceSlice: - createdObj, err = b.ClientV1().ResourceSlices().Create(ctx, obj, metav1.CreateOptions{}) - ginkgo.DeferCleanup(func(ctx context.Context) { - err := b.ClientV1().ResourceSlices().Delete(ctx, createdObj.GetName(), metav1.DeleteOptions{}) - framework.ExpectNoError(err, "delete node resource slice") + createdObj, err = b.ClientV1(tCtx).ResourceSlices().Create(tCtx, obj, metav1.CreateOptions{}) + cleanupCtx(func(tCtx ktesting.TContext) { + err := b.ClientV1(tCtx).ResourceSlices().Delete(tCtx, createdObj.GetName(), metav1.DeleteOptions{}) + tCtx.ExpectNoError(err, "delete node resource slice") }) case *resourcealphaapi.DeviceTaintRule: - createdObj, err = b.f.ClientSet.ResourceV1alpha3().DeviceTaintRules().Create(ctx, obj, metav1.CreateOptions{}) - ginkgo.DeferCleanup(func(ctx context.Context) { - err := b.f.ClientSet.ResourceV1alpha3().DeviceTaintRules().Delete(ctx, createdObj.GetName(), metav1.DeleteOptions{}) - framework.ExpectNoError(err, "delete DeviceTaintRule") + createdObj, err = tCtx.Client().ResourceV1alpha3().DeviceTaintRules().Create(tCtx, obj, metav1.CreateOptions{}) + cleanupCtx(func(tCtx ktesting.TContext) { + err := tCtx.Client().ResourceV1alpha3().DeviceTaintRules().Delete(tCtx, createdObj.GetName(), metav1.DeleteOptions{}) + tCtx.ExpectNoError(err, "delete DeviceTaintRule") }) case *appsv1.DaemonSet: - createdObj, err = b.f.ClientSet.AppsV1().DaemonSets(b.f.Namespace.Name).Create(ctx, obj, metav1.CreateOptions{}) + createdObj, err = tCtx.Client().AppsV1().DaemonSets(b.namespace).Create(tCtx, obj, metav1.CreateOptions{}) // Cleanup not really needed, but speeds up namespace shutdown. - ginkgo.DeferCleanup(func(ctx context.Context) { - err := b.f.ClientSet.AppsV1().DaemonSets(b.f.Namespace.Name).Delete(ctx, obj.Name, metav1.DeleteOptions{}) - framework.ExpectNoError(err, "delete daemonset") + cleanupCtx(func(tCtx ktesting.TContext) { + err := tCtx.Client().AppsV1().DaemonSets(b.namespace).Delete(tCtx, obj.Name, metav1.DeleteOptions{}) + tCtx.ExpectNoError(err, "delete daemonset") }) default: - framework.Fail(fmt.Sprintf("internal error, unsupported type %T", obj), 1) + tCtx.Fatalf("internal error, unsupported type %T", obj) } - framework.ExpectNoErrorWithOffset(1, err, "create %T", obj) + tCtx.ExpectNoError(err, "create %T", obj) createdObjs = append(createdObjs, createdObj) } return createdObjs } -func (b *Builder) DeletePodAndWaitForNotFound(ctx context.Context, pod *v1.Pod) { - err := b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{}) - framework.ExpectNoErrorWithOffset(1, err, "delete %T", pod) - err = e2epod.WaitForPodNotFoundInNamespace(ctx, b.f.ClientSet, pod.Name, pod.Namespace, b.f.Timeouts.PodDelete) - framework.ExpectNoErrorWithOffset(1, err, "terminate %T", pod) +func (b *Builder) DeletePodAndWaitForNotFound(tCtx ktesting.TContext, pod *v1.Pod) { + tCtx.Helper() + err := tCtx.Client().CoreV1().Pods(b.namespace).Delete(tCtx, pod.Name, metav1.DeleteOptions{}) + tCtx.ExpectNoError(err, "delete %T", pod) + /* TODO: add timeouts to TContext? */ + err = e2epod.WaitForPodNotFoundInNamespace(tCtx, tCtx.Client(), pod.Name, pod.Namespace, 5*time.Minute /* former b.f.Timeouts.PodDelete */) + tCtx.ExpectNoError(err, "terminate %T", pod) } // TestPod runs pod and checks if container logs contain expected environment variables -func (b *Builder) TestPod(ctx context.Context, f *framework.Framework, pod *v1.Pod, env ...string) { - ginkgo.GinkgoHelper() +func (b *Builder) TestPod(tCtx ktesting.TContext, pod *v1.Pod, env ...string) { + tCtx.Helper() if !b.driver.WithKubelet { // Less testing when we cannot rely on the kubelet to actually run the pod. - err := e2epod.WaitForPodScheduled(ctx, f.ClientSet, pod.Namespace, pod.Name) - framework.ExpectNoError(err, "schedule pod") + err := e2epod.WaitForPodScheduled(tCtx, tCtx.Client(), pod.Namespace, pod.Name) + tCtx.ExpectNoError(err, "schedule pod") return } - err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod) - framework.ExpectNoError(err, "start pod") + err := e2epod.WaitForPodRunningInNamespace(tCtx, tCtx.Client(), pod) + tCtx.ExpectNoError(err, "start pod") if len(env) == 0 { _, env = b.ParametersEnv() } for _, container := range pod.Spec.Containers { - TestContainerEnv(ctx, f, pod, container.Name, false, env...) + TestContainerEnv(tCtx, pod, container.Name, false, env...) } } // envLineRE matches env output with variables set by test/e2e/dra/test-driver. var envLineRE = regexp.MustCompile(`^(?:admin|user|claim)_[a-zA-Z0-9_]*=.*$`) -func TestContainerEnv(ctx context.Context, f *framework.Framework, pod *v1.Pod, containerName string, fullMatch bool, env ...string) { - ginkgo.GinkgoHelper() - stdout, stderr, err := e2epod.ExecWithOptionsContext(ctx, f, e2epod.ExecOptions{ +func TestContainerEnv(tCtx ktesting.TContext, pod *v1.Pod, containerName string, fullMatch bool, env ...string) { + tCtx.Helper() + stdout, stderr, err := e2epod.ExecWithOptionsTCtx(tCtx, e2epod.ExecOptions{ Command: []string{"env"}, Namespace: pod.Namespace, PodName: pod.Name, @@ -440,8 +451,8 @@ func TestContainerEnv(ctx context.Context, f *framework.Framework, pod *v1.Pod, CaptureStderr: true, Quiet: true, }) - framework.ExpectNoError(err, fmt.Sprintf("get env output for container %s", containerName)) - gomega.Expect(stderr).To(gomega.BeEmpty(), fmt.Sprintf("env stderr for container %s", containerName)) + tCtx.ExpectNoError(err, fmt.Sprintf("get env output for container %s", containerName)) + tCtx.Expect(stderr).To(gomega.BeEmpty(), fmt.Sprintf("env stderr for container %s", containerName)) if fullMatch { // Find all env variables set by the test driver. var actualEnv, expectEnv []string @@ -455,91 +466,96 @@ func TestContainerEnv(ctx context.Context, f *framework.Framework, pod *v1.Pod, } sort.Strings(actualEnv) sort.Strings(expectEnv) - gomega.Expect(actualEnv).To(gomega.Equal(expectEnv), fmt.Sprintf("container %s env output:\n%s", containerName, stdout)) + tCtx.Expect(actualEnv).To(gomega.Equal(expectEnv), fmt.Sprintf("container %s env output:\n%s", containerName, stdout)) } else { for i := 0; i < len(env); i += 2 { envStr := fmt.Sprintf("%s=%s\n", env[i], env[i+1]) - gomega.Expect(stdout).To(gomega.ContainSubstring(envStr), fmt.Sprintf("container %s env variables", containerName)) + tCtx.Expect(stdout).To(gomega.ContainSubstring(envStr), fmt.Sprintf("container %s env variables", containerName)) } } } func NewBuilder(f *framework.Framework, driver *Driver) *Builder { - b := &Builder{f: f, driver: driver} - ginkgo.BeforeEach(b.setUp) + b := &Builder{driver: driver} + ginkgo.BeforeEach(func() { + b.setUp(f.TContext(context.Background())) + }) return b } -func NewBuilderNow(ctx context.Context, f *framework.Framework, driver *Driver) *Builder { - b := &Builder{f: f, driver: driver} - b.setUp(ctx) +func NewBuilderNow(tCtx ktesting.TContext, driver *Driver) *Builder { + b := &Builder{driver: driver} + b.setUp(tCtx) return b } -func (b *Builder) setUp(ctx context.Context) { +func (b *Builder) setUp(tCtx ktesting.TContext) { + b.namespace = tCtx.Namespace() b.podCounter = 0 b.claimCounter = 0 - b.Create(ctx, b.Class(0)) - ginkgo.DeferCleanup(b.tearDown) + b.Create(tCtx, b.Class(0)) + tCtx.CleanupCtx(b.tearDown) } // ClientV1 returns a wrapper for client-go which provides the V1 API on top of whatever is enabled in the cluster. -func (b *Builder) ClientV1() cgoresource.ResourceV1Interface { - return draclient.New(b.f.ClientSet) +func (b *Builder) ClientV1(tCtx ktesting.TContext) cgoresource.ResourceV1Interface { + return draclient.New(tCtx.Client()) } -func (b *Builder) tearDown(ctx context.Context) { +func (b *Builder) tearDown(tCtx ktesting.TContext) { + client := b.ClientV1(tCtx) + // Before we allow the namespace and all objects in it do be deleted by // the framework, we must ensure that test pods and the claims that // they use are deleted. Otherwise the driver might get deleted first, // in which case deleting the claims won't work anymore. - ginkgo.By("delete pods and claims") - pods, err := b.listTestPods(ctx) - framework.ExpectNoError(err, "list pods") + tCtx.Log("delete pods and claims") + pods, err := b.listTestPods(tCtx) + tCtx.ExpectNoError(err, "list pods") for _, pod := range pods { if pod.DeletionTimestamp != nil { continue } - ginkgo.By(fmt.Sprintf("deleting %T %s", &pod, klog.KObj(&pod))) - err := b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).Delete(ctx, pod.Name, metav1.DeleteOptions{}) + tCtx.Logf("Deleting %T %s", &pod, klog.KObj(&pod)) + err := tCtx.Client().CoreV1().Pods(b.namespace).Delete(tCtx, pod.Name, metav1.DeleteOptions{}) if !apierrors.IsNotFound(err) { - framework.ExpectNoError(err, "delete pod") + tCtx.ExpectNoError(err, "delete pod") } } - gomega.Eventually(func() ([]v1.Pod, error) { - return b.listTestPods(ctx) + ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) []v1.Pod { + pods, err := b.listTestPods(tCtx) + tCtx.ExpectNoError(err) + return pods }).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "remaining pods despite deletion") - claims, err := b.ClientV1().ResourceClaims(b.f.Namespace.Name).List(ctx, metav1.ListOptions{}) - framework.ExpectNoError(err, "get resource claims") + claims, err := b.ClientV1(tCtx).ResourceClaims(b.namespace).List(tCtx, metav1.ListOptions{}) + tCtx.ExpectNoError(err, "get resource claims") for _, claim := range claims.Items { if claim.DeletionTimestamp != nil { continue } - ginkgo.By(fmt.Sprintf("deleting %T %s", &claim, klog.KObj(&claim))) - err := b.ClientV1().ResourceClaims(b.f.Namespace.Name).Delete(ctx, claim.Name, metav1.DeleteOptions{}) + tCtx.Logf("Deleting %T %s", &claim, klog.KObj(&claim)) + err := client.ResourceClaims(b.namespace).Delete(tCtx, claim.Name, metav1.DeleteOptions{}) if !apierrors.IsNotFound(err) { - framework.ExpectNoError(err, "delete claim") + tCtx.ExpectNoError(err, "delete claim") } } for host, plugin := range b.driver.Nodes { - ginkgo.By(fmt.Sprintf("waiting for resources on %s to be unprepared", host)) - gomega.Eventually(plugin.GetPreparedResources).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "prepared claims on host %s", host) + tCtx.Logf("Waiting for resources on %s to be unprepared", host) + ktesting.Eventually(tCtx, func(ktesting.TContext) []app.ClaimID { return plugin.GetPreparedResources() }).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "prepared claims on host %s", host) } - ginkgo.By("waiting for claims to be deallocated and deleted") - gomega.Eventually(func() ([]resourceapi.ResourceClaim, error) { - claims, err := b.ClientV1().ResourceClaims(b.f.Namespace.Name).List(ctx, metav1.ListOptions{}) - if err != nil { - return nil, err - } - return claims.Items, nil + tCtx.Log("waiting for claims to be deallocated and deleted") + ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) []resourceapi.ResourceClaim { + claims, err := client.ResourceClaims(tCtx.Namespace()).List(tCtx, metav1.ListOptions{}) + tCtx.ExpectNoError(err) + return claims.Items }).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "claims in the namespaces") } -func (b *Builder) listTestPods(ctx context.Context) ([]v1.Pod, error) { - pods, err := b.f.ClientSet.CoreV1().Pods(b.f.Namespace.Name).List(ctx, metav1.ListOptions{}) +func (b *Builder) listTestPods(tCtx ktesting.TContext) ([]v1.Pod, error) { + pods, err := tCtx.Client().CoreV1().Pods(b.namespace).List(tCtx, metav1.ListOptions{}) if err != nil { return nil, err } diff --git a/test/e2e/dra/utils/deploy.go b/test/e2e/dra/utils/deploy.go index ef2afa55279..776cced2efc 100644 --- a/test/e2e/dra/utils/deploy.go +++ b/test/e2e/dra/utils/deploy.go @@ -75,6 +75,7 @@ import ( e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper" "k8s.io/kubernetes/test/e2e/storage/drivers/proxy" "k8s.io/kubernetes/test/e2e/storage/utils" + "k8s.io/kubernetes/test/utils/ktesting" "k8s.io/utils/clock" "k8s.io/utils/ptr" "sigs.k8s.io/yaml" @@ -102,30 +103,31 @@ type Nodes struct { func NewNodes(f *framework.Framework, minNodes, maxNodes int) *Nodes { nodes := &Nodes{} ginkgo.BeforeEach(func(ctx context.Context) { - nodes.init(ctx, f, minNodes, maxNodes) + nodes.init(f.TContext(ctx), minNodes, maxNodes) }) return nodes } -// NewNodesNow is a variant of NewNodes which can be used inside a ginkgo.It. -func NewNodesNow(ctx context.Context, f *framework.Framework, minNodes, maxNodes int) *Nodes { +// NewNodesNow is a variant of NewNodes which can be used inside a ginkgo.It +// or a Go unit test. +func NewNodesNow(tCtx ktesting.TContext, minNodes, maxNodes int) *Nodes { nodes := &Nodes{} - nodes.init(ctx, f, minNodes, maxNodes) + nodes.init(tCtx, minNodes, maxNodes) return nodes } -func (nodes *Nodes) init(ctx context.Context, f *framework.Framework, minNodes, maxNodes int) { - nodes.tempDir = ginkgo.GinkgoT().TempDir() +func (nodes *Nodes) init(tCtx ktesting.TContext, minNodes, maxNodes int) { + nodes.tempDir = tCtx.TempDir() - ginkgo.By("selecting nodes") + tCtx.Log("selecting nodes") // The kubelet plugin is harder. We deploy the builtin manifest // after patching in the driver name and all nodes on which we // want the plugin to run. // // Only a subset of the nodes are picked to avoid causing // unnecessary load on a big cluster. - nodeList, err := e2enode.GetBoundedReadySchedulableNodes(ctx, f.ClientSet, maxNodes) - framework.ExpectNoError(err, "get nodes") + nodeList, err := e2enode.GetBoundedReadySchedulableNodes(tCtx, tCtx.Client(), maxNodes) + tCtx.ExpectNoError(err, "get nodes") numNodes := int32(len(nodeList.Items)) if int(numNodes) < minNodes { e2eskipper.Skipf("%d ready nodes required, only have %d", minNodes+nodes.NumReservedNodes, numNodes) @@ -139,17 +141,18 @@ func (nodes *Nodes) init(ctx context.Context, f *framework.Framework, minNodes, nodes.NodeNames = append(nodes.NodeNames, node.Name) } sort.Strings(nodes.NodeNames) - framework.Logf("testing on nodes %v", nodes.NodeNames) + tCtx.Logf("testing on nodes %v", nodes.NodeNames) // Watch claims in the namespace. This is useful for monitoring a test // and enables additional sanity checks. - resourceClaimLogger := klog.LoggerWithName(klog.FromContext(ctx), "ResourceClaimListWatch") + resourceClaimLogger := klog.LoggerWithName(klog.FromContext(tCtx), "ResourceClaimListWatch") var resourceClaimWatchCounter atomic.Int32 - resourceClient := draclient.New(f.ClientSet) + resourceClient := draclient.New(tCtx.Client()) claimInformer := cache.NewSharedIndexInformer( &cache.ListWatch{ ListWithContextFunc: func(ctx context.Context, options metav1.ListOptions) (runtime.Object, error) { - slices, err := resourceClient.ResourceClaims(f.Namespace.Name).List(ctx, options) + tCtx := tCtx.WithContext(ctx) + slices, err := resourceClient.ResourceClaims(tCtx.Namespace()).List(tCtx, options) if err == nil { resourceClaimLogger.Info("Listed ResourceClaims", "resourceAPI", resourceClient.CurrentAPI(), "numClaims", len(slices.Items), "listMeta", slices.ListMeta) } else { @@ -158,7 +161,8 @@ func (nodes *Nodes) init(ctx context.Context, f *framework.Framework, minNodes, return slices, err }, WatchFuncWithContext: func(ctx context.Context, options metav1.ListOptions) (watch.Interface, error) { - w, err := resourceClient.ResourceClaims(f.Namespace.Name).Watch(ctx, options) + tCtx := tCtx.WithContext(ctx) + w, err := resourceClient.ResourceClaims(tCtx.Namespace()).Watch(tCtx, options) if err == nil { resourceClaimLogger.Info("Started watching ResourceClaims", "resourceAPI", resourceClient.CurrentAPI()) wrapper := newWatchWrapper(klog.LoggerWithName(resourceClaimLogger, fmt.Sprintf("%d", resourceClaimWatchCounter.Load())), w) @@ -179,26 +183,23 @@ func (nodes *Nodes) init(ctx context.Context, f *framework.Framework, minNodes, ) cancelCtx, cancel := context.WithCancelCause(context.Background()) var wg sync.WaitGroup - ginkgo.DeferCleanup(func() { + tCtx.Cleanup(func() { cancel(errors.New("test has completed")) wg.Wait() }) _, err = claimInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(obj any) { - defer ginkgo.GinkgoRecover() claim := obj.(*resourceapi.ResourceClaim) resourceClaimLogger.Info("New claim", "claim", format.Object(claim, 0)) - validateClaim(claim) + validateClaim(tCtx, claim) }, UpdateFunc: func(oldObj, newObj any) { - defer ginkgo.GinkgoRecover() oldClaim := oldObj.(*resourceapi.ResourceClaim) newClaim := newObj.(*resourceapi.ResourceClaim) resourceClaimLogger.Info("Updated claim", "newClaim", format.Object(newClaim, 0), "diff", cmp.Diff(oldClaim, newClaim)) - validateClaim(newClaim) + validateClaim(tCtx, newClaim) }, DeleteFunc: func(obj any) { - defer ginkgo.GinkgoRecover() if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok { obj = tombstone.Obj } @@ -206,11 +207,11 @@ func (nodes *Nodes) init(ctx context.Context, f *framework.Framework, minNodes, resourceClaimLogger.Info("Deleted claim", "claim", format.Object(claim, 0)) }, }) - framework.ExpectNoError(err, "AddEventHandler") + tCtx.ExpectNoError(err, "AddEventHandler") wg.Add(1) go func() { defer wg.Done() - claimInformer.Run(cancelCtx.Done()) + claimInformer.RunWithContext(cancelCtx) }() } @@ -251,13 +252,13 @@ func (w *watchWrapper) ResultChan() <-chan watch.Event { return w.resultChan } -func validateClaim(claim *resourceapi.ResourceClaim) { +func validateClaim(tCtx ktesting.TContext, claim *resourceapi.ResourceClaim) { // The apiserver doesn't enforce that a claim always has a finalizer // while being allocated. This is a convention that whoever allocates a // claim has to follow to prevent using a claim that is at risk of // being deleted. if claim.Status.Allocation != nil && len(claim.Finalizers) == 0 { - framework.Failf("Invalid claim: allocated without any finalizer:\n%s", format.Object(claim, 1)) + tCtx.Errorf("Invalid claim: allocated without any finalizer:\n%s", format.Object(claim, 1)) } } @@ -282,23 +283,24 @@ type driverResourcesMutatorFunc func(map[string]resourceslice.DriverResources) // // Call this outside of ginkgo.It, then use the instance inside ginkgo.It. func NewDriver(f *framework.Framework, nodes *Nodes, driverResourcesGenerator driverResourcesGenFunc, driverResourcesMutators ...driverResourcesMutatorFunc) *Driver { - d := NewDriverInstance(f) + d := NewDriverInstance(nil) ginkgo.BeforeEach(func() { + tCtx := f.TContext(context.Background()) + d.initName(tCtx) driverResources := driverResourcesGenerator(nodes) for _, mutator := range driverResourcesMutators { mutator(driverResources) } - d.Run(nodes, driverResources) + d.Run(tCtx, framework.TestContext.KubeletRootDir, nodes, driverResources) }) return d } // NewDriverInstance is a variant of NewDriver where the driver is inactive and must -// be started explicitly with Run. May be used inside ginkgo.It. -func NewDriverInstance(f *framework.Framework) *Driver { +// be started explicitly with Run. May be used inside ginkgo.It or a Go unit test. +func NewDriverInstance(tCtx ktesting.TContext) *Driver { d := &Driver{ - f: f, fail: map[MethodInstance]bool{}, callCounts: map[MethodInstance]int64{}, // By default, test with all gRPC APIs. @@ -310,24 +312,30 @@ func NewDriverInstance(f *framework.Framework) *Driver { WithKubelet: true, ExpectResourceSliceRemoval: true, } - d.initName() + if tCtx != nil { + d.initName(tCtx) + } return d } // ClientV1 returns a wrapper for client-go which provides the V1 API on top of whatever is enabled in the cluster. -func (d *Driver) ClientV1() cgoresource.ResourceV1Interface { - return draclient.New(d.f.ClientSet) +func (d *Driver) ClientV1(tCtx ktesting.TContext) cgoresource.ResourceV1Interface { + return draclient.New(tCtx.Client()) } -func (d *Driver) Run(nodes *Nodes, driverResources map[string]resourceslice.DriverResources) { - d.SetUp(nodes, driverResources) - ginkgo.DeferCleanup(d.TearDown) +func (d *Driver) Run(tCtx ktesting.TContext, kubeletRootDir string, nodes *Nodes, driverResources map[string]resourceslice.DriverResources) { + d.SetUp(tCtx, kubeletRootDir, nodes, driverResources) + tCtx.CleanupCtx(d.TearDown) } -// NewGetSlices generates a function for gomega.Eventually/Consistently which +// NewGetSlices generates a function for ktesting.Eventually/Consistently which // returns the ResourceSliceList. -func (d *Driver) NewGetSlices() framework.GetFunc[*resourceapi.ResourceSliceList] { - return framework.ListObjects(d.ClientV1().ResourceSlices().List, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + d.Name}) +func (d *Driver) NewGetSlices() func(tCtx ktesting.TContext) *resourceapi.ResourceSliceList { + return func(tCtx ktesting.TContext) *resourceapi.ResourceSliceList { + slices, err := framework.ListObjects(d.ClientV1(tCtx).ResourceSlices().List, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + d.Name})(tCtx) + tCtx.ExpectNoError(err, "list ResourceSlices") + return slices + } } type MethodInstance struct { @@ -336,9 +344,7 @@ type MethodInstance struct { } type Driver struct { - f *framework.Framework - ctx context.Context - cleanup []func(context.Context) // executed first-in-first-out + cleanup []func(ktesting.TContext) // executed first-in-first-out wg sync.WaitGroup serviceAccountName string @@ -388,39 +394,37 @@ type KubeletPlugin struct { ClientSet kubernetes.Interface } -func (d *Driver) initName() { - d.Name = d.f.UniqueName + d.NameSuffix + ".k8s.io" +func (d *Driver) initName(tCtx ktesting.TContext) { + d.Name = tCtx.Namespace() + d.NameSuffix + ".k8s.io" } -func (d *Driver) SetUp(nodes *Nodes, driverResources map[string]resourceslice.DriverResources) { - d.initName() - ginkgo.By(fmt.Sprintf("deploying driver %s on nodes %v", d.Name, nodes.NodeNames)) +func (d *Driver) SetUp(tCtx ktesting.TContext, kubeletRootDir string, nodes *Nodes, driverResources map[string]resourceslice.DriverResources) { + tCtx.Logf("deploying driver %s on nodes %v", d.Name, nodes.NodeNames) d.Nodes = make(map[string]KubeletPlugin) - ctx, cancel := context.WithCancel(context.Background()) - logger := klog.FromContext(ctx) + tCtx = tCtx.WithCancel() + logger := klog.FromContext(tCtx) logger = klog.LoggerWithValues(logger, "driverName", d.Name) if d.InstanceSuffix != "" { instance, _ := strings.CutPrefix(d.InstanceSuffix, "-") logger = klog.LoggerWithValues(logger, "instance", instance) } - ctx = klog.NewContext(ctx, logger) - d.ctx = ctx - d.cleanup = append(d.cleanup, func(context.Context) { cancel() }) + tCtx = tCtx.WithLogger(logger) + d.cleanup = append(d.cleanup, func(ktesting.TContext) { tCtx.Cancel("cleaning up test") }) // After shutdown, check that all ResourceSlices were removed, either by the kubelet // or our own test code. This runs last because it gets registered first. if d.ExpectResourceSliceRemoval { - ginkgo.DeferCleanup(d.IsGone) + tCtx.CleanupCtx(d.IsGone) } driverResource, useMultiHostDriverResources := driverResources[multiHostDriverResources] if useMultiHostDriverResources || !d.WithKubelet { // We have to remove ResourceSlices ourselves. // Otherwise the kubelet does it after unregistering the driver. - ginkgo.DeferCleanup(func(ctx context.Context) { - err := d.f.ClientSet.ResourceV1().ResourceSlices().DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + d.Name}) - framework.ExpectNoError(err, "delete ResourceSlices of the driver") + tCtx.CleanupCtx(func(tCtx ktesting.TContext) { + err := tCtx.Client().ResourceV1().ResourceSlices().DeleteCollection(tCtx, metav1.DeleteOptions{}, metav1.ListOptions{FieldSelector: resourceapi.ResourceSliceSelectorDriver + "=" + d.Name}) + tCtx.ExpectNoError(err, "delete ResourceSlices of the driver") }) } @@ -445,8 +449,8 @@ func (d *Driver) SetUp(nodes *Nodes, driverResources map[string]resourceslice.Dr Devices: slice.Devices, }, } - _, err := d.f.ClientSet.ResourceV1().ResourceSlices().Create(ctx, resourceSlice, metav1.CreateOptions{}) - framework.ExpectNoError(err) + _, err := tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, resourceSlice, metav1.CreateOptions{}) + tCtx.ExpectNoError(err) } } } @@ -460,9 +464,9 @@ func (d *Driver) SetUp(nodes *Nodes, driverResources map[string]resourceslice.Dr // Create service account and corresponding RBAC rules. d.serviceAccountName = "dra-kubelet-plugin-" + d.Name + d.InstanceSuffix + "-service-account" content := example.PluginPermissions - content = strings.ReplaceAll(content, "dra-kubelet-plugin-namespace", d.f.Namespace.Name) + content = strings.ReplaceAll(content, "dra-kubelet-plugin-namespace", tCtx.Namespace()) content = strings.ReplaceAll(content, "dra-kubelet-plugin", "dra-kubelet-plugin-"+d.Name+d.InstanceSuffix) - d.createFromYAML(ctx, []byte(content), d.f.Namespace.Name) + d.createFromYAML(tCtx, []byte(content), tCtx.Namespace()) // Using a ReplicaSet instead of a DaemonSet has the advantage that we can control // the lifecycle explicitly, in particular run two pods per node long enough to @@ -470,10 +474,10 @@ func (d *Driver) SetUp(nodes *Nodes, driverResources map[string]resourceslice.Dr instanceKey := "app.kubernetes.io/instance" rsName := "" numNodes := int32(len(nodes.NodeNames)) - pluginDataDirectoryPath := path.Join(framework.TestContext.KubeletRootDir, "plugins", d.Name) - registrarDirectoryPath := path.Join(framework.TestContext.KubeletRootDir, "plugins_registry") + pluginDataDirectoryPath := path.Join(kubeletRootDir, "plugins", d.Name) + registrarDirectoryPath := path.Join(kubeletRootDir, "plugins_registry") instanceName := d.Name + d.InstanceSuffix - err := utils.CreateFromManifests(ctx, d.f, d.f.Namespace, func(item interface{}) error { + err := utils.CreateFromManifestsTCtx(tCtx, func(item interface{}) error { switch item := item.(type) { case *appsv1.ReplicaSet: item.Name += d.NameSuffix + d.InstanceSuffix @@ -514,21 +518,21 @@ func (d *Driver) SetUp(nodes *Nodes, driverResources map[string]resourceslice.Dr } return nil }, manifests...) - framework.ExpectNoError(err, "deploy kubelet plugin replicaset") + tCtx.ExpectNoError(err, "deploy kubelet plugin replicaset") - rs, err := d.f.ClientSet.AppsV1().ReplicaSets(d.f.Namespace.Name).Get(ctx, rsName, metav1.GetOptions{}) - framework.ExpectNoError(err, "get replicaset") + rs, err := tCtx.Client().AppsV1().ReplicaSets(tCtx.Namespace()).Get(tCtx, rsName, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get replicaset") // Wait for all pods to be running. - if err := e2ereplicaset.WaitForReplicaSetTargetAvailableReplicas(ctx, d.f.ClientSet, rs, numNodes); err != nil { - framework.ExpectNoError(err, "all kubelet plugin proxies running") + if err := e2ereplicaset.WaitForReplicaSetTargetAvailableReplicas(tCtx, tCtx.Client(), rs, numNodes); err != nil { + tCtx.ExpectNoError(err, "all kubelet plugin proxies running") } requirement, err := labels.NewRequirement(instanceKey, selection.Equals, []string{instanceName}) - framework.ExpectNoError(err, "create label selector requirement") + tCtx.ExpectNoError(err, "create label selector requirement") selector := labels.NewSelector().Add(*requirement) - pods, err := d.f.ClientSet.CoreV1().Pods(d.f.Namespace.Name).List(ctx, metav1.ListOptions{LabelSelector: selector.String()}) - framework.ExpectNoError(err, "list proxy pods") - gomega.Expect(numNodes).To(gomega.Equal(int32(len(pods.Items))), "number of proxy pods") + pods, err := tCtx.Client().CoreV1().Pods(tCtx.Namespace()).List(tCtx, metav1.ListOptions{LabelSelector: selector.String()}) + tCtx.ExpectNoError(err, "list proxy pods") + tCtx.Expect(numNodes).To(gomega.Equal(int32(len(pods.Items))), "number of proxy pods") sort.Slice(pods.Items, func(i, j int) bool { return pods.Items[i].Spec.NodeName < pods.Items[j].Spec.NodeName }) @@ -550,13 +554,13 @@ func (d *Driver) SetUp(nodes *Nodes, driverResources map[string]resourceslice.Dr // https://github.com/kubernetes/kubernetes/pull/124711). // // Here we merely use impersonation, which is faster. - driverClient := d.ImpersonateKubeletPlugin(&pod) + driverClient := d.ImpersonateKubeletPlugin(tCtx, &pod) logger := klog.LoggerWithValues(klog.LoggerWithName(logger, "kubelet-plugin"), "node", pod.Spec.NodeName, "pod", klog.KObj(&pod)) - loggerCtx := klog.NewContext(ctx, logger) + loggerCtx := klog.NewContext(tCtx, logger) fileOps := app.FileOperations{ Create: func(name string, content []byte) error { - klog.Background().Info("creating CDI file", "node", nodename, "filename", name, "content", string(content)) + logger.Info("creating CDI file", "node", nodename, "filename", name, "content", string(content)) if d.IsLocal { // Name starts with /cdi, which is how it is mapped in the container. // Here we need it under /var/run. @@ -570,19 +574,21 @@ func (d *Driver) SetUp(nodes *Nodes, driverResources map[string]resourceslice.Dr } return nil } - return d.createFile(&pod, name, content) + return d.createFile(tCtx, &pod, name, content) }, Remove: func(name string) error { - klog.Background().Info("deleting CDI file", "node", nodename, "filename", name) + logger.Info("deleting CDI file", "node", nodename, "filename", name) if d.IsLocal { name = path.Join("/var/run", name) return os.Remove(name) } - return d.removeFile(&pod, name) + return d.removeFile(tCtx, &pod, name) }, HandleError: func(ctx context.Context, err error, msg string) { // Record a failure, but don't kill the background goroutine. + // TODO: add to TContext or do it in Error/Assert/etc? defer ginkgo.GinkgoRecover() + // During tests when canceling the context it is possible to get all kinds of // follow-up errors for that, like: // processing ResourceSlice objects: retrieve node "127.0.0.1": client rate limiter Wait returned an error: context canceled @@ -592,7 +598,7 @@ func (d *Driver) SetUp(nodes *Nodes, driverResources map[string]resourceslice.Dr // treat errors as failures which definitely shouldn't occur: var droppedFields *resourceslice.DroppedFieldsError if errors.As(err, &droppedFields) { - framework.Failf("driver %s: %v", d.Name, err) + tCtx.Errorf("driver %s: %v", d.Name, err) } }, } @@ -628,25 +634,25 @@ func (d *Driver) SetUp(nodes *Nodes, driverResources map[string]resourceslice.Dr kubeletplugin.FlockDirectoryPath(nodes.tempDir), kubeletplugin.PluginDataDirectoryPath(pluginDataDirectoryPath), - kubeletplugin.PluginListener(d.listen(&pod, &listenerPort)), + kubeletplugin.PluginListener(d.listen(tCtx, &pod, &listenerPort)), kubeletplugin.RegistrarDirectoryPath(registrarDirectoryPath), - kubeletplugin.RegistrarListener(d.listen(&pod, &listenerPort)), + kubeletplugin.RegistrarListener(d.listen(tCtx, &pod, &listenerPort)), ) - framework.ExpectNoError(err, "start kubelet plugin for node %s", pod.Spec.NodeName) - d.cleanup = append(d.cleanup, func(ctx context.Context) { + tCtx.ExpectNoError(err, "start kubelet plugin for node %s", pod.Spec.NodeName) + d.cleanup = append(d.cleanup, func(tCtx ktesting.TContext) { // Depends on cancel being called first. plugin.Stop() // Also explicitly stop all pods. - ginkgo.By("scaling down driver proxy pods for " + d.Name) - rs, err := d.f.ClientSet.AppsV1().ReplicaSets(d.f.Namespace.Name).Get(ctx, rsName, metav1.GetOptions{}) - framework.ExpectNoError(err, "get ReplicaSet for driver "+d.Name) + tCtx.Log("scaling down driver proxy pods for", d.Name) + rs, err := tCtx.Client().AppsV1().ReplicaSets(tCtx.Namespace()).Get(tCtx, rsName, metav1.GetOptions{}) + tCtx.ExpectNoError(err, "get ReplicaSet for driver "+d.Name) rs.Spec.Replicas = ptr.To(int32(0)) - rs, err = d.f.ClientSet.AppsV1().ReplicaSets(d.f.Namespace.Name).Update(ctx, rs, metav1.UpdateOptions{}) - framework.ExpectNoError(err, "scale down ReplicaSet for driver "+d.Name) - if err := e2ereplicaset.WaitForReplicaSetTargetAvailableReplicas(ctx, d.f.ClientSet, rs, 0); err != nil { - framework.ExpectNoError(err, "all kubelet plugin proxies stopped") + rs, err = tCtx.Client().AppsV1().ReplicaSets(tCtx.Namespace()).Update(tCtx, rs, metav1.UpdateOptions{}) + tCtx.ExpectNoError(err, "scale down ReplicaSet for driver "+d.Name) + if err := e2ereplicaset.WaitForReplicaSetTargetAvailableReplicas(tCtx, tCtx.Client(), rs, 0); err != nil { + tCtx.ExpectNoError(err, "all kubelet plugin proxies stopped") } }) d.Nodes[nodename] = KubeletPlugin{ExamplePlugin: plugin, ClientSet: driverClient} @@ -657,8 +663,8 @@ func (d *Driver) SetUp(nodes *Nodes, driverResources map[string]resourceslice.Dr } // Wait for registration. - ginkgo.By("wait for plugin registration") - gomega.Eventually(func() map[string][]app.GRPCCall { + tCtx.Log("wait for plugin registration") + ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) map[string][]app.GRPCCall { notRegistered := make(map[string][]app.GRPCCall) for nodename, plugin := range d.Nodes { calls := plugin.GetGRPCCalls() @@ -670,8 +676,8 @@ func (d *Driver) SetUp(nodes *Nodes, driverResources map[string]resourceslice.Dr }).WithTimeout(time.Minute).Should(gomega.BeEmpty(), "hosts where the plugin has not been registered yet") } -func (d *Driver) ImpersonateKubeletPlugin(pod *v1.Pod) kubernetes.Interface { - ginkgo.GinkgoHelper() +func (d *Driver) ImpersonateKubeletPlugin(tCtx ktesting.TContext, pod *v1.Pod) kubernetes.Interface { + tCtx.Helper() driverUserInfo := (&serviceaccount.ServiceAccountInfo{ Name: d.serviceAccountName, Namespace: pod.Namespace, @@ -679,36 +685,36 @@ func (d *Driver) ImpersonateKubeletPlugin(pod *v1.Pod) kubernetes.Interface { PodName: pod.Name, PodUID: string(pod.UID), }).UserInfo() - driverClientConfig := d.f.ClientConfig() + driverClientConfig := tCtx.RESTConfig() driverClientConfig.Impersonate = rest.ImpersonationConfig{ UserName: driverUserInfo.GetName(), Groups: driverUserInfo.GetGroups(), Extra: driverUserInfo.GetExtra(), } driverClient, err := kubernetes.NewForConfig(driverClientConfig) - framework.ExpectNoError(err, "create client for driver") + tCtx.ExpectNoError(err, "create client for driver") return driverClient } -func (d *Driver) createFile(pod *v1.Pod, name string, content []byte) error { +func (d *Driver) createFile(tCtx ktesting.TContext, pod *v1.Pod, name string, content []byte) error { buffer := bytes.NewBuffer(content) // Writing the content can be slow. Better create a temporary file and // move it to the final destination once it is complete. tmpName := name + ".tmp" - if err := d.podIO(pod).CreateFile(tmpName, buffer); err != nil { - _ = d.podIO(pod).RemoveAll(tmpName) + if err := d.podIO(tCtx, pod).CreateFile(tmpName, buffer); err != nil { + _ = d.podIO(tCtx, pod).RemoveAll(tmpName) return err } - return d.podIO(pod).Rename(tmpName, name) + return d.podIO(tCtx, pod).Rename(tmpName, name) } -func (d *Driver) removeFile(pod *v1.Pod, name string) error { - return d.podIO(pod).RemoveAll(name) +func (d *Driver) removeFile(tCtx ktesting.TContext, pod *v1.Pod, name string) error { + return d.podIO(tCtx, pod).RemoveAll(name) } -func (d *Driver) createFromYAML(ctx context.Context, content []byte, namespace string) { +func (d *Driver) createFromYAML(tCtx ktesting.TContext, content []byte, namespace string) { // Not caching the discovery result isn't very efficient, but good enough. - discoveryCache := memory.NewMemCacheClient(d.f.ClientSet.Discovery()) + discoveryCache := memory.NewMemCacheClient(tCtx.Client().Discovery()) restMapper := restmapper.NewDeferredDiscoveryRESTMapper(discoveryCache) for _, content := range bytes.Split(content, []byte("---\n")) { @@ -717,16 +723,16 @@ func (d *Driver) createFromYAML(ctx context.Context, content []byte, namespace s } var obj *unstructured.Unstructured - framework.ExpectNoError(yaml.UnmarshalStrict(content, &obj), fmt.Sprintf("Full YAML:\n%s\n", string(content))) + tCtx.ExpectNoError(yaml.UnmarshalStrict(content, &obj), fmt.Sprintf("Full YAML:\n%s\n", string(content))) gv, err := schema.ParseGroupVersion(obj.GetAPIVersion()) - framework.ExpectNoError(err, fmt.Sprintf("extract group+version from object %q", klog.KObj(obj))) + tCtx.ExpectNoError(err, fmt.Sprintf("extract group+version from object %q", klog.KObj(obj))) gk := schema.GroupKind{Group: gv.Group, Kind: obj.GetKind()} mapping, err := restMapper.RESTMapping(gk, gv.Version) - framework.ExpectNoError(err, fmt.Sprintf("map %q to resource", gk)) + tCtx.ExpectNoError(err, fmt.Sprintf("map %q to resource", gk)) - resourceClient := d.f.DynamicClient.Resource(mapping.Resource) + resourceClient := tCtx.Dynamic().Resource(mapping.Resource) options := metav1.CreateOptions{ // If the YAML input is invalid, then we want the // apiserver to tell us via an error. This can @@ -736,31 +742,31 @@ func (d *Driver) createFromYAML(ctx context.Context, content []byte, namespace s } switch mapping.Scope.Name() { case meta.RESTScopeNameRoot: - _, err = resourceClient.Create(ctx, obj, options) + _, err = resourceClient.Create(tCtx, obj, options) case meta.RESTScopeNameNamespace: if namespace == "" { - framework.Failf("need namespace for object type %s", gk) + tCtx.Fatalf("need namespace for object type %s", gk) } - _, err = resourceClient.Namespace(namespace).Create(ctx, obj, options) + _, err = resourceClient.Namespace(namespace).Create(tCtx, obj, options) } - framework.ExpectNoError(err, "create object") - ginkgo.DeferCleanup(func(ctx context.Context) { + tCtx.ExpectNoError(err, "create object") + tCtx.CleanupCtx(func(tCtx ktesting.TContext) { del := resourceClient.Delete if mapping.Scope.Name() == meta.RESTScopeNameNamespace { del = resourceClient.Namespace(namespace).Delete } - err := del(ctx, obj.GetName(), metav1.DeleteOptions{}) + err := del(tCtx, obj.GetName(), metav1.DeleteOptions{}) if !apierrors.IsNotFound(err) { - framework.ExpectNoError(err, fmt.Sprintf("deleting %s.%s %s", obj.GetKind(), obj.GetAPIVersion(), klog.KObj(obj))) + tCtx.ExpectNoError(err, fmt.Sprintf("deleting %s.%s %s", obj.GetKind(), obj.GetAPIVersion(), klog.KObj(obj))) } }) } } -func (d *Driver) podIO(pod *v1.Pod) proxy.PodDirIO { - logger := klog.Background() +func (d *Driver) podIO(tCtx ktesting.TContext, pod *v1.Pod) proxy.PodDirIO { + logger := tCtx.Logger() return proxy.PodDirIO{ - F: d.f, + TCtx: tCtx, Namespace: pod.Namespace, PodName: pod.Name, ContainerName: pod.Spec.Containers[0].Name, @@ -775,7 +781,7 @@ var errListenerDone = errors.New("listener is shutting down") // listen returns the function which the kubeletplugin helper needs to open a listening socket. // For that it spins up hostpathplugin in the pod for the desired node // and connects to hostpathplugin via port forwarding. -func (d *Driver) listen(pod *v1.Pod, port *int32) func(ctx context.Context, endpoint string) (net.Listener, error) { +func (d *Driver) listen(tCtx ktesting.TContext, pod *v1.Pod, port *int32) func(ctx context.Context, endpoint string) (net.Listener, error) { return func(ctx context.Context, endpoint string) (l net.Listener, e error) { // No need create sockets, the kubelet is not expected to use them. if !d.WithKubelet { @@ -801,9 +807,9 @@ func (d *Driver) listen(pod *v1.Pod, port *int32) func(ctx context.Context, endp ctx = klog.NewContext(ctx, logger) // Start hostpathplugin in proxy mode and keep it running until the listener gets closed. - req := d.f.ClientSet.CoreV1().RESTClient().Post(). + req := tCtx.Client().CoreV1().RESTClient().Post(). Resource("pods"). - Namespace(d.f.Namespace.Name). + Namespace(tCtx.Namespace()). Name(pod.Name). SubResource("exec"). VersionedParams(&v1.PodExecOptions{ @@ -838,7 +844,7 @@ func (d *Driver) listen(pod *v1.Pod, port *int32) func(ctx context.Context, endp runHostpathPlugin := func(ctx context.Context) (bool, error) { // errors.Is(err, listenerDoneErr) would be nicer, but we don't get // that error from remotecommand. Instead forgo logging when we already shut down. - if err := execute(ctx, req.URL(), d.f.ClientConfig(), 5); err != nil && ctx.Err() == nil { + if err := execute(ctx, req.URL(), tCtx.RESTConfig(), 5); err != nil && ctx.Err() == nil { klog.FromContext(ctx).V(5).Info("execution failed, will retry", "err", err) } // There is no reason to stop except for context cancellation => @@ -848,9 +854,9 @@ func (d *Driver) listen(pod *v1.Pod, port *int32) func(ctx context.Context, endp _ = delayFn.Until(cmdCtx, true /* immediate */, true /* sliding */, runHostpathPlugin) // Killing hostpathplugin does not remove the socket. Need to do that manually. - req := d.f.ClientSet.CoreV1().RESTClient().Post(). + req := tCtx.Client().CoreV1().RESTClient().Post(). Resource("pods"). - Namespace(d.f.Namespace.Name). + Namespace(tCtx.Namespace()). Name(pod.Name). SubResource("exec"). VersionedParams(&v1.PodExecOptions{ @@ -865,7 +871,7 @@ func (d *Driver) listen(pod *v1.Pod, port *int32) func(ctx context.Context, endp }, scheme.ParameterCodec) cleanupLogger := klog.LoggerWithName(logger, "cleanup") cleanupCtx := klog.NewContext(ctx, cleanupLogger) - if err := execute(cleanupCtx, req.URL(), d.f.ClientConfig(), 0); err != nil { + if err := execute(cleanupCtx, req.URL(), tCtx.RESTConfig(), 0); err != nil { cleanupLogger.Error(err, "Socket removal failed") } }() @@ -881,12 +887,12 @@ func (d *Driver) listen(pod *v1.Pod, port *int32) func(ctx context.Context, endp } addr := proxy.Addr{ - Namespace: d.f.Namespace.Name, + Namespace: tCtx.Namespace(), PodName: pod.Name, ContainerName: pod.Spec.Containers[0].Name, Port: int(port), } - listener, err := proxy.Listen(ctx, d.f.ClientSet, d.f.ClientConfig(), addr) + listener, err := proxy.Listen(ctx, tCtx.Client(), tCtx.RESTConfig(), addr) if err != nil { return nil, fmt.Errorf("listen for connections from %+v: %w", addr, err) } @@ -973,9 +979,9 @@ func pipe(ctx context.Context, msg string, verbosity int) *io.PipeWriter { return writer } -func (d *Driver) TearDown(ctx context.Context) { +func (d *Driver) TearDown(tCtx ktesting.TContext) { for _, c := range d.cleanup { - c(ctx) + c(tCtx) } d.cleanup = nil d.wg.Wait() @@ -987,9 +993,9 @@ func (d *Driver) TearDown(ctx context.Context) { // because of the delay in the kubelet. // // Only use this in tests where kubelet support for DRA is guaranteed. -func (d *Driver) IsGone(ctx context.Context) { - ginkgo.By(fmt.Sprintf("Waiting for ResourceSlices of driver %s to be removed...", d.Name)) - gomega.Eventually(ctx, d.NewGetSlices()).WithTimeout(2 * time.Minute).Should(gomega.HaveField("Items", gomega.BeEmpty())) +func (d *Driver) IsGone(tCtx ktesting.TContext) { + tCtx.Logf("Waiting for ResourceSlices of driver %s to be removed...", d.Name) + ktesting.Eventually(tCtx, d.NewGetSlices()).WithTimeout(2 * time.Minute).Should(gomega.HaveField("Items", gomega.BeEmpty())) } func (d *Driver) interceptor(nodename string, ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { diff --git a/test/e2e/framework/pod/exec_util.go b/test/e2e/framework/pod/exec_util.go index d2d09c464b2..1394661e71b 100644 --- a/test/e2e/framework/pod/exec_util.go +++ b/test/e2e/framework/pod/exec_util.go @@ -29,10 +29,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/httpstream" "k8s.io/client-go/kubernetes/scheme" - restclient "k8s.io/client-go/rest" "k8s.io/client-go/tools/remotecommand" clientexec "k8s.io/client-go/util/exec" "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/utils/ktesting" "github.com/onsi/gomega" ) @@ -59,14 +59,18 @@ func ExecWithOptions(f *framework.Framework, options ExecOptions) (string, strin } func ExecWithOptionsContext(ctx context.Context, f *framework.Framework, options ExecOptions) (string, string, error) { + return ExecWithOptionsTCtx(f.TContext(ctx), options) +} + +func ExecWithOptionsTCtx(tCtx ktesting.TContext, options ExecOptions) (string, string, error) { if !options.Quiet { - framework.Logf("ExecWithOptions %+v", options) + tCtx.Logf("ExecWithOptions %+v", options) } const tty = false - framework.Logf("ExecWithOptions: Clientset creation") - req := f.ClientSet.CoreV1().RESTClient().Post(). + tCtx.Logf("ExecWithOptions: Clientset creation") + req := tCtx.Client().CoreV1().RESTClient().Post(). Resource("pods"). Name(options.PodName). Namespace(options.Namespace). @@ -81,8 +85,8 @@ func ExecWithOptionsContext(ctx context.Context, f *framework.Framework, options }, scheme.ParameterCodec) var stdout, stderr bytes.Buffer - framework.Logf("ExecWithOptions: execute(%s)", req.URL()) - err := execute(ctx, req.URL(), f.ClientConfig(), options.Stdin, &stdout, &stderr, tty) + tCtx.Logf("ExecWithOptions: execute(%s)", req.URL()) + err := execute(tCtx, req.URL(), options.Stdin, &stdout, &stderr, tty) if options.PreserveWhitespace { return stdout.String(), stderr.String(), err @@ -182,7 +186,8 @@ func VerifyExecInPodFail(ctx context.Context, f *framework.Framework, pod *v1.Po return fmt.Errorf("%q should fail with exit code %d, but exit without error", shExec, exitCode) } -func execute(ctx context.Context, url *url.URL, config *restclient.Config, stdin io.Reader, stdout, stderr io.Writer, tty bool) error { +func execute(tCtx ktesting.TContext, url *url.URL, stdin io.Reader, stdout, stderr io.Writer, tty bool) error { + config := tCtx.RESTConfig() // WebSocketExecutor executor is default // WebSocketExecutor must be "GET" method as described in RFC 6455 Sec. 4.1 (page 17). websocketExec, err := remotecommand.NewWebSocketExecutor(config, "GET", url.String()) @@ -205,7 +210,7 @@ func execute(ctx context.Context, url *url.URL, config *restclient.Config, stdin return err } - return exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + return exec.StreamWithContext(tCtx, remotecommand.StreamOptions{ Stdin: stdin, Stdout: stdout, Stderr: stderr, diff --git a/test/e2e/storage/drivers/csi.go b/test/e2e/storage/drivers/csi.go index 8d892e99bc5..f8f2384711c 100644 --- a/test/e2e/storage/drivers/csi.go +++ b/test/e2e/storage/drivers/csi.go @@ -657,7 +657,7 @@ func (m *mockCSIDriver) PrepareTest(ctx context.Context, f *framework.Framework) VolumeMountGroupRequired: m.enableVolumeMountGroup, EnableTopology: m.enableTopology, IO: proxy.PodDirIO{ - F: f, + TCtx: f.TContext(ctx), Namespace: m.driverNamespace.Name, PodName: podname, ContainerName: "busybox", diff --git a/test/e2e/storage/drivers/proxy/io.go b/test/e2e/storage/drivers/proxy/io.go index 0d645877556..163de4f230a 100644 --- a/test/e2e/storage/drivers/proxy/io.go +++ b/test/e2e/storage/drivers/proxy/io.go @@ -22,13 +22,13 @@ import ( "io" "k8s.io/klog/v2" - "k8s.io/kubernetes/test/e2e/framework" e2epod "k8s.io/kubernetes/test/e2e/framework/pod" "k8s.io/kubernetes/test/e2e/storage/drivers/csi-test/mock/service" + "k8s.io/kubernetes/test/utils/ktesting" ) type PodDirIO struct { - F *framework.Framework + TCtx ktesting.TContext Namespace string PodName string ContainerName string @@ -105,7 +105,7 @@ func (p PodDirIO) RemoveAll(path string) error { } func (p PodDirIO) execute(command []string, stdin io.Reader) (string, string, error) { - stdout, stderr, err := e2epod.ExecWithOptions(p.F, e2epod.ExecOptions{ + stdout, stderr, err := e2epod.ExecWithOptionsTCtx(p.TCtx, e2epod.ExecOptions{ Command: command, Namespace: p.Namespace, PodName: p.PodName, diff --git a/test/e2e/storage/utils/create.go b/test/e2e/storage/utils/create.go index a0cdafe18a6..394df21afd0 100644 --- a/test/e2e/storage/utils/create.go +++ b/test/e2e/storage/utils/create.go @@ -23,13 +23,12 @@ import ( "errors" "fmt" - "github.com/onsi/ginkgo/v2" - appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" storagev1 "k8s.io/api/storage/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -39,6 +38,7 @@ import ( "k8s.io/kubernetes/test/e2e/framework" e2etestfiles "k8s.io/kubernetes/test/e2e/framework/testfiles" imageutils "k8s.io/kubernetes/test/utils/image" + "k8s.io/kubernetes/test/utils/ktesting" ) // LoadFromManifests loads .yaml or .json manifest files and returns @@ -121,31 +121,31 @@ func visitManifests(cb func([]byte) error, files ...string) error { // - only some common items are supported, unknown ones trigger an error // - only the latest stable API version for each item is supported func PatchItems(f *framework.Framework, driverNamespace *v1.Namespace, items ...interface{}) error { + tCtx := f.TContext(context.Background()) + if driverNamespace != nil { + tCtx = tCtx.WithNamespace(driverNamespace.Name) + } + return PatchItemsTCtx(tCtx, items...) +} + +// PatchItemsTCtx is a variant of PatchItems where all parameters, including +// the namespace, are passed through a TContext. +func PatchItemsTCtx(tCtx ktesting.TContext, items ...interface{}) error { for _, item := range items { // Uncomment when debugging the loading and patching of items. // Logf("patching original content of %T:\n%s", item, PrettyPrint(item)) - if err := patchItemRecursively(f, driverNamespace, item); err != nil { + if err := patchItemRecursively(tCtx, item); err != nil { return err } } return nil } -// CreateItems creates the items. Each of them must be an API object +// createItems creates the items. Each of them must be an API object // of a type that is registered in Factory. // -// It returns either a cleanup function or an error, but never both. -// -// Cleaning up after a test can be triggered in two ways: -// - the test invokes the returned cleanup function, -// usually in an AfterEach -// - the test suite terminates, potentially after -// skipping the test's AfterEach (https://github.com/onsi/ginkgo/issues/222) -// -// PatchItems has the some limitations as LoadFromManifests: -// - only some common items are supported, unknown ones trigger an error -// - only the latest stable API version for each item is supported -func CreateItems(ctx context.Context, f *framework.Framework, ns *v1.Namespace, items ...interface{}) error { +// Object get deleted automatically during test cleanup. +func createItems(tCtx ktesting.TContext, items ...interface{}) error { var result error for _, item := range items { // Each factory knows which item(s) it supports, so try each one. @@ -153,11 +153,17 @@ func CreateItems(ctx context.Context, f *framework.Framework, ns *v1.Namespace, description := describeItem(item) // Uncomment this line to get a full dump of the entire item. // description = fmt.Sprintf("%s:\n%s", description, PrettyPrint(item)) - framework.Logf("creating %s", description) + tCtx.Logf("creating %s", description) for _, factory := range factories { - destructor, err := factory.Create(ctx, f, ns, item) + destructor, err := factory.Create(tCtx, item) if destructor != nil { - ginkgo.DeferCleanup(framework.IgnoreNotFound(destructor), framework.AnnotatedLocation(fmt.Sprintf("deleting %s", description))) + tCtx.CleanupCtx(func(tCtx ktesting.TContext) { + err := destructor(tCtx) + if apierrors.IsNotFound(err) { + return + } + tCtx.ExpectNoError(err, fmt.Sprintf("deleting %s", description)) + }) } if err == nil { done = true @@ -178,13 +184,25 @@ func CreateItems(ctx context.Context, f *framework.Framework, ns *v1.Namespace, // CreateFromManifests is a combination of LoadFromManifests, // PatchItems, patching with an optional custom function, -// and CreateItems. +// and creating the resulting items. +// +// Objects get deleted automatically during test cleanup. func CreateFromManifests(ctx context.Context, f *framework.Framework, driverNamespace *v1.Namespace, patch func(item interface{}) error, files ...string) error { + tCtx := f.TContext(ctx) + if driverNamespace != nil { + tCtx = tCtx.WithNamespace(driverNamespace.Name) + } + return CreateFromManifestsTCtx(tCtx, patch, files...) +} + +// CreateFromManifestsTCtx is a variant of CreateFromManifests where all parameters, including +// the driver namespace, are passed through a TContext. It is therefore usable from Go unit tests. +func CreateFromManifestsTCtx(tCtx ktesting.TContext, patch func(item interface{}) error, files ...string) error { items, err := LoadFromManifests(files...) if err != nil { return fmt.Errorf("CreateFromManifests: %w", err) } - if err := PatchItems(f, driverNamespace, items...); err != nil { + if err := PatchItemsTCtx(tCtx, items...); err != nil { return err } if patch != nil { @@ -194,7 +212,7 @@ func CreateFromManifests(ctx context.Context, f *framework.Framework, driverName } } } - return CreateItems(ctx, f, driverNamespace, items...) + return createItems(tCtx, items...) } // What is a subset of metav1.TypeMeta which (in contrast to @@ -234,7 +252,7 @@ type ItemFactory interface { // error or a cleanup function for the created item. // If the item is of an unsupported type, it must return // an error that has errorItemNotSupported as cause. - Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, item interface{}) (func(ctx context.Context) error, error) + Create(tCtx ktesting.TContext, item interface{}) (func(ctx context.Context) error, error) } // describeItem always returns a string that describes the item, @@ -271,82 +289,79 @@ var factories = map[What]ItemFactory{ {"CustomResourceDefinition"}: &customResourceDefinitionFactory{}, } -// PatchName makes the name of some item unique by appending the +// patchName makes the name of some item unique by appending the // generated unique name. -func PatchName(f *framework.Framework, item *string) { +func patchName(uniqueName string, item *string) { if *item != "" { - *item = *item + "-" + f.UniqueName + *item = *item + "-" + uniqueName } } -// PatchNamespace moves the item into the test's namespace. Not +// patchNamespace moves the item into the test's namespace. Not // all items can be namespaced. For those, the name also needs to be // patched. -func PatchNamespace(f *framework.Framework, driverNamespace *v1.Namespace, item *string) { - if driverNamespace != nil { - *item = driverNamespace.GetName() - return - } - - if f.Namespace != nil { - *item = f.Namespace.GetName() +func patchNamespace(tCtx ktesting.TContext, item *string) { + namespace := tCtx.Namespace() + if namespace != "" { + *item = namespace } } -func patchItemRecursively(f *framework.Framework, driverNamespace *v1.Namespace, item interface{}) error { +func patchItemRecursively(tCtx ktesting.TContext, item interface{}) error { + uniqueName := tCtx.Namespace() switch item := item.(type) { case *rbacv1.Subject: - PatchNamespace(f, driverNamespace, &item.Namespace) + patchNamespace(tCtx, &item.Namespace) case *rbacv1.RoleRef: // TODO: avoid hard-coding this special name. Perhaps add a Framework.PredefinedRoles // which contains all role names that are defined cluster-wide before the test starts? // All those names are exempt from renaming. That list could be populated by querying // and get extended by tests. if item.Name != "e2e-test-privileged-psp" { - PatchName(f, &item.Name) + patchName(uniqueName, &item.Name) } case *rbacv1.ClusterRole: - PatchName(f, &item.Name) + patchName(uniqueName, &item.Name) case *rbacv1.Role: - PatchNamespace(f, driverNamespace, &item.Namespace) + patchNamespace(tCtx, &item.Namespace) // Roles are namespaced, but because for RoleRef above we don't // know whether the referenced role is a ClusterRole or Role // and therefore always renames, we have to do the same here. - PatchName(f, &item.Name) + patchName(uniqueName, &item.Name) case *storagev1.StorageClass: - PatchName(f, &item.Name) + patchName(uniqueName, &item.Name) case *storagev1.VolumeAttributesClass: - PatchName(f, &item.Name) + patchName(uniqueName, &item.Name) case *storagev1.CSIDriver: - PatchName(f, &item.Name) + patchName(uniqueName, &item.Name) case *v1.ServiceAccount: - PatchNamespace(f, driverNamespace, &item.ObjectMeta.Namespace) + patchNamespace(tCtx, &item.ObjectMeta.Namespace) case *v1.Secret: - PatchNamespace(f, driverNamespace, &item.ObjectMeta.Namespace) + patchNamespace(tCtx, &item.ObjectMeta.Namespace) case *rbacv1.ClusterRoleBinding: - PatchName(f, &item.Name) + patchName(uniqueName, &item.Name) for i := range item.Subjects { - if err := patchItemRecursively(f, driverNamespace, &item.Subjects[i]); err != nil { - return fmt.Errorf("%T: %w", f, err) + if err := patchItemRecursively(tCtx, &item.Subjects[i]); err != nil { + return fmt.Errorf("%T: %w", &item.Subjects[i], err) } } - if err := patchItemRecursively(f, driverNamespace, &item.RoleRef); err != nil { - return fmt.Errorf("%T: %w", f, err) + if err := patchItemRecursively(tCtx, &item.RoleRef); err != nil { + return fmt.Errorf("%T: %w", &item.RoleRef, err) } case *rbacv1.RoleBinding: - PatchNamespace(f, driverNamespace, &item.Namespace) + patchNamespace(tCtx, &item.Namespace) for i := range item.Subjects { - if err := patchItemRecursively(f, driverNamespace, &item.Subjects[i]); err != nil { - return fmt.Errorf("%T: %w", f, err) + if err := patchItemRecursively(tCtx, &item.Subjects[i]); err != nil { + return fmt.Errorf("%T: %w", &item.Subjects[i], err) } } - if err := patchItemRecursively(f, driverNamespace, &item.RoleRef); err != nil { - return fmt.Errorf("%T: %w", f, err) + if err := patchItemRecursively(tCtx, &item.RoleRef); err != nil { + return fmt.Errorf("%T: %w", &item.RoleRef, err) } case *v1.Service: - PatchNamespace(f, driverNamespace, &item.ObjectMeta.Namespace) + patchNamespace(tCtx, &item.ObjectMeta.Namespace) case *appsv1.StatefulSet: - PatchNamespace(f, driverNamespace, &item.ObjectMeta.Namespace) + patchNamespace(tCtx, &item.ObjectMeta.Namespace) if err := patchContainerImages(item.Spec.Template.Spec.Containers); err != nil { return err } @@ -354,7 +369,7 @@ func patchItemRecursively(f *framework.Framework, driverNamespace *v1.Namespace, return err } case *appsv1.Deployment: - PatchNamespace(f, driverNamespace, &item.ObjectMeta.Namespace) + patchNamespace(tCtx, &item.ObjectMeta.Namespace) if err := patchContainerImages(item.Spec.Template.Spec.Containers); err != nil { return err } @@ -362,7 +377,7 @@ func patchItemRecursively(f *framework.Framework, driverNamespace *v1.Namespace, return err } case *appsv1.DaemonSet: - PatchNamespace(f, driverNamespace, &item.ObjectMeta.Namespace) + patchNamespace(tCtx, &item.ObjectMeta.Namespace) if err := patchContainerImages(item.Spec.Template.Spec.Containers); err != nil { return err } @@ -370,7 +385,7 @@ func patchItemRecursively(f *framework.Framework, driverNamespace *v1.Namespace, return err } case *appsv1.ReplicaSet: - PatchNamespace(f, driverNamespace, &item.ObjectMeta.Namespace) + patchNamespace(tCtx, &item.ObjectMeta.Namespace) if err := patchContainerImages(item.Spec.Template.Spec.Containers); err != nil { return err } @@ -396,13 +411,13 @@ func (f *serviceAccountFactory) New() runtime.Object { return &v1.ServiceAccount{} } -func (*serviceAccountFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*serviceAccountFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*v1.ServiceAccount) if !ok { return nil, errorItemNotSupported } - client := f.ClientSet.CoreV1().ServiceAccounts(ns.Name) - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + client := tCtx.Client().CoreV1().ServiceAccounts(tCtx.Namespace()) + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create ServiceAccount: %w", err) } return func(ctx context.Context) error { @@ -416,15 +431,15 @@ func (f *clusterRoleFactory) New() runtime.Object { return &rbacv1.ClusterRole{} } -func (*clusterRoleFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*clusterRoleFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*rbacv1.ClusterRole) if !ok { return nil, errorItemNotSupported } - framework.Logf("Define cluster role %v", item.GetName()) - client := f.ClientSet.RbacV1().ClusterRoles() - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + tCtx.Logf("define cluster role %v", item.GetName()) + client := tCtx.Client().RbacV1().ClusterRoles() + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create ClusterRole: %w", err) } return func(ctx context.Context) error { @@ -438,14 +453,14 @@ func (f *clusterRoleBindingFactory) New() runtime.Object { return &rbacv1.ClusterRoleBinding{} } -func (*clusterRoleBindingFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*clusterRoleBindingFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*rbacv1.ClusterRoleBinding) if !ok { return nil, errorItemNotSupported } - client := f.ClientSet.RbacV1().ClusterRoleBindings() - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + client := tCtx.Client().RbacV1().ClusterRoleBindings() + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create ClusterRoleBinding: %w", err) } return func(ctx context.Context) error { @@ -459,14 +474,14 @@ func (f *roleFactory) New() runtime.Object { return &rbacv1.Role{} } -func (*roleFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*roleFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*rbacv1.Role) if !ok { return nil, errorItemNotSupported } - client := f.ClientSet.RbacV1().Roles(ns.Name) - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + client := tCtx.Client().RbacV1().Roles(tCtx.Namespace()) + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create Role: %w", err) } return func(ctx context.Context) error { @@ -480,14 +495,14 @@ func (f *roleBindingFactory) New() runtime.Object { return &rbacv1.RoleBinding{} } -func (*roleBindingFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*roleBindingFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*rbacv1.RoleBinding) if !ok { return nil, errorItemNotSupported } - client := f.ClientSet.RbacV1().RoleBindings(ns.Name) - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + client := tCtx.Client().RbacV1().RoleBindings(tCtx.Namespace()) + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create RoleBinding: %w", err) } return func(ctx context.Context) error { @@ -501,14 +516,14 @@ func (f *serviceFactory) New() runtime.Object { return &v1.Service{} } -func (*serviceFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*serviceFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*v1.Service) if !ok { return nil, errorItemNotSupported } - client := f.ClientSet.CoreV1().Services(ns.Name) - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + client := tCtx.Client().CoreV1().Services(tCtx.Namespace()) + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create Service: %w", err) } return func(ctx context.Context) error { @@ -522,14 +537,14 @@ func (f *statefulSetFactory) New() runtime.Object { return &appsv1.StatefulSet{} } -func (*statefulSetFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*statefulSetFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*appsv1.StatefulSet) if !ok { return nil, errorItemNotSupported } - client := f.ClientSet.AppsV1().StatefulSets(ns.Name) - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + client := tCtx.Client().AppsV1().StatefulSets(tCtx.Namespace()) + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create StatefulSet: %w", err) } return func(ctx context.Context) error { @@ -543,14 +558,14 @@ func (f *deploymentFactory) New() runtime.Object { return &appsv1.Deployment{} } -func (*deploymentFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*deploymentFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*appsv1.Deployment) if !ok { return nil, errorItemNotSupported } - client := f.ClientSet.AppsV1().Deployments(ns.Name) - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + client := tCtx.Client().AppsV1().Deployments(tCtx.Namespace()) + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create Deployment: %w", err) } return func(ctx context.Context) error { @@ -564,14 +579,14 @@ func (f *daemonSetFactory) New() runtime.Object { return &appsv1.DaemonSet{} } -func (*daemonSetFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*daemonSetFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*appsv1.DaemonSet) if !ok { return nil, errorItemNotSupported } - client := f.ClientSet.AppsV1().DaemonSets(ns.Name) - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + client := tCtx.Client().AppsV1().DaemonSets(tCtx.Namespace()) + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create DaemonSet: %w", err) } return func(ctx context.Context) error { @@ -585,14 +600,14 @@ func (f *replicaSetFactory) New() runtime.Object { return &appsv1.ReplicaSet{} } -func (*replicaSetFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*replicaSetFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*appsv1.ReplicaSet) if !ok { return nil, errorItemNotSupported } - client := f.ClientSet.AppsV1().ReplicaSets(ns.Name) - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + client := tCtx.Client().AppsV1().ReplicaSets(tCtx.Namespace()) + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create ReplicaSet: %w", err) } return func(ctx context.Context) error { @@ -606,14 +621,14 @@ func (f *storageClassFactory) New() runtime.Object { return &storagev1.StorageClass{} } -func (*storageClassFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*storageClassFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*storagev1.StorageClass) if !ok { return nil, errorItemNotSupported } - client := f.ClientSet.StorageV1().StorageClasses() - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + client := tCtx.Client().StorageV1().StorageClasses() + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create StorageClass: %w", err) } return func(ctx context.Context) error { @@ -627,14 +642,14 @@ func (f *volumeAttributesClassFactory) New() runtime.Object { return &storagev1.VolumeAttributesClass{} } -func (*volumeAttributesClassFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*volumeAttributesClassFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*storagev1.VolumeAttributesClass) if !ok { return nil, errorItemNotSupported } - client := f.ClientSet.StorageV1().VolumeAttributesClasses() - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + client := tCtx.Client().StorageV1().VolumeAttributesClasses() + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create VolumeAttributesClass: %w", err) } return func(ctx context.Context) error { @@ -648,14 +663,14 @@ func (f *csiDriverFactory) New() runtime.Object { return &storagev1.CSIDriver{} } -func (*csiDriverFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*csiDriverFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*storagev1.CSIDriver) if !ok { return nil, errorItemNotSupported } - client := f.ClientSet.StorageV1().CSIDrivers() - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + client := tCtx.Client().StorageV1().CSIDrivers() + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create CSIDriver: %w", err) } return func(ctx context.Context) error { @@ -669,14 +684,14 @@ func (f *secretFactory) New() runtime.Object { return &v1.Secret{} } -func (*secretFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*secretFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { item, ok := i.(*v1.Secret) if !ok { return nil, errorItemNotSupported } - client := f.ClientSet.CoreV1().Secrets(ns.Name) - if _, err := client.Create(ctx, item, metav1.CreateOptions{}); err != nil { + client := tCtx.Client().CoreV1().Secrets(tCtx.Namespace()) + if _, err := client.Create(tCtx, item, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create Secret: %w", err) } return func(ctx context.Context) error { @@ -690,7 +705,7 @@ func (f *customResourceDefinitionFactory) New() runtime.Object { return &apiextensionsv1.CustomResourceDefinition{} } -func (*customResourceDefinitionFactory) Create(ctx context.Context, f *framework.Framework, ns *v1.Namespace, i interface{}) (func(ctx context.Context) error, error) { +func (*customResourceDefinitionFactory) Create(tCtx ktesting.TContext, i interface{}) (func(ctx context.Context) error, error) { var err error unstructCRD := &unstructured.Unstructured{} gvr := schema.GroupVersionResource{Group: "apiextensions.k8s.io", Version: "v1", Resource: "customresourcedefinitions"} @@ -705,11 +720,11 @@ func (*customResourceDefinitionFactory) Create(ctx context.Context, f *framework return nil, err } - if _, err = f.DynamicClient.Resource(gvr).Create(ctx, unstructCRD, metav1.CreateOptions{}); err != nil { + if _, err = tCtx.Dynamic().Resource(gvr).Create(tCtx, unstructCRD, metav1.CreateOptions{}); err != nil { return nil, fmt.Errorf("create CustomResourceDefinition: %w", err) } return func(ctx context.Context) error { - return f.DynamicClient.Resource(gvr).Delete(ctx, item.GetName(), metav1.DeleteOptions{}) + return tCtx.Dynamic().Resource(gvr).Delete(ctx, item.GetName(), metav1.DeleteOptions{}) }, nil } diff --git a/test/e2e_dra/coredra_test.go b/test/e2e_dra/coredra_test.go index c2024b28ed5..d5f0a928644 100644 --- a/test/e2e_dra/coredra_test.go +++ b/test/e2e_dra/coredra_test.go @@ -32,7 +32,7 @@ func coreDRA(tCtx ktesting.TContext, f *framework.Framework, b *drautils.Builder claim := b.ExternalClaim() pod := b.PodExternal() b.Create(tCtx, claim, pod) - b.TestPod(tCtx, f, pod) + b.TestPod(tCtx, pod) return func(tCtx ktesting.TContext) step3Func { // Remove pod prepared in step 1. @@ -45,7 +45,7 @@ func coreDRA(tCtx ktesting.TContext, f *framework.Framework, b *drautils.Builder pod = b.PodExternal() pod.Spec.ResourceClaims[0].ResourceClaimName = &claim.Name b.Create(tCtx, claim, pod) - b.TestPod(tCtx, f, pod) + b.TestPod(tCtx, pod) return func(tCtx ktesting.TContext) { // We need to clean up explicitly because the normal diff --git a/test/e2e_dra/gtesting_test.go b/test/e2e_dra/gtesting_test.go deleted file mode 100644 index db2c57c0ec5..00000000000 --- a/test/e2e_dra/gtesting_test.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package e2edra - -import ( - "context" - "fmt" - "os" - "strings" - "time" - - "github.com/onsi/ginkgo/v2" - ginkgotypes "github.com/onsi/ginkgo/v2/types" - - "k8s.io/kubernetes/test/utils/ktesting" -) - -var ( - TimeNow = time.Now // Can be stubbed out for testing. - Pid = os.Getpid() // Can be stubbed out for testing. -) - -// TODO: replace with helper code from https://github.com/kubernetes/kubernetes/pull/122481 should that get merged - or vice versa. -type ginkgoTB struct { - ktesting.TB -} - -var _ ktesting.ContextTB = &ginkgoTB{} - -func GinkgoContextTB() ktesting.ContextTB { - return &ginkgoTB{ - TB: ginkgo.GinkgoT(), - } -} - -// CleanupCtx implements [ktesting.ContextTB.CleanupCtx]. It's identical to -// ginkgo.DeferCleanup. -func (g *ginkgoTB) CleanupCtx(cb func(context.Context)) { - ginkgo.GinkgoHelper() - ginkgo.DeferCleanup(cb) -} - -// Log overrides the implementation from Ginkgo to ensure consistent output. -func (g *ginkgoTB) Log(args ...any) { - log(1, fmt.Sprint(args...)) -} - -// Logf overrides the implementation from Ginkgo to ensure consistent output. -func (g *ginkgoTB) Logf(format string, args ...any) { - log(1, fmt.Sprintf(format, args...)) -} - -// log re-implements klog.Info: same header, but stack unwinding -// with support for ginkgo.GinkgoWriter and skipping stack levels. -func log(offset int, msg string) { - now := TimeNow() - file, line := unwind(offset + 1) - if file == "" { - file = "???" - line = 1 - } else if slash := strings.LastIndex(file, "/"); slash >= 0 { - file = file[slash+1:] - } - _, month, day := now.Date() - hour, minute, second := now.Clock() - header := fmt.Sprintf("I%02d%02d %02d:%02d:%02d.%06d %d %s:%d]", - month, day, hour, minute, second, now.Nanosecond()/1000, Pid, file, line) - - _, _ = fmt.Fprintln(ginkgo.GinkgoWriter, header, msg) -} - -func unwind(skip int) (string, int) { - location := ginkgotypes.NewCodeLocation(skip + 1) - return location.FileName, location.LineNumber -} diff --git a/test/e2e_dra/upgradedowngrade_test.go b/test/e2e_dra/upgradedowngrade_test.go index 08d09cb349d..ea8d867a5b8 100644 --- a/test/e2e_dra/upgradedowngrade_test.go +++ b/test/e2e_dra/upgradedowngrade_test.go @@ -266,15 +266,15 @@ var _ = ginkgo.Describe("DRA upgrade/downgrade", func() { tCtx = ktesting.Begin(tCtx, fmt.Sprintf("v%d.%d", major, previousMinor)) tCtx.ExpectNoError(e2enode.WaitForAllNodesSchedulable(tCtx, tCtx.Client(), f.Timeouts.NodeSchedulable), "wait for all nodes to be schedulable") - nodes := drautils.NewNodesNow(tCtx, f, 1, 1) + nodes := drautils.NewNodesNow(tCtx, 1, 1) // Opening sockets locally avoids intermittent errors and delays caused by proxying through the restarted apiserver. // We could speed up testing by shortening the sync delay in the ResourceSlice controller, but let's better // test the defaults. - driver := drautils.NewDriverInstance(f) + driver := drautils.NewDriverInstance(tCtx) driver.IsLocal = true - driver.Run(nodes, drautils.DriverResourcesNow(nodes, 8)) - b := drautils.NewBuilderNow(ctx, f, driver) + driver.Run(tCtx, nodes, drautils.DriverResourcesNow(nodes, 8)) + b := drautils.NewBuilderNow(tCtx, driver) tCtx = ktesting.End(tCtx) @@ -305,7 +305,7 @@ var _ = ginkgo.Describe("DRA upgrade/downgrade", func() { // The kubelet wipes all ResourceSlices on a restart because it doesn't know which drivers were running. // Wait for the ResourceSlice controller in the driver to notice and recreate the ResourceSlices. tCtx = ktesting.Begin(tCtx, "wait for ResourceSlices") - gomega.Eventually(ctx, driver.NewGetSlices()).WithTimeout(5 * time.Minute).Should(gomega.HaveField("Items", gomega.HaveLen(len(nodes.NodeNames)))) + ktesting.Eventually(tCtx, driver.NewGetSlices()).WithTimeout(5 * time.Minute).Should(gomega.HaveField("Items", gomega.HaveLen(len(nodes.NodeNames)))) tCtx = ktesting.End(tCtx) steps3 := make(map[string]step3Func, len(subTests))