mirror of
https://github.com/kubernetes/kubernetes.git
synced 2026-06-08 16:30:57 -04:00
DRA integration: refactor helper functions
They now live in a separate file, use ktesting+Gomega consistently, and follow (mostly) best practices: - strict validation during create - wait for completion of delete The latter is not complete yet because it was found that some binding condition test keeps a ResourceCLaim in an undeletable state. That's okayish (namespace stays around until apiserver gets restarted), but is worth looking into later.
This commit is contained in:
Patrick Ohly
parent
03e337cfb7
commit
8d3bc085ce
2 changed files with 211 additions and 146 deletions
|
|
@ -22,7 +22,6 @@ import (
|
|||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
|
@ -51,7 +50,6 @@ import (
|
|||
utilfeature "k8s.io/apiserver/pkg/util/feature"
|
||||
resourceapiac "k8s.io/client-go/applyconfigurations/resource/v1"
|
||||
"k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/client-go/tools/cache"
|
||||
"k8s.io/client-go/util/retry"
|
||||
"k8s.io/component-base/featuregate"
|
||||
|
|
@ -121,105 +119,6 @@ var (
|
|||
numNodes = 2
|
||||
)
|
||||
|
||||
// createTestNamespace creates a namespace with a name that is derived from the
|
||||
// current test name:
|
||||
// - Non-alpha-numeric characters replaced by hyphen.
|
||||
// - Truncated in the middle to make it short enough for GenerateName.
|
||||
// - Hyphen plus random suffix added by the apiserver.
|
||||
func createTestNamespace(tCtx ktesting.TContext, labels map[string]string) string {
|
||||
tCtx.Helper()
|
||||
name := regexp.MustCompile(`[^[:alnum:]_-]`).ReplaceAllString(tCtx.Name(), "-")
|
||||
name = strings.ToLower(name)
|
||||
if len(name) > 63 {
|
||||
name = name[:30] + "--" + name[len(name)-30:]
|
||||
}
|
||||
ns := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: name + "-"}}
|
||||
ns.Labels = labels
|
||||
ns, err := tCtx.Client().CoreV1().Namespaces().Create(tCtx, ns, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create test namespace")
|
||||
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
|
||||
tCtx.ExpectNoError(tCtx.Client().CoreV1().Namespaces().Delete(tCtx, ns.Name, metav1.DeleteOptions{}), "delete test namespace")
|
||||
})
|
||||
return ns.Name
|
||||
}
|
||||
|
||||
// createSlice creates the given ResourceSlice and removes it when the test is done.
|
||||
func createSlice(tCtx ktesting.TContext, slice *resourceapi.ResourceSlice) *resourceapi.ResourceSlice {
|
||||
tCtx.Helper()
|
||||
slice, err := tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, slice, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create ResourceSlice")
|
||||
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
|
||||
tCtx.Log("Cleaning up ResourceSlice...")
|
||||
err := tCtx.Client().ResourceV1().ResourceSlices().Delete(tCtx, slice.Name, metav1.DeleteOptions{})
|
||||
tCtx.ExpectNoError(err, "delete ResourceSlice")
|
||||
})
|
||||
return slice
|
||||
}
|
||||
|
||||
// createTestClass creates a DeviceClass with a driver name derived from the test namespace
|
||||
func createTestClass(tCtx ktesting.TContext, namespace string) (*resourceapi.DeviceClass, string) {
|
||||
tCtx.Helper()
|
||||
driverName := namespace + ".driver"
|
||||
class := class.DeepCopy()
|
||||
class.Name = namespace + ".class"
|
||||
class.Spec.Selectors = []resourceapi.DeviceSelector{{
|
||||
CEL: &resourceapi.CELDeviceSelector{
|
||||
Expression: fmt.Sprintf("device.driver == %q", driverName),
|
||||
},
|
||||
}}
|
||||
_, err := tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, class, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create class")
|
||||
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
|
||||
tCtx.Log("Cleaning up DeviceClass...")
|
||||
err := tCtx.Client().ResourceV1().DeviceClasses().Delete(tCtx, class.Name, metav1.DeleteOptions{})
|
||||
tCtx.ExpectNoError(err, "delete class")
|
||||
})
|
||||
|
||||
return class, driverName
|
||||
}
|
||||
|
||||
// createClaim creates a claim and in the namespace.
|
||||
// The class must already exist and is used for all requests.
|
||||
func createClaim(tCtx ktesting.TContext, namespace string, suffix string, class *resourceapi.DeviceClass, claim *resourceapi.ResourceClaim) *resourceapi.ResourceClaim {
|
||||
tCtx.Helper()
|
||||
claim = claim.DeepCopy()
|
||||
claim.Namespace = namespace
|
||||
claim.Name += suffix
|
||||
claimName := claim.Name
|
||||
for i := range claim.Spec.Devices.Requests {
|
||||
request := &claim.Spec.Devices.Requests[i]
|
||||
if request.Exactly != nil && request.Exactly.DeviceClassName != "" {
|
||||
request.Exactly.DeviceClassName = class.Name
|
||||
continue
|
||||
}
|
||||
for e := range request.FirstAvailable {
|
||||
subRequest := &request.FirstAvailable[e]
|
||||
subRequest.DeviceClassName = class.Name
|
||||
}
|
||||
}
|
||||
claim, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Create(tCtx, claim, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create claim "+claimName)
|
||||
return claim
|
||||
}
|
||||
|
||||
// createPod create a pod in the namespace, referencing the given claim.
|
||||
func createPod(tCtx ktesting.TContext, namespace string, suffix string, claim *resourceapi.ResourceClaim, pod *v1.Pod) *v1.Pod {
|
||||
tCtx.Helper()
|
||||
pod = pod.DeepCopy()
|
||||
pod.Name += suffix
|
||||
podName := pod.Name
|
||||
pod.Namespace = namespace
|
||||
pod.Spec.ResourceClaims[0].ResourceClaimName = &claim.Name
|
||||
pod, err := tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create pod "+podName)
|
||||
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
|
||||
tCtx.Log("Cleaning up Pod...")
|
||||
err := tCtx.Client().CoreV1().Pods(namespace).Delete(tCtx, pod.Name, metav1.DeleteOptions{})
|
||||
tCtx.ExpectNoError(err, "delete Pod")
|
||||
})
|
||||
return pod
|
||||
}
|
||||
|
||||
func TestDRA(t *testing.T) {
|
||||
// Each sub-test brings up the API server in a certain
|
||||
// configuration. These sub-tests must run sequentially because they
|
||||
|
|
@ -375,7 +274,7 @@ func createNodes(tCtx ktesting.TContext) {
|
|||
Name: fmt.Sprintf("worker-%d", i),
|
||||
},
|
||||
}
|
||||
node, err := tCtx.Client().CoreV1().Nodes().Create(tCtx, node, metav1.CreateOptions{})
|
||||
node, err := tCtx.Client().CoreV1().Nodes().Create(tCtx, node, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, fmt.Sprintf("creating node #%d", i))
|
||||
|
||||
// Make the node ready.
|
||||
|
|
@ -523,7 +422,7 @@ func testPod(tCtx ktesting.TContext, draEnabled bool) {
|
|||
namespace := createTestNamespace(tCtx, nil)
|
||||
podWithClaimName := podWithClaimName.DeepCopy()
|
||||
podWithClaimName.Namespace = namespace
|
||||
pod, err := tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, podWithClaimName, metav1.CreateOptions{})
|
||||
pod, err := tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, podWithClaimName, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create pod")
|
||||
if draEnabled {
|
||||
assert.NotEmpty(tCtx, pod.Spec.ResourceClaims, "should store resource claims in pod spec")
|
||||
|
|
@ -535,7 +434,7 @@ func testPod(tCtx ktesting.TContext, draEnabled bool) {
|
|||
// testAPIDisabled checks that the resource.k8s.io API is disabled.
|
||||
func testAPIDisabled(tCtx ktesting.TContext) {
|
||||
tCtx.Parallel()
|
||||
_, err := tCtx.Client().ResourceV1().ResourceClaims(claim.Namespace).Create(tCtx, claim, metav1.CreateOptions{})
|
||||
_, err := tCtx.Client().ResourceV1().ResourceClaims(claim.Namespace).Create(tCtx, claim, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
if !apierrors.IsNotFound(err) {
|
||||
tCtx.Fatalf("expected 'resource not found' error, got %v", err)
|
||||
}
|
||||
|
|
@ -547,7 +446,7 @@ func testConvert(tCtx ktesting.TContext) {
|
|||
namespace := createTestNamespace(tCtx, nil)
|
||||
claim := claim.DeepCopy()
|
||||
claim.Namespace = namespace
|
||||
claim, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Create(tCtx, claim, metav1.CreateOptions{})
|
||||
claim, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Create(tCtx, claim, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create claim")
|
||||
claimBeta2, err := tCtx.Client().ResourceV1beta2().ResourceClaims(namespace).Get(tCtx, claim.Name, metav1.GetOptions{})
|
||||
tCtx.ExpectNoError(err, "get claim")
|
||||
|
|
@ -565,7 +464,7 @@ func testAdminAccess(tCtx ktesting.TContext, adminAccessEnabled bool) {
|
|||
claim1.Namespace = namespace
|
||||
claim1.Spec.Devices.Requests[0].Exactly.AdminAccess = ptr.To(true)
|
||||
// create claim with AdminAccess in non-admin namespace
|
||||
_, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Create(tCtx, claim1, metav1.CreateOptions{})
|
||||
_, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Create(tCtx, claim1, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
if adminAccessEnabled {
|
||||
if err != nil {
|
||||
// should result in validation error
|
||||
|
|
@ -581,7 +480,7 @@ func testAdminAccess(tCtx ktesting.TContext, adminAccessEnabled bool) {
|
|||
claim2.Namespace = adminNS
|
||||
claim2.Name = "claim2"
|
||||
claim2.Spec.Devices.Requests[0].Exactly.AdminAccess = ptr.To(true)
|
||||
claim2, err := tCtx.Client().ResourceV1().ResourceClaims(adminNS).Create(tCtx, claim2, metav1.CreateOptions{})
|
||||
claim2, err := tCtx.Client().ResourceV1().ResourceClaims(adminNS).Create(tCtx, claim2, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create claim")
|
||||
if !ptr.Deref(claim2.Spec.Devices.Requests[0].Exactly.AdminAccess, true) {
|
||||
tCtx.Fatalf("should store AdminAccess in ResourceClaim %v", claim2)
|
||||
|
|
@ -705,7 +604,7 @@ func testPrioritizedList(tCtx ktesting.TContext, enabled bool) {
|
|||
|
||||
func testExtendedResource(tCtx ktesting.TContext, enabled bool) {
|
||||
tCtx.Parallel()
|
||||
c, err := tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, classWithExtendedResource, metav1.CreateOptions{})
|
||||
c, err := tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, classWithExtendedResource, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create class")
|
||||
if enabled {
|
||||
require.NotEmpty(tCtx, c.Spec.ExtendedResourceName, "should store ExtendedResourceName")
|
||||
|
|
@ -716,7 +615,7 @@ func testExtendedResource(tCtx ktesting.TContext, enabled bool) {
|
|||
|
||||
pod := podWithExtendedResource.DeepCopy()
|
||||
pod.Namespace = namespace
|
||||
_, err := tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod, metav1.CreateOptions{})
|
||||
_, err := tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create pod")
|
||||
schedulingAttempted := gomega.HaveField("Status.Conditions", gomega.ContainElement(
|
||||
gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
|
||||
|
|
@ -1094,7 +993,7 @@ func testResourceClaimDeviceStatus(tCtx ktesting.TContext, enabled bool) {
|
|||
},
|
||||
}
|
||||
|
||||
claim, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Create(tCtx, claim, metav1.CreateOptions{})
|
||||
claim, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Create(tCtx, claim, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create ResourceClaim")
|
||||
|
||||
deviceStatus := []resourceapi.AllocatedDeviceStatus{{
|
||||
|
|
@ -1247,8 +1146,7 @@ func testResourceClaimDeviceStatus(tCtx ktesting.TContext, enabled bool) {
|
|||
// and prints some information about it.
|
||||
func testMaxResourceSlice(tCtx ktesting.TContext) {
|
||||
slice := NewMaxResourceSlice()
|
||||
createdSlice, err := tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, slice, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err)
|
||||
createdSlice := createSlice(tCtx, slice)
|
||||
totalSize := createdSlice.Size()
|
||||
var managedFieldsSize int
|
||||
for _, f := range createdSlice.ManagedFields {
|
||||
|
|
@ -1323,7 +1221,7 @@ func testControllerManagerMetrics(tCtx ktesting.TContext) {
|
|||
},
|
||||
}
|
||||
|
||||
_, err := tCtx.Client().ResourceV1().ResourceClaimTemplates(namespace).Create(tCtx, template1, metav1.CreateOptions{})
|
||||
_, err := tCtx.Client().ResourceV1().ResourceClaimTemplates(namespace).Create(tCtx, template1, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create ResourceClaimTemplate without admin access")
|
||||
|
||||
pod1 := &v1.Pod{
|
||||
|
|
@ -1343,7 +1241,7 @@ func testControllerManagerMetrics(tCtx ktesting.TContext) {
|
|||
},
|
||||
}
|
||||
|
||||
_, err = tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod1, metav1.CreateOptions{})
|
||||
_, err = tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod1, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create Pod with ResourceClaimTemplate without admin access")
|
||||
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
|
@ -1377,7 +1275,7 @@ func testControllerManagerMetrics(tCtx ktesting.TContext) {
|
|||
},
|
||||
}
|
||||
|
||||
_, err = tCtx.Client().ResourceV1().ResourceClaimTemplates(adminNS).Create(tCtx, template2, metav1.CreateOptions{})
|
||||
_, err = tCtx.Client().ResourceV1().ResourceClaimTemplates(adminNS).Create(tCtx, template2, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create ResourceClaimTemplate with admin access")
|
||||
|
||||
pod2 := &v1.Pod{
|
||||
|
|
@ -1397,7 +1295,7 @@ func testControllerManagerMetrics(tCtx ktesting.TContext) {
|
|||
},
|
||||
}
|
||||
|
||||
_, err = tCtx.Client().CoreV1().Pods(adminNS).Create(tCtx, pod2, metav1.CreateOptions{})
|
||||
_, err = tCtx.Client().CoreV1().Pods(adminNS).Create(tCtx, pod2, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create Pod with ResourceClaimTemplate with admin access in admin namespace")
|
||||
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
|
@ -1428,7 +1326,7 @@ func testControllerManagerMetrics(tCtx ktesting.TContext) {
|
|||
},
|
||||
}
|
||||
|
||||
_, err = tCtx.Client().ResourceV1().ResourceClaimTemplates(namespace).Create(tCtx, invalidTemplate, metav1.CreateOptions{})
|
||||
_, err = tCtx.Client().ResourceV1().ResourceClaimTemplates(namespace).Create(tCtx, invalidTemplate, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
require.Error(tCtx, err, "should fail to create ResourceClaimTemplate with AdminAccess in non-admin namespace")
|
||||
require.ErrorContains(tCtx, err, "admin access to devices requires the `resource.kubernetes.io/admin-access: true` label on the containing namespace")
|
||||
|
||||
|
|
@ -1452,7 +1350,7 @@ func testControllerManagerMetrics(tCtx ktesting.TContext) {
|
|||
},
|
||||
}
|
||||
|
||||
_, err = tCtx.Client().ResourceV1().ResourceClaimTemplates(namespace).Create(tCtx, template4, metav1.CreateOptions{})
|
||||
_, err = tCtx.Client().ResourceV1().ResourceClaimTemplates(namespace).Create(tCtx, template4, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create second ResourceClaimTemplate without admin access")
|
||||
|
||||
pod4 := &v1.Pod{
|
||||
|
|
@ -1472,7 +1370,7 @@ func testControllerManagerMetrics(tCtx ktesting.TContext) {
|
|||
},
|
||||
}
|
||||
|
||||
_, err = tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod4, metav1.CreateOptions{})
|
||||
_, err = tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod4, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create second Pod with ResourceClaimTemplate without admin access")
|
||||
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
|
|
@ -1529,7 +1427,7 @@ func testDeviceBindingConditions(tCtx ktesting.TContext, enabled bool) {
|
|||
},
|
||||
},
|
||||
}
|
||||
slice, err := tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, slice, metav1.CreateOptions{})
|
||||
slice, err := tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, slice, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create slice")
|
||||
|
||||
haveBindingConditionFields := len(slice.Spec.Devices[0].BindingConditions) > 0 || len(slice.Spec.Devices[0].BindingFailureConditions) > 0
|
||||
|
|
@ -1561,7 +1459,7 @@ func testDeviceBindingConditions(tCtx ktesting.TContext, enabled bool) {
|
|||
},
|
||||
},
|
||||
}
|
||||
_, err = tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, sliceWithoutBinding, metav1.CreateOptions{})
|
||||
_, err = tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, sliceWithoutBinding, metav1.CreateOptions{FieldValidation: "Strict"})
|
||||
tCtx.ExpectNoError(err, "create slice without binding conditions")
|
||||
|
||||
// Schedule first pod and wait for the scheduler to reach the binding phase, which marks the claim as allocated.
|
||||
|
|
@ -1590,8 +1488,7 @@ func testDeviceBindingConditions(tCtx ktesting.TContext, enabled bool) {
|
|||
)),
|
||||
}))), "first allocated claim")
|
||||
|
||||
err = waitForPodScheduled(tCtx, tCtx.Client(), namespace, pod.Name)
|
||||
tCtx.ExpectNoError(err, "first pod scheduled")
|
||||
waitForPodScheduled(tCtx, namespace, pod.Name)
|
||||
|
||||
// Second pod should get the device with binding conditions.
|
||||
claim2 := createClaim(tCtx, namespace, "-b", class, claim)
|
||||
|
|
@ -1684,27 +1581,5 @@ func testDeviceBindingConditions(tCtx ktesting.TContext, enabled bool) {
|
|||
return err
|
||||
})
|
||||
tCtx.ExpectNoError(err, "add binding condition to second claim")
|
||||
err = waitForPodScheduled(tCtx, tCtx.Client(), namespace, pod.Name)
|
||||
tCtx.ExpectNoError(err, "second pod scheduled")
|
||||
}
|
||||
|
||||
func waitForPodScheduled(ctx context.Context, client kubernetes.Interface, namespace, podName string) error {
|
||||
timeout := time.After(60 * time.Second)
|
||||
tick := time.Tick(1 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case <-timeout:
|
||||
return fmt.Errorf("timed out waiting for pod %s/%s to be scheduled", namespace, podName)
|
||||
case <-tick:
|
||||
pod, err := client.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{})
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, cond := range pod.Status.Conditions {
|
||||
if cond.Type == v1.PodScheduled && cond.Status == v1.ConditionTrue {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
waitForPodScheduled(tCtx, namespace, pod.Name)
|
||||
}
|
||||
|
|
|
|||
190
test/integration/dra/helpers_test.go
Normal file
190
test/integration/dra/helpers_test.go
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
/*
|
||||
Copyright 2025 The Kubernetes Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package dra
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/onsi/gomega"
|
||||
|
||||
v1 "k8s.io/api/core/v1"
|
||||
resourceapi "k8s.io/api/resource/v1"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/kubernetes/test/utils/ktesting"
|
||||
"k8s.io/utils/ptr"
|
||||
)
|
||||
|
||||
// must can be wrapped around a Create/Update/Patch/Get/Delete call and handles the error checking:
|
||||
//
|
||||
// pod = must(tCtx, tCtx.Client().CoreV1().Pods(namespace).Create, pod, metav1.CreateOptions{})
|
||||
func must[R, P, O any](tCtx ktesting.TContext, call func(context.Context, P, O) (*R, error), p P, o O) *R {
|
||||
tCtx.Helper()
|
||||
r, err := call(tCtx, p, o)
|
||||
tCtx.ExpectNoError(err)
|
||||
return r
|
||||
}
|
||||
|
||||
// createTestNamespace creates a namespace with a name that is derived from the
|
||||
// current test name:
|
||||
// - Non-alpha-numeric characters replaced by hyphen.
|
||||
// - Truncated in the middle to make it short enough for GenerateName.
|
||||
// - Hyphen plus random suffix added by the apiserver.
|
||||
func createTestNamespace(tCtx ktesting.TContext, labels map[string]string) string {
|
||||
tCtx.Helper()
|
||||
name := regexp.MustCompile(`[^[:alnum:]_-]`).ReplaceAllString(tCtx.Name(), "-")
|
||||
name = strings.ToLower(name)
|
||||
if len(name) > 63 {
|
||||
name = name[:30] + "--" + name[len(name)-30:]
|
||||
}
|
||||
ns := &v1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: name + "-"}}
|
||||
ns.Labels = labels
|
||||
ns, err := tCtx.Client().CoreV1().Namespaces().Create(tCtx, ns, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create test namespace")
|
||||
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
|
||||
tCtx.ExpectNoError(tCtx.Client().CoreV1().Namespaces().Delete(tCtx, ns.Name, metav1.DeleteOptions{}))
|
||||
// *Not* waiting here. Deleting namespaces is slow.
|
||||
})
|
||||
return ns.Name
|
||||
}
|
||||
|
||||
// createSlice creates the given ResourceSlice and removes it when the test is done.
|
||||
func createSlice(tCtx ktesting.TContext, slice *resourceapi.ResourceSlice) *resourceapi.ResourceSlice {
|
||||
tCtx.Helper()
|
||||
slice, err := tCtx.Client().ResourceV1().ResourceSlices().Create(tCtx, slice, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create ResourceSlice")
|
||||
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
|
||||
tCtx.Log("Cleaning up ResourceSlice...")
|
||||
deleteAndWait(tCtx, tCtx.Client().ResourceV1().ResourceSlices().Delete, tCtx.Client().ResourceV1().ResourceSlices().Get, slice.Name)
|
||||
})
|
||||
return slice
|
||||
}
|
||||
|
||||
// createTestClass creates a DeviceClass with a driver name derived from the test namespace
|
||||
func createTestClass(tCtx ktesting.TContext, namespace string) (*resourceapi.DeviceClass, string) {
|
||||
tCtx.Helper()
|
||||
driverName := namespace + ".driver"
|
||||
class := class.DeepCopy()
|
||||
class.Name = namespace + ".class"
|
||||
class.Spec.Selectors = []resourceapi.DeviceSelector{{
|
||||
CEL: &resourceapi.CELDeviceSelector{
|
||||
Expression: fmt.Sprintf("device.driver == %q", driverName),
|
||||
},
|
||||
}}
|
||||
_, err := tCtx.Client().ResourceV1().DeviceClasses().Create(tCtx, class, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create class")
|
||||
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
|
||||
tCtx.Log("Cleaning up DeviceClass...")
|
||||
deleteAndWait(tCtx, tCtx.Client().ResourceV1().DeviceClasses().Delete, tCtx.Client().ResourceV1().DeviceClasses().Get, class.Name)
|
||||
})
|
||||
|
||||
return class, driverName
|
||||
}
|
||||
|
||||
// createClaim creates a claim and in the namespace.
|
||||
// The class must already exist and is used for all requests.
|
||||
func createClaim(tCtx ktesting.TContext, namespace string, suffix string, class *resourceapi.DeviceClass, claim *resourceapi.ResourceClaim) *resourceapi.ResourceClaim {
|
||||
tCtx.Helper()
|
||||
claim = claim.DeepCopy()
|
||||
claim.Namespace = namespace
|
||||
claim.Name += suffix
|
||||
claimName := claim.Name
|
||||
for i := range claim.Spec.Devices.Requests {
|
||||
request := &claim.Spec.Devices.Requests[i]
|
||||
if request.Exactly != nil && request.Exactly.DeviceClassName != "" {
|
||||
request.Exactly.DeviceClassName = class.Name
|
||||
continue
|
||||
}
|
||||
for e := range request.FirstAvailable {
|
||||
subRequest := &request.FirstAvailable[e]
|
||||
subRequest.DeviceClassName = class.Name
|
||||
}
|
||||
}
|
||||
claim, err := tCtx.Client().ResourceV1().ResourceClaims(namespace).Create(tCtx, claim, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create claim "+claimName)
|
||||
// TODO: some tests leak claims. Probably they need to be fixed... later.
|
||||
// tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
|
||||
// // We want to know when tearing this down gets stuck.
|
||||
// deleteAndWait(tCtx, tCtx.Client().ResourceV1().ResourceClaims(namespace).Delete, tCtx.Client().ResourceV1().ResourceClaims(namespace).Get, claim.Name)
|
||||
// })
|
||||
return claim
|
||||
}
|
||||
|
||||
// createPod create a pod in the namespace, referencing the given claim.
|
||||
func createPod(tCtx ktesting.TContext, namespace string, suffix string, claim *resourceapi.ResourceClaim, pod *v1.Pod) *v1.Pod {
|
||||
tCtx.Helper()
|
||||
pod = pod.DeepCopy()
|
||||
pod.Name += suffix
|
||||
podName := pod.Name
|
||||
pod.Namespace = namespace
|
||||
pod.Spec.ResourceClaims[0].ResourceClaimName = &claim.Name
|
||||
pod, err := tCtx.Client().CoreV1().Pods(namespace).Create(tCtx, pod, metav1.CreateOptions{})
|
||||
tCtx.ExpectNoError(err, "create pod "+podName)
|
||||
tCtx.CleanupCtx(func(tCtx ktesting.TContext) {
|
||||
tCtx.Log("Cleaning up Pod...")
|
||||
// We must delete pods before uninstalling our driver.
|
||||
// Also, we want to know when stopping it gets stuck.
|
||||
deleteAndWait(tCtx, tCtx.Client().CoreV1().Pods(namespace).Delete, tCtx.Client().CoreV1().Pods(namespace).Get, pod.Name)
|
||||
})
|
||||
return pod
|
||||
}
|
||||
|
||||
func waitForPodScheduled(tCtx ktesting.TContext, namespace, podName string) {
|
||||
tCtx.Helper()
|
||||
|
||||
ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) *v1.Pod {
|
||||
return must(tCtx, tCtx.Client().CoreV1().Pods(namespace).Get, podName, metav1.GetOptions{})
|
||||
}).WithTimeout(60*time.Second).Should(
|
||||
gomega.HaveField("Status.Conditions", gomega.ContainElement(
|
||||
gomega.And(
|
||||
gomega.HaveField("Type", gomega.Equal(v1.PodScheduled)),
|
||||
gomega.HaveField("Status", gomega.Equal(v1.ConditionTrue)),
|
||||
),
|
||||
)),
|
||||
"Pod %s should have been scheduled.", podName,
|
||||
)
|
||||
}
|
||||
|
||||
func deleteAndWait[T any](tCtx ktesting.TContext, del func(context.Context, string, metav1.DeleteOptions) error, get func(context.Context, string, metav1.GetOptions) (T, error), name string) {
|
||||
tCtx.Helper()
|
||||
|
||||
var t T
|
||||
var anyT any = t
|
||||
var options metav1.DeleteOptions
|
||||
if _, ok := anyT.(*v1.Pod); ok {
|
||||
// Special case for pods: we don't have a kubelet which acknowledges
|
||||
// shutdown of a scheduled pod, so we have to force-delete.
|
||||
options.GracePeriodSeconds = ptr.To(int64(0))
|
||||
}
|
||||
|
||||
tCtx.ExpectNoError(del(tCtx, name, options), fmt.Sprintf("delete %T %s", t, name))
|
||||
waitForNotFound(tCtx, get, name)
|
||||
}
|
||||
|
||||
func waitForNotFound[T any](tCtx ktesting.TContext, get func(context.Context, string, metav1.GetOptions) (T, error), name string) {
|
||||
tCtx.Helper()
|
||||
|
||||
var t T
|
||||
ktesting.Eventually(tCtx, func(tCtx ktesting.TContext) error {
|
||||
_, err := get(tCtx, name, metav1.GetOptions{})
|
||||
return err
|
||||
}).WithTimeout(60*time.Second).Should(gomega.MatchError(apierrors.IsNotFound, "IsNotFound"), "Object %T %s should have been removed.", t, name)
|
||||
}
|
||||
Loading…
Reference in a new issue