upstream/kubernetes
1
0
Fork 0
mirror of https://github.com/kubernetes/kubernetes.git synced 2026-02-18 18:28:18 -05:00
Code Issues Projects Releases Packages Wiki Activity Actions

DRA: CRUD conformance tests

Browse source 
Promoting real tests turned out to be harder than expected (should be rewritten
to be self-contained, additional reviews, etc.).

They would not achieve 100% endpoint+operation coverage because real tests only
use some of the operations. Therefore each API type has to be covered with
CRUD-style tests which only exercise the apiserver, then maybe additional
functional tests can be added later (depending on time and motivation).

The machinery for testing different API types is meant to be reusable, so it
gets added in the new e2e/framework/conformance helper package.
This commit is contained in:
Patrick Ohly 2025-08-28 11:09:05 +02:00
parent 545526cfe0
commit f95d531b0a
10 changed files with 1728 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -118,6 +118,7 @@ require (
k8s.io/sample-apiserver v0.0.0
k8s.io/system-validators v1.11.1
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730
sigs.k8s.io/knftables v0.0.17
sigs.k8s.io/randfill v1.0.0
sigs.k8s.io/structured-merge-diff/v6 v6.3.0
@ -217,7 +218,6 @@ require (
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b // indirect
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect
sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect
sigs.k8s.io/kustomize/api v0.20.1 // indirect
sigs.k8s.io/kustomize/kustomize/v5 v5.7.1 // indirect
sigs.k8s.io/kustomize/kyaml v0.20.1 // indirect

151
test/e2e/dra/dra.go
View file

@ -52,6 +52,7 @@ import (
drautils "k8s.io/kubernetes/test/e2e/dra/utils"
"k8s.io/kubernetes/test/e2e/feature"
"k8s.io/kubernetes/test/e2e/framework"
e2econformance "k8s.io/kubernetes/test/e2e/framework/conformance"
e2edaemonset "k8s.io/kubernetes/test/e2e/framework/daemonset"
e2eevents "k8s.io/kubernetes/test/e2e/framework/events"
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@ -77,6 +78,156 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
// modify /var/lib/kubelet/plugins.
f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
f.Context("CRUD Tests", func() {
/*
Release: v1.?
Testname: CRUD operations for deviceclasses
Description: kube-apiserver must support create/update/list/patch/delete operations for resource.k8s.io/v1 DeviceClass.
*/
framework.It("resource.k8s.io/v1 DeviceClass", func(ctx context.Context) {
e2econformance.TestResource(ctx, f,
&e2econformance.ResourceTestcase[*resourceapi.DeviceClass]{
GVR: resourceapi.SchemeGroupVersion.WithResource("deviceclasses"),
Namespaced: ptr.To(false),
InitialSpec: &resourceapi.DeviceClass{
Spec: resourceapi.DeviceClassSpec{
Selectors: []resourceapi.DeviceSelector{{
CEL: &resourceapi.CELDeviceSelector{
Expression: "false", // Matches no devices
},
}},
},
},
UpdateSpec: func(obj *resourceapi.DeviceClass) *resourceapi.DeviceClass {
obj.Spec.Selectors[0].CEL.Expression = "1 == 0" // Still matches no devices.
return obj
},
StrategicMergePatchSpec: `{"spec": {"selectors": [{"cel": {"expression": "1 == 0"}}]}}`,
},
)
})
/*
Release: v1.?
Testname: CRUD operations for resourceclaims
Description: kube-apiserver must support create/update/list/patch/delete operations for resource.k8s.io/v1 ResourceClaim.
*/
framework.It("resource.k8s.io/v1 ResourceClaim", func(ctx context.Context) {
e2econformance.TestResource(ctx, f,
&e2econformance.ResourceTestcase[*resourceapi.ResourceClaim]{
GVR: resourceapi.SchemeGroupVersion.WithResource("resourceclaims"),
Namespaced: ptr.To(true),
InitialSpec: &resourceapi.ResourceClaim{
Spec: resourceapi.ResourceClaimSpec{
Devices: resourceapi.DeviceClaim{
Requests: []resourceapi.DeviceRequest{{
Name: "req-0",
Exactly: &resourceapi.ExactDeviceRequest{
DeviceClassName: "dra.example.com",
},
}},
},
},
},
UpdateSpec: func(obj *resourceapi.ResourceClaim) *resourceapi.ResourceClaim {
// The spec is immutable, so let's add a label instead.
if obj.Labels == nil {
obj.Labels = make(map[string]string)
}
obj.Labels["test.dra.example.com"] = "test"
return obj
},
UpdateStatus: func(obj *resourceapi.ResourceClaim) *resourceapi.ResourceClaim {
// Nothing allocated" is a valid allocation result.
obj.Status.Allocation = &resourceapi.AllocationResult{}
return obj
},
// This test case uses all available patch types to demonstrate what this looks like
// and that TestResource supports them. Strategic merge patch, apply patch, and JSON
// merge patch are identical for these simple modifications.
//
// Testing with only one patch type would be sufficient for conformance.
ApplyPatchSpec: `{"metadata": {"labels": {"test.dra.example.com": "test"}}}`,
StrategicMergePatchSpec: `{"metadata": {"labels": {"test.dra.example.com": "test"}}}`,
JSONMergePatchSpec: `{"metadata": {"labels": {"test.dra.example.com": "test"}}}`,
JSONPatchSpec: `[{"op": "add", "path": "/metadata/labels/test.dra.example.com", "value": "test"}]`,
ApplyPatchStatus: `{"status": {"allocation": {}}}`,
StrategicMergePatchStatus: `{"status": {"allocation": {}}}`,
JSONMergePatchStatus: `{"status": {"allocation": {}}}`,
JSONPatchStatus: `[{"op": "add", "path": "/status/allocation", "value": {}}]`,
})
})
/*
Release: v1.?
Testname: CRUD operations for resourceclaimtemplates
Description: kube-apiserver must support create/update/list/patch/delete operations for resource.k8s.io/v1 ResourceClaimTemplate.
*/
framework.It("resource.k8s.io/v1 ResourceClaimTemplate", func(ctx context.Context) {
e2econformance.TestResource(ctx, f,
&e2econformance.ResourceTestcase[*resourceapi.ResourceClaimTemplate]{
GVR: resourceapi.SchemeGroupVersion.WithResource("resourceclaimtemplates"),
Namespaced: ptr.To(true),
InitialSpec: &resourceapi.ResourceClaimTemplate{
Spec: resourceapi.ResourceClaimTemplateSpec{
Spec: resourceapi.ResourceClaimSpec{
Devices: resourceapi.DeviceClaim{
Requests: []resourceapi.DeviceRequest{{
Name: "req-0",
Exactly: &resourceapi.ExactDeviceRequest{
DeviceClassName: "dra.example.com",
},
}},
},
},
},
},
UpdateSpec: func(obj *resourceapi.ResourceClaimTemplate) *resourceapi.ResourceClaimTemplate {
// The spec is immutable, so let's add a label instead.
if obj.Labels == nil {
obj.Labels = make(map[string]string)
}
obj.Labels["test.dra.example.com"] = "test"
return obj
},
StrategicMergePatchSpec: `{"metadata": {"labels": {"test.dra.example.com": "test"}}}`,
},
)
})
/*
Release: v1.?
Testname: CRUD operations for resoureslices
Description: kube-apiserver must support create/update/list/patch/delete operations for resource.k8s.io/v1 ResourceSlice.
*/
framework.It("resource.k8s.io/v1 ResourceSlice", func(ctx context.Context) {
e2econformance.TestResource(ctx, f,
&e2econformance.ResourceTestcase[*resourceapi.ResourceSlice]{
GVR: resourceapi.SchemeGroupVersion.WithResource("resourceslices"),
Namespaced: ptr.To(false),
InitialSpec: &resourceapi.ResourceSlice{
Spec: resourceapi.ResourceSliceSpec{
Driver: "dra.example.com",
Pool: resourceapi.ResourcePool{
Name: "cluster",
Generation: 1,
ResourceSliceCount: 1,
},
NodeName: ptr.To("no-such-node"),
// The pool is empty -> no devices.
},
},
UpdateSpec: func(obj *resourceapi.ResourceSlice) *resourceapi.ResourceSlice {
obj.Spec.Devices = []resourceapi.Device{{Name: "device-0"}}
return obj
},
StrategicMergePatchSpec: `{"spec": {"devices": [{"name": "device-0"}]}}`,
},
)
})
})
f.Context("kubelet", feature.DynamicResourceAllocation, func() {
nodes := drautils.NewNodes(f, 1, 1)
driver := drautils.NewDriver(f, nodes, drautils.NetworkResources(10, false))

12
test/e2e/framework/conformance/.import-restrictions Normal file
View file

@ -0,0 +1,12 @@
# This E2E framework sub-package is currently allowed to use arbitrary
# dependencies except of k/k/pkg, therefore we need to override the
# restrictions from the parent .import-restrictions file.
#
# At some point it may become useful to also check this package's
# dependencies more careful.
rules:
- selectorRegexp: "^k8s[.]io/kubernetes/pkg"
allowedPrefixes: []
- selectorRegexp: ""
allowedPrefixes: [ "" ]

99
test/e2e/framework/conformance/compare.go Normal file
View file

@ -0,0 +1,99 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package architecture
import (
"github.com/google/go-cmp/cmp"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
// compareObjects checks that all expected fields are set as expected.
// The actual object may have additional fields, their values are ignored.
func compareObjects(expected, actual *unstructured.Unstructured) string {
diff := cmp.Diff(expected.Object, actual.Object,
// Fields which are not in the expected object can be ignored.
// Only existing fields need to be compared.
//
// A maybe (?) simpler approach would be to trim the actual object,
// then compare with go-cmp. The advantage of telling go-cmp to
// ignore fields is that they show up as truncated ("...") in the diff,
// which is a bit more correct.
cmp.FilterPath(func(path cmp.Path) bool {
return fieldIsMissing(expected.Object, path)
}, cmp.Ignore()),
)
return diff
}
// fieldIsMissing returns true if the field identified by the path is not
// present in the object. It works by recursively descending along the
// path and checking the corresponding content of the object along the way.
func fieldIsMissing(obj map[string]any, path cmp.Path) bool {
// First entry is a NOP.
missing := fieldIsMissingStep(obj, path[1:])
// Uncomment for debugging...
// fmt.Printf("fieldIsMissing: %s %v\n", path.GoString(), missing)
return missing
}
func fieldIsMissingStep(value any, path []cmp.PathStep) bool {
if len(path) == 0 {
// Done, full path was checked.
return false
}
// We only need to descend for certain lookup steps,
// everything else is treated as "not missing" and thus
// gets compared.
switch pathElement := path[0].(type) {
case cmp.MapIndex:
key := pathElement.Key().String()
value, ok := value.(map[string]any)
if !ok {
// Type mismatch.
return false
}
entry, found := value[key]
if !found {
return true
}
return fieldIsMissingStep(entry, path[1:])
case cmp.SliceIndex:
key := pathElement.Key()
value, ok := value.([]any)
if !ok {
// Type mismatch.
return false
}
if key < 0 {
// Not sure why go-cmp uses a negative index, so let's compare it.
return false
}
if key >= len(value) {
// Slice is smaller -> missing entry.
return true
}
entry := value[key]
return fieldIsMissingStep(entry, path[1:])
case cmp.TypeAssertion:
// Actual value type will be checked when needed,
// skip the assertion here.
return fieldIsMissingStep(value, path[1:])
default:
return false
}
}

113
test/e2e/framework/conformance/compare_test.go Normal file
View file

@ -0,0 +1,113 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package architecture
import (
"testing"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
)
func TestCompareObjects(t *testing.T) {
for name, tc := range map[string]struct {
expected, actual string
expectedDiff string
}{
"equal": {
expected: `{"kind":"test","hello":"world"}`,
actual: `{"kind":"test","hello":"world"}`,
},
"missing": {
expected: `{"kind":"test","hello":"world"}`,
actual: `{"kind":"test"}`,
expectedDiff: ` map[string]any{
- "hello": string("world"),
"kind": string("test"),
}
`,
},
"added": {
expected: `{"kind":"test","hello":"world"}`,
actual: `{"kind":"test","hello":"world","foo":"bar"}`,
},
"replaced": {
expected: `{"kind":"test","hello":"world"}`,
actual: `{"kind":"test","hello":1}`,
expectedDiff: ` map[string]any{
- "hello": string("world"),
+ "hello": int64(1),
"kind": string("test"),
}
`,
},
"recursive": {
expected: `{"kind":"test","spec":{"hello":"world","removed":42}}`,
actual: `{"kind":"test","spec":{"hello":1,"added":42}}`,
expectedDiff: ` map[string]any{
"kind": string("test"),
"spec": map[string]any{
... // 1 ignored entry
- "hello": string("world"),
+ "hello": int64(1),
- "removed": int64(42),
},
}
`,
},
"list": {
expected: `{"kind":"test","items":[{"index":0},{"hello":"world","removed":42}]}`,
actual: `{"kind":"test","items":[{"index":0,"added":true},{"hello":1,"added":42},{"new-entry": true},"new-non-object-entry"]}`,
expectedDiff: ` map[string]any{
"items": []any{
map[string]any{"index": int64(0), ...},
+ map[string]any{"added": int64(42), "hello": int64(1)},
+ map[string]any{"new-entry": bool(true)},
- map[string]any{"hello": string("world"), "removed": int64(42)},
+ string("new-non-object-entry"),
},
"kind": string("test"),
}`,
},
} {
t.Run(name, func(t *testing.T) {
var expected, actual unstructured.Unstructured
require.NoError(t, expected.UnmarshalJSON([]byte(tc.expected)), "unmarshal expected")
require.NoError(t, actual.UnmarshalJSON([]byte(tc.actual)), "unmarshal actual")
actualDiff := compareObjects(&expected, &actual)
t.Logf("Actual diff:\n%s", actualDiff)
// Upstream go-cmp does not want the diff output to be checked in
// tests because it is not stable. They intentionally randomly
// switch between space and non-break space to enforce that
// (https://github.com/google/go-cmp/issues/366).
//
// Therefore the expected diff is merely informative, we only check
// for empty vs. not empty.
if tc.expectedDiff == "" {
require.Empty(t, actualDiff)
} else {
require.NotEmpty(t, actualDiff)
}
})
}
}

946
test/e2e/framework/conformance/conformance.go Normal file
View file

@ -0,0 +1,946 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package architecture
import (
"context"
"encoding/json"
"fmt"
"slices"
"time"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
gtypes "github.com/onsi/gomega/types"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/resourceversion"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/dynamic/dynamicinformer"
"k8s.io/client-go/informers"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
apimachineryutils "k8s.io/kubernetes/test/e2e/common/apimachinery"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/utils/format"
"k8s.io/utils/ptr"
k8sjson "sigs.k8s.io/json"
)
// ResourceTestcaseInterface describes how to test one particular API endpoint
// by executing different operations against it.
//
// The content of created or patched objects is verified by ensuring that
// all fields are set as in the sent object. Extra fields or map entries
// are ignored.
//
// Basic create/read/update/delete (CRUD) semantic is covered, which
// is the minimum that is required for conformance testing of a
// GA feature. Actual functional testing is desirable, but not
// required.
//
// See [ResourceTestcase] for an implementation of this interface
// where test data is provided as Go objects and patch strings.
type ResourceTestcaseInterface interface {
// GetGroupVersionResource returns the API group, version, and resource (plural form, lower case).
GetGroupVersionResource() schema.GroupVersionResource
// IsNamespaced defines whether the object must be created inside a namespace.
IsNamespaced() bool
// HasStatus defines whether the resource has a "status" sub-resource.
//
// Other sub-resources are not supported by this common test code.
HasStatus() bool
// VerifyContent defines whether the content of objects returned by
// the apiserver gets compared against the content that was sent.
//
// If enabled, all field values that were sent must also be included
// in the returned object. Additional fields and list or map entries
// may get added (for example, because of defaulting or mutating
// admission).
//
// This should not be enabled in conformance tests because admission
// is allowed to modify what is being stored.
VerifyContent() bool
// GetInitialObject returns the data which is going to be used in a Create call.
//
// For cluster-scoped resources the test namespace can be used
// to create a name which does not conflict with other objects
// because it is unique while the test runs.
//
// It does not need to be set for namespaced resources because the
// caller will ensure that. The caller cannot do that for the name because
// different resources have different rules for what names are valid
GetInitialObject(namespace string) *unstructured.Unstructured
// GetUpdateSpec modifies an existing object.
// It gets called for the result of creating the initial object.
//
// Ideally it should change the spec (hence the name).
// If that is impossible, then adding some label is also okay.
// The goal is to add some fields that can be checked for
// after an Update.
//
// May modify and return the input object.
GetUpdateSpec(object *unstructured.Unstructured) *unstructured.Unstructured
// GetUpdateStatus modifies the status of an existing object.
// It gets called for the result of creating the initial object
// and then updating its spec.
//
// May modify and return the input object.
GetUpdateStatus(object *unstructured.Unstructured) *unstructured.Unstructured
// GetPatchSpec describes how to generate patches.
//
// Each patch is applied to the initial object by itself, without the other patches.
// An empty slice is valid and disables testing of patching. This may not be sufficient
// for full conformance testing of the resource.
//
// If content verification is enabled, then this must cause the same change as GetUpdateSpec
// because verification of the patch result uses the GetUpdateSpec result as reference.
GetPatchSpec() []Patch
// GetPatchStatus is like GetPatchSpec for the status.
//
// The initial object with the updated spec gets patched,
// so the result must match the result of GetUpdateStatus
// applied to GetInitialObject if content verification is
// enabled.
GetPatchStatus(object *unstructured.Unstructured) []Patch
}
// Patch contains the parameters for a Patch API call.
//
// The data must match an existing object.
//
// There's no retry loop because of conflicts, so the patch should not include
// a check of the ResourceVersion. Checking the UID in the patch is encouraged to prevent
// patching a replaced resource.
type Patch struct {
GetData func(object *unstructured.Unstructured) []byte
Type types.PatchType
}
// ResourceTestcase provides test data for testing operations for a resource.
// Test data is based on the native Go type of the resource.
// The template parameter must be a pointer to the native Go type.
//
// The data is used like this:
// - create InitialSpec -> update with UpdateSpec -> update status with UpdateStatus
// - create InitialSpec -> apply StrategicMergePatchSpec and compare against UpdateSpec -> apply StrategicMergePatchStatus and compare against UpdateStatus
type ResourceTestcase[T runtime.Object] struct {
// GroupResourceVersion identifies the API group, version and resource (plural form, lower case)
// within that API which is to be tested.
GVR schema.GroupVersionResource
// Namespaced must be true if the resource must be created in a
// namespace, false if it is cluster-scoped. Leaving it unset is
// an error.
//
// Namespaced resources get created in the test namespace.
//
// The name of cluster-scoped resources gets extended with
// `-<test namespace name>` to make it unique.
Namespaced *bool
// ContentVerificationEnabled defines whether the content of objects returned by
// the apiserver gets compared against the content that was sent.
//
// If enabled, all field values that were sent must also be included
// in the returned object. Additional fields and list or map entries
// may get added (for example, because of defaulting or mutating
// admission).
//
// This should not be enabled in conformance tests because admission
// is allowed to modify what is being stored.
ContentVerificationEnabled bool
// InitialSpec must contain the initial state of a valid resource, without a status.
InitialSpec T
// UpdateSpec gets called for the created initial object
// and must update something, ideally the spec (hence the name).
// If that is not possible, then adding some label also works
// for the sake of testing an update.
//
// May modify and return the input object.
UpdateSpec func(T) T
// UpdateStatus gets called for the updated object
// and must add a status.
//
// May be nil if no status is supported.
//
// May modify and return the input object.
UpdateStatus func(T) T
// StrategicMergePatchSpec may modify fields in InitialSpec
// with a strategic merge patch
// (https://github.com/kubernetes/community/blob/master/contributors/devel/sig-api-machinery/strategic-merge-patch.md).
// Muse use JSON encoding.
//
// If content verification is enabled, then this must contain the same change as UpdateSpec
// because verification of the patch result uses UpdateSpec as reference.
StrategicMergePatchSpec string
// StrategicMergePatchStatus may add status fields
// with a strategic merge patch
// (https://github.com/kubernetes/community/blob/master/contributors/devel/sig-api-machinery/strategic-merge-patch.md)
// Must use JSON encoding.
//
// The initial object with the updated spec gets patched,
// so the result must match GetUpdateStatus
// applied to GetInitialObject if content verification is
// enabled.
//
// If empty, the status sub-resource is not getting tested.
// May contain the name, but that's not required.
StrategicMergePatchStatus string
// ApplyPatchSpec corresponds to StrategicMergePatchSpec,
// using the JSON encoding of a server-side-apply (SSA) patch
// (https://kubernetes.io/docs/reference/using-api/server-side-apply).
ApplyPatchSpec string
// ApplyPatchStatus corresponds to StrategicMergePatchStatus,
// using the JSON encoding of a server-side-apply (SSA) patch
// (https://kubernetes.io/docs/reference/using-api/server-side-apply).
ApplyPatchStatus string
// JSONPatchSpec corresponds to StrategicMergePatchSpec,
// using a JSON patch (https://tools.ietf.org/html/rfc6902).
JSONPatchSpec string
// JSONPatchStatus corresponds to StrategicMergePatchStatus,
// using a JSON patch (https://tools.ietf.org/html/rfc6902).
JSONPatchStatus string
// JSONMergePatchSpec corresponds to StrategicMergePatchSpec,
// using a JSON merge patch (https://tools.ietf.org/html/rfc7386).
JSONMergePatchSpec string
// JSONMergePatchStatus corresponds to StrategicMergePatchStatus,
// using a JSON merge patch (https://tools.ietf.org/html/rfc7386).
JSONMergePatchStatus string
}
var _ ResourceTestcaseInterface = &ResourceTestcase[*v1.Pod]{}
func (tc *ResourceTestcase[T]) GetGroupVersionResource() schema.GroupVersionResource {
return tc.GVR
}
func (tc *ResourceTestcase[T]) IsNamespaced() bool {
if tc.Namespaced == nil {
framework.Fail("Test case error: Namespaced must be set")
}
return *tc.Namespaced
}
func (tc *ResourceTestcase[T]) HasStatus() bool {
return tc.UpdateStatus != nil
}
func (tc *ResourceTestcase[T]) VerifyContent() bool {
return tc.ContentVerificationEnabled
}
func (tc *ResourceTestcase[T]) GetInitialObject(namespace string) *unstructured.Unstructured {
object := tc.toUnstructured("InitialSpec", tc.InitialSpec)
if object.GetName() == "" {
object.SetName("test")
}
if !tc.IsNamespaced() {
object.SetName(object.GetName() + "-" + namespace)
}
return object
}
func (tc *ResourceTestcase[T]) GetUpdateSpec(in *unstructured.Unstructured) *unstructured.Unstructured {
out := tc.fromUnstructured("existing object", in)
out = tc.UpdateSpec(out)
return tc.toUnstructured("updated object", out)
}
func (tc *ResourceTestcase[T]) GetUpdateStatus(in *unstructured.Unstructured) *unstructured.Unstructured {
out := tc.fromUnstructured("updated object", in)
out = tc.UpdateStatus(out)
return tc.toUnstructured("updated object with status", out)
}
func (tc *ResourceTestcase[T]) GetPatchSpec() []Patch {
var patches []Patch
if tc.StrategicMergePatchSpec != "" {
patches = append(patches, Patch{
Type: types.StrategicMergePatchType,
GetData: func(existingObject *unstructured.Unstructured) []byte {
patch := tc.createPatchObject("StrategicMergePatchSpec", tc.StrategicMergePatchSpec, existingObject)
jsonData, err := patch.MarshalJSON()
framework.ExpectNoError(err, "re-encode spec patch as JSON")
return jsonData
},
})
}
if tc.ApplyPatchSpec != "" {
patches = append(patches, Patch{
Type: types.ApplyPatchType,
GetData: func(existingObject *unstructured.Unstructured) []byte {
patch := tc.createPatchObject("ApplyPatchSpec", tc.ApplyPatchSpec, existingObject)
jsonData, err := patch.MarshalJSON()
framework.ExpectNoError(err, "re-encode spec patch as JSON")
return jsonData
},
})
}
if tc.JSONMergePatchSpec != "" {
patches = append(patches, Patch{
Type: types.MergePatchType,
GetData: func(existingObject *unstructured.Unstructured) []byte {
patch := tc.createPatchObject("JSONMergePatchSpec", tc.JSONMergePatchSpec, existingObject)
jsonData, err := patch.MarshalJSON()
framework.ExpectNoError(err, "re-encode spec patch as JSON")
return jsonData
},
})
}
if tc.JSONPatchSpec != "" {
patches = append(patches, Patch{
Type: types.JSONPatchType,
GetData: func(existingObject *unstructured.Unstructured) []byte {
return []byte(tc.JSONPatchSpec)
},
})
}
return patches
}
func (tc *ResourceTestcase[T]) GetPatchStatus(object *unstructured.Unstructured) []Patch {
var patches []Patch
if tc.StrategicMergePatchStatus != "" {
patches = append(patches, Patch{
Type: types.StrategicMergePatchType,
GetData: func(existingObject *unstructured.Unstructured) []byte {
patch := tc.createPatchObject("StrategicMergePatchStatus", tc.StrategicMergePatchStatus, existingObject)
jsonData, err := patch.MarshalJSON()
framework.ExpectNoError(err, "re-encode status patch as JSON")
return jsonData
},
})
}
if tc.ApplyPatchStatus != "" {
patches = append(patches, Patch{
Type: types.ApplyPatchType,
GetData: func(existingObject *unstructured.Unstructured) []byte {
patch := tc.createPatchObject("ApplyPatchStatus", tc.ApplyPatchStatus, existingObject)
jsonData, err := patch.MarshalJSON()
framework.ExpectNoError(err, "re-encode status patch as JSON")
return jsonData
},
})
}
if tc.JSONMergePatchStatus != "" {
patches = append(patches, Patch{
Type: types.MergePatchType,
GetData: func(existingObject *unstructured.Unstructured) []byte {
patch := tc.createPatchObject("JSONMergePatchStatus", tc.JSONMergePatchStatus, existingObject)
jsonData, err := patch.MarshalJSON()
framework.ExpectNoError(err, "re-encode status patch as JSON")
return jsonData
},
})
}
if tc.JSONPatchStatus != "" {
patches = append(patches, Patch{
Type: types.JSONPatchType,
GetData: func(existingObject *unstructured.Unstructured) []byte {
return []byte(tc.JSONPatchStatus)
},
})
}
return patches
}
func (tc *ResourceTestcase[T]) toUnstructured(what string, in T) *unstructured.Unstructured {
data, err := json.Marshal(in)
framework.ExpectNoError(err, "encode %s as JSON", what)
out := tc.toUnstructuredFromJSON(what, data)
return out
}
func (tc *ResourceTestcase[T]) toUnstructuredFromJSON(what string, in []byte) *unstructured.Unstructured {
// UnmarshalCaseSensitivePreserveInts does not need kind (in contrast to unstructured.Unstructured.UnmarshalJSON)
// and matches the behavior of preserving ints that we get when receiving from the apiserver (in contrast to plain json.Unmarshal).
var out unstructured.Unstructured
err := k8sjson.UnmarshalCaseSensitivePreserveInts(in, &out.Object)
framework.ExpectNoError(err, "decode %s from JSON", what)
return &out
}
func (tc *ResourceTestcase[T]) fromUnstructured(what string, in *unstructured.Unstructured) T {
data, err := in.MarshalJSON()
framework.ExpectNoError(err, "encode %s as JSON", what)
var out T
err = k8sjson.UnmarshalCaseSensitivePreserveInts(data, &out)
framework.ExpectNoError(err, "decode %s from JSON", what)
return out
}
// createPatchObject parses JSON data and then copies namespace/name/uid/kind/apiVersion from the existing object
// to make the patch complete. This works for strategic merge patches, apply patches and JSON merge patches.
func (tc *ResourceTestcase[T]) createPatchObject(what string, data string, existingObject *unstructured.Unstructured) *unstructured.Unstructured {
object := tc.toUnstructuredFromJSON(what, []byte(data))
object.SetNamespace(existingObject.GetNamespace())
object.SetName(existingObject.GetName())
object.SetUID(existingObject.GetUID())
object.SetAPIVersion(existingObject.GetAPIVersion())
object.SetKind(existingObject.GetKind())
return object
}
// TestResource covers all the typical endpoints for a resource through
// dynamic client calls.
func TestResource(ctx context.Context, f *framework.Framework, tc ResourceTestcaseInterface) {
// Set up clients.
gvr := tc.GetGroupVersionResource()
gv := gvr.GroupVersion()
resource := gvr.Resource
resourceClient := f.DynamicClient.Resource(gvr)
var client dynamic.ResourceInterface
var resourceType string
if tc.IsNamespaced() {
client = resourceClient.Namespace(f.Namespace.Name)
resourceType = "namespaced resource"
} else {
client = resourceClient
resourceType = "cluster-scoped resource"
}
// e.g. `cluster-scoped resource "deviceclasses"`
// gvr.String() is too long and includes a comma ("resource.k8s.io/v1, Resource=deviceclasses").
resourceType = fmt.Sprintf("%s %q", resourceType, gvr.Resource)
config := dynamic.ConfigFor(f.ClientConfig())
httpClient, err := rest.HTTPClientFor(config)
framework.ExpectNoError(err, "construct HTTP client")
restClient, err := rest.UnversionedRESTClientForConfigAndClient(config, httpClient)
framework.ExpectNoError(err, "construct REST client")
// All objects get one label added by the test for List and DeleteCollection.
// The label must get added to all objects returned by ResourceTestcase
// because the implementation of that interface is unaware of the extra label.
labelName := "e2e-test.kubernetes.io"
labelValue := f.UniqueName
listOptions := metav1.ListOptions{LabelSelector: labelName + "=" + labelValue}
addLabel := func(obj *unstructured.Unstructured) *unstructured.Unstructured {
obj = obj.DeepCopy()
labels := obj.GetLabels()
if labels == nil {
labels = make(map[string]string)
}
labels[labelName] = labelValue
obj.SetLabels(labels)
return obj
}
// Prepare for Create, Get and List.
desiredInitialObject := addLabel(tc.GetInitialObject(f.Namespace.Name))
if tc.IsNamespaced() {
desiredInitialObject.SetNamespace(f.Namespace.Name)
}
getResource := func(ctx context.Context) (*unstructured.Unstructured, error) {
return client.Get(ctx, desiredInitialObject.GetName(), metav1.GetOptions{})
}
desiredUpdatedObject := tc.GetUpdateSpec(desiredInitialObject.DeepCopy())
var desiredUpdatedObjectWithStatus *unstructured.Unstructured
if tc.HasStatus() {
desiredUpdatedObjectWithStatus = addLabel(tc.GetUpdateStatus(desiredUpdatedObject))
}
// Get all resources in the API. The resulting list of resources must include what we are testing.
ginkgo.By(fmt.Sprintf("Get %s", gv))
path := "/apis/" + gv.String()
var api unstructured.Unstructured
err = restClient.
Get().
AbsPath(path).
Do(ctx).
Into(&api)
framework.ExpectNoError(err, "get resource API")
resources := api.Object["resources"].([]any)
index := slices.IndexFunc(resources, func(entry any) bool {
return entry.(map[string]any)["name"].(string) == resource
})
if index < 0 {
framework.Failf("API for %s does not include entry for %s, got:\n%s", gv, resource, format.Object(api, 1))
}
// Set up informers, optionally also in the namespace.
// After each step we check that the informers catch up
// and what events they received in the meantime.
// They get stopped through test context cancellation.
var resourceEvents, namespaceEvents eventRecorder
resourceInformer := dynamicinformer.NewFilteredDynamicInformer(f.DynamicClient, gvr, "", 0, nil, func(options *metav1.ListOptions) {
options.LabelSelector = listOptions.LabelSelector
})
_, err = resourceInformer.Informer().AddEventHandler(&resourceEvents)
framework.ExpectNoError(err, "register resource event handler")
listResource := func(_ context.Context) ([]runtime.Object, error) {
return resourceInformer.Lister().List(labels.Everything())
}
go resourceInformer.Informer().RunWithContext(ctx)
informersHaveSynced := []cache.InformerSynced{resourceInformer.Informer().HasSynced}
var namespaceInformer informers.GenericInformer
var listNamespace func(_ context.Context) ([]runtime.Object, error)
if tc.IsNamespaced() {
namespaceInformer = dynamicinformer.NewFilteredDynamicInformer(f.DynamicClient, gvr, f.Namespace.Name, 0, nil, func(options *metav1.ListOptions) {
options.LabelSelector = listOptions.LabelSelector
})
_, err = namespaceInformer.Informer().AddEventHandler(&namespaceEvents)
framework.ExpectNoError(err, "register namespace event handler")
listNamespace = func(_ context.Context) ([]runtime.Object, error) {
return namespaceInformer.Lister().List(labels.Everything())
}
go namespaceInformer.Informer().RunWithContext(ctx)
informersHaveSynced = append(informersHaveSynced, namespaceInformer.Informer().HasSynced)
}
if !cache.WaitForNamedCacheSyncWithContext(ctx, informersHaveSynced...) {
ginkgo.Fail("informers should have synced and didn't")
}
// matchObject generates a matcher which checks the result of a list operation
// against the expected object. Content verification is optional. Without it,
// only the namespace and name are checked.
matchObject := func(expectedObject *unstructured.Unstructured) gtypes.GomegaMatcher {
return &matchObjectList{expectedObject: expectedObject, verifyContent: tc.VerifyContent()}
}
gomega.Expect(listResource(ctx)).Should(matchObject(nil), "initial list of resources from informer cache")
gomega.Expect(resourceEvents.list()).To(gomega.HaveField("Events", gomega.BeEmpty()), "no events from resource informer yet")
if listNamespace != nil {
gomega.Expect(listNamespace(ctx)).Should(matchObject(nil), "initial list of namespace from informer cache")
gomega.Expect(resourceEvents.list()).To(gomega.HaveField("Events", gomega.BeEmpty()), "no events from namespace informer yet")
}
// matchEvents generates a matcher which checks the informer event list.
//
// The events are expected to involve only the given object.
// The ResourceVersion must not decrease.
// The sequence of valid events is given as a regular expression
// which is applied to the string returned by [EventList.Types].
matchEvents := func(obj *unstructured.Unstructured, regexp string) gtypes.GomegaMatcher {
// Verify namespace/name/uid of ids event.
ids := gomega.HaveField("Events", gomega.Or(gomega.BeEmpty(), gomega.HaveEach(gomega.HaveField("ID()", gomega.Equal(fmt.Sprintf("%s, %s", klog.KObj(obj), obj.GetUID()))))))
// Match the regexp against the result of Types().
order := gomega.HaveField("Types()", gomega.MatchRegexp(regexp))
// Include full object dump, HaveField itself doesn't.
return framework.GomegaObject(gomega.And(ids, order))
}
// ResourceVersion must not decrease. Delete events reset the sequence.
var resourceRV, namespaceRV string
nextEvents := func(rv *string, events eventList) {
ginkgo.GinkgoHelper()
checkNext := func(obj any) {
ginkgo.GinkgoHelper()
if obj == nil {
return
}
if tomb, ok := obj.(cache.DeletedFinalStateUnknown); ok {
obj = tomb.Obj
}
metaData, err := meta.Accessor(obj)
framework.ExpectNoError(err, "access meta data")
gomega.Expect(metaData).To(apimachineryutils.HaveValidResourceVersion())
nextRV := metaData.GetResourceVersion()
if *rv == "" {
// Nothing to compare yet, initial version.
*rv = nextRV
return
}
cmpResult, err := resourceversion.CompareResourceVersion(nextRV, *rv)
framework.ExpectNoError(err, "compare ResourceVersions")
if cmpResult < 0 {
framework.Failf("ResourceVersion %s in %s with UID %s is smaller than previous %s, should be equal or larger", nextRV, klog.KObj(metaData), metaData.GetUID(), *rv)
}
*rv = nextRV
}
for _, e := range events.Events {
checkNext(e.oldObj)
checkNext(e.newObj)
}
}
// Verification of the result after each step (= what).
//
// Can check the content of an object or be limited to just name/namespace.
//
// Also checks the informers. This is a bit redundant after read-only steps,
// but then it's also fast and thus can be done more often than strictly necessary.
// Collected informer events get reset, so each verify call must match
// events since the previous one.
verify := func(what string, expected, actual *unstructured.Unstructured, haveExpectedEvents gtypes.GomegaMatcher) {
ginkgo.GinkgoHelper()
// This captures several different failures before handing them to Ginkgo.
var failures gomegaFailures
if expected != nil {
if tc.VerifyContent() {
diff := compareObjects(expected, actual)
failures.Add(fmt.Sprintf("%s: unexpected actual object (- expected, + actual):\n%s", what, diff))
} else {
failures.G().Expect(actual.GetName()).Should(gomega.Equal(expected.GetName()), "%s: name in returned object", what)
failures.G().Expect(actual.GetNamespace()).Should(gomega.Equal(expected.GetNamespace()), "%s: namespace in returned object", what)
}
failures.G().Expect(actual).To(apimachineryutils.HaveValidResourceVersion())
}
// Abort checking now if there were failures, otherwise we just risk timining out slowly.
failures.Check()
failures.G().Eventually(ctx, listResource).Should(matchObject(expected), "list of resources from informer cache after %s", what)
if haveExpectedEvents != nil {
// Even if the cache is up-to-date we still need to wait for event delivery.
failures.G().Eventually(resourceEvents.list).
WithTimeout(5*time.Second).
Should(haveExpectedEvents, "list of resource informer events after %s", what)
}
nextEvents(&resourceRV, resourceEvents.reset())
if listNamespace != nil {
failures.G().Eventually(ctx, listNamespace).Should(matchObject(expected), "list of namespace from informer cache after %s", what)
if haveExpectedEvents != nil {
failures.G().Eventually(namespaceEvents.list).
WithTimeout(5*time.Second).
Should(haveExpectedEvents, "list of namespace informer events after %s", what)
}
nextEvents(&namespaceRV, namespaceEvents.reset())
}
failures.Check()
}
// Create the initial resource.
ginkgo.By(fmt.Sprintf("Creating:\n%s", format.Object(desiredInitialObject, 1)))
existingObject, err := client.Create(ctx, desiredInitialObject, metav1.CreateOptions{FieldValidation: "Strict"})
framework.ExpectNoError(err, "create initial %s", resourceType)
ginkgo.DeferCleanup(func(ctx context.Context) {
// Always clean up.
err = client.Delete(ctx, desiredInitialObject.GetName(), metav1.DeleteOptions{})
if apierrors.IsNotFound(err) {
return
}
framework.ExpectNoError(err, "delete %s", resourceType)
ensureNotFound(ctx, getResource)
})
verify("create", desiredInitialObject, existingObject,
// Initial creation of the object followed by some optional updates by cluster components.
matchEvents(existingObject, "^add,(update,)*$"),
)
createdResourceVersion := existingObject.GetResourceVersion()
// Get to check for existence.
ginkgo.By(fmt.Sprintf("Getting %s", klog.KObj(desiredInitialObject)))
existingObject, err = client.Get(ctx, desiredInitialObject.GetName(), metav1.GetOptions{})
framework.ExpectNoError(err, "get updated %s", resourceType)
verify("get", desiredInitialObject, existingObject,
// Optional updates by cluster components.
matchEvents(existingObject, "^(update,)*$"),
)
// Update the resource. Retry because the existing object might have been updated in the meantime.
mustGet := false
gomega.Eventually(ctx, func(ctx context.Context) error {
if mustGet {
ginkgo.By(fmt.Sprintf("Getting updated %s", klog.KObj(desiredInitialObject)))
existingObject, err = getResource(ctx)
if err != nil {
return fmt.Errorf("get existing %s: %w", resourceType, err)
}
}
object := tc.GetUpdateSpec(existingObject.DeepCopy())
ginkgo.By(fmt.Sprintf("Updating:\n%s", format.Object(object, 1)))
existingObject, err = client.Update(ctx, object, metav1.UpdateOptions{})
if err == nil {
return nil
}
mustGet = apierrors.IsConflict(err)
if mustGet {
// Retry immediately.
return fmt.Errorf("update %s: %w", resourceType, err)
}
if retry, retryAfter := framework.ShouldRetry(err); retry {
// Retry with a delay.
return gomega.TryAgainAfter(retryAfter)
}
// Give up, some other error occurred.
return gomega.StopTrying(fmt.Sprintf("update %s", resourceType)).Wrap(err)
}).Should(gomega.Succeed())
verify("update", desiredUpdatedObject, existingObject,
// At least one update.
matchEvents(existingObject, "^(update,)+$"),
)
updatedResourceVersion := existingObject.GetResourceVersion()
cmpResult, err := resourceversion.CompareResourceVersion(createdResourceVersion, updatedResourceVersion)
framework.ExpectNoError(err, "compare ResourceVersion after create against ResourceVersion after update")
if cmpResult >= 0 {
framework.Failf("ResourceVersion should have increased during update and didn't (before: %s, after: %s)", createdResourceVersion, updatedResourceVersion)
}
// Same for the status. In addition, read the status (same result, but different endpoint+method).
if tc.HasStatus() {
mustGet := false
gomega.Eventually(ctx, func(ctx context.Context) error {
if mustGet {
ginkgo.By(fmt.Sprintf("Getting updated %s", klog.KObj(desiredInitialObject)))
existingObject, err = client.Get(ctx, desiredInitialObject.GetName(), metav1.GetOptions{}, "status")
if err != nil {
return fmt.Errorf("get existing %s: %w", resourceType, err)
}
}
object := tc.GetUpdateStatus(existingObject)
ginkgo.By(fmt.Sprintf("Updating status:\n%s", format.Object(object, 1)))
existingObject, err = client.Update(ctx, object, metav1.UpdateOptions{}, "status")
if err == nil {
return nil
}
mustGet = apierrors.IsConflict(err)
if mustGet {
// Retry immediately.
return fmt.Errorf("update %s status: %w", resourceType, err)
}
if retry, retryAfter := framework.ShouldRetry(err); retry {
// Retry with a delay.
return gomega.TryAgainAfter(retryAfter)
}
// Give up, some other error occurred.
return gomega.StopTrying(fmt.Sprintf("update %s status", resourceType)).Wrap(err)
}).Should(gomega.Succeed())
verify("update status", desiredUpdatedObjectWithStatus, existingObject,
// At least one update.
matchEvents(existingObject, "^(update,)+$"),
)
ginkgo.By(fmt.Sprintf("Getting %s status", klog.KObj(desiredInitialObject)))
existingObject, err = client.Get(ctx, desiredInitialObject.GetName(), metav1.GetOptions{}, "status")
framework.ExpectNoError(err, "get updated %s", resourceType)
verify("get", desiredUpdatedObjectWithStatus, existingObject,
// Optional updates by cluster components.
matchEvents(existingObject, "^(update,)*$"),
)
}
// Patch the resource, potentially using multiple different patch types.
// The result must be the same each time if content verification is enabled.
for _, patch := range tc.GetPatchSpec() {
// Delete the resource to start anew.
ginkgo.By(fmt.Sprintf("Deleting %s", klog.KObj(desiredInitialObject)))
err = client.Delete(ctx, desiredInitialObject.GetName(), metav1.DeleteOptions{})
framework.ExpectNoError(err, "delete updated %s", resourceType)
ensureNotFound(ctx, getResource)
verify("delete", nil, nil,
// Optional updates, deletion.
//
// We have to verify this here because
// otherwise we have no guarantee that we see the delete event.
matchEvents(existingObject, "^(update,)*delete,$"),
)
// Recreate for patching.
ginkgo.By(fmt.Sprintf("Creating again:\n%s", format.Object(desiredInitialObject, 1)))
existingObject, err = client.Create(ctx, desiredInitialObject, metav1.CreateOptions{})
framework.ExpectNoError(err, "create %s again", resourceType)
patchData := patch.GetData(existingObject)
ginkgo.By(fmt.Sprintf("Patching with %s:\n%s", patch.Type, string(patchData)))
options := metav1.PatchOptions{FieldValidation: "Strict"}
switch patch.Type {
case types.ApplyYAMLPatchType, types.ApplyCBORPatchType:
options.FieldManager = "test-apply"
options.Force = ptr.To(true)
}
existingObject, err = client.Patch(ctx, desiredInitialObject.GetName(), patch.Type, patchData, options)
framework.ExpectNoError(err, "patch %s %s", patch.Type, resourceType)
verify(fmt.Sprintf("patch %s", patch.Type), desiredUpdatedObject, existingObject,
// Recreation and then at least one update.
matchEvents(existingObject, "^add,(update,)+$"),
)
}
// Same for status. The patches apply on top of the updated object.
for _, patch := range tc.GetPatchStatus(existingObject) {
// Delete the resource to start anew.
ginkgo.By(fmt.Sprintf("Deleting %s", klog.KObj(desiredInitialObject)))
err = client.Delete(ctx, desiredInitialObject.GetName(), metav1.DeleteOptions{})
framework.ExpectNoError(err, "delete updated %s", resourceType)
ensureNotFound(ctx, getResource)
verify("delete", nil, nil,
// Optional updates, deletion.
//
// We have to verify this here because
// otherwise we have no guarantee that we see the delete event.
matchEvents(existingObject, "^(update,)*delete,$"),
)
// Recreate for patching.
ginkgo.By(fmt.Sprintf("Creating again:\n%s", format.Object(desiredInitialObject, 1)))
existingObject, err = client.Create(ctx, desiredInitialObject, metav1.CreateOptions{})
framework.ExpectNoError(err, "create %s again", resourceType)
// Update again.
mustGet := false
gomega.Eventually(ctx, func(ctx context.Context) error {
if mustGet {
ginkgo.By(fmt.Sprintf("Getting updated %s", klog.KObj(desiredInitialObject)))
existingObject, err = getResource(ctx)
if err != nil {
return fmt.Errorf("get existing %s: %w", resourceType, err)
}
}
object := tc.GetUpdateSpec(existingObject.DeepCopy())
ginkgo.By(fmt.Sprintf("Updating:\n%s", format.Object(object, 1)))
existingObject, err = client.Update(ctx, object, metav1.UpdateOptions{})
if err == nil {
return nil
}
mustGet = apierrors.IsConflict(err)
if mustGet {
// Retry immediately.
return fmt.Errorf("update %s: %w", resourceType, err)
}
if retry, retryAfter := framework.ShouldRetry(err); retry {
// Retry with a delay.
return gomega.TryAgainAfter(retryAfter)
}
// Give up, some other error occurred.
return gomega.StopTrying(fmt.Sprintf("update %s", resourceType)).Wrap(err)
}).Should(gomega.Succeed())
patchData := patch.GetData(existingObject)
ginkgo.By(fmt.Sprintf("Patching status with %s:\n%s", patch.Type, string(patchData)))
options := metav1.PatchOptions{FieldValidation: "Strict"}
switch patch.Type {
case types.ApplyYAMLPatchType, types.ApplyCBORPatchType:
options.FieldManager = "test-apply"
options.Force = ptr.To(true)
}
existingObject, err = client.Patch(ctx, desiredInitialObject.GetName(), patch.Type, patchData, options, "status")
framework.ExpectNoError(err, "patch %s %s status", patch.Type, resourceType)
verify(fmt.Sprintf("patch %s status", patch.Type), desiredUpdatedObject, existingObject,
// Recreation and then at least one update.
matchEvents(existingObject, "^add,(update,)+$"),
)
}
// Use the label as selector in List and DeleteCollection calls.
ginkgo.By(fmt.Sprintf("Listing %s collection with label selector %s", gvr, listOptions.LabelSelector))
items, err := client.List(ctx, listOptions)
framework.ExpectNoError(err, "list %s", resourceType)
gomega.Expect(items.Items).Should(gomega.HaveLen(1), "Should have listed exactly the test resource.")
verify("list", desiredUpdatedObject, &items.Items[0],
// Optional updates by cluster components.
matchEvents(existingObject, "^(update,)*$"),
)
if tc.IsNamespaced() {
ginkgo.By(fmt.Sprintf("Listing %s without namespace and with label selector %s", gvr, listOptions.LabelSelector))
items, err := resourceClient.List(ctx, listOptions)
framework.ExpectNoError(err, "list %s in all namespaces", resourceType)
gomega.Expect(items.Items).Should(gomega.HaveLen(1), "Should have listed exactly the test resource in all namespaces.")
verify("list all namespaces", desiredUpdatedObject, &items.Items[0],
// Optional updates by cluster components.
matchEvents(existingObject, "^(update,)*$"),
)
}
ginkgo.By(fmt.Sprintf("Deleting %s collection with label selector %s", gvr, listOptions.LabelSelector))
err = client.DeleteCollection(ctx, metav1.DeleteOptions{}, listOptions)
framework.ExpectNoError(err, "delete collection of %s", resourceType)
ensureNotFound(ctx, getResource)
verify("list", nil, nil,
// Optional updates by cluster components, then deletion.
matchEvents(existingObject, "^(update,)*delete,$"),
)
}
// ensureNotFound ensures that the error returned by the get function is NotFound.
// This can be called after deleting an object to ensure that it really got removed
// and not just marked for deletion with a DeletionTimestamp. Deletion is not
// necessarily instantaneous e.g. because some cluster component might add its
// own finalizer.
func ensureNotFound(ctx context.Context, get func(context.Context) (*unstructured.Unstructured, error)) {
ginkgo.GinkgoHelper()
ginkgo.By("Checking for existence")
gomega.Eventually(ctx, func(ctx context.Context) error {
obj, err := framework.HandleRetry(get)(ctx)
switch {
case apierrors.IsNotFound(err):
return nil
case err != nil:
return fmt.Errorf("unexpected error after GET: %w", err)
default:
return fmt.Errorf("resource not removed yet:\n%s", format.Object(obj, 1))
}
}).WithTimeout(30 * time.Second /* From prior conformance tests, e.g. https://github.com/kubernetes/kubernetes/blame/be361a18dda0f2fab1f5e25f8067a9ed43fc3b89/test/e2e/storage/storageclass.go#L152 */).
Should(gomega.Succeed())
}

60
test/e2e/framework/conformance/failures.go Normal file
View file

@ -0,0 +1,60 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package architecture
import (
"fmt"
"strings"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
gtypes "github.com/onsi/gomega/types"
)
type gomegaFailures struct {
failures []string
}
var _ gtypes.GomegaTestingT = &gomegaFailures{}
// Helper implements [gtyppes.GomegaTestingT].
func (g *gomegaFailures) Helper() {}
// Fatalf implements [gtypes.GomegaTestingT].
func (g *gomegaFailures) Fatalf(format string, args ...any) {
g.Add(fmt.Sprintf(format, args...))
}
// Adds one failure.
func (g *gomegaFailures) Add(failure string) {
if !strings.HasSuffix(failure, "\n") {
failure += "\n"
}
g.failures = append(g.failures, failure)
}
// Check fails via [ginkgo.Fail] if there were any failures.
func (g *gomegaFailures) Check() {
if len(g.failures) > 0 {
ginkgo.GinkgoHelper()
ginkgo.Fail(strings.Join(g.failures, "\n\n"))
}
}
func (g *gomegaFailures) G() *gomega.WithT {
return gomega.NewWithT(g)
}

167
test/e2e/framework/conformance/informerevents.go Normal file
View file

@ -0,0 +1,167 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package architecture
import (
"fmt"
"slices"
"strings"
"sync"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"
)
// eventRecorder implements [cache.ResourceEventHandler] by recording all events.
// It is thread-safe.
type eventRecorder struct {
mutex sync.Mutex
events []event
}
// event describes one add/update/delete event.
// They can be distinguished based on which object(s) are set.
// Delete events may contain a tombstone instead of the actual
// deleted object.
type event struct {
oldObj, newObj any
isInitialList bool
}
func (e event) Type() string {
switch {
case e.oldObj == nil:
return "add"
case e.newObj == nil:
return "delete"
case e.oldObj != nil && e.newObj != nil:
return "update"
default:
return "null"
}
}
// ID returns "[namespace/]name, uid" for the object
// described in the event. If for whatever reason
// that is different for the old and new event (an error!),
// it returns both strings separated by semicolon.
// If meta data access is not possible, the error string is returned.
//
// This is meant to be used with gomega.HaveEach and a matcher
// which checks for the expected ID.
func (e event) ID() string {
oldID := id(e.oldObj)
newID := id(e.newObj)
if oldID == newID {
return newID
}
if oldID != "" && newID != "" {
return oldID + "; " + newID
}
if newID != "" {
return newID
}
return oldID
}
func id(obj any) string {
if obj == nil {
return ""
}
if tomb, ok := obj.(cache.DeletedFinalStateUnknown); ok {
obj = tomb.Obj
}
metaData, err := meta.Accessor(obj)
if err != nil {
return err.Error()
}
return fmt.Sprintf("%s, %s", klog.KObj(metaData).String(), metaData.GetUID())
}
var _ cache.ResourceEventHandler = &eventRecorder{}
func (er *eventRecorder) OnAdd(obj any, isInitialList bool) {
er.mutex.Lock()
defer er.mutex.Unlock()
er.events = append(er.events, event{
newObj: obj,
isInitialList: isInitialList,
})
}
func (er *eventRecorder) OnUpdate(oldObj, newObj any) {
er.mutex.Lock()
defer er.mutex.Unlock()
er.events = append(er.events, event{
oldObj: oldObj,
newObj: newObj,
})
}
func (er *eventRecorder) OnDelete(obj any) {
er.mutex.Lock()
defer er.mutex.Unlock()
er.events = append(er.events, event{
oldObj: obj,
})
}
// list returns a shallow copy of the current list of events.
func (er *eventRecorder) list() eventList {
er.mutex.Lock()
defer er.mutex.Unlock()
return eventList{Events: slices.Clone(er.events)}
}
// reset clears the current list of events.
// Should only be called during idle periods.
func (er *eventRecorder) reset() eventList {
er.mutex.Lock()
defer er.mutex.Unlock()
events := eventList{Events: er.events}
er.events = nil
return events
}
// eventList adds pretty-printing to a slice of events.
type eventList struct {
Events []event
}
// Types returns a comma-separated list of the type of each event.
// For the sake of simplicity the last entry also ends with a comma.
//
// This can be used in Gomega assertions like this:
//
// gomega.Expect(events).To(gomega.HaveField("Types()", gomega.MatchRegexp("^add,(update,)*$"))
// gomega.Expect(events).To(gomega.HaveField("Types()", gomega.MatchRegexp("^(update,)*$"))
func (el eventList) Types() string {
var buffer strings.Builder
for _, e := range el.Events {
buffer.WriteString(e.Type())
buffer.WriteRune(',')
}
return buffer.String()
}

114
test/e2e/framework/conformance/objectmatcher.go Normal file
View file

@ -0,0 +1,114 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package architecture
import (
"fmt"
"strings"
"github.com/onsi/gomega/format"
gtypes "github.com/onsi/gomega/types"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/klog/v2"
)
// matchObjectList is a custom matcher for the result of a generic lister List result
// ([]runtime.Object where each entry is *unstructured.Unstructed). The list is
// expected to have exactly one element or none.
type matchObjectList struct {
expectedObject *unstructured.Unstructured
verifyContent bool
lastDiff string
}
var _ gtypes.GomegaMatcher = &matchObjectList{}
func (m *matchObjectList) Match(actual any) (bool, error) {
// Reset state.
m.lastDiff = ""
actualObjects, ok := actual.([]runtime.Object)
if !ok {
return false, fmt.Errorf("must be passed a []runtime.Object, got %T", actual)
}
if m.expectedObject == nil {
// Must be empty,
return len(actualObjects) == 0, nil
}
// Must have exactly the expected object.
if len(actualObjects) != 1 {
return false, nil
}
actualObject, ok := actualObjects[0].(*unstructured.Unstructured)
if !ok {
// Shouldn't happen.
return false, fmt.Errorf("expected *unstructured.Unstructured, got %T", actualObjects[0])
}
if m.verifyContent {
// Remember diff for failure message.
m.lastDiff = compareObjects(m.expectedObject, actualObject)
if m.lastDiff != "" {
return false, nil
}
return true, nil
}
return m.expectedObject.GetName() == actualObject.GetName() && m.expectedObject.GetNamespace() == actualObject.GetNamespace(), nil
}
func (m *matchObjectList) FailureMessage(actual any) string {
return m.message(actual, "to")
}
func (m *matchObjectList) NegatedFailureMessage(actual any) string {
return m.message(actual, "not to")
}
func (m *matchObjectList) message(actual any, to string) string {
// Gomega renders []runtime.Object as nested maps.
// YAML is more readable.
var buffer strings.Builder
buffer.WriteString("Expected\n")
if actualObjects, ok := actual.([]runtime.Object); ok {
buffer.WriteString(fmt.Sprintf(" %T len:%d:\n", actualObjects, len(actualObjects)))
for _, object := range actualObjects {
buffer.WriteString(" ---\n")
if o, ok := object.(*unstructured.Unstructured); ok {
buffer.WriteString(format.Object(o, 2))
} else {
buffer.WriteString(format.Object(object, 2))
}
}
} else {
buffer.WriteString(format.Object(actual, 1))
}
buffer.WriteString(fmt.Sprintf("\n%s contain exactly the following element:\n", to))
if m.verifyContent {
buffer.WriteString(format.Object(m.expectedObject, 1))
} else {
buffer.WriteString(" " + klog.KObj(m.expectedObject).String())
}
if m.lastDiff != "" {
buffer.WriteString("\nDiff of checked fields (- expected, + actual):\n")
buffer.WriteString(m.lastDiff)
}
return buffer.String()
}

65
test/e2e/framework/gomega.go Normal file
View file

@ -0,0 +1,65 @@
/*
Copyright 2025 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package framework contains provider-independent helper code for
// building and running E2E tests with Ginkgo. The actual Ginkgo test
// suites gets assembled by combining this framework, the optional
// provider support code and specific tests via a separate .go file
// like Kubernetes' test/e2e.go.
package framework
import (
"fmt"
"strings"
"github.com/onsi/gomega/format"
gtypes "github.com/onsi/gomega/types"
)
// GomegaObject returns a matcher which appends a full dump of the actual value
// to the failure of the matcher that it wraps. This is useful e.g. for
// gomega.HaveField which otherwise only generates a message containing
// the field that it is checking, but not the object in which that field occurs.
func GomegaObject(shouldMatch gtypes.GomegaMatcher) gtypes.GomegaMatcher {
return &gomegaObjectMatcher{
shouldMatch: shouldMatch,
}
}
type gomegaObjectMatcher struct {
shouldMatch gtypes.GomegaMatcher
}
func (m *gomegaObjectMatcher) Match(actual interface{}) (success bool, err error) {
return m.shouldMatch.Match(actual)
}
func (m *gomegaObjectMatcher) FailureMessage(actual interface{}) (message string) {
return m.withDump(actual, m.shouldMatch.FailureMessage(actual))
}
func (m *gomegaObjectMatcher) NegatedFailureMessage(actual interface{}) (message string) {
return m.withDump(actual, m.shouldMatch.NegatedFailureMessage(actual))
}
func (m *gomegaObjectMatcher) withDump(actual any, message string) string {
dump := format.Object(actual, 1)
if !strings.HasSuffix(message, "\n") {
message += "\n"
}
message += fmt.Sprintf("\nFull object:\n%s", dump)
return message
}