fixed bug such that implicit extended resource name can always be used,

no matter the explicit extendedResourceName field in device class is set or not.
This commit is contained in:
yliao 2025-08-03 01:46:20 +00:00
parent ddd8e70b1e
commit 79f8d1b1c5
10 changed files with 353 additions and 52 deletions

View file

@ -39,7 +39,6 @@ import (
drahealthv1alpha1 "k8s.io/kubelet/pkg/apis/dra-health/v1alpha1"
drapb "k8s.io/kubelet/pkg/apis/dra/v1"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
kubefeatures "k8s.io/kubernetes/pkg/features"
draplugin "k8s.io/kubernetes/pkg/kubelet/cm/dra/plugin"
"k8s.io/kubernetes/pkg/kubelet/cm/dra/state"
@ -48,6 +47,7 @@ import (
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
kubeletmetrics "k8s.io/kubernetes/pkg/kubelet/metrics"
"k8s.io/kubernetes/pkg/kubelet/pluginmanager/cache"
schedutil "k8s.io/kubernetes/pkg/scheduler/util"
)
// draManagerStateFileName is the file name where dra manager stores its state
@ -526,7 +526,7 @@ func (m *Manager) GetResources(pod *v1.Pod, container *v1.Container) (*Container
// We only care about the resources requested by the pod
continue
}
if v1helper.IsExtendedResourceName(rName) {
if schedutil.IsDRAExtendedResourceName(rName) {
requestName := ""
for _, rm := range pod.Status.ExtendedResourceClaimStatus.RequestMappings {
if rm.ContainerName == container.Name && rm.ResourceName == rName.String() {

View file

@ -47,7 +47,6 @@ import (
"k8s.io/dynamic-resource-allocation/structured"
"k8s.io/klog/v2"
fwk "k8s.io/kube-scheduler/framework"
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
"k8s.io/kubernetes/pkg/scheduler/apis/config"
"k8s.io/kubernetes/pkg/scheduler/apis/config/validation"
"k8s.io/kubernetes/pkg/scheduler/framework"
@ -435,7 +434,7 @@ func hasDeviceClassMappedExtendedResource(reqs v1.ResourceList, deviceClassMappi
// We only care about the resources requested by the pod we are trying to schedule.
continue
}
if v1helper.IsExtendedResourceName(rName) {
if schedutil.IsDRAExtendedResourceName(rName) {
_, ok := deviceClassMapping[rName]
if ok {
return true
@ -766,7 +765,7 @@ func (pl *DynamicResources) filterExtendedResources(state *stateData, pod *v1.Po
extendedResources := make(map[v1.ResourceName]int64)
hasExtendedResource := false
for rName, rQuant := range state.draExtendedResource.podScalarResources {
if !v1helper.IsExtendedResourceName(rName) {
if !schedutil.IsDRAExtendedResourceName(rName) {
continue
}
// Skip in case request quantity is zero

View file

@ -60,21 +60,22 @@ import (
var (
podKind = v1.SchemeGroupVersion.WithKind("Pod")
nodeName = "worker"
node2Name = "worker-2"
node3Name = "worker-3"
driver = "some-driver"
driver2 = "some-driver-2"
podName = "my-pod"
podUID = "1234"
resourceName = "my-resource"
resourceName2 = resourceName + "-2"
claimName = podName + "-" + resourceName
claimName2 = podName + "-" + resourceName2
className = "my-resource-class"
namespace = "default"
attrName = resourceapi.QualifiedName("healthy") // device attribute only available on non-default node
extendedResourceName = "example.com/gpu"
nodeName = "worker"
node2Name = "worker-2"
node3Name = "worker-3"
driver = "some-driver"
driver2 = "some-driver-2"
podName = "my-pod"
podUID = "1234"
resourceName = "my-resource"
resourceName2 = resourceName + "-2"
claimName = podName + "-" + resourceName
claimName2 = podName + "-" + resourceName2
className = "my-resource-class"
namespace = "default"
attrName = resourceapi.QualifiedName("healthy") // device attribute only available on non-default node
extendedResourceName = "example.com/gpu"
implicitExtendedResourceName = "deviceclass.resource.kubernetes.io/my-resource-class"
deviceClass = &resourceapi.DeviceClass{
ObjectMeta: metav1.ObjectMeta{
@ -124,6 +125,22 @@ var (
v1.ResourceName(extendedResourceName): "1",
}).
Obj()
podWithImplicitExtendedResourceName = st.MakePod().Name(podName).Namespace(namespace).
UID(podUID).
Req(map[v1.ResourceName]string{
v1.ResourceName(implicitExtendedResourceName): "1",
v1.ResourceName(extendedResourceName): "2",
}).
Obj()
podWithImplicitExtendedResourceNameTwoContainers = st.MakePod().Name(podName).Namespace(namespace).
UID(podUID).
Req(map[v1.ResourceName]string{
v1.ResourceName(implicitExtendedResourceName): "1",
}).
Req(map[v1.ResourceName]string{
v1.ResourceName(extendedResourceName): "2",
}).
Obj()
// Node with "instance-1" device and no device attributes.
workerNode = &st.MakeNode().Name(nodeName).Label("kubernetes.io/hostname", nodeName).Node
@ -218,6 +235,60 @@ var (
return st.MakeNodeSelector().In("metadata.name", []string{nodeName}, st.NodeSelectorTypeMatchFields).Obj()
}(),
}
implicitExtendedResourceAllocationResult = &resourceapi.AllocationResult{
Devices: resourceapi.DeviceAllocationResult{
Results: []resourceapi.DeviceRequestAllocationResult{
{
Driver: driver,
Pool: nodeName,
Device: "instance-1",
Request: "container-0-request-0",
},
{
Driver: driver,
Pool: nodeName,
Device: "instance-2",
Request: "container-0-request-1",
},
{
Driver: driver,
Pool: nodeName,
Device: "instance-3",
Request: "container-0-request-1",
},
},
},
NodeSelector: func() *v1.NodeSelector {
return st.MakeNodeSelector().In("metadata.name", []string{nodeName}, st.NodeSelectorTypeMatchFields).Obj()
}(),
}
implicitExtendedResourceAllocationResultTwoContainers = &resourceapi.AllocationResult{
Devices: resourceapi.DeviceAllocationResult{
Results: []resourceapi.DeviceRequestAllocationResult{
{
Driver: driver,
Pool: nodeName,
Device: "instance-1",
Request: "container-0-request-0",
},
{
Driver: driver,
Pool: nodeName,
Device: "instance-2",
Request: "container-1-request-0",
},
{
Driver: driver,
Pool: nodeName,
Device: "instance-3",
Request: "container-1-request-0",
},
},
},
NodeSelector: func() *v1.NodeSelector {
return st.MakeNodeSelector().In("metadata.name", []string{nodeName}, st.NodeSelectorTypeMatchFields).Obj()
}(),
}
extendedResourceAllocationResultNode2 = &resourceapi.AllocationResult{
Devices: resourceapi.DeviceAllocationResult{
Results: []resourceapi.DeviceRequestAllocationResult{{
@ -311,7 +382,78 @@ var (
RequestWithName("container-0-request-0", className).
Allocation(extendedResourceAllocationResult).
Obj()
implicitExtendedResourceClaim = st.MakeResourceClaim().
Name("my-pod-extended-resources-0").
GenerateName("my-pod-extended-resources-").
Namespace(namespace).
Annotations(map[string]string{"resource.kubernetes.io/extended-resource-claim": "true"}).
OwnerRef(
metav1.OwnerReference{
APIVersion: "v1",
Kind: "Pod",
Name: podName,
UID: types.UID(podUID),
Controller: ptr.To(true),
BlockOwnerDeletion: ptr.To(true),
}).
RequestWithName("container-0-request-0", className).
RequestWithNameCount("container-0-request-1", className, 2).
Allocation(implicitExtendedResourceAllocationResult).
Obj()
implicitExtendedResourceClaimNoName = st.MakeResourceClaim().
Name(specialClaimInMemName).
GenerateName("my-pod-extended-resources-").
Namespace(namespace).
Annotations(map[string]string{"resource.kubernetes.io/extended-resource-claim": "true"}).
OwnerRef(
metav1.OwnerReference{
APIVersion: "v1",
Kind: "Pod",
Name: podName,
UID: types.UID(podUID),
Controller: ptr.To(true),
BlockOwnerDeletion: ptr.To(true),
}).
RequestWithName("container-0-request-0", className).
RequestWithNameCount("container-0-request-1", className, 2).
Allocation(implicitExtendedResourceAllocationResult).
Obj()
implicitExtendedResourceClaimTwoContainers = st.MakeResourceClaim().
Name("my-pod-extended-resources-0").
GenerateName("my-pod-extended-resources-").
Namespace(namespace).
Annotations(map[string]string{"resource.kubernetes.io/extended-resource-claim": "true"}).
OwnerRef(
metav1.OwnerReference{
APIVersion: "v1",
Kind: "Pod",
Name: podName,
UID: types.UID(podUID),
Controller: ptr.To(true),
BlockOwnerDeletion: ptr.To(true),
}).
RequestWithName("container-0-request-0", className).
RequestWithNameCount("container-1-request-0", className, 2).
Allocation(implicitExtendedResourceAllocationResultTwoContainers).
Obj()
implicitExtendedResourceClaimNoNameTwoContainers = st.MakeResourceClaim().
Name(specialClaimInMemName).
GenerateName("my-pod-extended-resources-").
Namespace(namespace).
Annotations(map[string]string{"resource.kubernetes.io/extended-resource-claim": "true"}).
OwnerRef(
metav1.OwnerReference{
APIVersion: "v1",
Kind: "Pod",
Name: podName,
UID: types.UID(podUID),
Controller: ptr.To(true),
BlockOwnerDeletion: ptr.To(true),
}).
RequestWithName("container-0-request-0", className).
RequestWithNameCount("container-1-request-0", className, 2).
Allocation(implicitExtendedResourceAllocationResultTwoContainers).
Obj()
extendedResourceClaimNode2 = st.MakeResourceClaim().
Name("my-pod-extended-resources-0").
GenerateName("my-pod-extended-resources-").
@ -1311,6 +1453,42 @@ func TestPlugin(t *testing.T) {
},
},
},
"implicit-extended-resource-name-with-resources": {
enableDRAExtendedResource: true,
pod: podWithImplicitExtendedResourceName,
classes: []*resourceapi.DeviceClass{deviceClassWithExtendResourceName},
objs: []apiruntime.Object{largeWorkerNodeSlice, podWithImplicitExtendedResourceName},
want: want{
reserve: result{
inFlightClaim: implicitExtendedResourceClaimNoName,
},
prebind: result{
assumedClaim: reserve(implicitExtendedResourceClaim, podWithImplicitExtendedResourceName),
added: []metav1.Object{reserve(implicitExtendedResourceClaim, podWithImplicitExtendedResourceName)},
},
postbind: result{
assumedClaim: reserve(implicitExtendedResourceClaim, podWithImplicitExtendedResourceName),
},
},
},
"implicit-extended-resource-name-two-containers-with-resources": {
enableDRAExtendedResource: true,
pod: podWithImplicitExtendedResourceNameTwoContainers,
classes: []*resourceapi.DeviceClass{deviceClassWithExtendResourceName},
objs: []apiruntime.Object{largeWorkerNodeSlice, podWithImplicitExtendedResourceNameTwoContainers},
want: want{
reserve: result{
inFlightClaim: implicitExtendedResourceClaimNoNameTwoContainers,
},
prebind: result{
assumedClaim: reserve(implicitExtendedResourceClaimTwoContainers, podWithImplicitExtendedResourceNameTwoContainers),
added: []metav1.Object{reserve(implicitExtendedResourceClaimTwoContainers, podWithImplicitExtendedResourceNameTwoContainers)},
},
postbind: result{
assumedClaim: reserve(implicitExtendedResourceClaimTwoContainers, podWithImplicitExtendedResourceNameTwoContainers),
},
},
},
"extended-resource-name-with-resources-fail-patch": {
enableDRAExtendedResource: true,
failPatch: true,
@ -1938,6 +2116,8 @@ func (tc *testContext) verify(t *testing.T, expected result, initialObjects []me
ignoreFieldsInResourceClaims := []cmp.Option{
cmpopts.IgnoreFields(metav1.ObjectMeta{}, "UID", "ResourceVersion"),
cmpopts.IgnoreFields(resourceapi.AllocationResult{}, "AllocationTimestamp"),
// It does not matter which specific device is allocated for the testing purpose.
cmpopts.IgnoreFields(resourceapi.DeviceRequestAllocationResult{}, "Device"),
}
if diff := cmp.Diff(wantObjects, objects, ignoreFieldsInResourceClaims...); diff != "" {
t.Errorf("Stored objects are different (- expected, + actual):\n%s", diff)
@ -1995,12 +2175,27 @@ func (tc *testContext) verify(t *testing.T, expected result, initialObjects []me
}
}
// sortClaim sorts given claim's Requests in spec, and Results in status
func sortClaim(claim *resourceapi.ResourceClaim) {
requests := claim.Spec.Devices.Requests
sort.Slice(requests, func(i, j int) bool {
return requests[i].Name < requests[j].Name
})
if claim.Status.Allocation != nil {
results := claim.Status.Allocation.Devices.Results
sort.Slice(results, func(i, j int) bool {
return results[i].Request < results[j].Request
})
}
}
func (tc *testContext) listAll(t *testing.T) (objects []metav1.Object) {
t.Helper()
claims, err := tc.client.ResourceV1().ResourceClaims("").List(tc.ctx, metav1.ListOptions{})
require.NoError(t, err, "list claims")
for _, claim := range claims.Items {
claim := claim
sortClaim(&claim)
objects = append(objects, &claim)
}
sortObjects(objects)
@ -2012,6 +2207,7 @@ func (tc *testContext) listAssumedClaims() ([]metav1.Object, []metav1.Object) {
var sameClaims []metav1.Object
for _, obj := range tc.draManager.resourceClaimTracker.cache.List(nil) {
claim := obj.(*resourceapi.ResourceClaim)
sortClaim(claim)
obj, _ := tc.draManager.resourceClaimTracker.cache.Get(claim.Namespace + "/" + claim.Name)
apiObj, _ := tc.draManager.resourceClaimTracker.cache.GetAPIObj(claim.Namespace + "/" + claim.Name)
if obj != apiObj {
@ -2028,7 +2224,9 @@ func (tc *testContext) listAssumedClaims() ([]metav1.Object, []metav1.Object) {
func (tc *testContext) listInFlightClaims() []metav1.Object {
var inFlightClaims []metav1.Object
tc.draManager.resourceClaimTracker.inFlightAllocations.Range(func(key, value any) bool {
inFlightClaims = append(inFlightClaims, value.(*resourceapi.ResourceClaim))
claim := value.(*resourceapi.ResourceClaim)
sortClaim(claim)
inFlightClaims = append(inFlightClaims, claim)
return true
})
sortObjects(inFlightClaims)

View file

@ -18,7 +18,7 @@ package extended
import (
v1 "k8s.io/api/core/v1"
"k8s.io/api/resource/v1beta1"
resourceapi "k8s.io/api/resource/v1"
fwk "k8s.io/kube-scheduler/framework"
)
@ -29,11 +29,10 @@ func DeviceClassMapping(draManager fwk.SharedDRAManager) (map[v1.ResourceName]st
return nil, err
}
for _, c := range classes {
if c.Spec.ExtendedResourceName == nil {
extendedResources[v1.ResourceName(v1beta1.ResourceDeviceClassPrefix+c.Name)] = c.Name
} else {
if c.Spec.ExtendedResourceName != nil {
extendedResources[v1.ResourceName(*c.Spec.ExtendedResourceName)] = c.Name
}
extendedResources[v1.ResourceName(resourceapi.ResourceDeviceClassPrefix+c.Name)] = c.Name
}
return extendedResources, nil
}

View file

@ -122,6 +122,15 @@ func TestDeviceClassMapping(t *testing.T) {
if c != "class2" {
t.Errorf("result does not match device class name %s", "class2")
}
for _, c := range []string{"class1", "class2", "class3"} {
n, ok := rm[v1.ResourceName("deviceclass.resource.kubernetes.io/"+c)]
if !ok {
t.Errorf("result does not contain implicit extended resource for device class %s", c)
}
if n != c {
t.Errorf("result %s does not match device class name %s", n, c)
}
}
}
func TestNoDeviceClassMapping(t *testing.T) {

View file

@ -241,7 +241,7 @@ func withDeviceClass(result *preFilterState, draManager fwk.SharedDRAManager) *f
continue
}
if v1helper.IsExtendedResourceName(rName) {
if schedutil.IsDRAExtendedResourceName(rName) {
hasExtendedResource = true
break
}

View file

@ -1908,3 +1908,63 @@ func TestHaveAnyRequestedResourcesIncreased(t *testing.T) {
})
}
}
func TestWithDeviceClass(t *testing.T) {
logger, ctx := ktesting.NewTestContext(t)
ctx, cancel := context.WithCancel(ctx)
defer cancel()
client := fake.NewClientset(deviceClassWithExtendResourceName)
informerFactory := informers.NewSharedInformerFactory(client, 0)
draManager := dynamicresources.NewDRAManager(ctx, assumecache.NewAssumeCache(logger, informerFactory.Resource().V1().ResourceClaims().Informer(), "resource claim", "", nil), nil, informerFactory)
informerFactory.Start(ctx.Done())
t.Cleanup(func() {
// Now we can wait for all goroutines to stop.
informerFactory.Shutdown()
})
informerFactory.WaitForCacheSync(ctx.Done())
testCases := map[string]struct {
state *preFilterState
expectedResourceToDeviceClass map[v1.ResourceName]string
}{
"regular extended resource name": {
state: &preFilterState{
Resource: framework.Resource{
ScalarResources: map[v1.ResourceName]int64{extendedResourceA: 1},
},
},
expectedResourceToDeviceClass: map[v1.ResourceName]string{
v1.ResourceName("deviceclass.resource.kubernetes.io/device-class-name"): deviceClassName,
v1.ResourceName("extended.resource.dra.io/something"): deviceClassName,
},
},
"implicit extended resource name": {
state: &preFilterState{
Resource: framework.Resource{
ScalarResources: map[v1.ResourceName]int64{v1.ResourceName("deviceclass.resource.kubernetes.io/" + deviceClassName): 1},
},
},
expectedResourceToDeviceClass: map[v1.ResourceName]string{
v1.ResourceName("deviceclass.resource.kubernetes.io/device-class-name"): deviceClassName,
v1.ResourceName("extended.resource.dra.io/something"): deviceClassName,
},
},
"no extended resource name": {
state: &preFilterState{
Resource: framework.Resource{
MilliCPU: 1000,
},
},
expectedResourceToDeviceClass: nil,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
withDeviceClass(tc.state, draManager)
if diff := cmp.Diff(tc.state.resourceToDeviceClass, tc.expectedResourceToDeviceClass); diff != "" {
t.Error("resourceToDeviceClass doesn't match (-expected +actual):\n", diff)
}
})
}
}

View file

@ -1173,6 +1173,21 @@ func (wrapper *ResourceClaimWrapper) RequestWithName(name, deviceClassName strin
return wrapper
}
// RequestWithNameCount adds one device request for the given device class with given request name and count.
func (wrapper *ResourceClaimWrapper) RequestWithNameCount(name, deviceClassName string, count int64) *ResourceClaimWrapper {
wrapper.Spec.Devices.Requests = append(wrapper.Spec.Devices.Requests,
resourceapi.DeviceRequest{
Name: name,
Exactly: &resourceapi.ExactDeviceRequest{
// Cannot rely on defaulting here, this is used in unit tests.
AllocationMode: resourceapi.DeviceAllocationModeExactCount,
Count: count,
DeviceClassName: deviceClassName,
},
})
return wrapper
}
// RequestWithPrioritizedList adds one device request with one subrequest
// per provided deviceClassName.
func (wrapper *ResourceClaimWrapper) RequestWithPrioritizedList(deviceClassNames ...string) *ResourceClaimWrapper {

View file

@ -20,9 +20,11 @@ import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
v1 "k8s.io/api/core/v1"
resourceapi "k8s.io/api/resource/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
@ -142,6 +144,12 @@ func IsScalarResourceName(name v1.ResourceName) bool {
v1helper.IsPrefixedNativeResource(name) || v1helper.IsAttachableVolumeResourceName(name)
}
// IsDRAExtendedResourceName returns true when name is an extended resource name, or an implicit extended resource name
// derived from device class name with the format of deviceclass.resource.kubernetes.io/<device class name>
func IsDRAExtendedResourceName(name v1.ResourceName) bool {
return v1helper.IsExtendedResourceName(name) || strings.HasPrefix(string(name), resourceapi.ResourceDeviceClassPrefix)
}
// As converts two objects to the given type.
// Both objects must be of the same type. If not, an error is returned.
// nil objects are allowed and will be converted to nil.

View file

@ -1909,51 +1909,64 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
})
})
extendedResourceTest := func(ctx context.Context, b *drautils.Builder, f *framework.Framework, resourceNames []string, containerEnv []string) {
pod := b.Pod()
res := v1.ResourceList{}
for _, resourceName := range resourceNames {
res[v1.ResourceName(resourceName)] = resource.MustParse("1")
}
pod.Spec.Containers[0].Resources.Requests = res
pod.Spec.Containers[0].Resources.Limits = res
b.Create(ctx, pod)
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
framework.ExpectNoError(err, "start pod")
drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[0].Name, false, containerEnv...)
}
framework.Context(f.WithFeatureGate(features.DRAExtendedResource), func() {
nodes := drautils.NewNodes(f, 1, 1)
driver := drautils.NewDriver(f, nodes, drautils.NetworkResources(10, false))
b := drautils.NewBuilder(f, driver)
b.UseExtendedResourceName = true
ginkgo.It("must run a pod with extended resource with one container one resource", func(ctx context.Context) {
pod := b.Pod()
res := v1.ResourceList{}
res[v1.ResourceName(b.ExtendedResourceName(0))] = resource.MustParse("1")
pod.Spec.Containers[0].Resources.Requests = res
pod.Spec.Containers[0].Resources.Limits = res
b.Create(ctx, pod)
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
framework.ExpectNoError(err, "start pod")
containerEnv := []string{
ginkgo.It("must run a pod with both implicit and explicit extended resource with one container two resources", func(ctx context.Context) {
extendedResourceTest(ctx, b, f, []string{
// implicit extended resource name
"deviceclass.resource.kubernetes.io/" + b.ClassName(),
// b.ExtendedResourceName(0) is added to the deivce class with name: b.ClassName()+"0"
b.ExtendedResourceName(0),
}, []string{
"container_0_request_0", "true",
}
drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[0].Name, false, containerEnv...)
"container_0_request_1", "true",
})
})
ginkgo.It("must run a pod with extended resource with one container one resource", func(ctx context.Context) {
extendedResourceTest(ctx, b, f, []string{
b.ExtendedResourceName(0),
}, []string{
"container_0_request_0", "true",
})
})
ginkgo.It("must run a pod with extended resource with one container three resources", func(ctx context.Context) {
var objects []klog.KMetadata
pod := b.Pod()
res := v1.ResourceList{}
for i := range 3 {
res[v1.ResourceName(b.ExtendedResourceName(i))] = resource.MustParse("1")
if i > 0 {
objects = append(objects, b.Class(i))
}
}
pod.Spec.Containers[0].Resources.Requests = res
pod.Spec.Containers[0].Resources.Limits = res
objects = append(objects, pod)
b.Create(ctx, objects...)
err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
framework.ExpectNoError(err, "start pod")
containerEnv := []string{
extendedResourceTest(ctx, b, f, []string{
b.ExtendedResourceName(0),
b.ExtendedResourceName(1),
b.ExtendedResourceName(2),
}, []string{
"container_0_request_0", "true",
"container_0_request_1", "true",
"container_0_request_2", "true",
}
drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[0].Name, false, containerEnv...)
})
})
ginkgo.It("must run a pod with extended resource with three containers one resource each", func(ctx context.Context) {
var objects []klog.KMetadata