diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 1e58879f4e5..7acd23f2af1 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -19546,6 +19546,20 @@ } ] }, + "io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingConstraints": { + "description": "PodGroupSchedulingConstraints defines scheduling constraints (e.g. topology) for a PodGroup.", + "properties": { + "topology": { + "description": "Topology defines the topology constraints for the pod group. Currently only a single topology constraint can be specified. This may change in the future.", + "items": { + "$ref": "#/definitions/io.k8s.api.scheduling.v1alpha2.TopologyConstraint" + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "type": "object" + }, "io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingPolicy": { "description": "PodGroupSchedulingPolicy defines the scheduling configuration for a PodGroup. Exactly one policy must be set.", "properties": { @@ -19575,6 +19589,10 @@ "$ref": "#/definitions/io.k8s.api.scheduling.v1alpha2.PodGroupTemplateReference", "description": "PodGroupTemplateRef references an optional PodGroup template within other object (e.g. Workload) that was used to create the PodGroup. This field is immutable." }, + "schedulingConstraints": { + "$ref": "#/definitions/io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingConstraints", + "description": "SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroup. Controllers are expected to fill this field by copying it from a PodGroupTemplate. This field is immutable. This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled." + }, "schedulingPolicy": { "$ref": "#/definitions/io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingPolicy", "description": "SchedulingPolicy defines the scheduling policy for this instance of the PodGroup. Controllers are expected to fill this field by copying it from a PodGroupTemplate. This field is immutable." @@ -19611,6 +19629,10 @@ "description": "Name is a unique identifier for the PodGroupTemplate within the Workload. It must be a DNS label. This field is immutable.", "type": "string" }, + "schedulingConstraints": { + "$ref": "#/definitions/io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingConstraints", + "description": "SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroupTemplate. This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled." + }, "schedulingPolicy": { "$ref": "#/definitions/io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingPolicy", "description": "SchedulingPolicy defines the scheduling policy for this PodGroupTemplate." @@ -19639,6 +19661,19 @@ } ] }, + "io.k8s.api.scheduling.v1alpha2.TopologyConstraint": { + "description": "TopologyConstraint defines a topology constraint for a PodGroup.", + "properties": { + "key": { + "description": "Key specifies the key of the node label representing the topology domain. All pods within the PodGroup must be colocated within the same domain instance. Different PodGroups can land on different domain instances even if they derive from the same PodGroupTemplate. Examples: \"topology.kubernetes.io/rack\"", + "type": "string" + } + }, + "required": [ + "key" + ], + "type": "object" + }, "io.k8s.api.scheduling.v1alpha2.TypedLocalObjectReference": { "description": "TypedLocalObjectReference allows to reference typed object inside the same namespace.", "properties": { diff --git a/api/openapi-spec/v3/apis__scheduling.k8s.io__v1alpha2_openapi.json b/api/openapi-spec/v3/apis__scheduling.k8s.io__v1alpha2_openapi.json index 02af3ee922d..93324bc4581 100644 --- a/api/openapi-spec/v3/apis__scheduling.k8s.io__v1alpha2_openapi.json +++ b/api/openapi-spec/v3/apis__scheduling.k8s.io__v1alpha2_openapi.json @@ -116,6 +116,25 @@ } ] }, + "io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingConstraints": { + "description": "PodGroupSchedulingConstraints defines scheduling constraints (e.g. topology) for a PodGroup.", + "properties": { + "topology": { + "description": "Topology defines the topology constraints for the pod group. Currently only a single topology constraint can be specified. This may change in the future.", + "items": { + "allOf": [ + { + "$ref": "#/components/schemas/io.k8s.api.scheduling.v1alpha2.TopologyConstraint" + } + ], + "default": {} + }, + "type": "array", + "x-kubernetes-list-type": "atomic" + } + }, + "type": "object" + }, "io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingPolicy": { "description": "PodGroupSchedulingPolicy defines the scheduling configuration for a PodGroup. Exactly one policy must be set.", "properties": { @@ -157,6 +176,14 @@ ], "description": "PodGroupTemplateRef references an optional PodGroup template within other object (e.g. Workload) that was used to create the PodGroup. This field is immutable." }, + "schedulingConstraints": { + "allOf": [ + { + "$ref": "#/components/schemas/io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingConstraints" + } + ], + "description": "SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroup. Controllers are expected to fill this field by copying it from a PodGroupTemplate. This field is immutable. This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled." + }, "schedulingPolicy": { "allOf": [ { @@ -204,6 +231,14 @@ "description": "Name is a unique identifier for the PodGroupTemplate within the Workload. It must be a DNS label. This field is immutable.", "type": "string" }, + "schedulingConstraints": { + "allOf": [ + { + "$ref": "#/components/schemas/io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingConstraints" + } + ], + "description": "SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroupTemplate. This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled." + }, "schedulingPolicy": { "allOf": [ { @@ -241,6 +276,20 @@ } ] }, + "io.k8s.api.scheduling.v1alpha2.TopologyConstraint": { + "description": "TopologyConstraint defines a topology constraint for a PodGroup.", + "properties": { + "key": { + "default": "", + "description": "Key specifies the key of the node label representing the topology domain. All pods within the PodGroup must be colocated within the same domain instance. Different PodGroups can land on different domain instances even if they derive from the same PodGroupTemplate. Examples: \"topology.kubernetes.io/rack\"", + "type": "string" + } + }, + "required": [ + "key" + ], + "type": "object" + }, "io.k8s.api.scheduling.v1alpha2.TypedLocalObjectReference": { "description": "TypedLocalObjectReference allows to reference typed object inside the same namespace.", "properties": { diff --git a/pkg/apis/scheduling/types.go b/pkg/apis/scheduling/types.go index 7562d1e2780..a700f42342f 100644 --- a/pkg/apis/scheduling/types.go +++ b/pkg/apis/scheduling/types.go @@ -178,6 +178,13 @@ type PodGroupTemplate struct { // // +required SchedulingPolicy PodGroupSchedulingPolicy + + // SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroupTemplate. + // This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled. + // + // +optional + // +featureGate=TopologyAwareWorkloadScheduling + SchedulingConstraints *PodGroupSchedulingConstraints } // PodGroupSchedulingPolicy defines the scheduling configuration for a PodGroup. @@ -269,6 +276,15 @@ type PodGroupSpec struct { // // +required SchedulingPolicy PodGroupSchedulingPolicy + + // SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroup. + // Controllers are expected to fill this field by copying it from a PodGroupTemplate. + // This field is immutable. + // This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled. + // + // +optional + // +featureGate=TopologyAwareWorkloadScheduling + SchedulingConstraints *PodGroupSchedulingConstraints } // PodGroupStatus represents information about the status of a pod group. @@ -343,3 +359,24 @@ type WorkloadPodGroupTemplateReference struct { // +required PodGroupTemplateName string } + +// PodGroupSchedulingConstraints defines scheduling constraints (e.g. topology) for a PodGroup. +type PodGroupSchedulingConstraints struct { + // Topology defines the topology constraints for the pod group. + // Currently only a single topology constraint can be specified. This may change in the future. + // + // +optional + // +listType=atomic + Topology []TopologyConstraint +} + +// TopologyConstraint defines a topology constraint for a PodGroup. +type TopologyConstraint struct { + // Key specifies the key of the node label representing the topology domain. + // All pods within the PodGroup must be colocated within the same domain instance. + // Different PodGroups can land on different domain instances even if they derive from the same PodGroupTemplate. + // Examples: "topology.kubernetes.io/rack" + // + // +required + Key string +} diff --git a/pkg/apis/scheduling/v1alpha2/zz_generated.conversion.go b/pkg/apis/scheduling/v1alpha2/zz_generated.conversion.go index 75383a99c76..ce2f030edb9 100644 --- a/pkg/apis/scheduling/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/scheduling/v1alpha2/zz_generated.conversion.go @@ -78,6 +78,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*schedulingv1alpha2.PodGroupSchedulingConstraints)(nil), (*scheduling.PodGroupSchedulingConstraints)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_PodGroupSchedulingConstraints_To_scheduling_PodGroupSchedulingConstraints(a.(*schedulingv1alpha2.PodGroupSchedulingConstraints), b.(*scheduling.PodGroupSchedulingConstraints), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.PodGroupSchedulingConstraints)(nil), (*schedulingv1alpha2.PodGroupSchedulingConstraints)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_PodGroupSchedulingConstraints_To_v1alpha2_PodGroupSchedulingConstraints(a.(*scheduling.PodGroupSchedulingConstraints), b.(*schedulingv1alpha2.PodGroupSchedulingConstraints), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*schedulingv1alpha2.PodGroupSchedulingPolicy)(nil), (*scheduling.PodGroupSchedulingPolicy)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_PodGroupSchedulingPolicy_To_scheduling_PodGroupSchedulingPolicy(a.(*schedulingv1alpha2.PodGroupSchedulingPolicy), b.(*scheduling.PodGroupSchedulingPolicy), scope) }); err != nil { @@ -128,6 +138,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddGeneratedConversionFunc((*schedulingv1alpha2.TopologyConstraint)(nil), (*scheduling.TopologyConstraint)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1alpha2_TopologyConstraint_To_scheduling_TopologyConstraint(a.(*schedulingv1alpha2.TopologyConstraint), b.(*scheduling.TopologyConstraint), scope) + }); err != nil { + return err + } + if err := s.AddGeneratedConversionFunc((*scheduling.TopologyConstraint)(nil), (*schedulingv1alpha2.TopologyConstraint)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_scheduling_TopologyConstraint_To_v1alpha2_TopologyConstraint(a.(*scheduling.TopologyConstraint), b.(*schedulingv1alpha2.TopologyConstraint), scope) + }); err != nil { + return err + } if err := s.AddGeneratedConversionFunc((*schedulingv1alpha2.TypedLocalObjectReference)(nil), (*scheduling.TypedLocalObjectReference)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1alpha2_TypedLocalObjectReference_To_scheduling_TypedLocalObjectReference(a.(*schedulingv1alpha2.TypedLocalObjectReference), b.(*scheduling.TypedLocalObjectReference), scope) }); err != nil { @@ -273,6 +293,26 @@ func Convert_scheduling_PodGroupList_To_v1alpha2_PodGroupList(in *scheduling.Pod return autoConvert_scheduling_PodGroupList_To_v1alpha2_PodGroupList(in, out, s) } +func autoConvert_v1alpha2_PodGroupSchedulingConstraints_To_scheduling_PodGroupSchedulingConstraints(in *schedulingv1alpha2.PodGroupSchedulingConstraints, out *scheduling.PodGroupSchedulingConstraints, s conversion.Scope) error { + out.Topology = *(*[]scheduling.TopologyConstraint)(unsafe.Pointer(&in.Topology)) + return nil +} + +// Convert_v1alpha2_PodGroupSchedulingConstraints_To_scheduling_PodGroupSchedulingConstraints is an autogenerated conversion function. +func Convert_v1alpha2_PodGroupSchedulingConstraints_To_scheduling_PodGroupSchedulingConstraints(in *schedulingv1alpha2.PodGroupSchedulingConstraints, out *scheduling.PodGroupSchedulingConstraints, s conversion.Scope) error { + return autoConvert_v1alpha2_PodGroupSchedulingConstraints_To_scheduling_PodGroupSchedulingConstraints(in, out, s) +} + +func autoConvert_scheduling_PodGroupSchedulingConstraints_To_v1alpha2_PodGroupSchedulingConstraints(in *scheduling.PodGroupSchedulingConstraints, out *schedulingv1alpha2.PodGroupSchedulingConstraints, s conversion.Scope) error { + out.Topology = *(*[]schedulingv1alpha2.TopologyConstraint)(unsafe.Pointer(&in.Topology)) + return nil +} + +// Convert_scheduling_PodGroupSchedulingConstraints_To_v1alpha2_PodGroupSchedulingConstraints is an autogenerated conversion function. +func Convert_scheduling_PodGroupSchedulingConstraints_To_v1alpha2_PodGroupSchedulingConstraints(in *scheduling.PodGroupSchedulingConstraints, out *schedulingv1alpha2.PodGroupSchedulingConstraints, s conversion.Scope) error { + return autoConvert_scheduling_PodGroupSchedulingConstraints_To_v1alpha2_PodGroupSchedulingConstraints(in, out, s) +} + func autoConvert_v1alpha2_PodGroupSchedulingPolicy_To_scheduling_PodGroupSchedulingPolicy(in *schedulingv1alpha2.PodGroupSchedulingPolicy, out *scheduling.PodGroupSchedulingPolicy, s conversion.Scope) error { out.Basic = (*scheduling.BasicSchedulingPolicy)(unsafe.Pointer(in.Basic)) out.Gang = (*scheduling.GangSchedulingPolicy)(unsafe.Pointer(in.Gang)) @@ -300,6 +340,7 @@ func autoConvert_v1alpha2_PodGroupSpec_To_scheduling_PodGroupSpec(in *scheduling if err := Convert_v1alpha2_PodGroupSchedulingPolicy_To_scheduling_PodGroupSchedulingPolicy(&in.SchedulingPolicy, &out.SchedulingPolicy, s); err != nil { return err } + out.SchedulingConstraints = (*scheduling.PodGroupSchedulingConstraints)(unsafe.Pointer(in.SchedulingConstraints)) return nil } @@ -313,6 +354,7 @@ func autoConvert_scheduling_PodGroupSpec_To_v1alpha2_PodGroupSpec(in *scheduling if err := Convert_scheduling_PodGroupSchedulingPolicy_To_v1alpha2_PodGroupSchedulingPolicy(&in.SchedulingPolicy, &out.SchedulingPolicy, s); err != nil { return err } + out.SchedulingConstraints = (*schedulingv1alpha2.PodGroupSchedulingConstraints)(unsafe.Pointer(in.SchedulingConstraints)) return nil } @@ -346,6 +388,7 @@ func autoConvert_v1alpha2_PodGroupTemplate_To_scheduling_PodGroupTemplate(in *sc if err := Convert_v1alpha2_PodGroupSchedulingPolicy_To_scheduling_PodGroupSchedulingPolicy(&in.SchedulingPolicy, &out.SchedulingPolicy, s); err != nil { return err } + out.SchedulingConstraints = (*scheduling.PodGroupSchedulingConstraints)(unsafe.Pointer(in.SchedulingConstraints)) return nil } @@ -359,6 +402,7 @@ func autoConvert_scheduling_PodGroupTemplate_To_v1alpha2_PodGroupTemplate(in *sc if err := Convert_scheduling_PodGroupSchedulingPolicy_To_v1alpha2_PodGroupSchedulingPolicy(&in.SchedulingPolicy, &out.SchedulingPolicy, s); err != nil { return err } + out.SchedulingConstraints = (*schedulingv1alpha2.PodGroupSchedulingConstraints)(unsafe.Pointer(in.SchedulingConstraints)) return nil } @@ -387,6 +431,26 @@ func Convert_scheduling_PodGroupTemplateReference_To_v1alpha2_PodGroupTemplateRe return autoConvert_scheduling_PodGroupTemplateReference_To_v1alpha2_PodGroupTemplateReference(in, out, s) } +func autoConvert_v1alpha2_TopologyConstraint_To_scheduling_TopologyConstraint(in *schedulingv1alpha2.TopologyConstraint, out *scheduling.TopologyConstraint, s conversion.Scope) error { + out.Key = in.Key + return nil +} + +// Convert_v1alpha2_TopologyConstraint_To_scheduling_TopologyConstraint is an autogenerated conversion function. +func Convert_v1alpha2_TopologyConstraint_To_scheduling_TopologyConstraint(in *schedulingv1alpha2.TopologyConstraint, out *scheduling.TopologyConstraint, s conversion.Scope) error { + return autoConvert_v1alpha2_TopologyConstraint_To_scheduling_TopologyConstraint(in, out, s) +} + +func autoConvert_scheduling_TopologyConstraint_To_v1alpha2_TopologyConstraint(in *scheduling.TopologyConstraint, out *schedulingv1alpha2.TopologyConstraint, s conversion.Scope) error { + out.Key = in.Key + return nil +} + +// Convert_scheduling_TopologyConstraint_To_v1alpha2_TopologyConstraint is an autogenerated conversion function. +func Convert_scheduling_TopologyConstraint_To_v1alpha2_TopologyConstraint(in *scheduling.TopologyConstraint, out *schedulingv1alpha2.TopologyConstraint, s conversion.Scope) error { + return autoConvert_scheduling_TopologyConstraint_To_v1alpha2_TopologyConstraint(in, out, s) +} + func autoConvert_v1alpha2_TypedLocalObjectReference_To_scheduling_TypedLocalObjectReference(in *schedulingv1alpha2.TypedLocalObjectReference, out *scheduling.TypedLocalObjectReference, s conversion.Scope) error { out.APIGroup = in.APIGroup out.Kind = in.Kind diff --git a/pkg/apis/scheduling/v1alpha2/zz_generated.validations.go b/pkg/apis/scheduling/v1alpha2/zz_generated.validations.go index 5371cb8c87b..56bc9ddc08a 100644 --- a/pkg/apis/scheduling/v1alpha2/zz_generated.validations.go +++ b/pkg/apis/scheduling/v1alpha2/zz_generated.validations.go @@ -106,6 +106,38 @@ func Validate_PodGroup(ctx context.Context, op operation.Operation, fldPath *fie return errs } +// Validate_PodGroupSchedulingConstraints validates an instance of PodGroupSchedulingConstraints according +// to declarative validation rules in the API schema. +func Validate_PodGroupSchedulingConstraints(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *schedulingv1alpha2.PodGroupSchedulingConstraints) (errs field.ErrorList) { + // field schedulingv1alpha2.PodGroupSchedulingConstraints.Topology + errs = append(errs, + func(fldPath *field.Path, obj, oldObj []schedulingv1alpha2.TopologyConstraint, oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update && equality.Semantic.DeepEqual(obj, oldObj) { + return nil + } + // call field-attached validations + earlyReturn := false + if e := validate.MaxItems(ctx, op, fldPath, obj, oldObj, 1); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.OptionalSlice(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // iterate the list and call the type's validation function + errs = append(errs, validate.EachSliceVal(ctx, op, fldPath, obj, oldObj, nil, nil, Validate_TopologyConstraint)...) + return + }(fldPath.Child("topology"), obj.Topology, safe.Field(oldObj, func(oldObj *schedulingv1alpha2.PodGroupSchedulingConstraints) []schedulingv1alpha2.TopologyConstraint { + return oldObj.Topology + }), oldObj != nil)...) + + return errs +} + var unionMembershipFor_k8s_io_api_scheduling_v1alpha2_PodGroupSchedulingPolicy_ = validate.NewUnionMembership(validate.NewUnionMember("basic"), validate.NewUnionMember("gang")) // Validate_PodGroupSchedulingPolicy validates an instance of PodGroupSchedulingPolicy according @@ -220,6 +252,39 @@ func Validate_PodGroupSpec(ctx context.Context, op operation.Operation, fldPath return &oldObj.SchedulingPolicy }), oldObj != nil)...) + // field schedulingv1alpha2.PodGroupSpec.SchedulingConstraints + errs = append(errs, + func(fldPath *field.Path, obj, oldObj *schedulingv1alpha2.PodGroupSchedulingConstraints, oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update && equality.Semantic.DeepEqual(obj, oldObj) { + return nil + } + // call field-attached validations + earlyReturn := false + if e := validate.IfOption(ctx, op, fldPath, obj, oldObj, "TopologyAwareWorkloadScheduling", false, validate.ForbiddenPointer); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.IfOption(ctx, op, fldPath, obj, oldObj, "TopologyAwareWorkloadScheduling", false, validate.OptionalPointer); len(e) != 0 { + earlyReturn = true + } + if e := validate.IfOption(ctx, op, fldPath, obj, oldObj, "TopologyAwareWorkloadScheduling", true, validate.OptionalPointer); len(e) != 0 { + earlyReturn = true + } + if e := validate.IfOption(ctx, op, fldPath, obj, oldObj, "TopologyAwareWorkloadScheduling", true, validate.Immutable); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_PodGroupSchedulingConstraints(ctx, op, fldPath, obj, oldObj)...) + return + }(fldPath.Child("schedulingConstraints"), obj.SchedulingConstraints, safe.Field(oldObj, func(oldObj *schedulingv1alpha2.PodGroupSpec) *schedulingv1alpha2.PodGroupSchedulingConstraints { + return oldObj.SchedulingConstraints + }), oldObj != nil)...) + return errs } @@ -260,6 +325,35 @@ func Validate_PodGroupTemplate(ctx context.Context, op operation.Operation, fldP return &oldObj.SchedulingPolicy }), oldObj != nil)...) + // field schedulingv1alpha2.PodGroupTemplate.SchedulingConstraints + errs = append(errs, + func(fldPath *field.Path, obj, oldObj *schedulingv1alpha2.PodGroupSchedulingConstraints, oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update && equality.Semantic.DeepEqual(obj, oldObj) { + return nil + } + // call field-attached validations + earlyReturn := false + if e := validate.IfOption(ctx, op, fldPath, obj, oldObj, "TopologyAwareWorkloadScheduling", false, validate.ForbiddenPointer); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if e := validate.IfOption(ctx, op, fldPath, obj, oldObj, "TopologyAwareWorkloadScheduling", false, validate.OptionalPointer); len(e) != 0 { + earlyReturn = true + } + if e := validate.IfOption(ctx, op, fldPath, obj, oldObj, "TopologyAwareWorkloadScheduling", true, validate.OptionalPointer); len(e) != 0 { + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + // call the type's validation function + errs = append(errs, Validate_PodGroupSchedulingConstraints(ctx, op, fldPath, obj, oldObj)...) + return + }(fldPath.Child("schedulingConstraints"), obj.SchedulingConstraints, safe.Field(oldObj, func(oldObj *schedulingv1alpha2.PodGroupTemplate) *schedulingv1alpha2.PodGroupSchedulingConstraints { + return oldObj.SchedulingConstraints + }), oldObj != nil)...) + return errs } @@ -300,6 +394,32 @@ func Validate_PodGroupTemplateReference(ctx context.Context, op operation.Operat return errs } +// Validate_TopologyConstraint validates an instance of TopologyConstraint according +// to declarative validation rules in the API schema. +func Validate_TopologyConstraint(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *schedulingv1alpha2.TopologyConstraint) (errs field.ErrorList) { + // field schedulingv1alpha2.TopologyConstraint.Key + errs = append(errs, + func(fldPath *field.Path, obj, oldObj *string, oldValueCorrelated bool) (errs field.ErrorList) { + // don't revalidate unchanged data + if oldValueCorrelated && op.Type == operation.Update && (obj == oldObj || (obj != nil && oldObj != nil && *obj == *oldObj)) { + return nil + } + // call field-attached validations + earlyReturn := false + if e := validate.RequiredValue(ctx, op, fldPath, obj, oldObj); len(e) != 0 { + errs = append(errs, e...) + earlyReturn = true + } + if earlyReturn { + return // do not proceed + } + errs = append(errs, validate.LabelKey(ctx, op, fldPath, obj, oldObj)...) + return + }(fldPath.Child("key"), &obj.Key, safe.Field(oldObj, func(oldObj *schedulingv1alpha2.TopologyConstraint) *string { return &oldObj.Key }), oldObj != nil)...) + + return errs +} + // Validate_TypedLocalObjectReference validates an instance of TypedLocalObjectReference according // to declarative validation rules in the API schema. func Validate_TypedLocalObjectReference(ctx context.Context, op operation.Operation, fldPath *field.Path, obj, oldObj *schedulingv1alpha2.TypedLocalObjectReference) (errs field.ErrorList) { diff --git a/pkg/apis/scheduling/validation/validation_test.go b/pkg/apis/scheduling/validation/validation_test.go index 262594c8c70..3221b145a1e 100644 --- a/pkg/apis/scheduling/validation/validation_test.go +++ b/pkg/apis/scheduling/validation/validation_test.go @@ -188,6 +188,9 @@ func TestValidateWorkload(t *testing.T) { "no controllerRef": mkWorkload(func(w *scheduling.Workload) { w.Spec.ControllerRef = nil }), + "no scheduling constraints": mkWorkload(func(w *scheduling.Workload) { + w.Spec.PodGroupTemplates[1].SchedulingConstraints = nil + }), } for name, workload := range successCases { errs := ValidateWorkload(workload) @@ -381,6 +384,11 @@ func mkWorkload(tweaks ...func(w *scheduling.Workload)) *scheduling.Workload { SchedulingPolicy: scheduling.PodGroupSchedulingPolicy{ Basic: &scheduling.BasicSchedulingPolicy{}, }, + SchedulingConstraints: &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{ + {Key: "foo"}, + }, + }, }, { Name: "group2", SchedulingPolicy: scheduling.PodGroupSchedulingPolicy{ @@ -388,6 +396,11 @@ func mkWorkload(tweaks ...func(w *scheduling.Workload)) *scheduling.Workload { MinCount: 2, }, }, + SchedulingConstraints: &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{ + {Key: "foo"}, + }, + }, }}, }, } @@ -400,6 +413,9 @@ func mkWorkload(tweaks ...func(w *scheduling.Workload)) *scheduling.Workload { func TestValidatePodGroup(t *testing.T) { successCases := map[string]*scheduling.PodGroup{ "gang policy": mkPodGroup(), + "no scheduling constraints": mkPodGroup(func(pg *scheduling.PodGroup) { + pg.Spec.SchedulingConstraints = nil + }), } for name, podGroup := range successCases { errs := ValidatePodGroup(podGroup) @@ -737,6 +753,11 @@ func mkPodGroup(tweaks ...func(pg *scheduling.PodGroup)) *scheduling.PodGroup { MinCount: 5, }, }, + SchedulingConstraints: &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{ + {Key: "foo"}, + }, + }, }, } for _, tweak := range tweaks { diff --git a/pkg/apis/scheduling/zz_generated.deepcopy.go b/pkg/apis/scheduling/zz_generated.deepcopy.go index 9d19e49b270..498a1796e9c 100644 --- a/pkg/apis/scheduling/zz_generated.deepcopy.go +++ b/pkg/apis/scheduling/zz_generated.deepcopy.go @@ -120,6 +120,27 @@ func (in *PodGroupList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodGroupSchedulingConstraints) DeepCopyInto(out *PodGroupSchedulingConstraints) { + *out = *in + if in.Topology != nil { + in, out := &in.Topology, &out.Topology + *out = make([]TopologyConstraint, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGroupSchedulingConstraints. +func (in *PodGroupSchedulingConstraints) DeepCopy() *PodGroupSchedulingConstraints { + if in == nil { + return nil + } + out := new(PodGroupSchedulingConstraints) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodGroupSchedulingPolicy) DeepCopyInto(out *PodGroupSchedulingPolicy) { *out = *in @@ -155,6 +176,11 @@ func (in *PodGroupSpec) DeepCopyInto(out *PodGroupSpec) { (*in).DeepCopyInto(*out) } in.SchedulingPolicy.DeepCopyInto(&out.SchedulingPolicy) + if in.SchedulingConstraints != nil { + in, out := &in.SchedulingConstraints, &out.SchedulingConstraints + *out = new(PodGroupSchedulingConstraints) + (*in).DeepCopyInto(*out) + } return } @@ -195,6 +221,11 @@ func (in *PodGroupStatus) DeepCopy() *PodGroupStatus { func (in *PodGroupTemplate) DeepCopyInto(out *PodGroupTemplate) { *out = *in in.SchedulingPolicy.DeepCopyInto(&out.SchedulingPolicy) + if in.SchedulingConstraints != nil { + in, out := &in.SchedulingConstraints, &out.SchedulingConstraints + *out = new(PodGroupSchedulingConstraints) + (*in).DeepCopyInto(*out) + } return } @@ -293,6 +324,22 @@ func (in *PriorityClassList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TopologyConstraint) DeepCopyInto(out *TopologyConstraint) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TopologyConstraint. +func (in *TopologyConstraint) DeepCopy() *TopologyConstraint { + if in == nil { + return nil + } + out := new(TopologyConstraint) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TypedLocalObjectReference) DeepCopyInto(out *TypedLocalObjectReference) { *out = *in diff --git a/pkg/generated/openapi/zz_generated.openapi.go b/pkg/generated/openapi/zz_generated.openapi.go index 95edd8b539b..fe345a646a0 100644 --- a/pkg/generated/openapi/zz_generated.openapi.go +++ b/pkg/generated/openapi/zz_generated.openapi.go @@ -1136,11 +1136,13 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA schedulingv1alpha2.GangSchedulingPolicy{}.OpenAPIModelName(): schema_k8sio_api_scheduling_v1alpha2_GangSchedulingPolicy(ref), schedulingv1alpha2.PodGroup{}.OpenAPIModelName(): schema_k8sio_api_scheduling_v1alpha2_PodGroup(ref), schedulingv1alpha2.PodGroupList{}.OpenAPIModelName(): schema_k8sio_api_scheduling_v1alpha2_PodGroupList(ref), + schedulingv1alpha2.PodGroupSchedulingConstraints{}.OpenAPIModelName(): schema_k8sio_api_scheduling_v1alpha2_PodGroupSchedulingConstraints(ref), schedulingv1alpha2.PodGroupSchedulingPolicy{}.OpenAPIModelName(): schema_k8sio_api_scheduling_v1alpha2_PodGroupSchedulingPolicy(ref), schedulingv1alpha2.PodGroupSpec{}.OpenAPIModelName(): schema_k8sio_api_scheduling_v1alpha2_PodGroupSpec(ref), schedulingv1alpha2.PodGroupStatus{}.OpenAPIModelName(): schema_k8sio_api_scheduling_v1alpha2_PodGroupStatus(ref), schedulingv1alpha2.PodGroupTemplate{}.OpenAPIModelName(): schema_k8sio_api_scheduling_v1alpha2_PodGroupTemplate(ref), schedulingv1alpha2.PodGroupTemplateReference{}.OpenAPIModelName(): schema_k8sio_api_scheduling_v1alpha2_PodGroupTemplateReference(ref), + schedulingv1alpha2.TopologyConstraint{}.OpenAPIModelName(): schema_k8sio_api_scheduling_v1alpha2_TopologyConstraint(ref), schedulingv1alpha2.TypedLocalObjectReference{}.OpenAPIModelName(): schema_k8sio_api_scheduling_v1alpha2_TypedLocalObjectReference(ref), schedulingv1alpha2.Workload{}.OpenAPIModelName(): schema_k8sio_api_scheduling_v1alpha2_Workload(ref), schedulingv1alpha2.WorkloadList{}.OpenAPIModelName(): schema_k8sio_api_scheduling_v1alpha2_WorkloadList(ref), @@ -54048,6 +54050,40 @@ func schema_k8sio_api_scheduling_v1alpha2_PodGroupList(ref common.ReferenceCallb } } +func schema_k8sio_api_scheduling_v1alpha2_PodGroupSchedulingConstraints(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "PodGroupSchedulingConstraints defines scheduling constraints (e.g. topology) for a PodGroup.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "topology": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Topology defines the topology constraints for the pod group. Currently only a single topology constraint can be specified. This may change in the future.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref(schedulingv1alpha2.TopologyConstraint{}.OpenAPIModelName()), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + schedulingv1alpha2.TopologyConstraint{}.OpenAPIModelName()}, + } +} + func schema_k8sio_api_scheduling_v1alpha2_PodGroupSchedulingPolicy(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -54107,12 +54143,18 @@ func schema_k8sio_api_scheduling_v1alpha2_PodGroupSpec(ref common.ReferenceCallb Ref: ref(schedulingv1alpha2.PodGroupSchedulingPolicy{}.OpenAPIModelName()), }, }, + "schedulingConstraints": { + SchemaProps: spec.SchemaProps{ + Description: "SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroup. Controllers are expected to fill this field by copying it from a PodGroupTemplate. This field is immutable. This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled.", + Ref: ref(schedulingv1alpha2.PodGroupSchedulingConstraints{}.OpenAPIModelName()), + }, + }, }, Required: []string{"schedulingPolicy"}, }, }, Dependencies: []string{ - schedulingv1alpha2.PodGroupSchedulingPolicy{}.OpenAPIModelName(), schedulingv1alpha2.PodGroupTemplateReference{}.OpenAPIModelName()}, + schedulingv1alpha2.PodGroupSchedulingConstraints{}.OpenAPIModelName(), schedulingv1alpha2.PodGroupSchedulingPolicy{}.OpenAPIModelName(), schedulingv1alpha2.PodGroupTemplateReference{}.OpenAPIModelName()}, } } @@ -54177,12 +54219,18 @@ func schema_k8sio_api_scheduling_v1alpha2_PodGroupTemplate(ref common.ReferenceC Ref: ref(schedulingv1alpha2.PodGroupSchedulingPolicy{}.OpenAPIModelName()), }, }, + "schedulingConstraints": { + SchemaProps: spec.SchemaProps{ + Description: "SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroupTemplate. This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled.", + Ref: ref(schedulingv1alpha2.PodGroupSchedulingConstraints{}.OpenAPIModelName()), + }, + }, }, Required: []string{"name", "schedulingPolicy"}, }, }, Dependencies: []string{ - schedulingv1alpha2.PodGroupSchedulingPolicy{}.OpenAPIModelName()}, + schedulingv1alpha2.PodGroupSchedulingConstraints{}.OpenAPIModelName(), schedulingv1alpha2.PodGroupSchedulingPolicy{}.OpenAPIModelName()}, } } @@ -54218,6 +54266,28 @@ func schema_k8sio_api_scheduling_v1alpha2_PodGroupTemplateReference(ref common.R } } +func schema_k8sio_api_scheduling_v1alpha2_TopologyConstraint(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "TopologyConstraint defines a topology constraint for a PodGroup.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "key": { + SchemaProps: spec.SchemaProps{ + Description: "Key specifies the key of the node label representing the topology domain. All pods within the PodGroup must be colocated within the same domain instance. Different PodGroups can land on different domain instances even if they derive from the same PodGroupTemplate. Examples: \"topology.kubernetes.io/rack\"", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"key"}, + }, + }, + } +} + func schema_k8sio_api_scheduling_v1alpha2_TypedLocalObjectReference(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/pkg/registry/scheduling/podgroup/declarative_validation_test.go b/pkg/registry/scheduling/podgroup/declarative_validation_test.go index 39ccd252529..19ef25be74d 100644 --- a/pkg/registry/scheduling/podgroup/declarative_validation_test.go +++ b/pkg/registry/scheduling/podgroup/declarative_validation_test.go @@ -22,9 +22,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apimachinery/pkg/util/version" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" apitesting "k8s.io/kubernetes/pkg/api/testing" "k8s.io/kubernetes/pkg/apis/scheduling" + "k8s.io/kubernetes/pkg/features" // Ensure all API groups are registered with the scheme _ "k8s.io/kubernetes/pkg/apis/scheduling/install" @@ -52,6 +56,7 @@ func testDeclarativeValidate(t *testing.T, apiVersion string) { testCases := map[string]struct { input scheduling.PodGroup expectedErrs field.ErrorList + tasEnabled bool }{ "valid": { input: mkValidPodGroup(), @@ -108,10 +113,64 @@ func testDeclarativeValidate(t *testing.T, apiVersion string) { input: mkValidPodGroup(setBothPolicies()), expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "schedulingPolicy"), nil, "").WithOrigin("union")}, }, + "with schedulingConstraints and TAS disabled": { + input: mkValidPodGroup(addTopologyConstraint("foo")), + expectedErrs: field.ErrorList{field.Forbidden(field.NewPath("spec", "schedulingConstraints"), "")}, + }, + "valid with schedulingConstraints": { + input: mkValidPodGroup(addTopologyConstraint("foo")), + tasEnabled: true, + }, + "schedulingConstraints with multiple topology constraints": { + input: mkValidPodGroup(addTopologyConstraint("foo"), addTopologyConstraint("bar")), + expectedErrs: field.ErrorList{field.TooMany(field.NewPath("spec", "schedulingConstraints", "topology"), 2, 1).WithOrigin("maxItems")}, + tasEnabled: true, + }, + "valid with empty schedulingConstraints": { + input: mkValidPodGroup(setSchedulingConstraints()), + tasEnabled: true, + }, + "topologyConstraint with empty topology key": { + input: mkValidPodGroup(addTopologyConstraint("")), + expectedErrs: field.ErrorList{field.Required(field.NewPath("spec", "schedulingConstraints", "topology").Index(0).Child("key"), "")}, + tasEnabled: true, + }, + "valid with topology key with DNS prefix": { + input: mkValidPodGroup(addTopologyConstraint("example.com/Foo")), + tasEnabled: true, + }, + "valid with topology key with prefix with max length": { + input: mkValidPodGroup(addTopologyConstraint(strings.Repeat("a", 253) + "/" + strings.Repeat("b", 63))), + tasEnabled: true, + }, + "with topology key with prefix exceending max prefix length": { + input: mkValidPodGroup(addTopologyConstraint(strings.Repeat("a", 254) + "/foo")), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "schedulingConstraints", "topology").Index(0).Child("key"), nil, "").WithOrigin("format=k8s-label-key")}, + tasEnabled: true, + }, + "with topology key with prefix exceending max name length": { + input: mkValidPodGroup(addTopologyConstraint("foo/" + strings.Repeat("b", 64))), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "schedulingConstraints", "topology").Index(0).Child("key"), nil, "").WithOrigin("format=k8s-label-key")}, + tasEnabled: true, + }, + "with topology key without prefix exceeding max length": { + input: mkValidPodGroup(addTopologyConstraint(strings.Repeat("b", 64))), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "schedulingConstraints", "topology").Index(0).Child("key"), nil, "").WithOrigin("format=k8s-label-key")}, + tasEnabled: true, + }, + "with topology key with invalid characters": { + input: mkValidPodGroup(addTopologyConstraint("Example.com/Foo")), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "schedulingConstraints", "topology").Index(0).Child("key"), nil, "").WithOrigin("format=k8s-label-key")}, + tasEnabled: true, + }, } for k, tc := range testCases { t.Run(k, func(t *testing.T) { - apitesting.VerifyValidationEquivalence(t, ctx, &tc.input, strategy.Validate, tc.expectedErrs) + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.GenericWorkload: tc.tasEnabled, + features.TopologyAwareWorkloadScheduling: tc.tasEnabled, + }) + apitesting.VerifyValidationEquivalence(t, ctx, &tc.input, strategy.Validate, tc.expectedErrs, apitesting.WithMinEmulationVersion(version.MustParse("1.36"))) }) } } @@ -139,6 +198,7 @@ func testDeclarativeValidateUpdate(t *testing.T, apiVersion string) { oldObj scheduling.PodGroup updateObj scheduling.PodGroup expectedErrs field.ErrorList + tasEnabled bool }{ "valid update": { oldObj: mkValidPodGroup(setResourceVersion("1")), @@ -184,11 +244,57 @@ func testDeclarativeValidateUpdate(t *testing.T, apiVersion string) { updateObj: mkValidPodGroup(setResourceVersion("1"), setBasicPolicy()), expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "schedulingPolicy"), nil, "field is immutable").WithOrigin("immutable").MarkAlpha()}, }, + "valid update with unchanged scheduling constraints and TAS disabled": { + oldObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("foo")), + updateObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("foo")), + }, + "valid update with unchanged scheduling constraints": { + oldObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("foo")), + updateObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("foo")), + tasEnabled: true, + }, + "invalid update to scheduling constraints": { + oldObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("foo")), + updateObj: mkValidPodGroup(setResourceVersion("1"), setSchedulingConstraints()), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "schedulingConstraints"), nil, "field is immutable").WithOrigin("immutable")}, + tasEnabled: true, + }, + "invalid update to topology constraints": { + oldObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("foo")), + updateObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("foo"), addTopologyConstraint("bar")), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "schedulingConstraints"), nil, "field is immutable").WithOrigin("immutable")}, + tasEnabled: true, + }, + "invalid update to topology key": { + oldObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("foo")), + updateObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("bar")), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "schedulingConstraints"), nil, "field is immutable").WithOrigin("immutable")}, + tasEnabled: true, + }, + "invalid update to scheduling constraints with TAS disabled": { + oldObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("foo")), + updateObj: mkValidPodGroup(setResourceVersion("1"), setSchedulingConstraints()), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "schedulingConstraints"), nil, "field is immutable").WithOrigin("immutable")}, + }, + "invalid update to topology constraints with TAS disabled": { + oldObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("foo")), + updateObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("foo"), addTopologyConstraint("bar")), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "schedulingConstraints"), nil, "field is immutable").WithOrigin("immutable")}, + }, + "invalid update to topology key with TAS disabled": { + oldObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("foo")), + updateObj: mkValidPodGroup(setResourceVersion("1"), addTopologyConstraint("bar")), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "schedulingConstraints"), nil, "field is immutable").WithOrigin("immutable")}, + }, } for k, tc := range testCases { t.Run(k, func(t *testing.T) { + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.GenericWorkload: tc.tasEnabled, + features.TopologyAwareWorkloadScheduling: tc.tasEnabled, + }) strategy := NewStrategy() - apitesting.VerifyUpdateValidationEquivalence(t, ctx, &tc.updateObj, &tc.oldObj, strategy.ValidateUpdate, tc.expectedErrs) + apitesting.VerifyUpdateValidationEquivalence(t, ctx, &tc.updateObj, &tc.oldObj, strategy.ValidateUpdate, tc.expectedErrs, apitesting.WithMinEmulationVersion(version.MustParse("1.36"))) }) } } @@ -333,3 +439,18 @@ func addCondition(conditionType string) func(obj *scheduling.PodGroup) { }) } } + +func setSchedulingConstraints() func(obj *scheduling.PodGroup) { + return func(obj *scheduling.PodGroup) { + obj.Spec.SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{} + } +} + +func addTopologyConstraint(value string) func(obj *scheduling.PodGroup) { + return func(obj *scheduling.PodGroup) { + if obj.Spec.SchedulingConstraints == nil { + setSchedulingConstraints()(obj) + } + obj.Spec.SchedulingConstraints.Topology = append(obj.Spec.SchedulingConstraints.Topology, scheduling.TopologyConstraint{Key: value}) + } +} diff --git a/pkg/registry/scheduling/podgroup/strategy.go b/pkg/registry/scheduling/podgroup/strategy.go index bc4975d5e41..ba58f0667a9 100644 --- a/pkg/registry/scheduling/podgroup/strategy.go +++ b/pkg/registry/scheduling/podgroup/strategy.go @@ -27,9 +27,11 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/registry/rest" "k8s.io/apiserver/pkg/storage/names" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/scheduling" "k8s.io/kubernetes/pkg/apis/scheduling/validation" + "k8s.io/kubernetes/pkg/features" ) // podGroupStrategy implements behavior for PodGroup objects. @@ -66,12 +68,17 @@ func (*podGroupStrategy) PrepareForCreate(ctx context.Context, obj runtime.Objec podGroup := obj.(*scheduling.PodGroup) // Status must not be set by user on create. podGroup.Status = scheduling.PodGroupStatus{} + dropDisabledPodGroupFields(podGroup, nil) } func (*podGroupStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { podGroup := obj.(*scheduling.PodGroup) allErrs := validation.ValidatePodGroup(podGroup) - return rest.ValidateDeclarativelyWithMigrationChecks(ctx, legacyscheme.Scheme, obj, nil, allErrs, operation.Create, rest.WithDeclarativeEnforcement()) + opts := []string{} + if schedulingConstraintsInUse(nil) { + opts = append(opts, string(features.TopologyAwareWorkloadScheduling)) + } + return rest.ValidateDeclarativelyWithMigrationChecks(ctx, legacyscheme.Scheme, obj, nil, allErrs, operation.Create, rest.WithDeclarativeEnforcement(), rest.WithOptions(opts)) } func (*podGroupStrategy) WarningsOnCreate(ctx context.Context, obj runtime.Object) []string { @@ -88,13 +95,18 @@ func (*podGroupStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime. newPodGroup := obj.(*scheduling.PodGroup) oldPodGroup := old.(*scheduling.PodGroup) newPodGroup.Status = oldPodGroup.Status + dropDisabledPodGroupFields(newPodGroup, oldPodGroup) } func (*podGroupStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { newPodGroup := obj.(*scheduling.PodGroup) oldPodGroup := old.(*scheduling.PodGroup) allErrs := validation.ValidatePodGroupUpdate(newPodGroup, oldPodGroup) - return rest.ValidateDeclarativelyWithMigrationChecks(ctx, legacyscheme.Scheme, newPodGroup, oldPodGroup, allErrs, operation.Update, rest.WithDeclarativeEnforcement()) + opts := []string{} + if schedulingConstraintsInUse(oldPodGroup) { + opts = append(opts, string(features.TopologyAwareWorkloadScheduling)) + } + return rest.ValidateDeclarativelyWithMigrationChecks(ctx, legacyscheme.Scheme, newPodGroup, oldPodGroup, allErrs, operation.Update, rest.WithDeclarativeEnforcement(), rest.WithOptions(opts)) } func (*podGroupStrategy) WarningsOnUpdate(ctx context.Context, obj, old runtime.Object) []string { @@ -145,3 +157,23 @@ func (r *podGroupStatusStrategy) ValidateUpdate(ctx context.Context, obj, old ru func (*podGroupStatusStrategy) WarningsOnUpdate(ctx context.Context, obj, old runtime.Object) []string { return nil } + +// dropDisabledPodGroupFields removes fields which are covered by a feature gate. +func dropDisabledPodGroupFields(newPodGroup, oldPodGroup *scheduling.PodGroup) { + dropDisabledSchedulingConstraintsFields(newPodGroup, oldPodGroup) +} + +// dropDisabledSchedulingConstraintsFields drops the SchedulingConstraints field +// from the new PodGroup if the TopologyAwareWorkloadScheduling feature gate is disabled +// and it was not used in the old PodGroup. +func dropDisabledSchedulingConstraintsFields(newPodGroup, oldPodGroup *scheduling.PodGroup) { + if schedulingConstraintsInUse(oldPodGroup) { + // No need to drop anything. + return + } + newPodGroup.Spec.SchedulingConstraints = nil +} + +func schedulingConstraintsInUse(pg *scheduling.PodGroup) bool { + return utilfeature.DefaultFeatureGate.Enabled(features.TopologyAwareWorkloadScheduling) || (pg != nil && pg.Spec.SchedulingConstraints != nil) +} diff --git a/pkg/registry/scheduling/podgroup/strategy_test.go b/pkg/registry/scheduling/podgroup/strategy_test.go index 2816fca477a..b5970c7ca12 100644 --- a/pkg/registry/scheduling/podgroup/strategy_test.go +++ b/pkg/registry/scheduling/podgroup/strategy_test.go @@ -18,13 +18,18 @@ package podgroup import ( "context" + "fmt" "strings" "testing" + "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/apis/scheduling" + "k8s.io/kubernetes/pkg/features" ) var podGroup = &scheduling.PodGroup{ @@ -47,6 +52,31 @@ var podGroup = &scheduling.PodGroup{ }, } +var podGroupWithSchedulingConstraints = &scheduling.PodGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Namespace: metav1.NamespaceDefault, + }, + Spec: scheduling.PodGroupSpec{ + PodGroupTemplateRef: &scheduling.PodGroupTemplateReference{ + Workload: &scheduling.WorkloadPodGroupTemplateReference{ + WorkloadName: "w", + PodGroupTemplateName: "t", + }, + }, + SchedulingPolicy: scheduling.PodGroupSchedulingPolicy{ + Gang: &scheduling.GangSchedulingPolicy{ + MinCount: 5, + }, + }, + SchedulingConstraints: &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{ + {Key: "foo"}, + }, + }, + }, +} + var ( fieldImmutableError = "field is immutable" minCountError = "must be greater than or equal to 1" @@ -76,11 +106,13 @@ func ctxWithRequestInfo() context.Context { func TestStrategyCreate(t *testing.T) { ctx := ctxWithRequestInfo() now := metav1.Now() - testCases := map[string]struct { + type testCase struct { obj *scheduling.PodGroup expectObj *scheduling.PodGroup expectValidationError string - }{ + tasEnabled bool + } + testCases := map[string]testCase{ "simple": { obj: podGroup, expectObj: podGroup, @@ -125,10 +157,52 @@ func TestStrategyCreate(t *testing.T) { }(), expectObj: podGroup, }, + "multiple topology constraints": { + obj: func() *scheduling.PodGroup { + newPodGroup := podGroupWithSchedulingConstraints.DeepCopy() + newPodGroup.Spec.SchedulingConstraints.Topology = []scheduling.TopologyConstraint{ + {Key: "foo"}, + {Key: "bar"}, + } + return newPodGroup + }(), + expectValidationError: "must have at most 1 item", + tasEnabled: true, + }, + "invalid topology key": { + obj: func() *scheduling.PodGroup { + newPodGroup := podGroupWithSchedulingConstraints.DeepCopy() + newPodGroup.Spec.SchedulingConstraints.Topology[0].Key = "foo-" + return newPodGroup + }(), + expectValidationError: "Invalid value: \"foo-\"", + tasEnabled: true, + }, + "with TAS feature gate disabled, drop scheduling constraints on creation": { + obj: podGroupWithSchedulingConstraints.DeepCopy(), + expectObj: podGroup, + tasEnabled: false, + }, } + allTestCases := make(map[string]testCase) for name, tc := range testCases { + allTestCases[name] = tc + if tc.tasEnabled { + newTc := testCase{ + obj: tc.obj, + expectObj: podGroup, + } + allTestCases[fmt.Sprintf("drops scheduling constraints, originally %s", name)] = newTc + } + } + + for name, tc := range allTestCases { t.Run(name, func(t *testing.T) { + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.GenericWorkload: tc.tasEnabled, + features.TopologyAwareWorkloadScheduling: tc.tasEnabled, + }) podGroup := tc.obj.DeepCopy() strategy := NewStrategy() @@ -151,6 +225,11 @@ func TestStrategyCreate(t *testing.T) { if warnings := strategy.WarningsOnCreate(ctx, podGroup); len(warnings) != 0 { t.Fatalf("unexpected warnings: %q", warnings) } + if tc.expectObj != nil { + if diff := cmp.Diff(tc.expectObj, podGroup); diff != "" { + t.Errorf("got unexpected podGroup object (-want, +got): %s", diff) + } + } }) } } @@ -161,6 +240,7 @@ func TestStrategyUpdate(t *testing.T) { oldObj *scheduling.PodGroup newObj *scheduling.PodGroup expectValidationError string + tasEnabled bool }{ "no changes": { oldObj: podGroup, @@ -209,10 +289,71 @@ func TestStrategyUpdate(t *testing.T) { }(), expectValidationError: fieldImmutableError, }, + "changing scheduling constraints not allowed": { + oldObj: podGroupWithSchedulingConstraints, + newObj: func() *scheduling.PodGroup { + newPodGroup := podGroupWithSchedulingConstraints.DeepCopy() + newPodGroup.Spec.SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{} + return newPodGroup + }(), + expectValidationError: fieldImmutableError, + tasEnabled: true, + }, + "changing topology constraints not allowed": { + oldObj: podGroupWithSchedulingConstraints, + newObj: func() *scheduling.PodGroup { + newPodGroup := podGroupWithSchedulingConstraints.DeepCopy() + newPodGroup.Spec.SchedulingConstraints.Topology = []scheduling.TopologyConstraint{} + return newPodGroup + }(), + expectValidationError: fieldImmutableError, + tasEnabled: true, + }, + "changing topology key not allowed": { + oldObj: podGroupWithSchedulingConstraints, + newObj: func() *scheduling.PodGroup { + newPodGroup := podGroupWithSchedulingConstraints.DeepCopy() + newPodGroup.Spec.SchedulingConstraints.Topology[0].Key = "foobar" + return newPodGroup + }(), + expectValidationError: fieldImmutableError, + tasEnabled: true, + }, + "changing scheduling constraints not allowed with TAS disabled": { + oldObj: podGroupWithSchedulingConstraints, + newObj: func() *scheduling.PodGroup { + newPodGroup := podGroupWithSchedulingConstraints.DeepCopy() + newPodGroup.Spec.SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{} + return newPodGroup + }(), + expectValidationError: fieldImmutableError, + }, + "changing topology constraints not allowed with TAS disabled": { + oldObj: podGroupWithSchedulingConstraints, + newObj: func() *scheduling.PodGroup { + newPodGroup := podGroupWithSchedulingConstraints.DeepCopy() + newPodGroup.Spec.SchedulingConstraints.Topology = []scheduling.TopologyConstraint{} + return newPodGroup + }(), + expectValidationError: fieldImmutableError, + }, + "changing topology key not allowed with TAS disabled": { + oldObj: podGroupWithSchedulingConstraints, + newObj: func() *scheduling.PodGroup { + newPodGroup := podGroupWithSchedulingConstraints.DeepCopy() + newPodGroup.Spec.SchedulingConstraints.Topology[0].Key = "foobar" + return newPodGroup + }(), + expectValidationError: fieldImmutableError, + }, } for name, tc := range testCases { t.Run(name, func(t *testing.T) { + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.GenericWorkload: tc.tasEnabled, + features.TopologyAwareWorkloadScheduling: tc.tasEnabled, + }) podGroup := tc.oldObj.DeepCopy() newPodGroup := tc.newObj.DeepCopy() newPodGroup.ResourceVersion = "4" diff --git a/pkg/registry/scheduling/workload/declarative_validation_test.go b/pkg/registry/scheduling/workload/declarative_validation_test.go index 6bebe7c5db8..a4a29a46cbf 100644 --- a/pkg/registry/scheduling/workload/declarative_validation_test.go +++ b/pkg/registry/scheduling/workload/declarative_validation_test.go @@ -23,9 +23,13 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/apimachinery/pkg/util/version" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" apitesting "k8s.io/kubernetes/pkg/api/testing" "k8s.io/kubernetes/pkg/apis/scheduling" + "k8s.io/kubernetes/pkg/features" // Ensure all API groups are registered with the scheme _ "k8s.io/kubernetes/pkg/apis/scheduling/install" @@ -52,6 +56,7 @@ func testDeclarativeValidate(t *testing.T, apiVersion string) { testCases := map[string]struct { input scheduling.Workload expectedErrs field.ErrorList + tasEnabled bool }{ "valid": { input: mkValidWorkload(), @@ -135,10 +140,60 @@ func testDeclarativeValidate(t *testing.T, apiVersion string) { "valid with basic policy": { input: mkValidWorkload(setBasicPolicy(0)), }, + "valid with schedulingConstraints": { + input: mkValidWorkload(addTopologyConstraint(0, "foo")), + tasEnabled: true, + }, + "valid with empty schedulingConstraints": { + input: mkValidWorkload(setSchedulingConstraints(0)), + tasEnabled: true, + }, + "with multiple topology constraints": { + input: mkValidWorkload(addTopologyConstraint(0, "foo"), addTopologyConstraint(0, "bar")), + expectedErrs: field.ErrorList{field.TooMany(field.NewPath("spec", "podGroupTemplates").Index(0).Child("schedulingConstraints", "topology"), 2, 1).WithOrigin("maxItems")}, + tasEnabled: true, + }, + "with empty topology key": { + input: mkValidWorkload(addTopologyConstraint(0, "")), + expectedErrs: field.ErrorList{field.Required(field.NewPath("spec", "podGroupTemplates").Index(0).Child("schedulingConstraints", "topology").Index(0).Child("key"), "")}, + tasEnabled: true, + }, + "valid with topology key with DNS prefix": { + input: mkValidWorkload(addTopologyConstraint(0, "example.com/Foo")), + tasEnabled: true, + }, + "valid with topology key with prefix with max length": { + input: mkValidWorkload(addTopologyConstraint(0, strings.Repeat("a", 253)+"/"+strings.Repeat("b", 63))), + tasEnabled: true, + }, + "with topology key with prefix exceending max prefix length": { + input: mkValidWorkload(addTopologyConstraint(0, strings.Repeat("a", 254)+"/foo")), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "podGroupTemplates").Index(0).Child("schedulingConstraints", "topology").Index(0).Child("key"), nil, "").WithOrigin("format=k8s-label-key")}, + tasEnabled: true, + }, + "with topology key with prefix exceending max name length": { + input: mkValidWorkload(addTopologyConstraint(0, "foo/"+strings.Repeat("b", 64))), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "podGroupTemplates").Index(0).Child("schedulingConstraints", "topology").Index(0).Child("key"), nil, "").WithOrigin("format=k8s-label-key")}, + tasEnabled: true, + }, + "with topology key without prefix exceeding max length": { + input: mkValidWorkload(addTopologyConstraint(0, strings.Repeat("b", 64))), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "podGroupTemplates").Index(0).Child("schedulingConstraints", "topology").Index(0).Child("key"), nil, "").WithOrigin("format=k8s-label-key")}, + tasEnabled: true, + }, + "with topology key with invalid characters": { + input: mkValidWorkload(addTopologyConstraint(0, "Example.com/Foo")), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "podGroupTemplates").Index(0).Child("schedulingConstraints", "topology").Index(0).Child("key"), nil, "").WithOrigin("format=k8s-label-key")}, + tasEnabled: true, + }, } for k, tc := range testCases { t.Run(k, func(t *testing.T) { - apitesting.VerifyValidationEquivalence(t, ctx, &tc.input, Strategy.Validate, tc.expectedErrs) + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.GenericWorkload: tc.tasEnabled, + features.TopologyAwareWorkloadScheduling: tc.tasEnabled, + }) + apitesting.VerifyValidationEquivalence(t, ctx, &tc.input, Strategy.Validate, tc.expectedErrs, apitesting.WithMinEmulationVersion(version.MustParse("1.36"))) }) } } @@ -156,6 +211,7 @@ func testDeclarativeValidateUpdate(t *testing.T, apiVersion string) { testCases := map[string]struct { oldObj scheduling.Workload updateObj scheduling.Workload + tasEnabled bool expectedErrs field.ErrorList }{ "valid update": { @@ -221,9 +277,55 @@ func testDeclarativeValidateUpdate(t *testing.T, apiVersion string) { updateObj: mkValidWorkload(setResourceVersion("1"), setBasicPolicy(0)), expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "podGroupTemplates"), nil, "field is immutable").WithOrigin("immutable").MarkAlpha()}, }, + "valid update with unchanged scheduling constraints": { + oldObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "foo")), + updateObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "foo")), + tasEnabled: true, + }, + "invalid update to scheduling constraints": { + oldObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "foo")), + updateObj: mkValidWorkload(setResourceVersion("1"), setSchedulingConstraints(0)), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "podGroupTemplates"), nil, "field is immutable").WithOrigin("immutable").MarkAlpha()}, + tasEnabled: true, + }, + "invalid update to topology constraints": { + oldObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "foo")), + updateObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "foo"), addTopologyConstraint(0, "bar")), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "podGroupTemplates"), nil, "field is immutable").WithOrigin("immutable").MarkAlpha()}, + tasEnabled: true, + }, + "invalid update to topology key": { + oldObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "foo")), + updateObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "bar")), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "podGroupTemplates"), nil, "field is immutable").WithOrigin("immutable").MarkAlpha()}, + tasEnabled: true, + }, + "valid update with unchanged scheduling constraints with TAS disabled": { + oldObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "foo")), + updateObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "foo")), + }, + "invalid update to scheduling constraints with TAS disabled": { + oldObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "foo")), + updateObj: mkValidWorkload(setResourceVersion("1"), setSchedulingConstraints(0)), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "podGroupTemplates"), nil, "field is immutable").WithOrigin("immutable").MarkAlpha()}, + }, + "invalid update to topology constraints with TAS disabled": { + oldObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "foo")), + updateObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "foo"), addTopologyConstraint(0, "bar")), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "podGroupTemplates"), nil, "field is immutable").WithOrigin("immutable").MarkAlpha()}, + }, + "invalid update to topology key with TAS disabled": { + oldObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "foo")), + updateObj: mkValidWorkload(setResourceVersion("1"), addTopologyConstraint(0, "bar")), + expectedErrs: field.ErrorList{field.Invalid(field.NewPath("spec", "podGroupTemplates"), nil, "field is immutable").WithOrigin("immutable").MarkAlpha()}, + }, } for k, tc := range testCases { t.Run(k, func(t *testing.T) { + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.GenericWorkload: tc.tasEnabled, + features.TopologyAwareWorkloadScheduling: tc.tasEnabled, + }) ctx := genericapirequest.WithRequestInfo(genericapirequest.NewDefaultContext(), &genericapirequest.RequestInfo{ APIPrefix: "apis", APIGroup: "scheduling.k8s.io", @@ -233,7 +335,7 @@ func testDeclarativeValidateUpdate(t *testing.T, apiVersion string) { IsResourceRequest: true, Verb: "update", }) - apitesting.VerifyUpdateValidationEquivalence(t, ctx, &tc.updateObj, &tc.oldObj, Strategy.ValidateUpdate, tc.expectedErrs) + apitesting.VerifyUpdateValidationEquivalence(t, ctx, &tc.updateObj, &tc.oldObj, Strategy.ValidateUpdate, tc.expectedErrs, apitesting.WithMinEmulationVersion(version.MustParse("1.36"))) }) } } @@ -354,3 +456,19 @@ func setControllerRef(apiGroup, kind, name string) func(obj *scheduling.Workload } } } + +func setSchedulingConstraints(pgIdx int) func(obj *scheduling.Workload) { + return func(obj *scheduling.Workload) { + obj.Spec.PodGroupTemplates[pgIdx].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{} + } +} + +func addTopologyConstraint(pgIdx int, topologyKey string) func(obj *scheduling.Workload) { + return func(obj *scheduling.Workload) { + if obj.Spec.PodGroupTemplates[pgIdx].SchedulingConstraints == nil { + setSchedulingConstraints(pgIdx)(obj) + } + obj.Spec.PodGroupTemplates[pgIdx].SchedulingConstraints.Topology = append(obj.Spec.PodGroupTemplates[pgIdx].SchedulingConstraints.Topology, + scheduling.TopologyConstraint{Key: topologyKey}) + } +} diff --git a/pkg/registry/scheduling/workload/strategy.go b/pkg/registry/scheduling/workload/strategy.go index cad7de27809..dcc3b212d70 100644 --- a/pkg/registry/scheduling/workload/strategy.go +++ b/pkg/registry/scheduling/workload/strategy.go @@ -24,9 +24,11 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/registry/rest" "k8s.io/apiserver/pkg/storage/names" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/kubernetes/pkg/api/legacyscheme" "k8s.io/kubernetes/pkg/apis/scheduling" "k8s.io/kubernetes/pkg/apis/scheduling/validation" + "k8s.io/kubernetes/pkg/features" ) // workloadStrategy implements behavior for Workload objects. @@ -42,12 +44,18 @@ func (workloadStrategy) NamespaceScoped() bool { return true } -func (workloadStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) {} +func (workloadStrategy) PrepareForCreate(ctx context.Context, obj runtime.Object) { + dropDisabledWorkloadFields(obj.(*scheduling.Workload), nil) +} func (workloadStrategy) Validate(ctx context.Context, obj runtime.Object) field.ErrorList { workloadScheduling := obj.(*scheduling.Workload) allErrs := validation.ValidateWorkload(workloadScheduling) - return rest.ValidateDeclarativelyWithMigrationChecks(ctx, legacyscheme.Scheme, obj, nil, allErrs, operation.Create, rest.WithDeclarativeEnforcement()) + opts := []string{} + if utilfeature.DefaultFeatureGate.Enabled(features.TopologyAwareWorkloadScheduling) || anySchedulingConstraintsInUse(nil) { + opts = append(opts, string(features.TopologyAwareWorkloadScheduling)) + } + return rest.ValidateDeclarativelyWithMigrationChecks(ctx, legacyscheme.Scheme, obj, nil, allErrs, operation.Create, rest.WithDeclarativeEnforcement(), rest.WithOptions(opts)) } func (workloadStrategy) WarningsOnCreate(ctx context.Context, obj runtime.Object) []string { @@ -60,11 +68,17 @@ func (workloadStrategy) AllowCreateOnUpdate() bool { return false } -func (workloadStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) {} +func (workloadStrategy) PrepareForUpdate(ctx context.Context, obj, old runtime.Object) { + dropDisabledWorkloadFields(obj.(*scheduling.Workload), old.(*scheduling.Workload)) +} func (workloadStrategy) ValidateUpdate(ctx context.Context, obj, old runtime.Object) field.ErrorList { allErrs := validation.ValidateWorkloadUpdate(obj.(*scheduling.Workload), old.(*scheduling.Workload)) - return rest.ValidateDeclarativelyWithMigrationChecks(ctx, legacyscheme.Scheme, obj, old, allErrs, operation.Update, rest.WithDeclarativeEnforcement()) + opts := []string{} + if utilfeature.DefaultFeatureGate.Enabled(features.TopologyAwareWorkloadScheduling) || anySchedulingConstraintsInUse(old.(*scheduling.Workload)) { + opts = append(opts, string(features.TopologyAwareWorkloadScheduling)) + } + return rest.ValidateDeclarativelyWithMigrationChecks(ctx, legacyscheme.Scheme, obj, old, allErrs, operation.Update, rest.WithDeclarativeEnforcement(), rest.WithOptions(opts)) } func (workloadStrategy) WarningsOnUpdate(ctx context.Context, obj, old runtime.Object) []string { @@ -74,3 +88,65 @@ func (workloadStrategy) WarningsOnUpdate(ctx context.Context, obj, old runtime.O func (workloadStrategy) AllowUnconditionalUpdate() bool { return true } + +// dropDisabledWorkloadFields removes fields which are covered by a feature gate. +func dropDisabledWorkloadFields(workload, oldWorkload *scheduling.Workload) { + var workloadSpec, oldWorkloadSpec *scheduling.WorkloadSpec + if workload != nil { + workloadSpec = &workload.Spec + } + if oldWorkload != nil { + oldWorkloadSpec = &oldWorkload.Spec + } + dropDisabledWorkloadSpecFields(workloadSpec, oldWorkloadSpec) +} + +func dropDisabledWorkloadSpecFields(workloadSpec, oldWorkloadSpec *scheduling.WorkloadSpec) { + var templates, oldTemplates []scheduling.PodGroupTemplate + if workloadSpec != nil { + templates = workloadSpec.PodGroupTemplates + } + if oldWorkloadSpec != nil { + oldTemplates = oldWorkloadSpec.PodGroupTemplates + } + dropDisabledPodGroupTemplatesFields(templates, oldTemplates) +} + +func dropDisabledPodGroupTemplatesFields(templates, oldTemplates []scheduling.PodGroupTemplate) { + m := len(oldTemplates) + for i := range templates { + var oldTemplate *scheduling.PodGroupTemplate + if i < m { + oldTemplate = &oldTemplates[i] + } + template := &templates[i] + dropDisabledSchedulingConstraintsFields(template, oldTemplate) + } +} + +// dropDisabledSchedulingConstraintsFields drops the SchedulingConstraints field +// from the PodGroupTemplate if the TopologyAwareWorkloadScheduling feature gate is disabled. +func dropDisabledSchedulingConstraintsFields(template, oldTemplate *scheduling.PodGroupTemplate) { + if utilfeature.DefaultFeatureGate.Enabled(features.TopologyAwareWorkloadScheduling) || schedulingConstraintsInUse(oldTemplate) { + return + } + + template.SchedulingConstraints = nil +} + +func anySchedulingConstraintsInUse(workload *scheduling.Workload) bool { + if workload == nil { + return false + } + + for i := range workload.Spec.PodGroupTemplates { + if schedulingConstraintsInUse(&workload.Spec.PodGroupTemplates[i]) { + return true + } + } + return false +} + +func schedulingConstraintsInUse(pgt *scheduling.PodGroupTemplate) bool { + return pgt != nil && pgt.SchedulingConstraints != nil +} diff --git a/pkg/registry/scheduling/workload/strategy_test.go b/pkg/registry/scheduling/workload/strategy_test.go index a69c0100aa7..80f94fbe991 100644 --- a/pkg/registry/scheduling/workload/strategy_test.go +++ b/pkg/registry/scheduling/workload/strategy_test.go @@ -18,11 +18,16 @@ package workload import ( "context" + "strings" "testing" + "github.com/google/go-cmp/cmp" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" + utilfeature "k8s.io/apiserver/pkg/util/feature" + featuregatetesting "k8s.io/component-base/featuregate/testing" "k8s.io/kubernetes/pkg/apis/scheduling" + "k8s.io/kubernetes/pkg/features" ) var workload = &scheduling.Workload{ @@ -87,6 +92,100 @@ func TestPodSchedulingStrategyCreate(t *testing.T) { }) } +func TestPodSchedulingStrategyCreate_SchedulingConstraints(t *testing.T) { + testCases := map[string]struct { + obj *scheduling.Workload + expectObj *scheduling.Workload + expectValidationError string + tasEnabled bool + }{ + "drops field with SchedulingConstraints set and TAS disabled": { + obj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + expectObj: workload, + tasEnabled: false, + }, + "valid with SchedulingConstraints set and TAS enabled": { + obj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + expectObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + tasEnabled: true, + }, + "invalid with multiple topology constraints": { + obj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{ + {Key: "foo"}, + {Key: "bar"}, + }, + } + return workload + }(), + expectValidationError: "must have at most 1 item", + tasEnabled: true, + }, + "invalid with invalid topology key": { + obj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{ + {Key: ""}, + }, + } + return workload + }(), + expectValidationError: "Required value", + tasEnabled: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.GenericWorkload: tc.tasEnabled, + features.TopologyAwareWorkloadScheduling: tc.tasEnabled, + }) + ctx := ctxWithRequestInfo() + workload := tc.obj.DeepCopy() + + Strategy.PrepareForCreate(ctx, workload) + if errs := Strategy.Validate(ctx, workload); len(errs) != 0 { + if tc.expectValidationError == "" { + t.Fatalf("unexpected error(s): %v", errs) + } + if len(errs) != 1 { + t.Fatalf("exactly one error expected") + } + if errMsg := errs[0].Error(); !strings.Contains(errMsg, tc.expectValidationError) { + t.Fatalf("error %#v does not contain the expected message %q", errMsg, tc.expectValidationError) + } + } + if tc.expectObj != nil { + if diff := cmp.Diff(tc.expectObj, workload); diff != "" { + t.Errorf("got unexpected workload object (-want, +got): %s", diff) + } + } + }) + } +} + func TestPodSchedulingStrategyUpdate(t *testing.T) { t.Run("no changes", func(t *testing.T) { ctx := ctxWithRequestInfo() @@ -146,3 +245,227 @@ func TestPodSchedulingStrategyUpdate(t *testing.T) { } }) } + +func TestPodSchedulingStrategyUpdate_SchedulingConstraints(t *testing.T) { + testCases := map[string]struct { + oldObj *scheduling.Workload + newObj *scheduling.Workload + expectObj *scheduling.Workload + expectValidationError string + tasEnabled bool + }{ + "valid update with scheduling constraints unchanged and TAS disabled": { + oldObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates = append(workload.Spec.PodGroupTemplates, *workload.Spec.PodGroupTemplates[0].DeepCopy()) + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + newObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates = append(workload.Spec.PodGroupTemplates, *workload.Spec.PodGroupTemplates[0].DeepCopy()) + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + expectObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates = append(workload.Spec.PodGroupTemplates, *workload.Spec.PodGroupTemplates[0].DeepCopy()) + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + tasEnabled: false, + }, + "valid update with scheduling constraints unchanged and TAS enabled": { + oldObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates = append(workload.Spec.PodGroupTemplates, *workload.Spec.PodGroupTemplates[0].DeepCopy()) + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + newObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates = append(workload.Spec.PodGroupTemplates, *workload.Spec.PodGroupTemplates[0].DeepCopy()) + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + expectObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates = append(workload.Spec.PodGroupTemplates, *workload.Spec.PodGroupTemplates[0].DeepCopy()) + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + tasEnabled: true, + }, + "changing topology key not allowed with TAS disabled": { + oldObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + newObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "bar"}}, + } + return workload + }(), + expectValidationError: "field is immutable", + tasEnabled: false, + }, + "changing topology key not allowed with TAS enabled": { + oldObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + newObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "bar"}}, + } + return workload + }(), + expectValidationError: "field is immutable", + tasEnabled: true, + }, + "changing topology constraints not allowed with TAS disabled": { + oldObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates = append(workload.Spec.PodGroupTemplates, *workload.Spec.PodGroupTemplates[0].DeepCopy()) + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + newObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates = append(workload.Spec.PodGroupTemplates, *workload.Spec.PodGroupTemplates[0].DeepCopy()) + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + workload.Spec.PodGroupTemplates[1].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + expectValidationError: "field is immutable", + tasEnabled: false, + }, + "changing topology constraints not allowed with TAS enabled": { + oldObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates = append(workload.Spec.PodGroupTemplates, *workload.Spec.PodGroupTemplates[0].DeepCopy()) + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + newObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates = append(workload.Spec.PodGroupTemplates, *workload.Spec.PodGroupTemplates[0].DeepCopy()) + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + workload.Spec.PodGroupTemplates[1].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + expectValidationError: "field is immutable", + tasEnabled: true, + }, + "adding scheduling constraints not allowed with TAS disabled": { + oldObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + newObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates = append(workload.Spec.PodGroupTemplates, *workload.Spec.PodGroupTemplates[0].DeepCopy()) + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + workload.Spec.PodGroupTemplates[1].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + expectValidationError: "field is immutable", + tasEnabled: false, + }, + "adding scheduling constraints not allowed with TAS enabled": { + oldObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + newObj: func() *scheduling.Workload { + workload := workload.DeepCopy() + workload.Spec.PodGroupTemplates = append(workload.Spec.PodGroupTemplates, *workload.Spec.PodGroupTemplates[0].DeepCopy()) + workload.Spec.PodGroupTemplates[0].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + workload.Spec.PodGroupTemplates[1].SchedulingConstraints = &scheduling.PodGroupSchedulingConstraints{ + Topology: []scheduling.TopologyConstraint{{Key: "foo"}}, + } + return workload + }(), + expectValidationError: "field is immutable", + tasEnabled: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.GenericWorkload: tc.tasEnabled, + features.TopologyAwareWorkloadScheduling: tc.tasEnabled, + }) + ctx := ctxWithRequestInfo() + oldWorkload := tc.oldObj.DeepCopy() + oldWorkload.ResourceVersion = "1" + newWorkload := tc.newObj.DeepCopy() + newWorkload.ResourceVersion = "2" + + Strategy.PrepareForUpdate(ctx, newWorkload, oldWorkload) + if errs := Strategy.ValidateUpdate(ctx, newWorkload, oldWorkload); len(errs) != 0 { + if tc.expectValidationError == "" { + t.Fatalf("unexpected error(s): %v", errs) + } + if len(errs) != 1 { + t.Fatalf("exactly one error expected") + } + if errMsg := errs[0].Error(); !strings.Contains(errMsg, tc.expectValidationError) { + t.Fatalf("error %#v does not contain the expected message %q", errMsg, tc.expectValidationError) + } + } + if tc.expectObj != nil { + tc.expectObj.ResourceVersion = newWorkload.ResourceVersion + if diff := cmp.Diff(tc.expectObj, newWorkload); diff != "" { + t.Errorf("got unexpected workload object (-want, +got): %s", diff) + } + } + }) + } +} diff --git a/pkg/scheduler/apis/config/v1/default_plugins.go b/pkg/scheduler/apis/config/v1/default_plugins.go index e2729776e63..4cd8f652500 100644 --- a/pkg/scheduler/apis/config/v1/default_plugins.go +++ b/pkg/scheduler/apis/config/v1/default_plugins.go @@ -67,6 +67,9 @@ func applyFeatureGates(config *v1.Plugins) { if utilfeature.DefaultFeatureGate.Enabled(features.GangScheduling) { applyGangScheduling(config) } + if utilfeature.DefaultFeatureGate.Enabled(features.TopologyAwareWorkloadScheduling) { + config.MultiPoint.Enabled = append(config.MultiPoint.Enabled, v1.Plugin{Name: names.TopologyPlacementGenerator}) + } } func applyDynamicResources(config *v1.Plugins) { diff --git a/pkg/scheduler/framework/plugins/names/names.go b/pkg/scheduler/framework/plugins/names/names.go index 60915c352ad..86b764c95a2 100644 --- a/pkg/scheduler/framework/plugins/names/names.go +++ b/pkg/scheduler/framework/plugins/names/names.go @@ -38,4 +38,5 @@ const ( VolumeBinding = "VolumeBinding" VolumeRestrictions = "VolumeRestrictions" VolumeZone = "VolumeZone" + TopologyPlacementGenerator = "TopologyPlacementGenerator" ) diff --git a/pkg/scheduler/framework/plugins/registry.go b/pkg/scheduler/framework/plugins/registry.go index 8d945f6319f..175300781e5 100644 --- a/pkg/scheduler/framework/plugins/registry.go +++ b/pkg/scheduler/framework/plugins/registry.go @@ -36,6 +36,7 @@ import ( "k8s.io/kubernetes/pkg/scheduler/framework/plugins/queuesort" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/schedulinggates" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/tainttoleration" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/topologyaware" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumebinding" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumerestrictions" "k8s.io/kubernetes/pkg/scheduler/framework/plugins/volumezone" @@ -69,6 +70,7 @@ func NewInTreeRegistry() runtime.Registry { defaultpreemption.Name: runtime.FactoryAdapter(fts, defaultpreemption.New), schedulinggates.Name: runtime.FactoryAdapter(fts, schedulinggates.New), gangscheduling.Name: runtime.FactoryAdapter(fts, gangscheduling.New), + topologyaware.Name: runtime.FactoryAdapter(fts, topologyaware.New), } return registry diff --git a/pkg/scheduler/framework/plugins/topologyaware/topology_placement.go b/pkg/scheduler/framework/plugins/topologyaware/topology_placement.go new file mode 100644 index 00000000000..ccf7978748b --- /dev/null +++ b/pkg/scheduler/framework/plugins/topologyaware/topology_placement.go @@ -0,0 +1,142 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package topologyaware + +import ( + "context" + "fmt" + + v1 "k8s.io/api/core/v1" + schedulingapi "k8s.io/api/scheduling/v1alpha2" + "k8s.io/apimachinery/pkg/runtime" + schedulinglisters "k8s.io/client-go/listers/scheduling/v1alpha2" + "k8s.io/klog/v2" + fwk "k8s.io/kube-scheduler/framework" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/names" +) + +const ( + // Name is the name of the plugin used in the plugin registry and configurations. + Name = names.TopologyPlacementGenerator +) + +// TopologyPlacement is a plugin that generates placements for a pod group based on its topology constraints. +type TopologyPlacement struct { + handle fwk.Handle + podGroupLister schedulinglisters.PodGroupLister +} + +var _ fwk.PlacementGeneratePlugin = &TopologyPlacement{} + +// New initializes a new plugin and returns it. +func New(_ context.Context, _ runtime.Object, fh fwk.Handle, fts feature.Features) (*TopologyPlacement, error) { + return &TopologyPlacement{ + handle: fh, + podGroupLister: fh.SharedInformerFactory().Scheduling().V1alpha2().PodGroups().Lister(), + }, nil +} + +// Name returns name of the plugin. +func (pl *TopologyPlacement) Name() string { + return Name +} + +// GeneratePlacements generates placements for a pod group based on the topology constraints in the pod group spec. +// It uses the parent placement to find the nodes that are available for placement. +func (pl *TopologyPlacement) GeneratePlacements(ctx context.Context, state fwk.PodGroupCycleState, podGroup fwk.PodGroupInfo, parentPlacement *fwk.Placement) (*fwk.GeneratePlacementsResult, *fwk.Status) { + podGroupResource, err := pl.podGroupLister.PodGroups(podGroup.GetNamespace()).Get(podGroup.GetName()) + if err != nil { + return nil, fwk.AsStatus(err) + } + topologyKey, ok := pl.getTopologyKey(podGroupResource) + if !ok { + // No topology constraints, return a single placement with no constraints. + return &fwk.GeneratePlacementsResult{Placements: []*fwk.Placement{parentPlacement}}, nil + } + + var requiredDomain *string + scheduledPods, err := pl.getScheduledPods(podGroup) + if err != nil { + return nil, fwk.AsStatus(err) + } + if len(scheduledPods) > 0 { + scheduledDomain, err := pl.getScheduledPodsTopologyDomain(topologyKey, scheduledPods) + if err != nil { + return nil, fwk.AsStatus(fmt.Errorf("cannot determine domain for already scheduled pods: %w", err)) + } + requiredDomain = &scheduledDomain + } + + nodesPerTopologyDomain := make(map[string][]fwk.NodeInfo) + for _, node := range parentPlacement.Nodes { + if domain, ok := node.Node().Labels[topologyKey]; ok { + if requiredDomain == nil || *requiredDomain == domain { + nodesPerTopologyDomain[domain] = append(nodesPerTopologyDomain[domain], node) + } + } + } + + placements := make([]*fwk.Placement, 0, len(nodesPerTopologyDomain)) + for topologyDomain, nodes := range nodesPerTopologyDomain { + if len(nodes) > 0 { + placements = append(placements, &fwk.Placement{ + Name: topologyDomain, + Nodes: nodes, + }) + } + } + + return &fwk.GeneratePlacementsResult{Placements: placements}, nil +} + +func (pl *TopologyPlacement) getScheduledPodsTopologyDomain(topologyKey string, scheduledPods []*v1.Pod) (string, error) { + topologyDomain := "" + for _, pod := range scheduledPods { + node, err := pl.handle.SnapshotSharedLister().NodeInfos().Get(pod.Spec.NodeName) + if err != nil { + return "", fmt.Errorf("getting node for pod %v: %w", klog.KObj(pod), err) + } + domain, ok := node.Node().Labels[topologyKey] + if !ok { + return "", fmt.Errorf("no topology domain found for pod %v", klog.KObj(pod)) + } + if topologyDomain != "" && topologyDomain != domain { + return "", fmt.Errorf("more than 1 domain found for pod group: %v, %v", topologyDomain, domain) + } + topologyDomain = domain + } + return topologyDomain, nil +} + +// getTopologyKey returns the topology key for the pod group if there's any specified. +func (pl *TopologyPlacement) getTopologyKey(podGroupResource *schedulingapi.PodGroup) (string, bool) { + if schedulingConstraints := podGroupResource.Spec.SchedulingConstraints; schedulingConstraints != nil && len(schedulingConstraints.Topology) > 0 { + // Right now, we only support a single topology constraint on the API level. + return schedulingConstraints.Topology[0].Key, true + } + return "", false +} + +func (pl *TopologyPlacement) getScheduledPods(podGroup fwk.PodGroupInfo) ([]*v1.Pod, error) { + name := podGroup.GetName() + podGroupState, err := pl.handle.SnapshotSharedLister().PodGroupStates().Get(podGroup.GetNamespace(), name) + if err != nil { + return nil, err + } + return podGroupState.ScheduledPods(), nil +} diff --git a/pkg/scheduler/framework/plugins/topologyaware/topology_placement_test.go b/pkg/scheduler/framework/plugins/topologyaware/topology_placement_test.go new file mode 100644 index 00000000000..f42498f7f39 --- /dev/null +++ b/pkg/scheduler/framework/plugins/topologyaware/topology_placement_test.go @@ -0,0 +1,275 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package topologyaware + +import ( + "testing" + + v1 "k8s.io/api/core/v1" + schedulingapi "k8s.io/api/scheduling/v1alpha2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilfeature "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/informers" + clientsetfake "k8s.io/client-go/kubernetes/fake" + featuregatetesting "k8s.io/component-base/featuregate/testing" + fwk "k8s.io/kube-scheduler/framework" + "k8s.io/kubernetes/pkg/features" + "k8s.io/kubernetes/pkg/scheduler/backend/cache" + "k8s.io/kubernetes/pkg/scheduler/framework" + "k8s.io/kubernetes/pkg/scheduler/framework/plugins/feature" + "k8s.io/kubernetes/pkg/scheduler/framework/runtime" + st "k8s.io/kubernetes/pkg/scheduler/testing" + "k8s.io/kubernetes/test/utils/ktesting" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +func TestGeneratePlacements(t *testing.T) { + featuregatetesting.SetFeatureGatesDuringTest(t, utilfeature.DefaultFeatureGate, featuregatetesting.FeatureOverrides{ + features.GenericWorkload: true, + features.TopologyAwareWorkloadScheduling: true, + }) + + initialPlacementName := "test-placement" + tests := map[string]struct { + podGroup *schedulingapi.PodGroup + scheduledPodGroupPods map[string]string + placementNodes []*v1.Node + otherNodes []*v1.Node + wantPlacementNodes map[string][]string + wantStatus fwk.Code + }{ + "without constraint returns placement matching all nodes": { + podGroup: &schedulingapi.PodGroup{ + Spec: schedulingapi.PodGroupSpec{}, + }, + placementNodes: []*v1.Node{ + st.MakeNode().Name("node1").Obj(), + st.MakeNode().Name("node2").Label("foo", "bar").Obj(), + }, + otherNodes: []*v1.Node{ + st.MakeNode().Name("node3").Obj(), + }, + wantPlacementNodes: map[string][]string{ + initialPlacementName: {"node1", "node2"}, + }, + wantStatus: fwk.Success, + }, + "with topology key constraint, returns placement for each topology domain": { + podGroup: makePodGroup("topology1"), + placementNodes: []*v1.Node{ + st.MakeNode().Name("node0").Label("topology2", "d1").Obj(), + st.MakeNode().Name("node1").Label("topology2", "d4").Obj(), + st.MakeNode().Name("node2").Label("topology1", "d1").Obj(), + st.MakeNode().Name("node3").Label("topology1", "d2").Obj(), + st.MakeNode().Name("node4").Label("topology1", "d1").Obj(), + st.MakeNode().Name("node5").Label("topology1", "d3").Obj(), + }, + wantPlacementNodes: map[string][]string{ + "d1": {"node2", "node4"}, + "d2": {"node3"}, + "d3": {"node5"}, + }, + wantStatus: fwk.Success, + }, + "without matching topology label, returns empty": { + podGroup: makePodGroup("topology3"), + placementNodes: []*v1.Node{ + st.MakeNode().Name("node0").Label("topology2", "d1").Obj(), + st.MakeNode().Name("node1").Label("topology2", "d4").Obj(), + st.MakeNode().Name("node2").Label("topology1", "d1").Obj(), + st.MakeNode().Name("node3").Label("topology1", "d2").Obj(), + st.MakeNode().Name("node4").Label("topology1", "d1").Obj(), + st.MakeNode().Name("node5").Label("topology1", "d3").Obj(), + }, + wantPlacementNodes: map[string][]string{}, + wantStatus: fwk.Success, + }, + "with pods already scheduled in a single domain, returns that domain": { + podGroup: makePodGroup("topology"), + scheduledPodGroupPods: map[string]string{ + "pod1": "node2", + "pod2": "node3", + }, + placementNodes: []*v1.Node{ + st.MakeNode().Name("node0").Label("topology", "d2").Obj(), + st.MakeNode().Name("node1").Label("topology", "d1").Obj(), + }, + otherNodes: []*v1.Node{ + st.MakeNode().Name("node2").Label("topology", "d1").Obj(), + st.MakeNode().Name("node3").Label("topology", "d1").Obj(), + }, + wantPlacementNodes: map[string][]string{ + "d1": {"node1"}, + }, + wantStatus: fwk.Success, + }, + "with pods already scheduled in a single domain not present in current placement, returns empty": { + podGroup: makePodGroup("topology"), + scheduledPodGroupPods: map[string]string{ + "pod1": "node2", + "pod2": "node3", + }, + placementNodes: []*v1.Node{ + st.MakeNode().Name("node0").Label("topology", "d2").Obj(), + }, + otherNodes: []*v1.Node{ + st.MakeNode().Name("node2").Label("topology", "d1").Obj(), + st.MakeNode().Name("node3").Label("topology", "d1").Obj(), + }, + wantPlacementNodes: map[string][]string{}, + wantStatus: fwk.Success, + }, + "with pods already scheduled in conflicting domains, returns error": { + podGroup: makePodGroup("topology"), + scheduledPodGroupPods: map[string]string{ + "pod1": "node2", + "pod2": "node3", + }, + placementNodes: []*v1.Node{ + st.MakeNode().Name("node0").Label("topology", "d2").Obj(), + st.MakeNode().Name("node1").Label("topology", "d1").Obj(), + }, + otherNodes: []*v1.Node{ + st.MakeNode().Name("node2").Label("topology", "d0").Obj(), + st.MakeNode().Name("node3").Label("topology", "d1").Obj(), + }, + wantStatus: fwk.Error, + }, + "with already scheduled pod on node outside of snapshot, returns error": { + podGroup: makePodGroup("topology"), + scheduledPodGroupPods: map[string]string{ + "pod1": "node2", + "pod2": "node4", + }, + placementNodes: []*v1.Node{ + st.MakeNode().Name("node0").Label("topology", "d2").Obj(), + st.MakeNode().Name("node1").Label("topology", "d1").Obj(), + }, + otherNodes: []*v1.Node{ + st.MakeNode().Name("node2").Label("topology", "d1").Obj(), + st.MakeNode().Name("node3").Label("topology", "d1").Obj(), + }, + wantStatus: fwk.Error, + }, + "with already scheduled pod on node without topology label, returns error": { + podGroup: makePodGroup("topology"), + scheduledPodGroupPods: map[string]string{ + "pod1": "node2", + }, + placementNodes: []*v1.Node{ + st.MakeNode().Name("node1").Label("topology", "d2").Obj(), + }, + otherNodes: []*v1.Node{ + st.MakeNode().Name("node2").Label("foo", "bar").Obj(), + }, + wantStatus: fwk.Error, + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + _, tCtx := ktesting.NewTestContext(t) + + nodes := make([]v1.Node, 0, len(tt.placementNodes)+len(tt.otherNodes)) + for _, node := range append(tt.placementNodes, tt.otherNodes...) { + nodes = append(nodes, *node) + } + + cs := clientsetfake.NewClientset( + &schedulingapi.PodGroupList{Items: []schedulingapi.PodGroup{*tt.podGroup}}, + &v1.NodeList{Items: nodes}, + ) + informerFactory := informers.NewSharedInformerFactory(cs, 0) + _ = informerFactory.Scheduling().V1alpha2().PodGroups().Informer() + _ = informerFactory.Core().V1().Nodes().Informer() + informerFactory.StartWithContext(tCtx) + informerFactory.WaitForCacheSyncWithContext(tCtx) + + pods := make([]*v1.Pod, 0, len(tt.scheduledPodGroupPods)+1) + pods = append(pods, st.MakePod().Name("unscheduled").UID("unscheduled").Namespace(tt.podGroup.Namespace).PodGroupName(tt.podGroup.Name).Obj()) + for podName, nodeName := range tt.scheduledPodGroupPods { + pod := st.MakePod().Name(podName).UID(podName).Node(nodeName).Namespace(tt.podGroup.Namespace).PodGroupName(tt.podGroup.Name).Obj() + pods = append(pods, pod) + } + snapshot := cache.NewSnapshot(pods, append(tt.placementNodes, tt.otherNodes...)) + + fh, _ := runtime.NewFramework(tCtx, nil, nil, + runtime.WithInformerFactory(informerFactory), + runtime.WithSnapshotSharedLister(snapshot), + ) + + pl, err := New(tCtx, nil, fh, feature.Features{}) + if err != nil { + t.Fatalf("failed when creating plugin: %v", err) + } + + placement := &fwk.Placement{ + Name: initialPlacementName, + Nodes: make([]fwk.NodeInfo, len(tt.placementNodes)), + } + for i, node := range tt.placementNodes { + ni := framework.NewNodeInfo() + ni.SetNode(node) + placement.Nodes[i] = ni + } + podGroupInfo := &framework.PodGroupInfo{ + Name: tt.podGroup.Name, + Namespace: tt.podGroup.Namespace, + } + + result, status := pl.GeneratePlacements(tCtx, framework.NewCycleState(), podGroupInfo, placement) + + if status.Code() != tt.wantStatus { + t.Fatalf("expected status %v, got %v", tt.wantStatus, status.AsError()) + } + + if status.IsSuccess() { + gotPlacementNodes := make(map[string][]string) + for _, placement := range result.Placements { + gotPlacementNodes[placement.Name] = make([]string, len(placement.Nodes)) + for i, node := range placement.Nodes { + gotPlacementNodes[placement.Name][i] = node.Node().Name + } + } + + if diff := cmp.Diff(tt.wantPlacementNodes, gotPlacementNodes, cmpopts.EquateEmpty()); diff != "" { + t.Errorf("Unexpected placements (-want,+got):\n%s", diff) + } + } + }) + } +} + +func makePodGroup(topologyKey string) *schedulingapi.PodGroup { + return &schedulingapi.PodGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pg1", + Namespace: "default", + }, + Spec: schedulingapi.PodGroupSpec{ + SchedulingConstraints: &schedulingapi.PodGroupSchedulingConstraints{ + Topology: []schedulingapi.TopologyConstraint{ + { + Key: topologyKey, + }, + }, + }, + }, + } +} diff --git a/staging/src/k8s.io/api/scheduling/v1alpha2/generated.pb.go b/staging/src/k8s.io/api/scheduling/v1alpha2/generated.pb.go index 6b90317aa17..808e12c6c1a 100644 --- a/staging/src/k8s.io/api/scheduling/v1alpha2/generated.pb.go +++ b/staging/src/k8s.io/api/scheduling/v1alpha2/generated.pb.go @@ -39,6 +39,8 @@ func (m *PodGroup) Reset() { *m = PodGroup{} } func (m *PodGroupList) Reset() { *m = PodGroupList{} } +func (m *PodGroupSchedulingConstraints) Reset() { *m = PodGroupSchedulingConstraints{} } + func (m *PodGroupSchedulingPolicy) Reset() { *m = PodGroupSchedulingPolicy{} } func (m *PodGroupSpec) Reset() { *m = PodGroupSpec{} } @@ -49,6 +51,8 @@ func (m *PodGroupTemplate) Reset() { *m = PodGroupTemplate{} } func (m *PodGroupTemplateReference) Reset() { *m = PodGroupTemplateReference{} } +func (m *TopologyConstraint) Reset() { *m = TopologyConstraint{} } + func (m *TypedLocalObjectReference) Reset() { *m = TypedLocalObjectReference{} } func (m *Workload) Reset() { *m = Workload{} } @@ -208,6 +212,43 @@ func (m *PodGroupList) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *PodGroupSchedulingConstraints) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PodGroupSchedulingConstraints) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *PodGroupSchedulingConstraints) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Topology) > 0 { + for iNdEx := len(m.Topology) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Topology[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func (m *PodGroupSchedulingPolicy) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -275,6 +316,18 @@ func (m *PodGroupSpec) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.SchedulingConstraints != nil { + { + size, err := m.SchedulingConstraints.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } { size, err := m.SchedulingPolicy.MarshalToSizedBuffer(dAtA[:i]) if err != nil { @@ -357,6 +410,18 @@ func (m *PodGroupTemplate) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.SchedulingConstraints != nil { + { + size, err := m.SchedulingConstraints.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } { size, err := m.SchedulingPolicy.MarshalToSizedBuffer(dAtA[:i]) if err != nil { @@ -410,6 +475,34 @@ func (m *PodGroupTemplateReference) MarshalToSizedBuffer(dAtA []byte) (int, erro return len(dAtA) - i, nil } +func (m *TopologyConstraint) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *TopologyConstraint) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *TopologyConstraint) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + i -= len(m.Key) + copy(dAtA[i:], m.Key) + i = encodeVarintGenerated(dAtA, i, uint64(len(m.Key))) + i-- + dAtA[i] = 0xa + return len(dAtA) - i, nil +} + func (m *TypedLocalObjectReference) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -682,6 +775,21 @@ func (m *PodGroupList) Size() (n int) { return n } +func (m *PodGroupSchedulingConstraints) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Topology) > 0 { + for _, e := range m.Topology { + l = e.Size() + n += 1 + l + sovGenerated(uint64(l)) + } + } + return n +} + func (m *PodGroupSchedulingPolicy) Size() (n int) { if m == nil { return 0 @@ -711,6 +819,10 @@ func (m *PodGroupSpec) Size() (n int) { } l = m.SchedulingPolicy.Size() n += 1 + l + sovGenerated(uint64(l)) + if m.SchedulingConstraints != nil { + l = m.SchedulingConstraints.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -739,6 +851,10 @@ func (m *PodGroupTemplate) Size() (n int) { n += 1 + l + sovGenerated(uint64(l)) l = m.SchedulingPolicy.Size() n += 1 + l + sovGenerated(uint64(l)) + if m.SchedulingConstraints != nil { + l = m.SchedulingConstraints.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -755,6 +871,17 @@ func (m *PodGroupTemplateReference) Size() (n int) { return n } +func (m *TopologyConstraint) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Key) + n += 1 + l + sovGenerated(uint64(l)) + return n +} + func (m *TypedLocalObjectReference) Size() (n int) { if m == nil { return 0 @@ -885,6 +1012,21 @@ func (this *PodGroupList) String() string { }, "") return s } +func (this *PodGroupSchedulingConstraints) String() string { + if this == nil { + return "nil" + } + repeatedStringForTopology := "[]TopologyConstraint{" + for _, f := range this.Topology { + repeatedStringForTopology += strings.Replace(strings.Replace(f.String(), "TopologyConstraint", "TopologyConstraint", 1), `&`, ``, 1) + "," + } + repeatedStringForTopology += "}" + s := strings.Join([]string{`&PodGroupSchedulingConstraints{`, + `Topology:` + repeatedStringForTopology + `,`, + `}`, + }, "") + return s +} func (this *PodGroupSchedulingPolicy) String() string { if this == nil { return "nil" @@ -903,6 +1045,7 @@ func (this *PodGroupSpec) String() string { s := strings.Join([]string{`&PodGroupSpec{`, `PodGroupTemplateRef:` + strings.Replace(this.PodGroupTemplateRef.String(), "PodGroupTemplateReference", "PodGroupTemplateReference", 1) + `,`, `SchedulingPolicy:` + strings.Replace(strings.Replace(this.SchedulingPolicy.String(), "PodGroupSchedulingPolicy", "PodGroupSchedulingPolicy", 1), `&`, ``, 1) + `,`, + `SchedulingConstraints:` + strings.Replace(this.SchedulingConstraints.String(), "PodGroupSchedulingConstraints", "PodGroupSchedulingConstraints", 1) + `,`, `}`, }, "") return s @@ -929,6 +1072,7 @@ func (this *PodGroupTemplate) String() string { s := strings.Join([]string{`&PodGroupTemplate{`, `Name:` + fmt.Sprintf("%v", this.Name) + `,`, `SchedulingPolicy:` + strings.Replace(strings.Replace(this.SchedulingPolicy.String(), "PodGroupSchedulingPolicy", "PodGroupSchedulingPolicy", 1), `&`, ``, 1) + `,`, + `SchedulingConstraints:` + strings.Replace(this.SchedulingConstraints.String(), "PodGroupSchedulingConstraints", "PodGroupSchedulingConstraints", 1) + `,`, `}`, }, "") return s @@ -943,6 +1087,16 @@ func (this *PodGroupTemplateReference) String() string { }, "") return s } +func (this *TopologyConstraint) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&TopologyConstraint{`, + `Key:` + fmt.Sprintf("%v", this.Key) + `,`, + `}`, + }, "") + return s +} func (this *TypedLocalObjectReference) String() string { if this == nil { return "nil" @@ -1402,6 +1556,90 @@ func (m *PodGroupList) Unmarshal(dAtA []byte) error { } return nil } +func (m *PodGroupSchedulingConstraints) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PodGroupSchedulingConstraints: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PodGroupSchedulingConstraints: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topology", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Topology = append(m.Topology, TopologyConstraint{}) + if err := m.Topology[len(m.Topology)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *PodGroupSchedulingPolicy) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -1622,6 +1860,42 @@ func (m *PodGroupSpec) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SchedulingConstraints", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SchedulingConstraints == nil { + m.SchedulingConstraints = &PodGroupSchedulingConstraints{} + } + if err := m.SchedulingConstraints.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -1821,6 +2095,42 @@ func (m *PodGroupTemplate) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SchedulingConstraints", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.SchedulingConstraints == nil { + m.SchedulingConstraints = &PodGroupSchedulingConstraints{} + } + if err := m.SchedulingConstraints.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) @@ -1928,6 +2238,88 @@ func (m *PodGroupTemplateReference) Unmarshal(dAtA []byte) error { } return nil } +func (m *TopologyConstraint) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: TopologyConstraint: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: TopologyConstraint: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Key = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipGenerated(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthGenerated + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *TypedLocalObjectReference) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/staging/src/k8s.io/api/scheduling/v1alpha2/generated.proto b/staging/src/k8s.io/api/scheduling/v1alpha2/generated.proto index e2de388e1a1..31de37c1a5f 100644 --- a/staging/src/k8s.io/api/scheduling/v1alpha2/generated.proto +++ b/staging/src/k8s.io/api/scheduling/v1alpha2/generated.proto @@ -49,6 +49,7 @@ message GangSchedulingPolicy { // PodGroups are created by workload controllers (Job, LWS, JobSet, etc...) from // Workload.podGroupTemplates. // PodGroup API enablement is toggled by the GenericWorkload feature gate. +// +k8s:validation-gen-nolint // to allow pre-GA tags while this API is pre-GA message PodGroup { // Standard object's metadata. // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata @@ -78,6 +79,19 @@ message PodGroupList { repeated PodGroup items = 2; } +// PodGroupSchedulingConstraints defines scheduling constraints (e.g. topology) for a PodGroup. +message PodGroupSchedulingConstraints { + // Topology defines the topology constraints for the pod group. + // Currently only a single topology constraint can be specified. This may change in the future. + // + // +optional + // +k8s:optional + // +k8s:maxItems=1 + // +listType=atomic + // +k8s:listType=atomic + repeated TopologyConstraint topology = 1; +} + // PodGroupSchedulingPolicy defines the scheduling configuration for a PodGroup. // Exactly one policy must be set. // +union @@ -116,6 +130,18 @@ message PodGroupSpec { // +required // +k8s:alpha(since:"1.36")=+k8s:immutable optional PodGroupSchedulingPolicy schedulingPolicy = 2; + + // SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroup. + // Controllers are expected to fill this field by copying it from a PodGroupTemplate. + // This field is immutable. + // This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled. + // + // +featureGate=TopologyAwareWorkloadScheduling + // +optional + // +k8s:ifDisabled(TopologyAwareWorkloadScheduling)=+k8s:forbidden + // +k8s:ifEnabled(TopologyAwareWorkloadScheduling)=+k8s:optional + // +k8s:ifEnabled(TopologyAwareWorkloadScheduling)=+k8s:immutable + optional PodGroupSchedulingConstraints schedulingConstraints = 3; } // PodGroupStatus represents information about the status of a pod group. @@ -159,6 +185,15 @@ message PodGroupTemplate { // // +required optional PodGroupSchedulingPolicy schedulingPolicy = 2; + + // SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroupTemplate. + // This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled. + // + // +featureGate=TopologyAwareWorkloadScheduling + // +optional + // +k8s:ifDisabled(TopologyAwareWorkloadScheduling)=+k8s:forbidden + // +k8s:ifEnabled(TopologyAwareWorkloadScheduling)=+k8s:optional + optional PodGroupSchedulingConstraints schedulingConstraints = 3; } // PodGroupTemplateReference references a PodGroup template defined in some object (e.g. Workload). @@ -174,6 +209,19 @@ message PodGroupTemplateReference { optional WorkloadPodGroupTemplateReference workload = 1; } +// TopologyConstraint defines a topology constraint for a PodGroup. +message TopologyConstraint { + // Key specifies the key of the node label representing the topology domain. + // All pods within the PodGroup must be colocated within the same domain instance. + // Different PodGroups can land on different domain instances even if they derive from the same PodGroupTemplate. + // Examples: "topology.kubernetes.io/rack" + // + // +required + // +k8s:required + // +k8s:format=k8s-label-key + optional string key = 1; +} + // TypedLocalObjectReference allows to reference typed object inside the same namespace. message TypedLocalObjectReference { // APIGroup is the group for the resource being referenced. @@ -207,6 +255,7 @@ message TypedLocalObjectReference { // when managing the lifecycle of workloads from the scheduling perspective, // including scheduling, preemption, eviction and other phases. // Workload API enablement is toggled by the GenericWorkload feature gate. +// +k8s:validation-gen-nolint // to allow pre-GA tags while this API is pre-GA message Workload { // Standard object's metadata. // More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata diff --git a/staging/src/k8s.io/api/scheduling/v1alpha2/types.go b/staging/src/k8s.io/api/scheduling/v1alpha2/types.go index f66ea146e53..d663761bf90 100644 --- a/staging/src/k8s.io/api/scheduling/v1alpha2/types.go +++ b/staging/src/k8s.io/api/scheduling/v1alpha2/types.go @@ -27,6 +27,7 @@ import ( // when managing the lifecycle of workloads from the scheduling perspective, // including scheduling, preemption, eviction and other phases. // Workload API enablement is toggled by the GenericWorkload feature gate. +// +k8s:validation-gen-nolint // to allow pre-GA tags while this API is pre-GA type Workload struct { metav1.TypeMeta `json:",inline"` // Standard object's metadata. @@ -125,6 +126,15 @@ type PodGroupTemplate struct { // // +required SchedulingPolicy PodGroupSchedulingPolicy `json:"schedulingPolicy" protobuf:"bytes,2,opt,name=schedulingPolicy"` + + // SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroupTemplate. + // This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled. + // + // +featureGate=TopologyAwareWorkloadScheduling + // +optional + // +k8s:ifDisabled(TopologyAwareWorkloadScheduling)=+k8s:forbidden + // +k8s:ifEnabled(TopologyAwareWorkloadScheduling)=+k8s:optional + SchedulingConstraints *PodGroupSchedulingConstraints `json:"schedulingConstraints" protobuf:"bytes,3,opt,name=schedulingConstraints"` } // PodGroupSchedulingPolicy defines the scheduling configuration for a PodGroup. @@ -177,6 +187,7 @@ type GangSchedulingPolicy struct { // PodGroups are created by workload controllers (Job, LWS, JobSet, etc...) from // Workload.podGroupTemplates. // PodGroup API enablement is toggled by the GenericWorkload feature gate. +// +k8s:validation-gen-nolint // to allow pre-GA tags while this API is pre-GA type PodGroup struct { metav1.TypeMeta `json:",inline"` // Standard object's metadata. @@ -227,6 +238,18 @@ type PodGroupSpec struct { // +required // +k8s:alpha(since:"1.36")=+k8s:immutable SchedulingPolicy PodGroupSchedulingPolicy `json:"schedulingPolicy" protobuf:"bytes,2,opt,name=schedulingPolicy"` + + // SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroup. + // Controllers are expected to fill this field by copying it from a PodGroupTemplate. + // This field is immutable. + // This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled. + // + // +featureGate=TopologyAwareWorkloadScheduling + // +optional + // +k8s:ifDisabled(TopologyAwareWorkloadScheduling)=+k8s:forbidden + // +k8s:ifEnabled(TopologyAwareWorkloadScheduling)=+k8s:optional + // +k8s:ifEnabled(TopologyAwareWorkloadScheduling)=+k8s:immutable + SchedulingConstraints *PodGroupSchedulingConstraints `json:"schedulingConstraints,omitempty" protobuf:"bytes,3,opt,name=schedulingConstraints"` } // PodGroupStatus represents information about the status of a pod group. @@ -307,3 +330,29 @@ type WorkloadPodGroupTemplateReference struct { // +k8s:format=k8s-short-name PodGroupTemplateName string `json:"podGroupTemplateName" protobuf:"bytes,2,opt,name=podGroupTemplateName"` } + +// PodGroupSchedulingConstraints defines scheduling constraints (e.g. topology) for a PodGroup. +type PodGroupSchedulingConstraints struct { + // Topology defines the topology constraints for the pod group. + // Currently only a single topology constraint can be specified. This may change in the future. + // + // +optional + // +k8s:optional + // +k8s:maxItems=1 + // +listType=atomic + // +k8s:listType=atomic + Topology []TopologyConstraint `json:"topology,omitempty" protobuf:"bytes,1,rep,name=topology"` +} + +// TopologyConstraint defines a topology constraint for a PodGroup. +type TopologyConstraint struct { + // Key specifies the key of the node label representing the topology domain. + // All pods within the PodGroup must be colocated within the same domain instance. + // Different PodGroups can land on different domain instances even if they derive from the same PodGroupTemplate. + // Examples: "topology.kubernetes.io/rack" + // + // +required + // +k8s:required + // +k8s:format=k8s-label-key + Key string `json:"key" protobuf:"bytes,1,opt,name=key"` +} diff --git a/staging/src/k8s.io/api/scheduling/v1alpha2/types_swagger_doc_generated.go b/staging/src/k8s.io/api/scheduling/v1alpha2/types_swagger_doc_generated.go index 7adb7200829..1a28048d906 100644 --- a/staging/src/k8s.io/api/scheduling/v1alpha2/types_swagger_doc_generated.go +++ b/staging/src/k8s.io/api/scheduling/v1alpha2/types_swagger_doc_generated.go @@ -65,6 +65,15 @@ func (PodGroupList) SwaggerDoc() map[string]string { return map_PodGroupList } +var map_PodGroupSchedulingConstraints = map[string]string{ + "": "PodGroupSchedulingConstraints defines scheduling constraints (e.g. topology) for a PodGroup.", + "topology": "Topology defines the topology constraints for the pod group. Currently only a single topology constraint can be specified. This may change in the future.", +} + +func (PodGroupSchedulingConstraints) SwaggerDoc() map[string]string { + return map_PodGroupSchedulingConstraints +} + var map_PodGroupSchedulingPolicy = map[string]string{ "": "PodGroupSchedulingPolicy defines the scheduling configuration for a PodGroup. Exactly one policy must be set.", "basic": "Basic specifies that the pods in this group should be scheduled using standard Kubernetes scheduling behavior.", @@ -76,9 +85,10 @@ func (PodGroupSchedulingPolicy) SwaggerDoc() map[string]string { } var map_PodGroupSpec = map[string]string{ - "": "PodGroupSpec defines the desired state of a PodGroup.", - "podGroupTemplateRef": "PodGroupTemplateRef references an optional PodGroup template within other object (e.g. Workload) that was used to create the PodGroup. This field is immutable.", - "schedulingPolicy": "SchedulingPolicy defines the scheduling policy for this instance of the PodGroup. Controllers are expected to fill this field by copying it from a PodGroupTemplate. This field is immutable.", + "": "PodGroupSpec defines the desired state of a PodGroup.", + "podGroupTemplateRef": "PodGroupTemplateRef references an optional PodGroup template within other object (e.g. Workload) that was used to create the PodGroup. This field is immutable.", + "schedulingPolicy": "SchedulingPolicy defines the scheduling policy for this instance of the PodGroup. Controllers are expected to fill this field by copying it from a PodGroupTemplate. This field is immutable.", + "schedulingConstraints": "SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroup. Controllers are expected to fill this field by copying it from a PodGroupTemplate. This field is immutable. This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled.", } func (PodGroupSpec) SwaggerDoc() map[string]string { @@ -95,9 +105,10 @@ func (PodGroupStatus) SwaggerDoc() map[string]string { } var map_PodGroupTemplate = map[string]string{ - "": "PodGroupTemplate represents a template for a set of pods with a scheduling policy.", - "name": "Name is a unique identifier for the PodGroupTemplate within the Workload. It must be a DNS label. This field is immutable.", - "schedulingPolicy": "SchedulingPolicy defines the scheduling policy for this PodGroupTemplate.", + "": "PodGroupTemplate represents a template for a set of pods with a scheduling policy.", + "name": "Name is a unique identifier for the PodGroupTemplate within the Workload. It must be a DNS label. This field is immutable.", + "schedulingPolicy": "SchedulingPolicy defines the scheduling policy for this PodGroupTemplate.", + "schedulingConstraints": "SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroupTemplate. This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled.", } func (PodGroupTemplate) SwaggerDoc() map[string]string { @@ -113,6 +124,15 @@ func (PodGroupTemplateReference) SwaggerDoc() map[string]string { return map_PodGroupTemplateReference } +var map_TopologyConstraint = map[string]string{ + "": "TopologyConstraint defines a topology constraint for a PodGroup.", + "key": "Key specifies the key of the node label representing the topology domain. All pods within the PodGroup must be colocated within the same domain instance. Different PodGroups can land on different domain instances even if they derive from the same PodGroupTemplate. Examples: \"topology.kubernetes.io/rack\"", +} + +func (TopologyConstraint) SwaggerDoc() map[string]string { + return map_TopologyConstraint +} + var map_TypedLocalObjectReference = map[string]string{ "": "TypedLocalObjectReference allows to reference typed object inside the same namespace.", "apiGroup": "APIGroup is the group for the resource being referenced. If APIGroup is empty, the specified Kind must be in the core API group. For any other third-party types, setting APIGroup is required. It must be a DNS subdomain.", diff --git a/staging/src/k8s.io/api/scheduling/v1alpha2/zz_generated.deepcopy.go b/staging/src/k8s.io/api/scheduling/v1alpha2/zz_generated.deepcopy.go index 7371f207340..e1ac6687811 100644 --- a/staging/src/k8s.io/api/scheduling/v1alpha2/zz_generated.deepcopy.go +++ b/staging/src/k8s.io/api/scheduling/v1alpha2/zz_generated.deepcopy.go @@ -119,6 +119,27 @@ func (in *PodGroupList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PodGroupSchedulingConstraints) DeepCopyInto(out *PodGroupSchedulingConstraints) { + *out = *in + if in.Topology != nil { + in, out := &in.Topology, &out.Topology + *out = make([]TopologyConstraint, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodGroupSchedulingConstraints. +func (in *PodGroupSchedulingConstraints) DeepCopy() *PodGroupSchedulingConstraints { + if in == nil { + return nil + } + out := new(PodGroupSchedulingConstraints) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PodGroupSchedulingPolicy) DeepCopyInto(out *PodGroupSchedulingPolicy) { *out = *in @@ -154,6 +175,11 @@ func (in *PodGroupSpec) DeepCopyInto(out *PodGroupSpec) { (*in).DeepCopyInto(*out) } in.SchedulingPolicy.DeepCopyInto(&out.SchedulingPolicy) + if in.SchedulingConstraints != nil { + in, out := &in.SchedulingConstraints, &out.SchedulingConstraints + *out = new(PodGroupSchedulingConstraints) + (*in).DeepCopyInto(*out) + } return } @@ -194,6 +220,11 @@ func (in *PodGroupStatus) DeepCopy() *PodGroupStatus { func (in *PodGroupTemplate) DeepCopyInto(out *PodGroupTemplate) { *out = *in in.SchedulingPolicy.DeepCopyInto(&out.SchedulingPolicy) + if in.SchedulingConstraints != nil { + in, out := &in.SchedulingConstraints, &out.SchedulingConstraints + *out = new(PodGroupSchedulingConstraints) + (*in).DeepCopyInto(*out) + } return } @@ -228,6 +259,22 @@ func (in *PodGroupTemplateReference) DeepCopy() *PodGroupTemplateReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TopologyConstraint) DeepCopyInto(out *TopologyConstraint) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TopologyConstraint. +func (in *TopologyConstraint) DeepCopy() *TopologyConstraint { + if in == nil { + return nil + } + out := new(TopologyConstraint) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TypedLocalObjectReference) DeepCopyInto(out *TypedLocalObjectReference) { *out = *in diff --git a/staging/src/k8s.io/api/scheduling/v1alpha2/zz_generated.model_name.go b/staging/src/k8s.io/api/scheduling/v1alpha2/zz_generated.model_name.go index 59f44737c7d..6cc7a2ccbec 100644 --- a/staging/src/k8s.io/api/scheduling/v1alpha2/zz_generated.model_name.go +++ b/staging/src/k8s.io/api/scheduling/v1alpha2/zz_generated.model_name.go @@ -41,6 +41,11 @@ func (in PodGroupList) OpenAPIModelName() string { return "io.k8s.api.scheduling.v1alpha2.PodGroupList" } +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in PodGroupSchedulingConstraints) OpenAPIModelName() string { + return "io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingConstraints" +} + // OpenAPIModelName returns the OpenAPI model name for this type. func (in PodGroupSchedulingPolicy) OpenAPIModelName() string { return "io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingPolicy" @@ -66,6 +71,11 @@ func (in PodGroupTemplateReference) OpenAPIModelName() string { return "io.k8s.api.scheduling.v1alpha2.PodGroupTemplateReference" } +// OpenAPIModelName returns the OpenAPI model name for this type. +func (in TopologyConstraint) OpenAPIModelName() string { + return "io.k8s.api.scheduling.v1alpha2.TopologyConstraint" +} + // OpenAPIModelName returns the OpenAPI model name for this type. func (in TypedLocalObjectReference) OpenAPIModelName() string { return "io.k8s.api.scheduling.v1alpha2.TypedLocalObjectReference" diff --git a/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.PodGroup.json b/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.PodGroup.json index 9937b07fc75..5ac9031568f 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.PodGroup.json +++ b/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.PodGroup.json @@ -55,6 +55,13 @@ "gang": { "minCount": 1 } + }, + "schedulingConstraints": { + "topology": [ + { + "key": "keyValue" + } + ] } }, "status": { diff --git a/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.PodGroup.pb b/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.PodGroup.pb index 9ea20af0c38..60cd1188fc2 100644 Binary files a/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.PodGroup.pb and b/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.PodGroup.pb differ diff --git a/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.PodGroup.yaml b/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.PodGroup.yaml index 6bcdfed65cf..578ad8e56c4 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.PodGroup.yaml +++ b/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.PodGroup.yaml @@ -37,6 +37,9 @@ spec: workload: podGroupTemplateName: podGroupTemplateNameValue workloadName: workloadNameValue + schedulingConstraints: + topology: + - key: keyValue schedulingPolicy: basic: {} gang: diff --git a/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.Workload.json b/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.Workload.json index a96fa767539..7b68e64c1e8 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.Workload.json +++ b/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.Workload.json @@ -57,6 +57,13 @@ "gang": { "minCount": 1 } + }, + "schedulingConstraints": { + "topology": [ + { + "key": "keyValue" + } + ] } } ] diff --git a/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.Workload.pb b/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.Workload.pb index 6882dcf3a5f..28d68dc7b66 100644 Binary files a/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.Workload.pb and b/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.Workload.pb differ diff --git a/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.Workload.yaml b/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.Workload.yaml index 4ad32f9f022..92991f1c434 100644 --- a/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.Workload.yaml +++ b/staging/src/k8s.io/api/testdata/HEAD/scheduling.k8s.io.v1alpha2.Workload.yaml @@ -39,6 +39,9 @@ spec: name: nameValue podGroupTemplates: - name: nameValue + schedulingConstraints: + topology: + - key: keyValue schedulingPolicy: basic: {} gang: diff --git a/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go b/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go index 7a66deaf482..bd1b7cc4644 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/internal/internal.go @@ -14526,6 +14526,15 @@ var schemaYAML = typed.YAMLObject(`types: type: namedType: io.k8s.api.scheduling.v1alpha2.PodGroupStatus default: {} +- name: io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingConstraints + map: + fields: + - name: topology + type: + list: + elementType: + namedType: io.k8s.api.scheduling.v1alpha2.TopologyConstraint + elementRelationship: atomic - name: io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingPolicy map: fields: @@ -14547,6 +14556,9 @@ var schemaYAML = typed.YAMLObject(`types: - name: podGroupTemplateRef type: namedType: io.k8s.api.scheduling.v1alpha2.PodGroupTemplateReference + - name: schedulingConstraints + type: + namedType: io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingConstraints - name: schedulingPolicy type: namedType: io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingPolicy @@ -14569,6 +14581,9 @@ var schemaYAML = typed.YAMLObject(`types: type: scalar: string default: "" + - name: schedulingConstraints + type: + namedType: io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingConstraints - name: schedulingPolicy type: namedType: io.k8s.api.scheduling.v1alpha2.PodGroupSchedulingPolicy @@ -14583,6 +14598,13 @@ var schemaYAML = typed.YAMLObject(`types: - fields: - fieldName: workload discriminatorValue: Workload +- name: io.k8s.api.scheduling.v1alpha2.TopologyConstraint + map: + fields: + - name: key + type: + scalar: string + default: "" - name: io.k8s.api.scheduling.v1alpha2.TypedLocalObjectReference map: fields: diff --git a/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/podgroupschedulingconstraints.go b/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/podgroupschedulingconstraints.go new file mode 100644 index 00000000000..eda72ea50e5 --- /dev/null +++ b/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/podgroupschedulingconstraints.go @@ -0,0 +1,48 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1alpha2 + +// PodGroupSchedulingConstraintsApplyConfiguration represents a declarative configuration of the PodGroupSchedulingConstraints type for use +// with apply. +// +// PodGroupSchedulingConstraints defines scheduling constraints (e.g. topology) for a PodGroup. +type PodGroupSchedulingConstraintsApplyConfiguration struct { + // Topology defines the topology constraints for the pod group. + // Currently only a single topology constraint can be specified. This may change in the future. + Topology []TopologyConstraintApplyConfiguration `json:"topology,omitempty"` +} + +// PodGroupSchedulingConstraintsApplyConfiguration constructs a declarative configuration of the PodGroupSchedulingConstraints type for use with +// apply. +func PodGroupSchedulingConstraints() *PodGroupSchedulingConstraintsApplyConfiguration { + return &PodGroupSchedulingConstraintsApplyConfiguration{} +} + +// WithTopology adds the given value to the Topology field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Topology field. +func (b *PodGroupSchedulingConstraintsApplyConfiguration) WithTopology(values ...*TopologyConstraintApplyConfiguration) *PodGroupSchedulingConstraintsApplyConfiguration { + for i := range values { + if values[i] == nil { + panic("nil value passed to WithTopology") + } + b.Topology = append(b.Topology, *values[i]) + } + return b +} diff --git a/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/podgroupspec.go b/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/podgroupspec.go index c9354d06ddf..656c2203321 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/podgroupspec.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/podgroupspec.go @@ -30,6 +30,11 @@ type PodGroupSpecApplyConfiguration struct { // Controllers are expected to fill this field by copying it from a PodGroupTemplate. // This field is immutable. SchedulingPolicy *PodGroupSchedulingPolicyApplyConfiguration `json:"schedulingPolicy,omitempty"` + // SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroup. + // Controllers are expected to fill this field by copying it from a PodGroupTemplate. + // This field is immutable. + // This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled. + SchedulingConstraints *PodGroupSchedulingConstraintsApplyConfiguration `json:"schedulingConstraints,omitempty"` } // PodGroupSpecApplyConfiguration constructs a declarative configuration of the PodGroupSpec type for use with @@ -53,3 +58,11 @@ func (b *PodGroupSpecApplyConfiguration) WithSchedulingPolicy(value *PodGroupSch b.SchedulingPolicy = value return b } + +// WithSchedulingConstraints sets the SchedulingConstraints field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the SchedulingConstraints field is set to the value of the last call. +func (b *PodGroupSpecApplyConfiguration) WithSchedulingConstraints(value *PodGroupSchedulingConstraintsApplyConfiguration) *PodGroupSpecApplyConfiguration { + b.SchedulingConstraints = value + return b +} diff --git a/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/podgrouptemplate.go b/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/podgrouptemplate.go index c4428def9a2..85e30ac5033 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/podgrouptemplate.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/podgrouptemplate.go @@ -28,6 +28,9 @@ type PodGroupTemplateApplyConfiguration struct { Name *string `json:"name,omitempty"` // SchedulingPolicy defines the scheduling policy for this PodGroupTemplate. SchedulingPolicy *PodGroupSchedulingPolicyApplyConfiguration `json:"schedulingPolicy,omitempty"` + // SchedulingConstraints defines optional scheduling constraints (e.g. topology) for this PodGroupTemplate. + // This field is only available when the TopologyAwareWorkloadScheduling feature gate is enabled. + SchedulingConstraints *PodGroupSchedulingConstraintsApplyConfiguration `json:"schedulingConstraints,omitempty"` } // PodGroupTemplateApplyConfiguration constructs a declarative configuration of the PodGroupTemplate type for use with @@ -51,3 +54,11 @@ func (b *PodGroupTemplateApplyConfiguration) WithSchedulingPolicy(value *PodGrou b.SchedulingPolicy = value return b } + +// WithSchedulingConstraints sets the SchedulingConstraints field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the SchedulingConstraints field is set to the value of the last call. +func (b *PodGroupTemplateApplyConfiguration) WithSchedulingConstraints(value *PodGroupSchedulingConstraintsApplyConfiguration) *PodGroupTemplateApplyConfiguration { + b.SchedulingConstraints = value + return b +} diff --git a/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/topologyconstraint.go b/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/topologyconstraint.go new file mode 100644 index 00000000000..92727eb2bb3 --- /dev/null +++ b/staging/src/k8s.io/client-go/applyconfigurations/scheduling/v1alpha2/topologyconstraint.go @@ -0,0 +1,45 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1alpha2 + +// TopologyConstraintApplyConfiguration represents a declarative configuration of the TopologyConstraint type for use +// with apply. +// +// TopologyConstraint defines a topology constraint for a PodGroup. +type TopologyConstraintApplyConfiguration struct { + // Key specifies the key of the node label representing the topology domain. + // All pods within the PodGroup must be colocated within the same domain instance. + // Different PodGroups can land on different domain instances even if they derive from the same PodGroupTemplate. + // Examples: "topology.kubernetes.io/rack" + Key *string `json:"key,omitempty"` +} + +// TopologyConstraintApplyConfiguration constructs a declarative configuration of the TopologyConstraint type for use with +// apply. +func TopologyConstraint() *TopologyConstraintApplyConfiguration { + return &TopologyConstraintApplyConfiguration{} +} + +// WithKey sets the Key field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Key field is set to the value of the last call. +func (b *TopologyConstraintApplyConfiguration) WithKey(value string) *TopologyConstraintApplyConfiguration { + b.Key = &value + return b +} diff --git a/staging/src/k8s.io/client-go/applyconfigurations/utils.go b/staging/src/k8s.io/client-go/applyconfigurations/utils.go index 48fc5ed0f5e..59d56b2fd1c 100644 --- a/staging/src/k8s.io/client-go/applyconfigurations/utils.go +++ b/staging/src/k8s.io/client-go/applyconfigurations/utils.go @@ -1830,6 +1830,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &applyconfigurationsschedulingv1alpha2.GangSchedulingPolicyApplyConfiguration{} case schedulingv1alpha2.SchemeGroupVersion.WithKind("PodGroup"): return &applyconfigurationsschedulingv1alpha2.PodGroupApplyConfiguration{} + case schedulingv1alpha2.SchemeGroupVersion.WithKind("PodGroupSchedulingConstraints"): + return &applyconfigurationsschedulingv1alpha2.PodGroupSchedulingConstraintsApplyConfiguration{} case schedulingv1alpha2.SchemeGroupVersion.WithKind("PodGroupSchedulingPolicy"): return &applyconfigurationsschedulingv1alpha2.PodGroupSchedulingPolicyApplyConfiguration{} case schedulingv1alpha2.SchemeGroupVersion.WithKind("PodGroupSpec"): @@ -1840,6 +1842,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &applyconfigurationsschedulingv1alpha2.PodGroupTemplateApplyConfiguration{} case schedulingv1alpha2.SchemeGroupVersion.WithKind("PodGroupTemplateReference"): return &applyconfigurationsschedulingv1alpha2.PodGroupTemplateReferenceApplyConfiguration{} + case schedulingv1alpha2.SchemeGroupVersion.WithKind("TopologyConstraint"): + return &applyconfigurationsschedulingv1alpha2.TopologyConstraintApplyConfiguration{} case schedulingv1alpha2.SchemeGroupVersion.WithKind("TypedLocalObjectReference"): return &applyconfigurationsschedulingv1alpha2.TypedLocalObjectReferenceApplyConfiguration{} case schedulingv1alpha2.SchemeGroupVersion.WithKind("Workload"):