From d1f4cfc16983ccd75d1c900673bd16f6639de1fc Mon Sep 17 00:00:00 2001 From: Varsha Prasad Narsing <varshaprasad96@gmail.com> Date: Tue, 15 Oct 2024 15:56:11 -0700 Subject: [PATCH] DROP: Expose available ResourceFlavors from the ClusterQueue in the LocalQueue status. Commit to be dropped later. Taken from PR: https://github.com/kubernetes-sigs/kueue/pull/3143 --- apis/kueue/v1beta1/localqueue_types.go | 32 +++++ apis/kueue/v1beta1/zz_generated.deepcopy.go | 41 ++++++ .../crd/kueue.x-k8s.io_localqueues.yaml | 72 +++++++++++ .../kueue/v1beta1/localqueueflavorstatus.go | 80 ++++++++++++ .../kueue/v1beta1/localqueuestatus.go | 26 +++- client-go/applyconfiguration/utils.go | 2 + .../crd/bases/kueue.x-k8s.io_localqueues.yaml | 72 +++++++++++ pkg/cache/cache.go | 33 +++++ pkg/controller/core/localqueue_controller.go | 1 + pkg/features/kube_features.go | 9 ++ site/content/en/docs/installation/_index.md | 27 ++-- .../en/docs/reference/kueue.v1beta1.md | 56 ++++++++ .../core/localqueue_controller_test.go | 121 +++++++++++++----- 13 files changed, 521 insertions(+), 51 deletions(-) create mode 100644 client-go/applyconfiguration/kueue/v1beta1/localqueueflavorstatus.go diff --git a/apis/kueue/v1beta1/localqueue_types.go b/apis/kueue/v1beta1/localqueue_types.go index 622014953a..bf7046c391 100644 --- a/apis/kueue/v1beta1/localqueue_types.go +++ b/apis/kueue/v1beta1/localqueue_types.go @@ -48,6 +48,31 @@ type LocalQueueSpec struct { // +kubebuilder:validation:Pattern="^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$" type ClusterQueueReference string +type LocalQueueFlavorStatus struct { + // name of the flavor. + Name ResourceFlavorReference `json:"name"` + + // resources used in the flavor. + // +listType=set + // +kubebuilder:validation:MaxItems=16 + // +optional + Resources []corev1.ResourceName `json:"resources,omitempty"` + + // nodeLabels are labels that associate the ResourceFlavor with Nodes that + // have the same labels. + // +mapType=atomic + // +kubebuilder:validation:MaxProperties=8 + // +optional + NodeLabels map[string]string `json:"nodeLabels,omitempty"` + + // nodeTaints are taints that the nodes associated with this ResourceFlavor + // have. + // +listType=atomic + // +kubebuilder:validation:MaxItems=8 + // +optional + NodeTaints []corev1.Taint `json:"nodeTaints,omitempty"` +} + // LocalQueueStatus defines the observed state of LocalQueue type LocalQueueStatus struct { // PendingWorkloads is the number of Workloads in the LocalQueue not yet admitted to a ClusterQueue @@ -88,6 +113,13 @@ type LocalQueueStatus struct { // +kubebuilder:validation:MaxItems=16 // +optional FlavorUsage []LocalQueueFlavorUsage `json:"flavorUsage"` + + // flavors lists all currently available ResourceFlavors in specified ClusterQueue. + // +listType=map + // +listMapKey=name + // +kubebuilder:validation:MaxItems=16 + // +optional + Flavors []LocalQueueFlavorStatus `json:"flavors,omitempty"` } const ( diff --git a/apis/kueue/v1beta1/zz_generated.deepcopy.go b/apis/kueue/v1beta1/zz_generated.deepcopy.go index cf773364d5..32ef8ecc6a 100644 --- a/apis/kueue/v1beta1/zz_generated.deepcopy.go +++ b/apis/kueue/v1beta1/zz_generated.deepcopy.go @@ -593,6 +593,40 @@ func (in *LocalQueue) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LocalQueueFlavorStatus) DeepCopyInto(out *LocalQueueFlavorStatus) { + *out = *in + if in.Resources != nil { + in, out := &in.Resources, &out.Resources + *out = make([]corev1.ResourceName, len(*in)) + copy(*out, *in) + } + if in.NodeLabels != nil { + in, out := &in.NodeLabels, &out.NodeLabels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.NodeTaints != nil { + in, out := &in.NodeTaints, &out.NodeTaints + *out = make([]corev1.Taint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalQueueFlavorStatus. +func (in *LocalQueueFlavorStatus) DeepCopy() *LocalQueueFlavorStatus { + if in == nil { + return nil + } + out := new(LocalQueueFlavorStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LocalQueueFlavorUsage) DeepCopyInto(out *LocalQueueFlavorUsage) { *out = *in @@ -707,6 +741,13 @@ func (in *LocalQueueStatus) DeepCopyInto(out *LocalQueueStatus) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.Flavors != nil { + in, out := &in.Flavors, &out.Flavors + *out = make([]LocalQueueFlavorStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LocalQueueStatus. diff --git a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml index ce39a3f9ae..614e12dcdd 100644 --- a/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml +++ b/charts/kueue/templates/crd/kueue.x-k8s.io_localqueues.yaml @@ -226,6 +226,78 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + flavors: + description: flavors lists all currently available ResourceFlavors + in specified ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + nodeLabels: + additionalProperties: + type: string + description: |- + nodeLabels are labels that associate the ResourceFlavor with Nodes that + have the same labels. + maxProperties: 8 + type: object + x-kubernetes-map-type: atomic + nodeTaints: + description: |- + nodeTaints are taints that the nodes associated with this ResourceFlavor + have. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to + a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + type: string + required: + - effect + - key + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + resources: + description: resources used in the flavor. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map flavorsReservation: description: |- flavorsReservation are the reserved quotas, by flavor currently in use by the diff --git a/client-go/applyconfiguration/kueue/v1beta1/localqueueflavorstatus.go b/client-go/applyconfiguration/kueue/v1beta1/localqueueflavorstatus.go new file mode 100644 index 0000000000..aac84ddfc4 --- /dev/null +++ b/client-go/applyconfiguration/kueue/v1beta1/localqueueflavorstatus.go @@ -0,0 +1,80 @@ +/* +Copyright The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +// Code generated by applyconfiguration-gen. DO NOT EDIT. + +package v1beta1 + +import ( + v1 "k8s.io/api/core/v1" + v1beta1 "sigs.k8s.io/kueue/apis/kueue/v1beta1" +) + +// LocalQueueFlavorStatusApplyConfiguration represents an declarative configuration of the LocalQueueFlavorStatus type for use +// with apply. +type LocalQueueFlavorStatusApplyConfiguration struct { + Name *v1beta1.ResourceFlavorReference `json:"name,omitempty"` + Resources []v1.ResourceName `json:"resources,omitempty"` + NodeLabels map[string]string `json:"nodeLabels,omitempty"` + NodeTaints []v1.Taint `json:"nodeTaints,omitempty"` +} + +// LocalQueueFlavorStatusApplyConfiguration constructs an declarative configuration of the LocalQueueFlavorStatus type for use with +// apply. +func LocalQueueFlavorStatus() *LocalQueueFlavorStatusApplyConfiguration { + return &LocalQueueFlavorStatusApplyConfiguration{} +} + +// WithName sets the Name field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the Name field is set to the value of the last call. +func (b *LocalQueueFlavorStatusApplyConfiguration) WithName(value v1beta1.ResourceFlavorReference) *LocalQueueFlavorStatusApplyConfiguration { + b.Name = &value + return b +} + +// WithResources adds the given value to the Resources field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Resources field. +func (b *LocalQueueFlavorStatusApplyConfiguration) WithResources(values ...v1.ResourceName) *LocalQueueFlavorStatusApplyConfiguration { + for i := range values { + b.Resources = append(b.Resources, values[i]) + } + return b +} + +// WithNodeLabels puts the entries into the NodeLabels field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, the entries provided by each call will be put on the NodeLabels field, +// overwriting an existing map entries in NodeLabels field with the same key. +func (b *LocalQueueFlavorStatusApplyConfiguration) WithNodeLabels(entries map[string]string) *LocalQueueFlavorStatusApplyConfiguration { + if b.NodeLabels == nil && len(entries) > 0 { + b.NodeLabels = make(map[string]string, len(entries)) + } + for k, v := range entries { + b.NodeLabels[k] = v + } + return b +} + +// WithNodeTaints adds the given value to the NodeTaints field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the NodeTaints field. +func (b *LocalQueueFlavorStatusApplyConfiguration) WithNodeTaints(values ...v1.Taint) *LocalQueueFlavorStatusApplyConfiguration { + for i := range values { + b.NodeTaints = append(b.NodeTaints, values[i]) + } + return b +} diff --git a/client-go/applyconfiguration/kueue/v1beta1/localqueuestatus.go b/client-go/applyconfiguration/kueue/v1beta1/localqueuestatus.go index 9b77e15c4f..6431b527ab 100644 --- a/client-go/applyconfiguration/kueue/v1beta1/localqueuestatus.go +++ b/client-go/applyconfiguration/kueue/v1beta1/localqueuestatus.go @@ -24,12 +24,13 @@ import ( // LocalQueueStatusApplyConfiguration represents an declarative configuration of the LocalQueueStatus type for use // with apply. type LocalQueueStatusApplyConfiguration struct { - PendingWorkloads *int32 `json:"pendingWorkloads,omitempty"` - ReservingWorkloads *int32 `json:"reservingWorkloads,omitempty"` - AdmittedWorkloads *int32 `json:"admittedWorkloads,omitempty"` - Conditions []v1.Condition `json:"conditions,omitempty"` - FlavorsReservation []LocalQueueFlavorUsageApplyConfiguration `json:"flavorsReservation,omitempty"` - FlavorUsage []LocalQueueFlavorUsageApplyConfiguration `json:"flavorUsage,omitempty"` + PendingWorkloads *int32 `json:"pendingWorkloads,omitempty"` + ReservingWorkloads *int32 `json:"reservingWorkloads,omitempty"` + AdmittedWorkloads *int32 `json:"admittedWorkloads,omitempty"` + Conditions []v1.Condition `json:"conditions,omitempty"` + FlavorsReservation []LocalQueueFlavorUsageApplyConfiguration `json:"flavorsReservation,omitempty"` + FlavorUsage []LocalQueueFlavorUsageApplyConfiguration `json:"flavorUsage,omitempty"` + Flavors []LocalQueueFlavorStatusApplyConfiguration `json:"flavors,omitempty"` } // LocalQueueStatusApplyConfiguration constructs an declarative configuration of the LocalQueueStatus type for use with @@ -97,3 +98,16 @@ func (b *LocalQueueStatusApplyConfiguration) WithFlavorUsage(values ...*LocalQue } return b } + +// WithFlavors adds the given value to the Flavors field in the declarative configuration +// and returns the receiver, so that objects can be build by chaining "With" function invocations. +// If called multiple times, values provided by each call will be appended to the Flavors field. +func (b *LocalQueueStatusApplyConfiguration) WithFlavors(values ...*LocalQueueFlavorStatusApplyConfiguration) *LocalQueueStatusApplyConfiguration { + for i := range values { + if values[i] == nil { + panic("nil value passed to WithFlavors") + } + b.Flavors = append(b.Flavors, *values[i]) + } + return b +} diff --git a/client-go/applyconfiguration/utils.go b/client-go/applyconfiguration/utils.go index 58c3961c1b..57134c8ace 100644 --- a/client-go/applyconfiguration/utils.go +++ b/client-go/applyconfiguration/utils.go @@ -88,6 +88,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} { return &kueuev1beta1.FlavorUsageApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("LocalQueue"): return &kueuev1beta1.LocalQueueApplyConfiguration{} + case v1beta1.SchemeGroupVersion.WithKind("LocalQueueFlavorStatus"): + return &kueuev1beta1.LocalQueueFlavorStatusApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("LocalQueueFlavorUsage"): return &kueuev1beta1.LocalQueueFlavorUsageApplyConfiguration{} case v1beta1.SchemeGroupVersion.WithKind("LocalQueueResourceUsage"): diff --git a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml index 74c64365a0..cd5639adfc 100644 --- a/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml +++ b/config/components/crd/bases/kueue.x-k8s.io_localqueues.yaml @@ -211,6 +211,78 @@ spec: x-kubernetes-list-map-keys: - name x-kubernetes-list-type: map + flavors: + description: flavors lists all currently available ResourceFlavors + in specified ClusterQueue. + items: + properties: + name: + description: name of the flavor. + maxLength: 253 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + nodeLabels: + additionalProperties: + type: string + description: |- + nodeLabels are labels that associate the ResourceFlavor with Nodes that + have the same labels. + maxProperties: 8 + type: object + x-kubernetes-map-type: atomic + nodeTaints: + description: |- + nodeTaints are taints that the nodes associated with this ResourceFlavor + have. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to + a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + type: string + required: + - effect + - key + type: object + maxItems: 8 + type: array + x-kubernetes-list-type: atomic + resources: + description: resources used in the flavor. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + maxItems: 16 + type: array + x-kubernetes-list-type: set + required: + - name + type: object + maxItems: 16 + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map flavorsReservation: description: |- flavorsReservation are the reserved quotas, by flavor currently in use by the diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index c9f5a2f99e..1ccac96397 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -23,6 +23,8 @@ import ( "sort" "sync" + corev1 "k8s.io/api/core/v1" + "github.com/go-logr/logr" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,8 +36,10 @@ import ( kueue "sigs.k8s.io/kueue/apis/kueue/v1beta1" utilindexer "sigs.k8s.io/kueue/pkg/controller/core/indexer" + "sigs.k8s.io/kueue/pkg/features" "sigs.k8s.io/kueue/pkg/metrics" "sigs.k8s.io/kueue/pkg/resources" + "sigs.k8s.io/kueue/pkg/util/maps" "sigs.k8s.io/kueue/pkg/workload" ) @@ -657,6 +661,7 @@ type LocalQueueUsageStats struct { ReservingWorkloads int AdmittedResources []kueue.LocalQueueFlavorUsage AdmittedWorkloads int + Flavors []kueue.LocalQueueFlavorStatus } func (c *Cache) LocalQueueUsage(qObj *kueue.LocalQueue) (*LocalQueueUsageStats, error) { @@ -672,11 +677,39 @@ func (c *Cache) LocalQueueUsage(qObj *kueue.LocalQueue) (*LocalQueueUsageStats, return nil, errQNotFound } + flavors := make(map[kueue.ResourceFlavorReference]kueue.LocalQueueFlavorStatus) + if features.Enabled(features.ExposeFlavorsInLocalQueue) { + resourcesInFlavor := make(map[kueue.ResourceFlavorReference]sets.Set[corev1.ResourceName]) + for _, rg := range cqImpl.ResourceGroups { + for _, rgFlavor := range rg.Flavors { + if _, ok := resourcesInFlavor[rgFlavor]; !ok { + resourcesInFlavor[rgFlavor] = sets.New[corev1.ResourceName]() + } + resourcesInFlavor[rgFlavor].Insert(rg.CoveredResources.UnsortedList()...) + } + } + + for _, rg := range cqImpl.ResourceGroups { + for _, rgFlavor := range rg.Flavors { + flavor := kueue.LocalQueueFlavorStatus{Name: rgFlavor} + if rif, ok := resourcesInFlavor[rgFlavor]; ok { + flavor.Resources = rif.UnsortedList() + } + if rf, ok := c.resourceFlavors[rgFlavor]; ok { + flavor.NodeLabels = rf.Spec.NodeLabels + flavor.NodeTaints = rf.Spec.NodeTaints + } + flavors[rgFlavor] = flavor + } + } + } + return &LocalQueueUsageStats{ ReservedResources: filterLocalQueueUsage(qImpl.usage, cqImpl.ResourceGroups), ReservingWorkloads: qImpl.reservingWorkloads, AdmittedResources: filterLocalQueueUsage(qImpl.admittedUsage, cqImpl.ResourceGroups), AdmittedWorkloads: qImpl.admittedWorkloads, + Flavors: maps.Values(flavors), }, nil } diff --git a/pkg/controller/core/localqueue_controller.go b/pkg/controller/core/localqueue_controller.go index df17859dad..a22216d960 100644 --- a/pkg/controller/core/localqueue_controller.go +++ b/pkg/controller/core/localqueue_controller.go @@ -328,6 +328,7 @@ func (r *LocalQueueReconciler) UpdateStatusIfChanged( queue.Status.AdmittedWorkloads = int32(stats.AdmittedWorkloads) queue.Status.FlavorsReservation = stats.ReservedResources queue.Status.FlavorUsage = stats.AdmittedResources + queue.Status.Flavors = stats.Flavors if len(conditionStatus) != 0 && len(reason) != 0 && len(msg) != 0 { meta.SetStatusCondition(&queue.Status.Conditions, metav1.Condition{ Type: kueue.LocalQueueActive, diff --git a/pkg/features/kube_features.go b/pkg/features/kube_features.go index 9002046124..b72a013af3 100644 --- a/pkg/features/kube_features.go +++ b/pkg/features/kube_features.go @@ -97,6 +97,14 @@ const ( // Enable more than one workload sharing flavors to preempt within a Cohort, // as long as the preemption targets don't overlap. MultiplePreemptions featuregate.Feature = "MultiplePreemptions" + + // **Note** - This is taken from upstream commit, keeping the details intact. + // owner: @mbobrovskyi + // beta: v0.9 + // + // Enable the Flavors status field in the LocalQueue, allowing users to view + // all currently available ResourceFlavors in the LocalQueue. + ExposeFlavorsInLocalQueue featuregate.Feature = "ExposeFlavorsInLocalQueue" ) func init() { @@ -120,6 +128,7 @@ var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ LendingLimit: {Default: false, PreRelease: featuregate.Alpha}, MultiKueueBatchJobWithManagedBy: {Default: false, PreRelease: featuregate.Alpha}, MultiplePreemptions: {Default: false, PreRelease: featuregate.Alpha}, + ExposeFlavorsInLocalQueue: {Default: true, PreRelease: featuregate.Beta}, } func SetFeatureGateDuringTest(tb testing.TB, f featuregate.Feature, value bool) func() { diff --git a/site/content/en/docs/installation/_index.md b/site/content/en/docs/installation/_index.md index 2bb5340e19..a73d5aa298 100644 --- a/site/content/en/docs/installation/_index.md +++ b/site/content/en/docs/installation/_index.md @@ -239,20 +239,21 @@ spec: The currently supported features are: -| Feature | Default | Stage | Since | Until | -|---------|---------|-------|-------|-------| -| `FlavorFungibility` | `true` | Beta | 0.5 | | -| `MultiKueue` | `false` | Alpha | 0.6 | | +| Feature | Default | Stage | Since | Until | +|-----------------------------------|---------|-------|-------|-------| +| `FlavorFungibility` | `true` | Beta | 0.5 | | +| `MultiKueue` | `false` | Alpha | 0.6 | | | `MultiKueueBatchJobWithManagedBy` | `false` | Alpha | 0.8 | | -| `PartialAdmission` | `false` | Alpha | 0.4 | 0.4 | -| `PartialAdmission` | `true` | Beta | 0.5 | | -| `ProvisioningACC` | `false` | Alpha | 0.5 | 0.6 | -| `ProvisioningACC` | `true` | Beta | 0.7 | | -| `QueueVisibility` | `false` | Alpha | 0.5 | | -| `VisibilityOnDemand` | `false` | Alpha | 0.6 | | -| `PrioritySortingWithinCohort` | `true` | Beta | 0.6 | | -| `LendingLimit` | `false` | Alpha | 0.6 | | -| `MultiplePreemptions` | `false` | Alpha | 0.8 | | +| `PartialAdmission` | `false` | Alpha | 0.4 | 0.4 | +| `PartialAdmission` | `true` | Beta | 0.5 | | +| `ProvisioningACC` | `false` | Alpha | 0.5 | 0.6 | +| `ProvisioningACC` | `true` | Beta | 0.7 | | +| `QueueVisibility` | `false` | Alpha | 0.5 | | +| `VisibilityOnDemand` | `false` | Alpha | 0.6 | | +| `PrioritySortingWithinCohort` | `true` | Beta | 0.6 | | +| `LendingLimit` | `false` | Alpha | 0.6 | | +| `MultiplePreemptions` | `false` | Alpha | 0.8 | | +| `ExposeFlavorsInLocalQueue` | `true` | Beta | 0.9 | | ## What's next diff --git a/site/content/en/docs/reference/kueue.v1beta1.md b/site/content/en/docs/reference/kueue.v1beta1.md index 876e60ab33..24a9793672 100644 --- a/site/content/en/docs/reference/kueue.v1beta1.md +++ b/site/content/en/docs/reference/kueue.v1beta1.md @@ -1121,6 +1121,53 @@ There could be up to 16 resources.</p> </tbody> </table> +## `LocalQueueFlavorStatus` {#kueue-x-k8s-io-v1beta1-LocalQueueFlavorStatus} + + +**Appears in:** + +- [LocalQueueStatus](#kueue-x-k8s-io-v1beta1-LocalQueueStatus) + + + +<table class="table"> +<thead><tr><th width="30%">Field</th><th>Description</th></tr></thead> +<tbody> + + +<tr><td><code>name</code> <B>[Required]</B><br/> +<a href="#kueue-x-k8s-io-v1beta1-ResourceFlavorReference"><code>ResourceFlavorReference</code></a> +</td> +<td> + <p>name of the flavor.</p> +</td> +</tr> +<tr><td><code>resources</code><br/> +<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#resourcename-v1-core"><code>[]k8s.io/api/core/v1.ResourceName</code></a> +</td> +<td> + <p>resources used in the flavor.</p> +</td> +</tr> +<tr><td><code>nodeLabels</code><br/> +<code>map[string]string</code> +</td> +<td> + <p>nodeLabels are labels that associate the ResourceFlavor with Nodes that +have the same labels.</p> +</td> +</tr> +<tr><td><code>nodeTaints</code><br/> +<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#taint-v1-core"><code>[]k8s.io/api/core/v1.Taint</code></a> +</td> +<td> + <p>nodeTaints are taints that the nodes associated with this ResourceFlavor +have.</p> +</td> +</tr> +</tbody> +</table> + ## `LocalQueueFlavorUsage` {#kueue-x-k8s-io-v1beta1-LocalQueueFlavorUsage} @@ -1286,6 +1333,13 @@ workloads assigned to this LocalQueue.</p> workloads assigned to this LocalQueue.</p> </td> </tr> +<tr><td><code>flavors</code><br/> +<a href="#kueue-x-k8s-io-v1beta1-LocalQueueFlavorStatus"><code>[]LocalQueueFlavorStatus</code></a> +</td> +<td> + <p>flavors lists all currently available ResourceFlavors in specified ClusterQueue.</p> +</td> +</tr> </tbody> </table> @@ -1614,6 +1668,8 @@ this time would be reset to null.</p> - [FlavorUsage](#kueue-x-k8s-io-v1beta1-FlavorUsage) +- [LocalQueueFlavorStatus](#kueue-x-k8s-io-v1beta1-LocalQueueFlavorStatus) + - [LocalQueueFlavorUsage](#kueue-x-k8s-io-v1beta1-LocalQueueFlavorUsage) - [PodSetAssignment](#kueue-x-k8s-io-v1beta1-PodSetAssignment) diff --git a/test/integration/controller/core/localqueue_controller_test.go b/test/integration/controller/core/localqueue_controller_test.go index 571717b044..96307a1df1 100644 --- a/test/integration/controller/core/localqueue_controller_test.go +++ b/test/integration/controller/core/localqueue_controller_test.go @@ -93,31 +93,62 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }) ginkgo.It("Should update conditions when clusterQueues that its localQueue references are updated", func() { - gomega.Eventually(func() []metav1.Condition { + gomega.Eventually(func() kueue.LocalQueueStatus { var updatedQueue kueue.LocalQueue gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(queue), &updatedQueue)).To(gomega.Succeed()) - return updatedQueue.Status.Conditions - }, util.Timeout, util.Interval).Should(gomega.BeComparableTo([]metav1.Condition{ - { - Type: kueue.LocalQueueActive, - Status: metav1.ConditionFalse, - Reason: "ClusterQueueDoesNotExist", - Message: "Can't submit new workloads to clusterQueue", + return updatedQueue.Status + }, util.Timeout, util.Interval).Should(gomega.BeComparableTo(kueue.LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: kueue.LocalQueueActive, + Status: metav1.ConditionFalse, + Reason: "ClusterQueueDoesNotExist", + Message: "Can't submit new workloads to clusterQueue", + }, }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) + emptyUsage := []kueue.LocalQueueFlavorUsage{ + { + Name: flavorModelC, + Resources: []kueue.LocalQueueResourceUsage{ + { + Name: resourceGPU, + Total: resource.MustParse("0"), + }, + }, + }, + { + Name: flavorModelD, + Resources: []kueue.LocalQueueResourceUsage{ + { + Name: resourceGPU, + Total: resource.MustParse("0"), + }, + }, + }, + } + ginkgo.By("Creating a clusterQueue") gomega.Expect(k8sClient.Create(ctx, clusterQueue)).To(gomega.Succeed()) - gomega.Eventually(func() []metav1.Condition { + gomega.Eventually(func() kueue.LocalQueueStatus { var updatedQueue kueue.LocalQueue gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(queue), &updatedQueue)).To(gomega.Succeed()) - return updatedQueue.Status.Conditions - }, util.Timeout, util.Interval).Should(gomega.BeComparableTo([]metav1.Condition{ - { - Type: kueue.LocalQueueActive, - Status: metav1.ConditionFalse, - Reason: "ClusterQueueIsInactive", - Message: "Can't submit new workloads to clusterQueue", + return updatedQueue.Status + }, util.Timeout, util.Interval).Should(gomega.BeComparableTo(kueue.LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: kueue.LocalQueueActive, + Status: metav1.ConditionFalse, + Reason: "ClusterQueueIsInactive", + Message: "Can't submit new workloads to clusterQueue", + }, + }, + FlavorUsage: emptyUsage, + FlavorsReservation: emptyUsage, + Flavors: []kueue.LocalQueueFlavorStatus{ + {Name: flavorModelC}, + {Name: flavorModelD}, }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) @@ -137,31 +168,41 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai Message: "Can admit new workloads", }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) - gomega.Eventually(func() []metav1.Condition { + gomega.Eventually(func() kueue.LocalQueueStatus { var updatedQueue kueue.LocalQueue gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(queue), &updatedQueue)).To(gomega.Succeed()) - return updatedQueue.Status.Conditions - }, util.Timeout, util.Interval).Should(gomega.BeComparableTo([]metav1.Condition{ - { - Type: kueue.LocalQueueActive, - Status: metav1.ConditionTrue, - Reason: "Ready", - Message: "Can submit new workloads to clusterQueue", + return updatedQueue.Status + }, util.Timeout, util.Interval).Should(gomega.BeComparableTo(kueue.LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: kueue.LocalQueueActive, + Status: metav1.ConditionTrue, + Reason: "Ready", + Message: "Can submit new workloads to clusterQueue", + }, + }, + FlavorsReservation: emptyUsage, + FlavorUsage: emptyUsage, + Flavors: []kueue.LocalQueueFlavorStatus{ + {Name: flavorModelC, NodeLabels: map[string]string{"example.com/gpu": "model-c"}}, + {Name: flavorModelD, NodeLabels: map[string]string{"example.com/gpu": "model-d"}}, }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) ginkgo.By("Deleting a clusterQueue") gomega.Expect(k8sClient.Delete(ctx, clusterQueue)).To(gomega.Succeed()) - gomega.Eventually(func() []metav1.Condition { + gomega.Eventually(func() kueue.LocalQueueStatus { var updatedQueue kueue.LocalQueue gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(queue), &updatedQueue)).To(gomega.Succeed()) - return updatedQueue.Status.Conditions - }, util.Timeout, util.Interval).Should(gomega.BeComparableTo([]metav1.Condition{ - { - Type: kueue.LocalQueueActive, - Status: metav1.ConditionFalse, - Reason: "ClusterQueueDoesNotExist", - Message: "Can't submit new workloads to clusterQueue", + return updatedQueue.Status + }, util.Timeout, util.Interval).Should(gomega.BeComparableTo(kueue.LocalQueueStatus{ + Conditions: []metav1.Condition{ + { + Type: kueue.LocalQueueActive, + Status: metav1.ConditionFalse, + Reason: "ClusterQueueDoesNotExist", + Message: "Can't submit new workloads to clusterQueue", + }, }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) }) @@ -241,6 +282,10 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: emptyUsage, FlavorUsage: emptyUsage, + Flavors: []kueue.LocalQueueFlavorStatus{ + {Name: flavorModelC}, + {Name: flavorModelD}, + }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) ginkgo.By("Setting the workloads quota reservation") @@ -291,6 +336,10 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: fullUsage, FlavorUsage: emptyUsage, + Flavors: []kueue.LocalQueueFlavorStatus{ + {Name: flavorModelC}, + {Name: flavorModelD}, + }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) ginkgo.By("Setting the workloads admission checks") @@ -316,6 +365,10 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: fullUsage, FlavorUsage: fullUsage, + Flavors: []kueue.LocalQueueFlavorStatus{ + {Name: flavorModelC}, + {Name: flavorModelD}, + }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) ginkgo.By("Finishing workloads") @@ -335,6 +388,10 @@ var _ = ginkgo.Describe("Queue controller", ginkgo.Ordered, ginkgo.ContinueOnFai }, FlavorsReservation: emptyUsage, FlavorUsage: emptyUsage, + Flavors: []kueue.LocalQueueFlavorStatus{ + {Name: flavorModelC}, + {Name: flavorModelD}, + }, }, util.IgnoreConditionTimestampsAndObservedGeneration)) }) })