From 2b16a8070fdbda83bd61b6922062b2ab64bec6a6 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat <86148689+yaroslavborbat@users.noreply.github.com> Date: Tue, 29 Oct 2024 14:32:33 +0300 Subject: [PATCH] feat(vmclass): add max allocatable resources (#480) add max allocatable resources --------- Signed-off-by: yaroslavborbat --- api/core/v1alpha2/virtual_machine_class.go | 9 ++-- api/core/v1alpha2/zz_generated.deepcopy.go | 7 ++++ .../generated/openapi/zz_generated.openapi.go | 18 +++++++- crds/doc-ru-virtualmachineclasses.yaml | 3 ++ crds/virtualmachineclasses.yaml | 15 ++++++- .../pkg/controller/common/filter.go | 40 ++++++++++++++++++ .../controller/vmclass/internal/discovery.go | 38 +++++++++++------ .../vmclass/internal/state/state.go | 28 ++++--------- .../controller/vmclass/vmclass_reconciler.go | 42 +++++++++++++++++-- 9 files changed, 158 insertions(+), 42 deletions(-) create mode 100644 images/virtualization-artifact/pkg/controller/common/filter.go diff --git a/api/core/v1alpha2/virtual_machine_class.go b/api/core/v1alpha2/virtual_machine_class.go index e3f0cf815..d1877f883 100644 --- a/api/core/v1alpha2/virtual_machine_class.go +++ b/api/core/v1alpha2/virtual_machine_class.go @@ -202,9 +202,12 @@ type VirtualMachineClassStatus struct { // It is not displayed for the types: `Host`, `HostPassthrough` // // +kubebuilder:example={node-1, node-2} - AvailableNodes []string `json:"availableNodes,omitempty"` - Conditions []metav1.Condition `json:"conditions,omitempty"` - // The generation last processed by the controller + AvailableNodes []string `json:"availableNodes,omitempty"` + // The maximum amount of free CPU and Memory resources observed among all available nodes. + // +kubebuilder:example={"maxAllocatableResources: {\"cpu\": 1, \"memory\": \"10Gi\"}"} + MaxAllocatableResources corev1.ResourceList `json:"maxAllocatableResources,omitempty"` + Conditions []metav1.Condition `json:"conditions,omitempty"` + // The generation last processed by the controller. ObservedGeneration int64 `json:"observedGeneration,omitempty"` } diff --git a/api/core/v1alpha2/zz_generated.deepcopy.go b/api/core/v1alpha2/zz_generated.deepcopy.go index 2af22c0e9..931d86fef 100644 --- a/api/core/v1alpha2/zz_generated.deepcopy.go +++ b/api/core/v1alpha2/zz_generated.deepcopy.go @@ -1718,6 +1718,13 @@ func (in *VirtualMachineClassStatus) DeepCopyInto(out *VirtualMachineClassStatus *out = make([]string, len(*in)) copy(*out, *in) } + if in.MaxAllocatableResources != nil { + in, out := &in.MaxAllocatableResources, &out.MaxAllocatableResources + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) diff --git a/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go b/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go index b3f6ed199..02e229009 100644 --- a/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go +++ b/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go @@ -3521,6 +3521,20 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineClassStatus(ref commo }, }, }, + "maxAllocatableResources": { + SchemaProps: spec.SchemaProps{ + Description: "The maximum amount of free CPU and Memory resources observed among all available nodes.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + }, + }, "conditions": { SchemaProps: spec.SchemaProps{ Type: []string{"array"}, @@ -3536,7 +3550,7 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineClassStatus(ref commo }, "observedGeneration": { SchemaProps: spec.SchemaProps{ - Description: "The generation last processed by the controller", + Description: "The generation last processed by the controller.", Type: []string{"integer"}, Format: "int64", }, @@ -3546,7 +3560,7 @@ func schema_virtualization_api_core_v1alpha2_VirtualMachineClassStatus(ref commo }, }, Dependencies: []string{ - "github.com/deckhouse/virtualization/api/core/v1alpha2.CpuFeatures", "k8s.io/apimachinery/pkg/apis/meta/v1.Condition"}, + "github.com/deckhouse/virtualization/api/core/v1alpha2.CpuFeatures", "k8s.io/apimachinery/pkg/api/resource.Quantity", "k8s.io/apimachinery/pkg/apis/meta/v1.Condition"}, } } diff --git a/crds/doc-ru-virtualmachineclasses.yaml b/crds/doc-ru-virtualmachineclasses.yaml index dc061ecab..f63017052 100644 --- a/crds/doc-ru-virtualmachineclasses.yaml +++ b/crds/doc-ru-virtualmachineclasses.yaml @@ -176,6 +176,9 @@ spec: description: | Список узлов, поддерживающих эту модель процессора. Не отображается для типов: `Host`, `HostPassthrough`. + maxAllocatableResources: + description: | + Максимальные размеры свободных ресурсов процессора и памяти, найденные среди всех доступных узлов. conditions: description: | Последнее подтвержденное состояние данного ресурса. diff --git a/crds/virtualmachineclasses.yaml b/crds/virtualmachineclasses.yaml index 3692f3f95..9c3bb9a96 100644 --- a/crds/virtualmachineclasses.yaml +++ b/crds/virtualmachineclasses.yaml @@ -479,8 +479,21 @@ spec: type: string type: array type: object + maxAllocatableResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: + The maximum amount of free CPU and Memory resources observed + among all available nodes. + example: + - 'maxAllocatableResources: {"cpu": 1, "memory": "10Gi"}' + type: object observedGeneration: - description: The generation last processed by the controller + description: The generation last processed by the controller. format: int64 type: integer phase: diff --git a/images/virtualization-artifact/pkg/controller/common/filter.go b/images/virtualization-artifact/pkg/controller/common/filter.go new file mode 100644 index 000000000..40bc506de --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/common/filter.go @@ -0,0 +1,40 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "slices" +) + +type FilterFunc[T any] func(obj *T) (skip bool) + +func Filter[T any](objs []T, skips ...FilterFunc[T]) []T { + if len(skips) == 0 { + return slices.Clone(objs) + } + var filtered []T +loop: + for _, o := range objs { + for _, skip := range skips { + if skip(&o) { + continue loop + } + } + filtered = append(filtered, o) + } + return filtered +} diff --git a/images/virtualization-artifact/pkg/controller/vmclass/internal/discovery.go b/images/virtualization-artifact/pkg/controller/vmclass/internal/discovery.go index 6cc79989d..cd248d5fc 100644 --- a/images/virtualization-artifact/pkg/controller/vmclass/internal/discovery.go +++ b/images/virtualization-artifact/pkg/controller/vmclass/internal/discovery.go @@ -24,6 +24,7 @@ import ( "strings" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" virtv1 "kubevirt.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -59,27 +60,17 @@ func (h *DiscoveryHandler) Handle(ctx context.Context, s state.VirtualMachineCla cpuType := current.Spec.CPU.Type - if cpuType == virtv2.CPUTypeHostPassthrough || cpuType == virtv2.CPUTypeHost { - cb := conditions.NewConditionBuilder(vmclasscondition.TypeDiscovered). - Generation(current.GetGeneration()). - Message(fmt.Sprintf("Discovery not needed for cpu.type %q", cpuType)). - Reason(vmclasscondition.ReasonDiscoverySkip). - Status(metav1.ConditionFalse) - - conditions.SetCondition(cb, &changed.Status.Conditions) - return reconcile.Result{}, nil - } - nodes, err := s.Nodes(ctx) if err != nil { return reconcile.Result{}, err } + availableNodes, err := s.AvailableNodes(nodes) if err != nil { return reconcile.Result{}, err } - availableNodeNames := make([]string, len(availableNodes)) + availableNodeNames := make([]string, len(availableNodes)) for i, n := range availableNodes { availableNodeNames[i] = n.GetName() } @@ -123,7 +114,6 @@ func (h *DiscoveryHandler) Handle(ctx context.Context, s state.VirtualMachineCla Reason(vmclasscondition.ReasonDiscoverySkip). Status(metav1.ConditionFalse) } - conditions.SetCondition(cb, &changed.Status.Conditions) sort.Strings(availableNodeNames) @@ -131,6 +121,7 @@ func (h *DiscoveryHandler) Handle(ctx context.Context, s state.VirtualMachineCla sort.Strings(featuresNotEnabled) changed.Status.AvailableNodes = availableNodeNames + changed.Status.MaxAllocatableResources = h.maxAllocatableResources(availableNodes) changed.Status.CpuFeatures = virtv2.CpuFeatures{ Enabled: featuresEnabled, NotEnabledCommon: featuresNotEnabled, @@ -163,3 +154,24 @@ func (h *DiscoveryHandler) discoveryCommonFeatures(nodes []corev1.Node) []string } return features } + +func (h *DiscoveryHandler) maxAllocatableResources(nodes []corev1.Node) corev1.ResourceList { + var ( + resourceList corev1.ResourceList = make(map[corev1.ResourceName]resource.Quantity) + resourceNames = []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory} + ) + + for _, node := range nodes { + for _, resourceName := range resourceNames { + newQ := node.Status.Allocatable[resourceName] + if newQ.IsZero() { + continue + } + oldQ := resourceList[resourceName] + if newQ.Cmp(oldQ) == 1 { + resourceList[resourceName] = newQ + } + } + } + return resourceList +} diff --git a/images/virtualization-artifact/pkg/controller/vmclass/internal/state/state.go b/images/virtualization-artifact/pkg/controller/vmclass/internal/state/state.go index 5253a8e0c..21f56a57b 100644 --- a/images/virtualization-artifact/pkg/controller/vmclass/internal/state/state.go +++ b/images/virtualization-artifact/pkg/controller/vmclass/internal/state/state.go @@ -67,23 +67,8 @@ func (s *state) VirtualMachines(ctx context.Context) ([]virtv2.VirtualMachine, e return vms.Items, nil } -type filterFunc func(node *corev1.Node) (skip bool) - -func nodeFilter(nodes []corev1.Node, filters ...filterFunc) []corev1.Node { - if len(filters) == 0 { - return nodes - } - var filtered []corev1.Node -loop: - for _, node := range nodes { - for _, f := range filters { - if f(&node) { - continue loop - } - } - filtered = append(filtered, node) - } - return filtered +func nodeFilter(nodes []corev1.Node, filters ...common.FilterFunc[corev1.Node]) []corev1.Node { + return common.Filter[corev1.Node](nodes, filters...) } func (s *state) Nodes(ctx context.Context) ([]corev1.Node, error) { @@ -94,12 +79,12 @@ func (s *state) Nodes(ctx context.Context) ([]corev1.Node, error) { var ( curr = s.vmClass.Current() matchLabels map[string]string - filters []filterFunc + filters []common.FilterFunc[corev1.Node] ) switch curr.Spec.CPU.Type { case virtv2.CPUTypeHost, virtv2.CPUTypeHostPassthrough: - return nil, nil + // each node case virtv2.CPUTypeDiscovery: matchLabels = curr.Spec.CPU.Discovery.NodeSelector.MatchLabels filters = append(filters, func(node *corev1.Node) bool { @@ -132,10 +117,13 @@ func (s *state) AvailableNodes(nodes []corev1.Node) ([]corev1.Node, error) { if s.vmClass == nil || s.vmClass.IsEmpty() { return nil, nil } + if len(nodes) == 0 { + return nodes, nil + } nodeSelector := s.vmClass.Current().Spec.NodeSelector - filters := []filterFunc{ + filters := []common.FilterFunc[corev1.Node]{ func(node *corev1.Node) bool { return !common.MatchLabels(node.GetLabels(), nodeSelector.MatchLabels) }, diff --git a/images/virtualization-artifact/pkg/controller/vmclass/vmclass_reconciler.go b/images/virtualization-artifact/pkg/controller/vmclass/vmclass_reconciler.go index 0a12ba636..7fde556a3 100644 --- a/images/virtualization-artifact/pkg/controller/vmclass/vmclass_reconciler.go +++ b/images/virtualization-artifact/pkg/controller/vmclass/vmclass_reconciler.go @@ -20,10 +20,13 @@ import ( "context" "errors" "fmt" + "slices" corev1 "k8s.io/api/core/v1" + "k8s.io/component-helpers/scheduling/corev1/nodeaffinity" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -74,14 +77,47 @@ func (r *Reconciler) SetupController(_ context.Context, mgr manager.Manager, ctr if err != nil { return nil } + for _, class := range classList.Items { - if common.MatchLabelSelector(node.GetLabels(), class.Spec.CPU.Discovery.NodeSelector) { - result = append(result, reconcile.Request{NamespacedName: common.NamespacedName(&class)}) + if slices.Contains(class.Status.AvailableNodes, node.GetName()) { + result = append(result, reconcile.Request{ + NamespacedName: common.NamespacedName(&class), + }) + continue + } + if !common.MatchLabels(node.GetLabels(), class.Spec.NodeSelector.MatchLabels) { + continue + } + ns, err := nodeaffinity.NewNodeSelector(&corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: class.Spec.NodeSelector.MatchExpressions}}, + }) + if err != nil || !ns.Match(node) { + continue } + result = append(result, reconcile.Request{ + NamespacedName: common.NamespacedName(&class), + }) } return result }), - predicate.LabelChangedPredicate{}, + predicate.Or( + predicate.LabelChangedPredicate{}, + predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { return true }, + DeleteFunc: func(e event.DeleteEvent) bool { return true }, + UpdateFunc: func(e event.UpdateEvent) bool { + oldNode := e.ObjectOld.(*corev1.Node) + newNode := e.ObjectNew.(*corev1.Node) + if !oldNode.Status.Allocatable[corev1.ResourceCPU].Equal(newNode.Status.Allocatable[corev1.ResourceCPU]) { + return true + } + if !oldNode.Status.Allocatable[corev1.ResourceMemory].Equal(newNode.Status.Allocatable[corev1.ResourceMemory]) { + return true + } + return false + }, + }, + ), ); err != nil { return fmt.Errorf("error setting watch on Node: %w", err) }