feat(vmclass): add max allocatable resources (#481)

add max allocatable resources --------- Signed-off-by: yaroslavborbat <[email protected]>
deckhouse · Oct 29, 2024 · ba17f4c · ba17f4c
1 parent 1d286ca
commit ba17f4c
Show file tree

Hide file tree

Showing 9 changed files with 161 additions and 45 deletions.
diff --git a/api/core/v1alpha2/virtual_machine_class.go b/api/core/v1alpha2/virtual_machine_class.go
@@ -202,9 +202,12 @@ type VirtualMachineClassStatus struct {
 	// It is not displayed for the types: `Host`, `HostPassthrough`
 	//
 	// +kubebuilder:example={node-1, node-2}
-	AvailableNodes []string           `json:"availableNodes,omitempty"`
-	Conditions     []metav1.Condition `json:"conditions,omitempty"`
-	// The generation last processed by the controller
+	AvailableNodes []string `json:"availableNodes,omitempty"`
+	// The maximum amount of free CPU and Memory resources observed among all available nodes.
+	// +kubebuilder:example={"maxAllocatableResources: {\"cpu\": 1, \"memory\": \"10Gi\"}"}
+	MaxAllocatableResources corev1.ResourceList `json:"maxAllocatableResources,omitempty"`
+	Conditions              []metav1.Condition  `json:"conditions,omitempty"`
+	// The generation last processed by the controller.
 	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 }
 

diff --git a/api/core/v1alpha2/zz_generated.deepcopy.go b/api/core/v1alpha2/zz_generated.deepcopy.go
diff --git a/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go b/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go
diff --git a/crds/doc-ru-virtualmachineclasses.yaml b/crds/doc-ru-virtualmachineclasses.yaml
@@ -176,6 +176,9 @@ spec:
                   description: |
                     Список узлов, поддерживающих эту модель процессора.
                     Не отображается для типов: `Host`, `HostPassthrough`.
+                maxAllocatableResources:
+                  description: |
+                    Максимальные размеры свободных ресурсов процессора и памяти, найденные среди всех доступных узлов.
                 conditions:
                   description: |
                     Последнее подтвержденное состояние данного ресурса.

diff --git a/crds/virtualmachineclasses.yaml b/crds/virtualmachineclasses.yaml
@@ -479,8 +479,21 @@ spec:
                         type: string
                       type: array
                   type: object
+                maxAllocatableResources:
+                  additionalProperties:
+                    anyOf:
+                      - type: integer
+                      - type: string
+                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                    x-kubernetes-int-or-string: true
+                  description:
+                    The maximum amount of free CPU and Memory resources observed
+                    among all available nodes.
+                  example:
+                    - 'maxAllocatableResources: {"cpu": 1, "memory": "10Gi"}'
+                  type: object
                 observedGeneration:
-                  description: The generation last processed by the controller
+                  description: The generation last processed by the controller.
                   format: int64
                   type: integer
                 phase:

diff --git a/images/virtualization-artifact/pkg/controller/common/filter.go b/images/virtualization-artifact/pkg/controller/common/filter.go
@@ -0,0 +1,40 @@
+/*
+Copyright 2024 Flant JSC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package common
+
+import (
+	"slices"
+)
+
+type FilterFunc[T any] func(obj *T) (keep bool)
+
+func Filter[T any](objs []T, keeps ...FilterFunc[T]) []T {
+	if len(keeps) == 0 {
+		return slices.Clone(objs)
+	}
+	var filtered []T
+loop:
+	for _, o := range objs {
+		for _, keep := range keeps {
+			if !keep(&o) {
+				continue loop
+			}
+		}
+		filtered = append(filtered, o)
+	}
+	return filtered
+}
diff --git a/images/virtualization-artifact/pkg/controller/vmclass/internal/discovery.go b/images/virtualization-artifact/pkg/controller/vmclass/internal/discovery.go
@@ -24,6 +24,7 @@ import (
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	virtv1 "kubevirt.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -59,27 +60,17 @@ func (h *DiscoveryHandler) Handle(ctx context.Context, s state.VirtualMachineCla
 
 	cpuType := current.Spec.CPU.Type
 
-	if cpuType == virtv2.CPUTypeHostPassthrough || cpuType == virtv2.CPUTypeHost {
-		cb := conditions.NewConditionBuilder(vmclasscondition.TypeDiscovered).
-			Generation(current.GetGeneration()).
-			Message(fmt.Sprintf("Discovery not needed for cpu.type %q", cpuType)).
-			Reason(vmclasscondition.ReasonDiscoverySkip).
-			Status(metav1.ConditionFalse)
-
-		conditions.SetCondition(cb, &changed.Status.Conditions)
-		return reconcile.Result{}, nil
-	}
-
 	nodes, err := s.Nodes(ctx)
 	if err != nil {
 		return reconcile.Result{}, err
 	}
+
 	availableNodes, err := s.AvailableNodes(nodes)
 	if err != nil {
 		return reconcile.Result{}, err
 	}
-	availableNodeNames := make([]string, len(availableNodes))
 
+	availableNodeNames := make([]string, len(availableNodes))
 	for i, n := range availableNodes {
 		availableNodeNames[i] = n.GetName()
 	}
@@ -123,14 +114,14 @@ func (h *DiscoveryHandler) Handle(ctx context.Context, s state.VirtualMachineCla
 			Reason(vmclasscondition.ReasonDiscoverySkip).
 			Status(metav1.ConditionFalse)
 	}
-
 	conditions.SetCondition(cb, &changed.Status.Conditions)
 
 	sort.Strings(availableNodeNames)
 	sort.Strings(featuresEnabled)
 	sort.Strings(featuresNotEnabled)
 
 	changed.Status.AvailableNodes = availableNodeNames
+	changed.Status.MaxAllocatableResources = h.maxAllocatableResources(availableNodes)
 	changed.Status.CpuFeatures = virtv2.CpuFeatures{
 		Enabled:          featuresEnabled,
 		NotEnabledCommon: featuresNotEnabled,
@@ -163,3 +154,24 @@ func (h *DiscoveryHandler) discoveryCommonFeatures(nodes []corev1.Node) []string
 	}
 	return features
 }
+
+func (h *DiscoveryHandler) maxAllocatableResources(nodes []corev1.Node) corev1.ResourceList {
+	var (
+		resourceList  corev1.ResourceList = make(map[corev1.ResourceName]resource.Quantity)
+		resourceNames                     = []corev1.ResourceName{corev1.ResourceCPU, corev1.ResourceMemory}
+	)
+
+	for _, node := range nodes {
+		for _, resourceName := range resourceNames {
+			newQ := node.Status.Allocatable[resourceName]
+			if newQ.IsZero() {
+				continue
+			}
+			oldQ := resourceList[resourceName]
+			if newQ.Cmp(oldQ) == 1 {
+				resourceList[resourceName] = newQ
+			}
+		}
+	}
+	return resourceList
+}
diff --git a/images/virtualization-artifact/pkg/controller/vmclass/internal/state/state.go b/images/virtualization-artifact/pkg/controller/vmclass/internal/state/state.go
@@ -67,23 +67,8 @@ func (s *state) VirtualMachines(ctx context.Context) ([]virtv2.VirtualMachine, e
 	return vms.Items, nil
 }
 
-type filterFunc func(node *corev1.Node) (skip bool)
-
-func nodeFilter(nodes []corev1.Node, filters ...filterFunc) []corev1.Node {
-	if len(filters) == 0 {
-		return nodes
-	}
-	var filtered []corev1.Node
-loop:
-	for _, node := range nodes {
-		for _, f := range filters {
-			if f(&node) {
-				continue loop
-			}
-		}
-		filtered = append(filtered, node)
-	}
-	return filtered
+func nodeFilter(nodes []corev1.Node, filters ...common.FilterFunc[corev1.Node]) []corev1.Node {
+	return common.Filter[corev1.Node](nodes, filters...)
 }
 
 func (s *state) Nodes(ctx context.Context) ([]corev1.Node, error) {
@@ -94,16 +79,16 @@ func (s *state) Nodes(ctx context.Context) ([]corev1.Node, error) {
 	var (
 		curr        = s.vmClass.Current()
 		matchLabels map[string]string
-		filters     []filterFunc
+		filters     []common.FilterFunc[corev1.Node]
 	)
 
 	switch curr.Spec.CPU.Type {
 	case virtv2.CPUTypeHost, virtv2.CPUTypeHostPassthrough:
-		return nil, nil
+		// each node
 	case virtv2.CPUTypeDiscovery:
 		matchLabels = curr.Spec.CPU.Discovery.NodeSelector.MatchLabels
 		filters = append(filters, func(node *corev1.Node) bool {
-			return !common.MatchExpressions(node.GetLabels(), curr.Spec.CPU.Discovery.NodeSelector.MatchExpressions)
+			return common.MatchExpressions(node.GetLabels(), curr.Spec.CPU.Discovery.NodeSelector.MatchExpressions)
 		})
 	case virtv2.CPUTypeModel:
 		matchLabels = map[string]string{virtv1.CPUModelLabel + curr.Spec.CPU.Model: "true"}
@@ -132,12 +117,15 @@ func (s *state) AvailableNodes(nodes []corev1.Node) ([]corev1.Node, error) {
 	if s.vmClass == nil || s.vmClass.IsEmpty() {
 		return nil, nil
 	}
+	if len(nodes) == 0 {
+		return nodes, nil
+	}
 
 	nodeSelector := s.vmClass.Current().Spec.NodeSelector
 
-	filters := []filterFunc{
+	filters := []common.FilterFunc[corev1.Node]{
 		func(node *corev1.Node) bool {
-			return !common.MatchLabels(node.GetLabels(), nodeSelector.MatchLabels)
+			return common.MatchLabels(node.GetLabels(), nodeSelector.MatchLabels)
 		},
 	}
 
@@ -150,7 +138,7 @@ func (s *state) AvailableNodes(nodes []corev1.Node) ([]corev1.Node, error) {
 		}
 
 		filters = append(filters, func(node *corev1.Node) bool {
-			return !ns.Match(node)
+			return ns.Match(node)
 		})
 	}
 	return nodeFilter(nodes, filters...), nil

diff --git a/images/virtualization-artifact/pkg/controller/vmclass/vmclass_reconciler.go b/images/virtualization-artifact/pkg/controller/vmclass/vmclass_reconciler.go
@@ -20,10 +20,13 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/component-helpers/scheduling/corev1/nodeaffinity"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
+	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
@@ -74,14 +77,47 @@ func (r *Reconciler) SetupController(_ context.Context, mgr manager.Manager, ctr
 			if err != nil {
 				return nil
 			}
+
 			for _, class := range classList.Items {
-				if common.MatchLabelSelector(node.GetLabels(), class.Spec.CPU.Discovery.NodeSelector) {
-					result = append(result, reconcile.Request{NamespacedName: common.NamespacedName(&class)})
+				if slices.Contains(class.Status.AvailableNodes, node.GetName()) {
+					result = append(result, reconcile.Request{
+						NamespacedName: common.NamespacedName(&class),
+					})
+					continue
+				}
+				if !common.MatchLabels(node.GetLabels(), class.Spec.NodeSelector.MatchLabels) {
+					continue
+				}
+				ns, err := nodeaffinity.NewNodeSelector(&corev1.NodeSelector{
+					NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: class.Spec.NodeSelector.MatchExpressions}},
+				})
+				if err != nil || !ns.Match(node) {
+					continue
 				}
+				result = append(result, reconcile.Request{
+					NamespacedName: common.NamespacedName(&class),
+				})
 			}
 			return result
 		}),
-		predicate.LabelChangedPredicate{},
+		predicate.Or(
+			predicate.LabelChangedPredicate{},
+			predicate.Funcs{
+				CreateFunc: func(e event.CreateEvent) bool { return true },
+				DeleteFunc: func(e event.DeleteEvent) bool { return true },
+				UpdateFunc: func(e event.UpdateEvent) bool {
+					oldNode := e.ObjectOld.(*corev1.Node)
+					newNode := e.ObjectNew.(*corev1.Node)
+					if !oldNode.Status.Allocatable[corev1.ResourceCPU].Equal(newNode.Status.Allocatable[corev1.ResourceCPU]) {
+						return true
+					}
+					if !oldNode.Status.Allocatable[corev1.ResourceMemory].Equal(newNode.Status.Allocatable[corev1.ResourceMemory]) {
+						return true
+					}
+					return false
+				},
+			},
+		),
 	); err != nil {
 		return fmt.Errorf("error setting watch on Node: %w", err)
 	}