feat(vmclass): improve vmclass (#476)

Add vmclass features: - freeze features after first discovery - add tolerations - generate available nodes with nodeSelector - immutable .spec.cpu - add maxAllocatableResources --------- Signed-off-by: yaroslavborbat <[email protected]> Co-authored-by: Ivan Mikheykin <[email protected]>
deckhouse · Nov 5, 2024 · a058abd · a058abd
1 parent 69d6e5c
commit a058abd
Show file tree

Hide file tree

Showing 14 changed files with 347 additions and 107 deletions.
diff --git a/api/core/v1alpha2/virtual_machine_class.go b/api/core/v1alpha2/virtual_machine_class.go
@@ -61,6 +61,9 @@ type VirtualMachineClassList struct {
 
 type VirtualMachineClassSpec struct {
 	NodeSelector NodeSelector `json:"nodeSelector,omitempty"`
+	// Tolerations are the same as `spec.tolerations` in the [Pod](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/).
+	// These tolerations will be merged with tolerations specified in VirtualMachine resource. VirtualMachine tolerations have higher priority.
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
 	// +kubebuilder:validation:Required
 	CPU            CPU            `json:"cpu"`
 	SizingPolicies []SizingPolicy `json:"sizingPolicies,omitempty"`
@@ -76,6 +79,7 @@ type NodeSelector struct {
 }
 
 // CPU defines the requirements for the virtual CPU model.
+// +kubebuilder:validation:XValidation:rule="self == oldSelf",message=".spec.cpu is immutable"
 // +kubebuilder:validation:XValidation:rule="self.type == 'HostPassthrough' || self.type == 'Host' ? !has(self.model) && !has(self.features) && !has(self.discovery) : true",message="HostPassthrough and Host cannot have model, features or discovery"
 // +kubebuilder:validation:XValidation:rule="self.type == 'Discovery' ? !has(self.model) && !has(self.features) : true",message="Discovery cannot have model or features"
 // +kubebuilder:validation:XValidation:rule="self.type == 'Model' ? has(self.model) && !has(self.features) && !has(self.discovery) : true",message="Model requires model and cannot have features or discovery"
@@ -198,9 +202,12 @@ type VirtualMachineClassStatus struct {
 	// It is not displayed for the types: `Host`, `HostPassthrough`
 	//
 	// +kubebuilder:example={node-1, node-2}
-	AvailableNodes []string           `json:"availableNodes,omitempty"`
-	Conditions     []metav1.Condition `json:"conditions,omitempty"`
-	// The generation last processed by the controller
+	AvailableNodes []string `json:"availableNodes,omitempty"`
+	// The maximum amount of free CPU and Memory resources observed among all available nodes.
+	// +kubebuilder:example={"maxAllocatableResources: {\"cpu\": 1, \"memory\": \"10Gi\"}"}
+	MaxAllocatableResources corev1.ResourceList `json:"maxAllocatableResources,omitempty"`
+	Conditions              []metav1.Condition  `json:"conditions,omitempty"`
+	// The generation last processed by the controller.
 	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 }
 

diff --git a/api/core/v1alpha2/zz_generated.deepcopy.go b/api/core/v1alpha2/zz_generated.deepcopy.go
diff --git a/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go b/api/pkg/apiserver/api/generated/openapi/zz_generated.openapi.go
diff --git a/crds/doc-ru-virtualmachineclasses.yaml b/crds/doc-ru-virtualmachineclasses.yaml
@@ -106,7 +106,23 @@ spec:
                         Карта пар `ключ=значение`. Один `ключ=значение` для matchLabels эквивалентен элементу matchExpressions, ключевым полем которого является «ключ», оператором - «In», а список значений содержит только «значение».
 
                         Для выборки по данному критерию все значения логически суммируются.
-
+                tolerations:
+                  description: |
+                    Параметр аналогичен параметру `spec.tolerations` у [Pod](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/).
+                    Tolerations из этого параметра будут объединены с tolerations, указанными в ресурсе VirtualMachine. При этом tolerations у VirtualMachine имеют больший приоритет.
+                  items:
+                    description: ""
+                    properties:
+                      effect:
+                        description: ""
+                      key:
+                        description: ""
+                      operator:
+                        description: ""
+                      tolerationSeconds:
+                        description: ""
+                      value:
+                        description: ""
                 sizingPolicies:
                   description: |
                     Политика выделения вычислительных ресурсов ВМ. Представлена в виде списка. Диапазоны cores.min - cores.max для разных элементов списка не должны пересекаться.
@@ -160,6 +176,9 @@ spec:
                   description: |
                     Список узлов, поддерживающих эту модель процессора.
                     Не отображается для типов: `Host`, `HostPassthrough`.
+                maxAllocatableResources:
+                  description: |
+                    Максимальные размеры свободных ресурсов процессора и памяти, найденные среди всех доступных узлов.
                 conditions:
                   description: |
                     Последнее подтвержденное состояние данного ресурса.

diff --git a/crds/virtualmachineclasses.yaml b/crds/virtualmachineclasses.yaml
@@ -159,6 +159,8 @@ spec:
                     - type
                   type: object
                   x-kubernetes-validations:
+                    - message: .spec.cpu is immutable
+                      rule: self == oldSelf
                     - message:
                         HostPassthrough and Host cannot have model, features or
                         discovery
@@ -325,6 +327,47 @@ spec:
                         type: object
                     type: object
                   type: array
+                tolerations:
+                  description: |-
+                    Tolerations are the same as `spec.tolerations` in the [Pod](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/).
+                    These tolerations will be merged with tolerations specified in VirtualMachine resource. VirtualMachine tolerations have higher priority.
+                  items:
+                    description: |-
+                      The pod this Toleration is attached to tolerates any taint that matches
+                      the triple <key,value,effect> using the matching operator <operator>.
+                    properties:
+                      effect:
+                        description: |-
+                          Effect indicates the taint effect to match. Empty means match all taint effects.
+                          When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
+                        type: string
+                      key:
+                        description: |-
+                          Key is the taint key that the toleration applies to. Empty means match all taint keys.
+                          If the key is empty, operator must be Exists; this combination means to match all values and all keys.
+                        type: string
+                      operator:
+                        description: |-
+                          Operator represents a key's relationship to the value.
+                          Valid operators are Exists and Equal. Defaults to Equal.
+                          Exists is equivalent to wildcard for value, so that a pod can
+                          tolerate all taints of a particular category.
+                        type: string
+                      tolerationSeconds:
+                        description: |-
+                          TolerationSeconds represents the period of time the toleration (which must be
+                          of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
+                          it is not set, which means tolerate the taint forever (do not evict). Zero and
+                          negative values will be treated as 0 (evict immediately) by the system.
+                        format: int64
+                        type: integer
+                      value:
+                        description: |-
+                          Value is the taint value the toleration matches to.
+                          If the operator is Exists, the value should be empty, otherwise just a regular string.
+                        type: string
+                    type: object
+                  type: array
               required:
                 - cpu
               type: object
@@ -436,8 +479,21 @@ spec:
                         type: string
                       type: array
                   type: object
+                maxAllocatableResources:
+                  additionalProperties:
+                    anyOf:
+                      - type: integer
+                      - type: string
+                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                    x-kubernetes-int-or-string: true
+                  description:
+                    The maximum amount of free CPU and Memory resources observed
+                    among all available nodes.
+                  example:
+                    - 'maxAllocatableResources: {"cpu": 1, "memory": "10Gi"}'
+                  type: object
                 observedGeneration:
-                  description: The generation last processed by the controller
+                  description: The generation last processed by the controller.
                   format: int64
                   type: integer
                 phase:

diff --git a/images/virtualization-artifact/cmd/virtualization-controller/main.go b/images/virtualization-artifact/cmd/virtualization-controller/main.go
@@ -253,7 +253,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	if _, err = vmclass.NewController(ctx, mgr, log); err != nil {
+	if _, err = vmclass.NewController(ctx, mgr, controllerNamespace, log); err != nil {
 		log.Error(err.Error())
 		os.Exit(1)
 	}

diff --git a/images/virtualization-artifact/pkg/controller/common/filter.go b/images/virtualization-artifact/pkg/controller/common/filter.go
@@ -0,0 +1,41 @@
+/*
+Copyright 2024 Flant JSC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package common
+
+import (
+	"slices"
+)
+
+type FilterFunc[T any] func(obj *T) (keep bool)
+
+// Filter removes an item from the "objs" array if at least one of the "keeps" functions return false for the item.
+func Filter[T any](objs []T, keeps ...FilterFunc[T]) []T {
+	if len(keeps) == 0 {
+		return slices.Clone(objs)
+	}
+	var filtered []T
+loop:
+	for _, o := range objs {
+		for _, keep := range keeps {
+			if !keep(&o) {
+				continue loop
+			}
+		}
+		filtered = append(filtered, o)
+	}
+	return filtered
+}
diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm.go b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm.go
@@ -153,8 +153,19 @@ func (b *KVVM) SetNodeSelector(vmNodeSelector, classNodeSelector map[string]stri
 	b.Resource.Spec.Template.Spec.NodeSelector = selector
 }
 
-func (b *KVVM) SetTolerations(tolerations []corev1.Toleration) {
-	b.Resource.Spec.Template.Spec.Tolerations = tolerations
+func (b *KVVM) SetTolerations(vmTolerations, classTolerations []corev1.Toleration) {
+	tolerationsMap := make(map[string]corev1.Toleration)
+	for _, toleration := range classTolerations {
+		tolerationsMap[toleration.Key] = toleration
+	}
+	for _, toleration := range vmTolerations {
+		tolerationsMap[toleration.Key] = toleration
+	}
+	resultTolerations := make([]corev1.Toleration, 0, len(tolerationsMap))
+	for _, toleration := range tolerationsMap {
+		resultTolerations = append(resultTolerations, toleration)
+	}
+	b.Resource.Spec.Template.Spec.Tolerations = resultTolerations
 }
 
 func (b *KVVM) SetPriorityClassName(priorityClassName string) {

diff --git a/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go b/images/virtualization-artifact/pkg/controller/kvbuilder/kvvm_utils.go
@@ -87,7 +87,7 @@ func ApplyVirtualMachineSpec(
 	kvvm.SetNetworkInterface(NetworkInterfaceName)
 	kvvm.SetTablet("default-0")
 	kvvm.SetNodeSelector(vm.Spec.NodeSelector, class.Spec.NodeSelector.MatchLabels)
-	kvvm.SetTolerations(vm.Spec.Tolerations)
+	kvvm.SetTolerations(vm.Spec.Tolerations, class.Spec.Tolerations)
 	kvvm.SetAffinity(virtv2.NewAffinityFromVMAffinity(vm.Spec.Affinity), class.Spec.NodeSelector.MatchExpressions)
 	kvvm.SetPriorityClassName(vm.Spec.PriorityClassName)
 	kvvm.SetTerminationGracePeriod(vm.Spec.TerminationGracePeriodSeconds)