-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(vd): set tolerations for provisioners
Signed-off-by: Isteb4k <[email protected]>
- Loading branch information
Showing
33 changed files
with
1,027 additions
and
243 deletions.
There are no files selected for viewing
220 changes: 220 additions & 0 deletions
220
images/cdi-artifact/patches/019-manage-provisioner-tolerations.patch
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,220 @@ | ||
diff --git a/pkg/controller/clone-controller.go b/pkg/controller/clone-controller.go | ||
index 59ee5fd3f..046d1f916 100644 | ||
--- a/pkg/controller/clone-controller.go | ||
+++ b/pkg/controller/clone-controller.go | ||
@@ -500,6 +500,11 @@ func (r *CloneReconciler) CreateCloneSourcePod(image, pullPolicy string, pvc *co | ||
return nil, err | ||
} | ||
|
||
+ workloadNodePlacement, err = cc.AdjustWorkloadNodePlacement(context.TODO(), r.client, workloadNodePlacement, pvc) | ||
+ if err != nil { | ||
+ return nil, fmt.Errorf("failed to adjust workload node placement: %w", err) | ||
+ } | ||
+ | ||
sourcePvc, err := r.getCloneRequestSourcePVC(pvc) | ||
if err != nil { | ||
return nil, err | ||
diff --git a/pkg/controller/clone/prep-claim.go b/pkg/controller/clone/prep-claim.go | ||
index 9317b7429..68a249b77 100644 | ||
--- a/pkg/controller/clone/prep-claim.go | ||
+++ b/pkg/controller/clone/prep-claim.go | ||
@@ -139,6 +139,11 @@ func (p *PrepClaimPhase) createPod(ctx context.Context, name string, pvc *corev1 | ||
return err | ||
} | ||
|
||
+ workloadNodePlacement, err = cc.AdjustWorkloadNodePlacement(context.TODO(), p.Client, workloadNodePlacement, pvc) | ||
+ if err != nil { | ||
+ return fmt.Errorf("failed to adjust workload node placement: %w", err) | ||
+ } | ||
+ | ||
pod := &corev1.Pod{ | ||
ObjectMeta: metav1.ObjectMeta{ | ||
Name: name, | ||
diff --git a/pkg/controller/common/util.go b/pkg/controller/common/util.go | ||
index 48c73628d..f2a751805 100644 | ||
--- a/pkg/controller/common/util.go | ||
+++ b/pkg/controller/common/util.go | ||
@@ -21,6 +21,7 @@ import ( | ||
"crypto/rand" | ||
"crypto/rsa" | ||
"crypto/tls" | ||
+ "encoding/json" | ||
"fmt" | ||
"io" | ||
"math" | ||
@@ -95,6 +96,11 @@ const ( | ||
// AnnExternalPopulation annotation marks a PVC as "externally populated", allowing the import-controller to skip it | ||
AnnExternalPopulation = AnnAPIGroup + "/externalPopulation" | ||
|
||
+ // AnnProvisionerTolerations annotation specifies tolerations to use for provisioners. | ||
+ AnnProvisionerTolerations = "virt.deckhouse.io/provisioner-tolerations" | ||
+ // AnnProvisionerName provides a name of data volume provisioner. | ||
+ AnnProvisionerName = "virt.deckhouse.io/provisioner-name" | ||
+ | ||
// AnnDeleteAfterCompletion is PVC annotation for deleting DV after completion | ||
AnnDeleteAfterCompletion = AnnAPIGroup + "/storage.deleteAfterCompletion" | ||
// AnnPodRetainAfterCompletion is PVC annotation for retaining transfer pods after completion | ||
@@ -780,6 +786,50 @@ func GetWorkloadNodePlacement(ctx context.Context, c client.Client) (*sdkapi.Nod | ||
return &cr.Spec.Workloads, nil | ||
} | ||
|
||
+// AdjustWorkloadNodePlacement adds tolerations specified in prime pvc annotation. | ||
+func AdjustWorkloadNodePlacement(ctx context.Context, c client.Client, nodePlacement *sdkapi.NodePlacement, primePVC *corev1.PersistentVolumeClaim) (*sdkapi.NodePlacement, error) { | ||
+ targetPVCKey := types.NamespacedName{ | ||
+ Namespace: primePVC.Namespace, | ||
+ } | ||
+ | ||
+ for _, ref := range primePVC.OwnerReferences { | ||
+ if ref.Kind == "PersistentVolumeClaim" { | ||
+ targetPVCKey.Name = ref.Name | ||
+ } | ||
+ } | ||
+ | ||
+ var targetPVC corev1.PersistentVolumeClaim | ||
+ err := c.Get(ctx, targetPVCKey, &targetPVC) | ||
+ if err != nil { | ||
+ return nil, fmt.Errorf("failed to get target pvc %s: %w", targetPVCKey, err) | ||
+ } | ||
+ | ||
+ provisionerTolerations, err := ExtractProvisionerTolerations(&targetPVC) | ||
+ if err != nil { | ||
+ return nil, fmt.Errorf("failed to extract provisioner tolerations: %w", err) | ||
+ } | ||
+ | ||
+ nodePlacement.Tolerations = append(nodePlacement.Tolerations, provisionerTolerations...) | ||
+ | ||
+ return nodePlacement, nil | ||
+} | ||
+ | ||
+func ExtractProvisionerTolerations(obj client.Object) ([]corev1.Toleration, error) { | ||
+ rawTolerations := obj.GetAnnotations()[AnnProvisionerTolerations] | ||
+ | ||
+ if rawTolerations == "" { | ||
+ return nil, nil | ||
+ } | ||
+ | ||
+ var tolerations []corev1.Toleration | ||
+ err := json.Unmarshal([]byte(rawTolerations), &tolerations) | ||
+ if err != nil { | ||
+ return nil, fmt.Errorf("failed to unmarshal provisioner tolerations %s: %w", rawTolerations, err) | ||
+ } | ||
+ | ||
+ return tolerations, nil | ||
+} | ||
+ | ||
// GetActiveCDI returns the active CDI CR | ||
func GetActiveCDI(ctx context.Context, c client.Client) (*cdiv1.CDI, error) { | ||
crList := &cdiv1.CDIList{} | ||
diff --git a/pkg/controller/datavolume/controller-base.go b/pkg/controller/datavolume/controller-base.go | ||
index b8c9f893e..99f8501be 100644 | ||
--- a/pkg/controller/datavolume/controller-base.go | ||
+++ b/pkg/controller/datavolume/controller-base.go | ||
@@ -1145,6 +1145,11 @@ func (r *ReconcilerBase) newPersistentVolumeClaim(dataVolume *cdiv1.DataVolume, | ||
annotations[k] = v | ||
} | ||
annotations[cc.AnnPodRestarts] = "0" | ||
+ | ||
+ if dataVolume.Annotations[cc.AnnProvisionerTolerations] != "" { | ||
+ annotations[cc.AnnProvisionerTolerations] = dataVolume.Annotations[cc.AnnProvisionerTolerations] | ||
+ } | ||
+ | ||
annotations[cc.AnnContentType] = string(cc.GetContentType(dataVolume.Spec.ContentType)) | ||
if dataVolume.Spec.PriorityClassName != "" { | ||
annotations[cc.AnnPriorityClassName] = dataVolume.Spec.PriorityClassName | ||
diff --git a/pkg/controller/datavolume/pvc-clone-controller.go b/pkg/controller/datavolume/pvc-clone-controller.go | ||
index e9d18ef30..f879408fe 100644 | ||
--- a/pkg/controller/datavolume/pvc-clone-controller.go | ||
+++ b/pkg/controller/datavolume/pvc-clone-controller.go | ||
@@ -42,6 +42,7 @@ import ( | ||
"sigs.k8s.io/controller-runtime/pkg/source" | ||
|
||
cdiv1 "kubevirt.io/containerized-data-importer-api/pkg/apis/core/v1beta1" | ||
+ | ||
"kubevirt.io/containerized-data-importer/pkg/common" | ||
cc "kubevirt.io/containerized-data-importer/pkg/controller/common" | ||
featuregates "kubevirt.io/containerized-data-importer/pkg/feature-gates" | ||
@@ -683,6 +684,12 @@ func (r *PvcCloneReconciler) makeSizeDetectionPodSpec( | ||
if err != nil { | ||
return nil | ||
} | ||
+ | ||
+ workloadNodePlacement.Tolerations, err = cc.ExtractProvisionerTolerations(dv) | ||
+ if err != nil { | ||
+ return nil | ||
+ } | ||
+ | ||
// Generate individual specs | ||
objectMeta := makeSizeDetectionObjectMeta(sourcePvc) | ||
volume := makeSizeDetectionVolumeSpec(sourcePvc.Name) | ||
diff --git a/pkg/controller/import-controller.go b/pkg/controller/import-controller.go | ||
index 49f1ff898..ba9fcb531 100644 | ||
--- a/pkg/controller/import-controller.go | ||
+++ b/pkg/controller/import-controller.go | ||
@@ -859,6 +859,11 @@ func createImporterPod(ctx context.Context, log logr.Logger, client client.Clien | ||
return nil, err | ||
} | ||
|
||
+ args.workloadNodePlacement, err = cc.AdjustWorkloadNodePlacement(context.TODO(), client, args.workloadNodePlacement, args.pvc) | ||
+ if err != nil { | ||
+ return nil, fmt.Errorf("failed to adjust workload node placement: %w", err) | ||
+ } | ||
+ | ||
if isRegistryNodeImport(args) { | ||
args.importImage, err = getRegistryImportImage(args.pvc) | ||
if err != nil { | ||
diff --git a/pkg/controller/populators/populator-base.go b/pkg/controller/populators/populator-base.go | ||
index 6c6fd8f8a..a69ce4f2a 100644 | ||
--- a/pkg/controller/populators/populator-base.go | ||
+++ b/pkg/controller/populators/populator-base.go | ||
@@ -223,7 +223,7 @@ type updatePVCAnnotationsFunc func(pvc, pvcPrime *corev1.PersistentVolumeClaim) | ||
|
||
var desiredAnnotations = []string{cc.AnnPodPhase, cc.AnnPodReady, cc.AnnPodRestarts, | ||
cc.AnnPreallocationRequested, cc.AnnPreallocationApplied, cc.AnnCurrentCheckpoint, cc.AnnMultiStageImportDone, | ||
- cc.AnnRunningCondition, cc.AnnRunningConditionMessage, cc.AnnRunningConditionReason} | ||
+ cc.AnnRunningCondition, cc.AnnRunningConditionMessage, cc.AnnRunningConditionReason, cc.AnnProvisionerName} | ||
|
||
func (r *ReconcilerBase) updatePVCWithPVCPrimeAnnotations(pvc, pvcPrime *corev1.PersistentVolumeClaim, updateFunc updatePVCAnnotationsFunc) (*corev1.PersistentVolumeClaim, error) { | ||
pvcCopy := pvc.DeepCopy() | ||
diff --git a/pkg/controller/upload-controller.go b/pkg/controller/upload-controller.go | ||
index f251cae5d..ed4420fb9 100644 | ||
--- a/pkg/controller/upload-controller.go | ||
+++ b/pkg/controller/upload-controller.go | ||
@@ -623,6 +623,11 @@ func (r *UploadReconciler) createUploadPod(args UploadPodArgs) (*corev1.Pod, err | ||
return nil, err | ||
} | ||
|
||
+ workloadNodePlacement, err = cc.AdjustWorkloadNodePlacement(context.TODO(), r.client, workloadNodePlacement, args.PVC) | ||
+ if err != nil { | ||
+ return nil, fmt.Errorf("failed to adjust workload node placement: %w", err) | ||
+ } | ||
+ | ||
pod := r.makeUploadPodSpec(args, podResourceRequirements, imagePullSecrets, workloadNodePlacement) | ||
util.SetRecommendedLabels(pod, r.installerLabels, "cdi-controller") | ||
|
||
diff --git a/pkg/controller/util.go b/pkg/controller/util.go | ||
index 81e050464..367b5453b 100644 | ||
--- a/pkg/controller/util.go | ||
+++ b/pkg/controller/util.go | ||
@@ -298,7 +298,21 @@ func podSucceededFromPVC(pvc *corev1.PersistentVolumeClaim) bool { | ||
} | ||
|
||
func setAnnotationsFromPodWithPrefix(anno map[string]string, pod *corev1.Pod, termMsg *common.TerminationMessage, prefix string) { | ||
- if pod == nil || pod.Status.ContainerStatuses == nil { | ||
+ if pod == nil { | ||
+ return | ||
+ } | ||
+ | ||
+ for _, cond := range pod.Status.Conditions { | ||
+ if cond.Type != corev1.PodScheduled { | ||
+ continue | ||
+ } | ||
+ | ||
+ anno[cc.AnnProvisionerName] = pod.Name | ||
+ | ||
+ break | ||
+ } | ||
+ | ||
+ if pod.Status.ContainerStatuses == nil { | ||
return | ||
} | ||
annPodRestarts, _ := strconv.Atoi(anno[cc.AnnPodRestarts]) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
87 changes: 87 additions & 0 deletions
87
images/virtualization-artifact/pkg/common/provisioner/node_placement.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,87 @@ | ||
/* | ||
Copyright 2024 Flant JSC | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package provisioner | ||
|
||
import ( | ||
"encoding/base64" | ||
"encoding/json" | ||
|
||
corev1 "k8s.io/api/core/v1" | ||
"sigs.k8s.io/controller-runtime/pkg/client" | ||
|
||
"github.com/deckhouse/virtualization-controller/pkg/common/annotations" | ||
) | ||
|
||
type NodePlacement struct { | ||
// tolerations is a list of tolerations applied to the relevant kind of pods | ||
// See https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ for more info. | ||
// These are additional tolerations other than default ones. | ||
Tolerations []corev1.Toleration `json:"tolerations,omitempty"` | ||
} | ||
|
||
func IsNodePlacementChanged(nodePlacement *NodePlacement, obj client.Object) (bool, error) { | ||
oldHash, exists := obj.GetAnnotations()[annotations.AnnTolerationsHash] | ||
|
||
if nodePlacement == nil && exists { | ||
return true, nil | ||
} | ||
|
||
if (nodePlacement == nil || len(nodePlacement.Tolerations) == 0) && !exists { | ||
return false, nil | ||
} | ||
|
||
JSON, err := json.Marshal(nodePlacement.Tolerations) | ||
if err != nil { | ||
return false, err | ||
} | ||
|
||
newHash := base64.StdEncoding.EncodeToString(JSON) | ||
|
||
return oldHash != newHash, nil | ||
} | ||
|
||
func KeepNodePlacementTolerations(nodePlacement *NodePlacement, obj client.Object) error { | ||
anno := obj.GetAnnotations() | ||
|
||
if nodePlacement == nil || len(nodePlacement.Tolerations) == 0 { | ||
_, ok := anno[annotations.AnnTolerationsHash] | ||
if !ok { | ||
return nil | ||
} | ||
|
||
delete(anno, annotations.AnnTolerationsHash) | ||
|
||
obj.SetAnnotations(anno) | ||
|
||
return nil | ||
} | ||
|
||
JSON, err := json.Marshal(nodePlacement.Tolerations) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
if anno == nil { | ||
anno = make(map[string]string) | ||
} | ||
|
||
anno[annotations.AnnTolerationsHash] = base64.StdEncoding.EncodeToString(JSON) | ||
|
||
obj.SetAnnotations(anno) | ||
|
||
return nil | ||
} |
Oops, something went wrong.