Skip to content

Commit

Permalink
fix(api): set pending phase for vd if provisioner is not scheduled
Browse files Browse the repository at this point in the history
Signed-off-by: Isteb4k <[email protected]>
  • Loading branch information
Isteb4k committed Dec 5, 2024
1 parent 64effd2 commit 51d4025
Show file tree
Hide file tree
Showing 19 changed files with 520 additions and 128 deletions.
137 changes: 137 additions & 0 deletions images/cdi-artifact/patches/019-add-provisioner-tolerations-anno.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
diff --git a/pkg/controller/clone-controller.go b/pkg/controller/clone-controller.go
index 59ee5fd3f..046d1f916 100644
--- a/pkg/controller/clone-controller.go
+++ b/pkg/controller/clone-controller.go
@@ -500,6 +500,11 @@ func (r *CloneReconciler) CreateCloneSourcePod(image, pullPolicy string, pvc *co
return nil, err
}

+ workloadNodePlacement, err = cc.AdjustWorkloadNodePlacement(context.TODO(), r.client, workloadNodePlacement, pvc)
+ if err != nil {
+ return nil, fmt.Errorf("failed to adjust workload node placement: %w", err)
+ }
+
sourcePvc, err := r.getCloneRequestSourcePVC(pvc)
if err != nil {
return nil, err
diff --git a/pkg/controller/common/util.go b/pkg/controller/common/util.go
index 48c73628d..a2f3eb7e2 100644
--- a/pkg/controller/common/util.go
+++ b/pkg/controller/common/util.go
@@ -21,6 +21,7 @@ import (
"crypto/rand"
"crypto/rsa"
"crypto/tls"
+ "encoding/json"
"fmt"
"io"
"math"
@@ -95,6 +96,9 @@ const (
// AnnExternalPopulation annotation marks a PVC as "externally populated", allowing the import-controller to skip it
AnnExternalPopulation = AnnAPIGroup + "/externalPopulation"

+ // AnnProvisionerTolerations annotation specifies tolerations to use for provisioners.
+ AnnProvisionerTolerations = "virt.deckhouse.io/provisioner-tolerations"
+
// AnnDeleteAfterCompletion is PVC annotation for deleting DV after completion
AnnDeleteAfterCompletion = AnnAPIGroup + "/storage.deleteAfterCompletion"
// AnnPodRetainAfterCompletion is PVC annotation for retaining transfer pods after completion
@@ -780,6 +784,50 @@ func GetWorkloadNodePlacement(ctx context.Context, c client.Client) (*sdkapi.Nod
return &cr.Spec.Workloads, nil
}

+// AdjustWorkloadNodePlacement adds tolerations specified in prime pvc annotation.
+func AdjustWorkloadNodePlacement(ctx context.Context, c client.Client, nodePlacement *sdkapi.NodePlacement, primePVC *corev1.PersistentVolumeClaim) (*sdkapi.NodePlacement, error) {
+ targetPVCKey := types.NamespacedName{
+ Namespace: primePVC.Namespace,
+ }
+
+ for _, ref := range primePVC.OwnerReferences {
+ if ref.Kind == "PersistentVolumeClaim" {
+ targetPVCKey.Name = ref.Name
+ }
+ }
+
+ var targetPVC corev1.PersistentVolumeClaim
+ err := c.Get(ctx, targetPVCKey, &targetPVC)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get target pvc %s: %w", targetPVCKey, err)
+ }
+
+ provisionerTolerations, err := ExtractProvisionerTolerations(&targetPVC)
+ if err != nil {
+ return nil, fmt.Errorf("failed to extract provisioner tolerations: %w", err)
+ }
+
+ nodePlacement.Tolerations = append(nodePlacement.Tolerations, provisionerTolerations...)
+
+ return nodePlacement, nil
+}
+
+func ExtractProvisionerTolerations(obj client.Object) ([]corev1.Toleration, error) {
+ rawTolerations := obj.GetAnnotations()[AnnProvisionerTolerations]
+
+ if rawTolerations == "" {
+ return nil, nil
+ }
+
+ var tolerations []corev1.Toleration
+ err := json.Unmarshal([]byte(rawTolerations), &tolerations)
+ if err != nil {
+ return nil, fmt.Errorf("failed to unmarshal provisioner tolerations %s: %w", rawTolerations, err)
+ }
+
+ return tolerations, nil
+}
+
// GetActiveCDI returns the active CDI CR
func GetActiveCDI(ctx context.Context, c client.Client) (*cdiv1.CDI, error) {
crList := &cdiv1.CDIList{}
diff --git a/pkg/controller/datavolume/controller-base.go b/pkg/controller/datavolume/controller-base.go
index b8c9f893e..99f8501be 100644
--- a/pkg/controller/datavolume/controller-base.go
+++ b/pkg/controller/datavolume/controller-base.go
@@ -1145,6 +1145,11 @@ func (r *ReconcilerBase) newPersistentVolumeClaim(dataVolume *cdiv1.DataVolume,
annotations[k] = v
}
annotations[cc.AnnPodRestarts] = "0"
+
+ if dataVolume.Annotations[cc.AnnProvisionerTolerations] != "" {
+ annotations[cc.AnnProvisionerTolerations] = dataVolume.Annotations[cc.AnnProvisionerTolerations]
+ }
+
annotations[cc.AnnContentType] = string(cc.GetContentType(dataVolume.Spec.ContentType))
if dataVolume.Spec.PriorityClassName != "" {
annotations[cc.AnnPriorityClassName] = dataVolume.Spec.PriorityClassName
diff --git a/pkg/controller/import-controller.go b/pkg/controller/import-controller.go
index 49f1ff898..ba9fcb531 100644
--- a/pkg/controller/import-controller.go
+++ b/pkg/controller/import-controller.go
@@ -859,6 +859,11 @@ func createImporterPod(ctx context.Context, log logr.Logger, client client.Clien
return nil, err
}

+ args.workloadNodePlacement, err = cc.AdjustWorkloadNodePlacement(context.TODO(), client, args.workloadNodePlacement, args.pvc)
+ if err != nil {
+ return nil, fmt.Errorf("failed to adjust workload node placement: %w", err)
+ }
+
if isRegistryNodeImport(args) {
args.importImage, err = getRegistryImportImage(args.pvc)
if err != nil {
diff --git a/pkg/controller/upload-controller.go b/pkg/controller/upload-controller.go
index f251cae5d..ed4420fb9 100644
--- a/pkg/controller/upload-controller.go
+++ b/pkg/controller/upload-controller.go
@@ -623,6 +623,11 @@ func (r *UploadReconciler) createUploadPod(args UploadPodArgs) (*corev1.Pod, err
return nil, err
}

+ workloadNodePlacement, err = cc.AdjustWorkloadNodePlacement(context.TODO(), r.client, workloadNodePlacement, args.PVC)
+ if err != nil {
+ return nil, fmt.Errorf("failed to adjust workload node placement: %w", err)
+ }
+
pod := r.makeUploadPodSpec(args, podResourceRequirements, imagePullSecrets, workloadNodePlacement)
util.SetRecommendedLabels(pod, r.installerLabels, "cdi-controller")

4 changes: 4 additions & 0 deletions images/cdi-artifact/patches/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,3 +76,7 @@ This is necessary for ensuring that the metrics can be accessed only by Promethe
Currently covered metrics:
- cdi-controller
- cdi-deployment

#### `019-add-provisioner-tolerations-anno.patch`

Add annotation to manage provisioner tolerations to avoid unschedulable error.
10 changes: 8 additions & 2 deletions images/virtualization-artifact/pkg/controller/common/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,21 @@ const (
// AnnAPIGroup is the APIGroup for virtualization-controller.
AnnAPIGroup = "virt.deckhouse.io"

// AnnCreatedBy is a pod annotation indicating if the pod was created by the PVC
// AnnCreatedBy is a pod annotation indicating if the pod was created by the PVC.
AnnCreatedBy = AnnAPIGroup + "/storage.createdByController"

// AnnPodRetainAfterCompletion is PVC annotation for retaining transfer pods after completion
AnnPodRetainAfterCompletion = AnnAPIGroup + "/storage.pod.retainAfterCompletion"

// AnnUploadURL provides a const for CVMI/VMI/VMD uploadURL annotation
// AnnUploadURL provides a const for CVMI/VMI/VMD uploadURL annotation.
AnnUploadURL = AnnAPIGroup + "/upload.url"

// AnnTolerationsHash provides a const for annotation with hash of applied tolerations.
AnnTolerationsHash = AnnAPIGroup + "/tolerations-hash"

// AnnProvisionerTolerations provides a const for tolerations to use for provisioners.
AnnProvisionerTolerations = AnnAPIGroup + "/provisioner-tolerations"

// AnnDefaultStorageClass is the annotation indicating that a storage class is the default one.
AnnDefaultStorageClass = "storageclass.kubernetes.io/is-default-class"

Expand Down
13 changes: 13 additions & 0 deletions images/virtualization-artifact/pkg/controller/conditions/getter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package conditions

import corev1 "k8s.io/api/core/v1"

func GetPodCondition(condType corev1.PodConditionType, conds []corev1.PodCondition) (corev1.PodCondition, bool) {
for _, cond := range conds {
if cond.Type == condType {
return cond, true
}
}

return corev1.PodCondition{}, false
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ import (
//go:generate moq -rm -out mock.go . Importer Uploader Stat

type Importer interface {
Start(ctx context.Context, settings *importer.Settings, obj service.ObjectKind, sup *supplements.Generator, caBundle *datasource.CABundle) error
Start(ctx context.Context, settings *importer.Settings, obj service.ObjectKind, sup *supplements.Generator, caBundle *datasource.CABundle, opts ...service.Option) error
StartWithPodSetting(ctx context.Context, settings *importer.Settings, sup *supplements.Generator, caBundle *datasource.CABundle, podSettings *importer.PodSettings) error
CleanUp(ctx context.Context, sup *supplements.Generator) (bool, error)
CleanUpSupplements(ctx context.Context, sup *supplements.Generator) (bool, error)
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 51d4025

Please sign in to comment.