clean: purge jobs if their pods are not going away

During the destroy-cluster, if there are any osd-prepare jobs they will be purged, similar to deployments that are not being purged automatically. Signed-off-by: Travis Nielsen <[email protected]>
rook · Apr 5, 2024 · 9622be2 · 9622be2
1 parent a9dad50
commit 9622be2
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 2 deletions.
diff --git a/pkg/crds/crds.go b/pkg/crds/crds.go
@@ -22,10 +22,12 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/rook/kubectl-rook-ceph/pkg/k8sutil"
 	"github.com/rook/kubectl-rook-ceph/pkg/logging"
 	corev1 "k8s.io/api/core/v1"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/types"
@@ -201,7 +203,11 @@ func ensureClusterIsEmpty(ctx context.Context, k8sClientSet kubernetes.Interface
 				appLabel := getPodLabel(ctx, pod, "app")
 				err := pruneDeployments(ctx, k8sClientSet, clusterNamespace, appLabel)
 				if err != nil {
-					logging.Fatal(err)
+					logging.Warning("failed to prune deployments. %v", err)
+				}
+				err = pruneJobs(ctx, k8sClientSet, clusterNamespace, appLabel)
+				if err != nil {
+					logging.Warning("failed to prune jobs. %v", err)
 				}
 			}
 
@@ -240,3 +246,29 @@ func pruneDeployments(ctx context.Context, k8sClientSet kubernetes.Interface, cl
 	}
 	return nil
 }
+
+func pruneJobs(ctx context.Context, k8sClientSet kubernetes.Interface, clusterNamespace, labelApp string) error {
+	selector := "rook_cluster=" + clusterNamespace + ",app=" + labelApp
+	jobs, err := k8sClientSet.BatchV1().Jobs(clusterNamespace).List(ctx, v1.ListOptions{
+		LabelSelector: selector,
+	})
+	if err != nil {
+		if !k8sErrors.IsNotFound(err) {
+			return err
+		}
+		return nil
+	}
+
+	for _, job := range jobs.Items {
+		logging.Info("job %s exists removing....", job.Name)
+		var gracePeriod int64
+		propagation := metav1.DeletePropagationForeground
+		options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
+		if err := k8sClientSet.BatchV1().Jobs(clusterNamespace).Delete(ctx, job.Name, *options); err != nil {
+			if !k8sErrors.IsNotFound(err) {
+				return errors.Wrapf(err, "failed to delete job %q", job.Name)
+			}
+		}
+	}
+	return nil
+}
diff --git a/pkg/k8sutil/k8sutil.go b/pkg/k8sutil/k8sutil.go
@@ -27,6 +27,7 @@ import (
 	autoscalingv1 "k8s.io/api/autoscaling/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
@@ -112,8 +113,11 @@ func GetDeployment(ctx context.Context, k8sclientset kubernetes.Interface, clust
 
 func DeleteDeployment(ctx context.Context, k8sclientset kubernetes.Interface, clusterNamespace string, deployment string) {
 	logging.Info("removing deployment %s", deployment)
+	var gracePeriod int64
+	propagation := metav1.DeletePropagationForeground
+	options := &metav1.DeleteOptions{GracePeriodSeconds: &gracePeriod, PropagationPolicy: &propagation}
 
-	err := k8sclientset.AppsV1().Deployments(clusterNamespace).Delete(ctx, deployment, v1.DeleteOptions{})
+	err := k8sclientset.AppsV1().Deployments(clusterNamespace).Delete(ctx, deployment, *options)
 	if err != nil {
 		if k8sErrors.IsNotFound(err) {
 			logging.Info("the server could not find the requested deployment: %s", deployment)