Merge pull request #740 from red-hat-storage/sync_us--master

Syncing latest changes from upstream master for rook

Documentation/Helm-Charts/operator-chart.md

@@ -149,6 +149,7 @@ The following table lists the configurable parameters of the rook-operator chart
 | `discoveryDaemonInterval` | Set the discovery daemon device discovery interval (default to 60m) | `"60m"` |
 | `enableDiscoveryDaemon` | Enable discovery daemon | `false` |
 | `enableOBCWatchOperatorNamespace` | Whether the OBC provisioner should watch on the operator namespace or not, if not the namespace of the cluster will be used | `true` |
+| `enforceHostNetwork` | Whether to create all Rook pods to run on the host network, for example in environments where a CNI is not enabled | `false` |
 | `hostpathRequiresPrivileged` | Runs Ceph Pods as privileged to be able to write to `hostPaths` in OpenShift with SELinux restrictions. | `false` |
 | `image.pullPolicy` | Image pull policy | `"IfNotPresent"` |
 | `image.repository` | Image | `"docker.io/rook/ceph"` |
@@ -163,6 +164,7 @@ The following table lists the configurable parameters of the rook-operator chart
 | `rbacAggregate.enableOBCs` | If true, create a ClusterRole aggregated to [user facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) for objectbucketclaims | `false` |
 | `rbacEnable` | If true, create & use RBAC resources | `true` |
 | `resources` | Pod resource requests & limits | `{"limits":{"memory":"512Mi"},"requests":{"cpu":"200m","memory":"128Mi"}}` |
+| `revisionHistoryLimit` | The revision history limit for all pods created by Rook. If blank, the K8s default is 10. | `nil` |
 | `scaleDownOperator` | If true, scale down the rook operator. This is useful for administrative actions where the rook operator must be scaled down, while using gitops style tooling to deploy your helm charts. | `false` |
 | `tolerations` | List of Kubernetes [`tolerations`](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) to add to the Deployment. | `[]` |
 | `unreachableNodeTolerationSeconds` | Delay to use for the `node.kubernetes.io/unreachable` pod failure toleration to override the Kubernetes default of 5 minutes | `5` |

deploy/charts/rook-ceph/templates/configmap.yaml

@@ -20,6 +20,13 @@ data:
 {{- if .Values.discoverDaemonUdev }}
   DISCOVER_DAEMON_UDEV_BLACKLIST: {{ .Values.discoverDaemonUdev | quote }}
 {{- end }}
+{{- if .Values.revisionHistoryLimit }}
+  ROOK_REVISION_HISTORY_LIMIT: {{ .Values.revisionHistoryLimit | quote }}
+{{- end }}
+{{- if .Values.enforceHostNetwork }}
+  ROOK_ENFORCE_HOST_NETWORK: {{ .Values.enforceHostNetwork | quote }}
+{{- end }}
+
 {{- if .Values.csi }}
   ROOK_CSI_ENABLE_RBD: {{ .Values.csi.enableRbdDriver | quote }}
   ROOK_CSI_ENABLE_CEPHFS: {{ .Values.csi.enableCephfsDriver | quote }}

deploy/charts/rook-ceph/values.yaml

@@ -633,9 +633,15 @@ discover:
 # -- Runs Ceph Pods as privileged to be able to write to `hostPaths` in OpenShift with SELinux restrictions.
 hostpathRequiresPrivileged: false
 
+# -- Whether to create all Rook pods to run on the host network, for example in environments where a CNI is not enabled
+enforceHostNetwork: false
+
 # -- Disable automatic orchestration when new devices are discovered.
 disableDeviceHotplug: false
 
+# -- The revision history limit for all pods created by Rook. If blank, the K8s default is 10.
+revisionHistoryLimit: 
+
 # -- Blacklist certain disks according to the regex provided.
 discoverDaemonUdev:
 

deploy/examples/operator-openshift.yaml

@@ -638,6 +638,12 @@ data:
 
   # (Optional) QPS to use while communicating with the kubernetes apiserver.
   # CSI_KUBE_API_QPS: "5.0"
+
+  # Whether to create all Rook pods to run on the host network, for example in environments where a CNI is not enabled
+  ROOK_ENFORCE_HOST_NETWORK: "false"
+
+  # RevisionHistoryLimit value for all deployments created by rook.
+  # ROOK_REVISION_HISTORY_LIMIT: "3"
 ---
 # The deployment for the rook operator
 # OLM: BEGIN OPERATOR DEPLOYMENT

deploy/examples/operator.yaml

@@ -568,6 +568,12 @@ data:
 
   # (Optional) QPS to use while communicating with the kubernetes apiserver.
   # CSI_KUBE_API_QPS: "5.0"
+
+  # Whether to create all Rook pods to run on the host network, for example in environments where a CNI is not enabled
+  ROOK_ENFORCE_HOST_NETWORK: "false"
+
+  # RevisionHistoryLimit value for all deployments created by rook.
+  # ROOK_REVISION_HISTORY_LIMIT: "3"
 ---
 # OLM: BEGIN OPERATOR DEPLOYMENT
 apiVersion: apps/v1

pkg/operator/ceph/cluster/mgr/spec.go

@@ -128,6 +128,7 @@ func (c *Cluster) makeDeployment(mgrConfig *mgrConfig) (*apps.Deployment, error)
 			Labels:    c.getPodLabels(mgrConfig, true),
 		},
 		Spec: apps.DeploymentSpec{
+			RevisionHistoryLimit: controller.RevisionHistoryLimit(),
 			Selector: &metav1.LabelSelector{
 				MatchLabels: c.getPodLabels(mgrConfig, false),
 			},

pkg/operator/ceph/cluster/mon/spec.go

@@ -108,6 +108,7 @@ func (c *Cluster) makeDeployment(monConfig *monConfig, canary bool) (*apps.Deplo
 	}
 	replicaCount := int32(1)
 	d.Spec = apps.DeploymentSpec{
+		RevisionHistoryLimit: controller.RevisionHistoryLimit(),
 		Selector: &metav1.LabelSelector{
 			MatchLabels: c.getLabels(monConfig, canary, false),
 		},

pkg/operator/ceph/cluster/nodedaemon/exporter.go

@@ -77,6 +77,7 @@ func (r *ReconcileNode) createOrUpdateCephExporter(node corev1.Node, tolerations
 			Namespace: cephCluster.GetNamespace(),
 		},
 	}
+	deploy.Spec.RevisionHistoryLimit = controller.RevisionHistoryLimit()
 	err := controllerutil.SetControllerReference(&cephCluster, deploy, r.scheme)
 	if err != nil {
 		return controllerutil.OperationResultNone, errors.Errorf("failed to set owner reference of ceph-exporter deployment %q", deploy.Name)

pkg/operator/ceph/cluster/osd/spec.go

@@ -713,6 +713,7 @@ func (c *Cluster) makeDeployment(osdProps osdProperties, osd *OSDInfo, provision
 					OsdIdLabelKey:       fmt.Sprintf("%d", osd.ID),
 				},
 			},
+			RevisionHistoryLimit: controller.RevisionHistoryLimit(),
 			Strategy: apps.DeploymentStrategy{
 				Type: apps.RecreateDeploymentStrategyType,
 			},

pkg/operator/ceph/cluster/rbd/spec.go

@@ -82,6 +82,7 @@ func (r *ReconcileCephRBDMirror) makeDeployment(daemonConfig *daemonConfig, rbdM
 			Labels:      controller.CephDaemonAppLabels(AppName, rbdMirror.Namespace, config.RbdMirrorType, daemonConfig.DaemonID, rbdMirror.Name, "cephrbdmirrors.ceph.rook.io", true),
 		},
 		Spec: apps.DeploymentSpec{
+			RevisionHistoryLimit: controller.RevisionHistoryLimit(),
 			Selector: &metav1.LabelSelector{
 				MatchLabels: podSpec.Labels,
 			},

pkg/operator/ceph/controller.go

@@ -133,6 +133,7 @@ func (r *ReconcileConfig) reconcile(request reconcile.Request) (reconcile.Result
 
 	opcontroller.SetAllowLoopDevices(r.config.Parameters)
 	opcontroller.SetEnforceHostNetwork(r.config.Parameters)
+	opcontroller.SetRevisionHistoryLimit(r.config.Parameters)
 
 	logger.Infof("%s done reconciling", controllerName)
 	return reconcile.Result{}, nil

pkg/operator/ceph/controller/controller_utils.go

@@ -54,6 +54,9 @@ const (
 	enforceHostNetworkSettingName  string = "ROOK_ENFORCE_HOST_NETWORK"
 	enforceHostNetworkDefaultValue string = "false"
 
+	revisionHistoryLimitSettingName  string = "ROOK_REVISION_HISTORY_LIMIT"
+	revisionHistoryLimitDefaultValue string = ""
+
 	// UninitializedCephConfigError refers to the error message printed by the Ceph CLI when there is no ceph configuration file
 	// This typically is raised when the operator has not finished initializing
 	UninitializedCephConfigError = "error calling conf_read_file"
@@ -86,7 +89,8 @@ var (
 	OperatorCephBaseImageVersion string
 
 	// loopDevicesAllowed indicates whether loop devices are allowed to be used
-	loopDevicesAllowed = false
+	loopDevicesAllowed          = false
+	revisionHistoryLimit *int32 = nil
 )
 
 func DiscoveryDaemonEnabled(data map[string]string) bool {
@@ -133,6 +137,26 @@ func EnforceHostNetwork() bool {
 	return cephv1.EnforceHostNetwork()
 }
 
+func SetRevisionHistoryLimit(data map[string]string) {
+	strval := k8sutil.GetValue(data, revisionHistoryLimitSettingName, revisionHistoryLimitDefaultValue)
+	if strval != "" {
+		numval, err := strconv.ParseInt(strval, 10, 32)
+		if err != nil {
+			logger.Warningf("failed to parse value %q for %q. assuming default value.", strval, revisionHistoryLimitSettingName)
+			revisionHistoryLimit = nil
+			return
+
+		}
+		limit := int32(numval)
+		revisionHistoryLimit = &limit
+	}
+
+}
+
+func RevisionHistoryLimit() *int32 {
+	return revisionHistoryLimit
+}
+
 // canIgnoreHealthErrStatusInReconcile determines whether a status of HEALTH_ERR in the CephCluster can be ignored safely.
 func canIgnoreHealthErrStatusInReconcile(cephCluster cephv1.CephCluster, controllerName string) bool {
 	// Get a list of all the keys causing the HEALTH_ERR status.

pkg/operator/ceph/controller/controller_utils_test.go

@@ -133,6 +133,33 @@ func TestSetEnforceHostNetwork(t *testing.T) {
 	assert.False(t, EnforceHostNetwork())
 }
 
+func TestSetRevisionHistoryLimit(t *testing.T) {
+	opConfig := map[string]string{}
+	t.Run("ROOK_REVISION_HISTORY_LIMIT: test default value", func(t *testing.T) {
+		SetRevisionHistoryLimit(opConfig)
+		assert.Nil(t, RevisionHistoryLimit())
+	})
+
+	var value string = "foo"
+	t.Run("ROOK_REVISION_HISTORY_LIMIT: test invalid value 'foo'", func(t *testing.T) {
+		opConfig[revisionHistoryLimitSettingName] = value
+		SetRevisionHistoryLimit(opConfig)
+		assert.Nil(t, RevisionHistoryLimit())
+	})
+
+	t.Run("ROOK_REVISION_HISTORY_LIMIT: test empty string value", func(t *testing.T) {
+		value = ""
+		opConfig[revisionHistoryLimitSettingName] = value
+		SetRevisionHistoryLimit(opConfig)
+		assert.Nil(t, RevisionHistoryLimit())
+	})
+	t.Run("ROOK_REVISION_HISTORY_LIMIT:  test valig value '10'", func(t *testing.T) {
+		value = "10"
+		opConfig[revisionHistoryLimitSettingName] = value
+		SetRevisionHistoryLimit(opConfig)
+		assert.Equal(t, int32(10), *RevisionHistoryLimit())
+	})
+}
 func TestIsReadyToReconcile(t *testing.T) {
 	scheme := scheme.Scheme
 	scheme.AddKnownTypes(cephv1.SchemeGroupVersion, &cephv1.CephCluster{}, &cephv1.CephClusterList{})

pkg/operator/ceph/file/mds/spec.go

@@ -94,8 +94,9 @@ func (c *Cluster) makeDeployment(mdsConfig *mdsConfig, fsNamespacedname types.Na
 			Selector: &metav1.LabelSelector{
 				MatchLabels: c.podLabels(mdsConfig, false),
 			},
-			Template: podSpec,
-			Replicas: &replicas,
+			RevisionHistoryLimit: controller.RevisionHistoryLimit(),
+			Template:             podSpec,
+			Replicas:             &replicas,
 			Strategy: apps.DeploymentStrategy{
 				Type: apps.RecreateDeploymentStrategyType,
 			},

pkg/operator/ceph/file/mirror/spec.go

@@ -79,6 +79,7 @@ func (r *ReconcileFilesystemMirror) makeDeployment(daemonConfig *daemonConfig, f
 			Annotations: fsMirror.Spec.Annotations,
 			Labels:      controller.CephDaemonAppLabels(AppName, fsMirror.Namespace, config.FilesystemMirrorType, userID, fsMirror.Name, "cephfilesystemmirrors.ceph.rook.io", true)},
 		Spec: apps.DeploymentSpec{
+			RevisionHistoryLimit: controller.RevisionHistoryLimit(),
 			Selector: &metav1.LabelSelector{
 				MatchLabels: podSpec.Labels,
 			},

pkg/operator/ceph/nfs/spec.go

@@ -191,6 +191,7 @@ func (r *ReconcileCephNFS) makeDeployment(nfs *cephv1.CephNFS, cfg daemonConfig)
 	// Multiple replicas of the nfs service would be handled by creating a service and a new deployment for each one, rather than increasing the pod count here
 	replicas := int32(1)
 	deployment.Spec = apps.DeploymentSpec{
+		RevisionHistoryLimit: controller.RevisionHistoryLimit(),
 		Selector: &metav1.LabelSelector{
 			MatchLabels: getLabels(nfs, cfg.ID, false),
 		},

pkg/operator/ceph/object/cosi/spec.go

@@ -43,7 +43,6 @@ func createCephCOSIDriverDeployment(cephCOSIDriver *cephv1.CephCOSIDriver) (*app
 	replica := int32(1)
 	minReadySeconds := int32(30)
 	progressDeadlineSeconds := int32(600)
-	revisionHistoryLimit := int32(3)
 
 	cephcosidriverDeployment := &appsv1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
@@ -52,15 +51,15 @@ func createCephCOSIDriverDeployment(cephCOSIDriver *cephv1.CephCOSIDriver) (*app
 			Labels:    getCOSILabels(cephCOSIDriver.Name, cephCOSIDriver.Namespace),
 		},
 		Spec: appsv1.DeploymentSpec{
-			Replicas: &replica,
+			RevisionHistoryLimit: controller.RevisionHistoryLimit(),
+			Replicas:             &replica,
 			Selector: &metav1.LabelSelector{
 				MatchLabels: getCOSILabels(cephCOSIDriver.Name, cephCOSIDriver.Namespace),
 			},
 			Template:                cosiPodSpec,
 			Strategy:                strategy,
 			MinReadySeconds:         minReadySeconds,
 			ProgressDeadlineSeconds: &progressDeadlineSeconds,
-			RevisionHistoryLimit:    &revisionHistoryLimit,
 		},
 	}
 

pkg/operator/ceph/object/spec.go

@@ -117,6 +117,7 @@ func (c *clusterConfig) createDeployment(rgwConfig *rgwConfig) (*apps.Deployment
 			Labels:    getLabels(c.store.Name, c.store.Namespace, true),
 		},
 		Spec: apps.DeploymentSpec{
+			RevisionHistoryLimit: controller.RevisionHistoryLimit(),
 			Selector: &metav1.LabelSelector{
 				MatchLabels: getLabels(c.store.Name, c.store.Namespace, false),
 			},

pkg/operator/discover/discover.go

@@ -102,6 +102,7 @@ func (d *Discover) createDiscoverDaemonSet(ctx context.Context, namespace, disco
 			Labels: getLabels(),
 		},
 		Spec: apps.DaemonSetSpec{
+			RevisionHistoryLimit: opcontroller.RevisionHistoryLimit(),
 			Selector: &metav1.LabelSelector{
 				MatchLabels: map[string]string{
 					"app": discoverDaemonsetName,

tests/framework/installer/ceph_helm_installer.go

@@ -57,6 +57,8 @@ func (h *CephInstaller) configureRookOperatorViaHelm(upgrade bool) error {
 		"enableDiscoveryDaemon": h.settings.EnableDiscovery,
 		"image":                 map[string]interface{}{"tag": h.settings.RookVersion},
 		"monitoring":            map[string]interface{}{"enabled": true},
+		"revisionHistoryLimit":  "3",
+		"enforceHostNetwork":    "false",
 	}
 	values["csi"] = map[string]interface{}{
 		"csiRBDProvisionerResource":    nil,