diff --git a/deploy/ocs-operator/manifests/provider-role.yaml b/deploy/ocs-operator/manifests/provider-role.yaml index 26d6589e50..40f39e7187 100644 --- a/deploy/ocs-operator/manifests/provider-role.yaml +++ b/deploy/ocs-operator/manifests/provider-role.yaml @@ -7,10 +7,22 @@ rules: - "" resources: - configmaps - - secrets - services verbs: - get +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +- apiGroups: + - ceph.rook.io + resources: + - cephblockpools/finalizers + verbs: + - update - apiGroups: - ceph.rook.io resources: @@ -18,6 +30,14 @@ rules: verbs: - get - list +- apiGroups: + - ceph.rook.io + resources: + - cephblockpools + verbs: + - get + - update + - create - apiGroups: - ocs.openshift.io resources: @@ -53,3 +73,10 @@ rules: - list - create - delete +- apiGroups: + - ceph.rook.io + resources: + - cephrbdmirrors + verbs: + - get + - create diff --git a/rbac/provider-role.yaml b/rbac/provider-role.yaml index 26d6589e50..40f39e7187 100644 --- a/rbac/provider-role.yaml +++ b/rbac/provider-role.yaml @@ -7,10 +7,22 @@ rules: - "" resources: - configmaps - - secrets - services verbs: - get +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +- apiGroups: + - ceph.rook.io + resources: + - cephblockpools/finalizers + verbs: + - update - apiGroups: - ceph.rook.io resources: @@ -18,6 +30,14 @@ rules: verbs: - get - list +- apiGroups: + - ceph.rook.io + resources: + - cephblockpools + verbs: + - get + - update + - create - apiGroups: - ocs.openshift.io resources: @@ -53,3 +73,10 @@ rules: - list - create - delete +- apiGroups: + - ceph.rook.io + resources: + - cephrbdmirrors + verbs: + - get + - create diff --git a/services/provider/server/cephblockpool.go b/services/provider/server/cephblockpool.go new file mode 100644 index 0000000000..2539a6b572 --- /dev/null +++ b/services/provider/server/cephblockpool.go @@ -0,0 +1,112 @@ +package server + +import ( + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "slices" + + rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type cephBlockPoolManager struct { + client client.Client + namespace string +} + +func newCephBlockPoolManager(cl client.Client, namespace string) (*cephBlockPoolManager, error) { + return &cephBlockPoolManager{ + client: cl, + namespace: namespace, + }, nil +} + +func (c *cephBlockPoolManager) EnableBlockPoolMirroring(ctx context.Context, blockPoolName string) error { + + cephBlockPool := &rookCephv1.CephBlockPool{} + cephBlockPool.Name = blockPoolName + cephBlockPool.Namespace = c.namespace + + _, err := ctrl.CreateOrUpdate(ctx, c.client, cephBlockPool, func() error { + cephBlockPool.Spec.Mirroring.Enabled = true + cephBlockPool.Spec.Mirroring.Mode = "image" + return nil + }) + if err != nil { + return fmt.Errorf("failed to enable mirroring on CephBlockPool resource with name %q. %v", blockPoolName, err) + } + + return nil + +} + +func (c *cephBlockPoolManager) SetBootstrapSecretRef(ctx context.Context, blockPoolName string, secretData map[string][]byte) error { + + cephBlockPool, err := c.GetBlockPoolByName(ctx, blockPoolName) + if err != nil { + return err + } + + // create the secret + secretJSON, err := json.Marshal(secretData) + if err != nil { + return err + } + secretNameMd5Sum := md5.Sum(secretJSON) + secretName := hex.EncodeToString(secretNameMd5Sum[:16]) + + bootstrapSecret := &corev1.Secret{} + bootstrapSecret.Name = secretName + bootstrapSecret.Namespace = c.namespace + + _, err = ctrl.CreateOrUpdate(ctx, c.client, bootstrapSecret, func() error { + bootstrapSecret.Data = secretData + err := ctrl.SetControllerReference(cephBlockPool, bootstrapSecret, c.client.Scheme()) + if err != nil { + return err + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to create/update the bootstrap secret %q. %v", secretName, err) + } + + _, err = ctrl.CreateOrUpdate(ctx, c.client, cephBlockPool, func() error { + if cephBlockPool.Spec.Mirroring.Peers == nil { + cephBlockPool.Spec.Mirroring.Peers = &rookCephv1.MirroringPeerSpec{SecretNames: []string{secretName}} + } else { + index := slices.IndexFunc(cephBlockPool.Spec.Mirroring.Peers.SecretNames, func(s string) bool { + return s == secretName + }) + if index < 0 { + cephBlockPool.Spec.Mirroring.Peers.SecretNames = append(cephBlockPool.Spec.Mirroring.Peers.SecretNames, secretName) + } + } + return nil + }) + if err != nil { + return fmt.Errorf("failed to set bootstrap secret ref on CephBlockPool resource with name %q. %v", blockPoolName, err) + } + + return nil +} + +func (c *cephBlockPoolManager) GetBlockPoolByName(ctx context.Context, blockPoolName string) (*rookCephv1.CephBlockPool, error) { + blockPoolObj := &rookCephv1.CephBlockPool{} + err := c.client.Get(ctx, types.NamespacedName{Name: blockPoolName, Namespace: c.namespace}, blockPoolObj) + if err != nil { + if kerrors.IsNotFound(err) { + return nil, fmt.Errorf("CephBlockPool resource %q not found. %v", blockPoolName, err) + } + return nil, fmt.Errorf("failed to get CephBlockPool resource with name %q. %v", blockPoolName, err) + } + return blockPoolObj, nil +} diff --git a/services/provider/server/cephrbdmirror.go b/services/provider/server/cephrbdmirror.go new file mode 100644 index 0000000000..474b158c2d --- /dev/null +++ b/services/provider/server/cephrbdmirror.go @@ -0,0 +1,49 @@ +package server + +import ( + "context" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + + rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const rBDMirrorName = "rbd-mirror" + +type cephRBDMirrorManager struct { + client client.Client + namespace string +} + +func newCephRBDMirrorManager(cl client.Client, namespace string) (*cephRBDMirrorManager, error) { + return &cephRBDMirrorManager{ + client: cl, + namespace: namespace, + }, nil +} + +func (c *cephRBDMirrorManager) Create(ctx context.Context) error { + + cephRBDMirrorObj := &rookCephv1.CephRBDMirror{} + err := c.client.Get(ctx, types.NamespacedName{Name: rBDMirrorName, Namespace: c.namespace}, cephRBDMirrorObj) + if err == nil { + return nil + } + + if err != nil && !kerrors.IsNotFound(err) { + return err + } + + cephRBDMirrorObj.Name = rBDMirrorName + cephRBDMirrorObj.Namespace = c.namespace + cephRBDMirrorObj.Spec = rookCephv1.RBDMirroringSpec{Count: 1} + + err = c.client.Create(ctx, cephRBDMirrorObj) + if err != nil { + return err + } + + return nil +} diff --git a/services/provider/server/server.go b/services/provider/server/server.go index 28b63f0aff..5ba49aad22 100644 --- a/services/provider/server/server.go +++ b/services/provider/server/server.go @@ -58,6 +58,8 @@ type OCSProviderServer struct { client client.Client consumerManager *ocsConsumerManager storageClassRequestManager *storageClassRequestManager + cephBlockPoolManager *cephBlockPoolManager + cephRBDMirrorManager *cephRBDMirrorManager namespace string } @@ -77,10 +79,23 @@ func NewOCSProviderServer(ctx context.Context, namespace string) (*OCSProviderSe return nil, fmt.Errorf("failed to create new StorageClassRequest instance. %v", err) } + cephBlockPoolManager, err := newCephBlockPoolManager(client, namespace) + if err != nil { + return nil, fmt.Errorf("failed to create new CephBlockPool instance. %v", err) + } + + cephRBDMirrorManager, err := newCephRBDMirrorManager(client, namespace) + if err != nil { + return nil, fmt.Errorf("failed to create new CephRBDMirror instance. %v", err) + + } + return &OCSProviderServer{ client: client, consumerManager: consumerManager, storageClassRequestManager: storageClassRequestManager, + cephBlockPoolManager: cephBlockPoolManager, + cephRBDMirrorManager: cephRBDMirrorManager, namespace: namespace, }, nil } @@ -690,3 +705,33 @@ func (s *OCSProviderServer) ReportStatus(ctx context.Context, req *pb.ReportStat return &pb.ReportStatusResponse{}, nil } + +// PeerBlockPool RPC call to send the bootstrap secret for the pool +func (s *OCSProviderServer) PeerBlockPool(ctx context.Context, req *pb.PeerBlockPoolRequest) (*pb.PeerBlockPoolResponse, error) { + + _, err := s.cephBlockPoolManager.GetBlockPoolByName(ctx, req.BlockPoolName) + if err != nil { + return nil, status.Errorf(codes.NotFound, "Failed to find CephBlockPool resource %s: %v", req.BlockPoolName, err) + } + + if err := s.cephRBDMirrorManager.Create(ctx); err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create RBDMirror instance: %v", err) + } + + // enable mirroring on blockPool in the req + if err := s.cephBlockPoolManager.EnableBlockPoolMirroring(ctx, req.BlockPoolName); err != nil { + return nil, status.Errorf(codes.Internal, "Failed to enable mirroring for CephBlockPool resource %s: %v", req.BlockPoolName, err) + } + + // create and set secret ref on the blockPool + if err := s.cephBlockPoolManager.SetBootstrapSecretRef( + ctx, + req.BlockPoolName, + map[string][]byte{ + "pool": req.Pool, + "token": req.Token, + }); err != nil { + return nil, status.Errorf(codes.Internal, "Failed to set bootstrap secret ref for CephBlockPool resource %s: %v", req.BlockPoolName, err) + } + return &pb.PeerBlockPoolResponse{}, nil +}