Skip to content

Commit

Permalink
add server implementation for peerblockpool rpc call
Browse files Browse the repository at this point in the history
Signed-off-by: Rewant Soni <[email protected]>
  • Loading branch information
rewantsoni committed Mar 4, 2024
1 parent ea05750 commit 42c6b87
Show file tree
Hide file tree
Showing 5 changed files with 262 additions and 2 deletions.
29 changes: 28 additions & 1 deletion deploy/ocs-operator/manifests/provider-role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,37 @@ rules:
- ""
resources:
- configmaps
- secrets
- services
verbs:
- get
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- create
- apiGroups:
- ceph.rook.io
resources:
- cephblockpools/finalizers
verbs:
- update
- apiGroups:
- ceph.rook.io
resources:
- cephfilesystemsubvolumegroups
verbs:
- get
- list
- apiGroups:
- ceph.rook.io
resources:
- cephblockpools
verbs:
- get
- update
- create
- apiGroups:
- ocs.openshift.io
resources:
Expand Down Expand Up @@ -53,3 +73,10 @@ rules:
- list
- create
- delete
- apiGroups:
- ceph.rook.io
resources:
- cephrbdmirrors
verbs:
- get
- create
29 changes: 28 additions & 1 deletion rbac/provider-role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,37 @@ rules:
- ""
resources:
- configmaps
- secrets
- services
verbs:
- get
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- create
- apiGroups:
- ceph.rook.io
resources:
- cephblockpools/finalizers
verbs:
- update
- apiGroups:
- ceph.rook.io
resources:
- cephfilesystemsubvolumegroups
verbs:
- get
- list
- apiGroups:
- ceph.rook.io
resources:
- cephblockpools
verbs:
- get
- update
- create
- apiGroups:
- ocs.openshift.io
resources:
Expand Down Expand Up @@ -53,3 +73,10 @@ rules:
- list
- create
- delete
- apiGroups:
- ceph.rook.io
resources:
- cephrbdmirrors
verbs:
- get
- create
112 changes: 112 additions & 0 deletions services/provider/server/cephblockpool.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
package server

import (
"context"
"crypto/md5"
"encoding/hex"
"encoding/json"
"fmt"
"slices"

rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"

corev1 "k8s.io/api/core/v1"
kerrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)

type cephBlockPoolManager struct {
client client.Client
namespace string
}

func newCephBlockPoolManager(cl client.Client, namespace string) (*cephBlockPoolManager, error) {
return &cephBlockPoolManager{
client: cl,
namespace: namespace,
}, nil
}

func (c *cephBlockPoolManager) EnableBlockPoolMirroring(ctx context.Context, blockPoolName string) error {

cephBlockPool := &rookCephv1.CephBlockPool{}
cephBlockPool.Name = blockPoolName
cephBlockPool.Namespace = c.namespace

_, err := ctrl.CreateOrUpdate(ctx, c.client, cephBlockPool, func() error {
cephBlockPool.Spec.Mirroring.Enabled = true
cephBlockPool.Spec.Mirroring.Mode = "image"
return nil
})
if err != nil {
return fmt.Errorf("failed to enable mirroring on CephBlockPool resource with name %q. %v", blockPoolName, err)
}

return nil

}

func (c *cephBlockPoolManager) SetBootstrapSecretRef(ctx context.Context, blockPoolName string, secretData map[string][]byte) error {

cephBlockPool, err := c.GetBlockPoolByName(ctx, blockPoolName)
if err != nil {
return err
}

// create the secret
secretJSON, err := json.Marshal(secretData)
if err != nil {
return err
}
secretNameMd5Sum := md5.Sum(secretJSON)
secretName := hex.EncodeToString(secretNameMd5Sum[:16])

bootstrapSecret := &corev1.Secret{}
bootstrapSecret.Name = secretName
bootstrapSecret.Namespace = c.namespace

_, err = ctrl.CreateOrUpdate(ctx, c.client, bootstrapSecret, func() error {
bootstrapSecret.Data = secretData
err := ctrl.SetControllerReference(cephBlockPool, bootstrapSecret, c.client.Scheme())
if err != nil {
return err
}
return nil
})
if err != nil {
return fmt.Errorf("failed to create/update the bootstrap secret %q. %v", secretName, err)
}

_, err = ctrl.CreateOrUpdate(ctx, c.client, cephBlockPool, func() error {
if cephBlockPool.Spec.Mirroring.Peers == nil {
cephBlockPool.Spec.Mirroring.Peers = &rookCephv1.MirroringPeerSpec{SecretNames: []string{secretName}}
} else {
index := slices.IndexFunc(cephBlockPool.Spec.Mirroring.Peers.SecretNames, func(s string) bool {
return s == secretName
})
if index < 0 {
cephBlockPool.Spec.Mirroring.Peers.SecretNames = append(cephBlockPool.Spec.Mirroring.Peers.SecretNames, secretName)
}
}
return nil
})
if err != nil {
return fmt.Errorf("failed to set bootstrap secret ref on CephBlockPool resource with name %q. %v", blockPoolName, err)
}

return nil
}

func (c *cephBlockPoolManager) GetBlockPoolByName(ctx context.Context, blockPoolName string) (*rookCephv1.CephBlockPool, error) {
blockPoolObj := &rookCephv1.CephBlockPool{}
err := c.client.Get(ctx, types.NamespacedName{Name: blockPoolName, Namespace: c.namespace}, blockPoolObj)
if err != nil {
if kerrors.IsNotFound(err) {
return nil, fmt.Errorf("CephBlockPool resource %q not found. %v", blockPoolName, err)
}
return nil, fmt.Errorf("failed to get CephBlockPool resource with name %q. %v", blockPoolName, err)
}
return blockPoolObj, nil
}
49 changes: 49 additions & 0 deletions services/provider/server/cephrbdmirror.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
package server

import (
"context"
kerrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"

rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"

"sigs.k8s.io/controller-runtime/pkg/client"
)

const rBDMirrorName = "rbd-mirror"

type cephRBDMirrorManager struct {
client client.Client
namespace string
}

func newCephRBDMirrorManager(cl client.Client, namespace string) (*cephRBDMirrorManager, error) {
return &cephRBDMirrorManager{
client: cl,
namespace: namespace,
}, nil
}

func (c *cephRBDMirrorManager) Create(ctx context.Context) error {

cephRBDMirrorObj := &rookCephv1.CephRBDMirror{}
err := c.client.Get(ctx, types.NamespacedName{Name: rBDMirrorName, Namespace: c.namespace}, cephRBDMirrorObj)
if err == nil {
return nil
}

if err != nil && !kerrors.IsNotFound(err) {
return err
}

cephRBDMirrorObj.Name = rBDMirrorName
cephRBDMirrorObj.Namespace = c.namespace
cephRBDMirrorObj.Spec = rookCephv1.RBDMirroringSpec{Count: 1}

err = c.client.Create(ctx, cephRBDMirrorObj)
if err != nil {
return err
}

return nil
}
45 changes: 45 additions & 0 deletions services/provider/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ type OCSProviderServer struct {
client client.Client
consumerManager *ocsConsumerManager
storageClassRequestManager *storageClassRequestManager
cephBlockPoolManager *cephBlockPoolManager
cephRBDMirrorManager *cephRBDMirrorManager
namespace string
}

Expand All @@ -77,10 +79,23 @@ func NewOCSProviderServer(ctx context.Context, namespace string) (*OCSProviderSe
return nil, fmt.Errorf("failed to create new StorageClassRequest instance. %v", err)
}

cephBlockPoolManager, err := newCephBlockPoolManager(client, namespace)
if err != nil {
return nil, fmt.Errorf("failed to create new CephBlockPool instance. %v", err)
}

cephRBDMirrorManager, err := newCephRBDMirrorManager(client, namespace)
if err != nil {
return nil, fmt.Errorf("failed to create new CephRBDMirror instance. %v", err)

}

return &OCSProviderServer{
client: client,
consumerManager: consumerManager,
storageClassRequestManager: storageClassRequestManager,
cephBlockPoolManager: cephBlockPoolManager,
cephRBDMirrorManager: cephRBDMirrorManager,
namespace: namespace,
}, nil
}
Expand Down Expand Up @@ -690,3 +705,33 @@ func (s *OCSProviderServer) ReportStatus(ctx context.Context, req *pb.ReportStat

return &pb.ReportStatusResponse{}, nil
}

// PeerBlockPool RPC call to send the bootstrap secret for the pool
func (s *OCSProviderServer) PeerBlockPool(ctx context.Context, req *pb.PeerBlockPoolRequest) (*pb.PeerBlockPoolResponse, error) {

_, err := s.cephBlockPoolManager.GetBlockPoolByName(ctx, req.BlockPoolName)
if err != nil {
return nil, status.Errorf(codes.NotFound, "Failed to find CephBlockPool resource %s: %v", req.BlockPoolName, err)
}

if err := s.cephRBDMirrorManager.Create(ctx); err != nil {
return nil, status.Errorf(codes.Internal, "Failed to create RBDMirror instance: %v", err)
}

// enable mirroring on blockPool in the req
if err := s.cephBlockPoolManager.EnableBlockPoolMirroring(ctx, req.BlockPoolName); err != nil {
return nil, status.Errorf(codes.Internal, "Failed to enable mirroring for CephBlockPool resource %s: %v", req.BlockPoolName, err)
}

// create and set secret ref on the blockPool
if err := s.cephBlockPoolManager.SetBootstrapSecretRef(
ctx,
req.BlockPoolName,
map[string][]byte{
"pool": req.Pool,
"token": req.Token,
}); err != nil {
return nil, status.Errorf(codes.Internal, "Failed to set bootstrap secret ref for CephBlockPool resource %s: %v", req.BlockPoolName, err)
}
return &pb.PeerBlockPoolResponse{}, nil
}

0 comments on commit 42c6b87

Please sign in to comment.