feat(orchestration): nokube v2 (#391)

* wip(orchestration): add new inspector pkg

* wip(orchestration): use enablek8s flag

* wip(orchestration): rename inspector pkg to nokube

* wip(orchestration): extract cluster option methods

* wip(orchestration): make duplicate of k8s in nokube and remove k8s dependencies

* refactor(orchestration): simplify node logic

* wip(orchestration): extract NodeOrchestrator interface

* wip(orchestration): init NodeOrchestrator in cluster

* wip(orchestration): remove nokube package

* refactor(orchestration): extract node orchestrator to separate folder

* fix(orchestration): add check for ErrNotSet

* fix(orchestration): return expected results for RunningNodes and StoppedNodes

* refactor(orchestration): remove isK8senabled check

* feat(config): enabile usage of static endpoints

* chore(config): set flag to false for use-static-endpoints

* fix: check if UseStaticEndpoints option enabled and add WithNoOptions

* fix: add GetUseStaticEndpoints function

* fix(endpoints): rename variables

* fix(cluster): split code for statit endpoint init

cmd/beekeeper/cmd/check.go

@@ -41,7 +41,11 @@ func (c *command) initCheckCmd() (err error) {
 			}
 
 			// setup cluster
-			cluster, err := c.setupCluster(ctx, c.globalConfig.GetString(optionNameClusterName), c.config, c.globalConfig.GetBool(optionNameCreateCluster))
+			cluster, err := c.setupCluster(ctx,
+				c.globalConfig.GetString(optionNameClusterName),
+				c.config,
+				c.globalConfig.GetBool(optionNameCreateCluster),
+			)
 			if err != nil {
 				return fmt.Errorf("cluster setup: %w", err)
 			}

cmd/beekeeper/cmd/cluster.go

@@ -119,6 +119,9 @@ func (c *command) setupCluster(ctx context.Context, clusterName string, cfg *con
 	var fundOpts orchestration.FundingOptions
 
 	if startCluster {
+		if clusterConfig.IsUsingStaticEndpoints() {
+			return nil, errors.New("static endpoints are not supported for starting the cluster")
+		}
 		if chainNodeEndpoint = c.globalConfig.GetString(optionNameChainNodeEndpoint); chainNodeEndpoint == "" {
 			return nil, errors.New("chain node endpoint (geth-url) not provided")
 		}
@@ -182,13 +185,14 @@ func ensureFundingDefaults(fundOpts orchestration.FundingOptions, log logging.Lo
 
 func configureCluster(clusterConfig config.Cluster, c *command) orchestration.Cluster {
 	clusterOpts := clusterConfig.Export()
-	clusterOpts.K8SClient = c.k8sClient
 	clusterOpts.SwapClient = c.swapClient
+	clusterOpts.K8SClient = c.k8sClient
 	return orchestrationK8S.NewCluster(clusterConfig.GetName(), clusterOpts, c.log)
 }
 
 func setupNodes(ctx context.Context, clusterConfig config.Cluster, cfg *config.Config, bootnode bool, cluster orchestration.Cluster, startCluster bool, bootnodesIn string, nodeResultCh chan nodeResult) (fundAddresses []string, bootnodesOut string, err error) {
 	var nodeCount uint32
+
 	for ngName, v := range clusterConfig.GetNodeGroups() {
 
 		if (v.Mode != bootnodeMode && bootnode) || (v.Mode == bootnodeMode && !bootnode) {
@@ -220,6 +224,17 @@ func setupNodes(ctx context.Context, clusterConfig config.Cluster, cfg *config.C
 			return nil, "", fmt.Errorf("get node group: %w", err)
 		}
 
+		if clusterConfig.IsUsingStaticEndpoints() {
+			for nodeName, endpoint := range v.GetEndpoints() {
+				beeOpt := orchestration.WithURLs(endpoint.APIURL, endpoint.DebugAPIURL)
+				nodeCount++
+				go setupOrAddNode(ctx, false, ng, nodeName, orchestration.NodeOptions{
+					Config: &bConfig,
+				}, nodeResultCh, beeOpt)
+			}
+			continue
+		}
+
 		for i, node := range v.Nodes {
 			// set node name
 			nodeName := fmt.Sprintf("%s-%d", ngName, i)
@@ -228,7 +243,6 @@ func setupNodes(ctx context.Context, clusterConfig config.Cluster, cfg *config.C
 			}
 
 			var nodeOpts orchestration.NodeOptions
-
 			if bootnode {
 				// set bootnodes
 				bConfig.Bootnodes = fmt.Sprintf(node.Bootnodes, clusterConfig.GetNamespace()) // TODO: improve bootnode management, support more than 2 bootnodes
@@ -237,17 +251,16 @@ func setupNodes(ctx context.Context, clusterConfig config.Cluster, cfg *config.C
 			} else {
 				nodeOpts = setupNodeOptions(node, nil)
 			}
-
 			nodeCount++
-			go setupOrAddNode(ctx, startCluster, ng, nodeName, nodeOpts, nodeResultCh)
+			go setupOrAddNode(ctx, startCluster, ng, nodeName, nodeOpts, nodeResultCh, orchestration.WithNoOptions())
 		}
 
 		if len(v.Nodes) == 0 && !bootnode {
 			for i := 0; i < v.Count; i++ {
 				// set node name
 				nodeName := fmt.Sprintf("%s-%d", ngName, i)
 				nodeCount++
-				go setupOrAddNode(ctx, startCluster, ng, nodeName, orchestration.NodeOptions{}, nodeResultCh)
+				go setupOrAddNode(ctx, startCluster, ng, nodeName, orchestration.NodeOptions{}, nodeResultCh, orchestration.WithNoOptions())
 			}
 		}
 	}
@@ -267,12 +280,12 @@ func setupNodes(ctx context.Context, clusterConfig config.Cluster, cfg *config.C
 	return fundAddresses, bootnodesOut, nil
 }
 
-func setupOrAddNode(ctx context.Context, startCluster bool, ng orchestration.NodeGroup, nName string, nodeOpts orchestration.NodeOptions, ch chan<- nodeResult) {
+func setupOrAddNode(ctx context.Context, startCluster bool, ng orchestration.NodeGroup, nName string, nodeOpts orchestration.NodeOptions, ch chan<- nodeResult, beeOpt orchestration.BeeClientOption) {
 	if startCluster {
 		ethAddress, err := ng.SetupNode(ctx, nName, nodeOpts)
 		ch <- nodeResult{ethAddress: ethAddress, err: err}
 	} else {
-		err := ng.AddNode(ctx, nName, nodeOpts)
+		err := ng.AddNode(ctx, nName, nodeOpts, beeOpt)
 		ch <- nodeResult{err: err}
 	}
 }

cmd/beekeeper/cmd/cmd.go

@@ -36,6 +36,9 @@ const (
 	optionNameTracingHost        = "tracing-host"
 	optionNameTracingPort        = "tracing-port"
 	optionNameTracingServiceName = "tracing-service-name"
+	optionNameEnableK8S          = "enable-k8s"
+	optionNameInCluster          = "in-cluster"
+	optionNameKubeconfig         = "kubeconfig"
 )
 
 func init() {
@@ -309,10 +312,9 @@ func (c *command) preRunE(cmd *cobra.Command, args []string) (err error) {
 }
 
 func (c *command) setK8S() (err error) {
-	if c.globalConfig.GetBool("enable-k8s") {
-
-		inCluster := c.globalConfig.GetBool("in-cluster")
-		kubeconfigPath := c.globalConfig.GetString("kubeconfig")
+	if c.globalConfig.GetBool(optionNameEnableK8S) {
+		inCluster := c.globalConfig.GetBool(optionNameInCluster)
+		kubeconfigPath := c.globalConfig.GetString(optionNameKubeconfig)
 
 		options := []k8s.ClientOption{
 			k8s.WithLogger(c.log),

cmd/beekeeper/cmd/simulate.go

@@ -41,7 +41,11 @@ func (c *command) initSimulateCmd() (err error) {
 			}
 
 			// setup cluster
-			cluster, err := c.setupCluster(ctx, c.globalConfig.GetString(optionNameClusterName), c.config, c.globalConfig.GetBool(optionNameCreateCluster))
+			cluster, err := c.setupCluster(ctx,
+				c.globalConfig.GetString(optionNameClusterName),
+				c.config,
+				c.globalConfig.GetBool(optionNameCreateCluster),
+			)
 			if err != nil {
 				return fmt.Errorf("cluster setup: %w", err)
 			}

config/config.yaml

@@ -6,6 +6,7 @@ clusters:
     name: bee
     namespace: beekeeper
     disable-namespace: false
+    use-static-endpoints: false
     api-domain: staging.internal
     api-insecure-tls: true
     api-scheme: https
@@ -19,6 +20,10 @@ clusters:
     node-groups:
       bootnode:
         mode: bootnode
+        endpoints:
+          - name: bootnode-0
+            api-url: https://bootnode-0.beekeeper.testnet.internal
+            debug-api-url: https://bootnode-0-debug.beekeeper.testnet.internal
         bee-config: bootnode
         config: default
         nodes:
@@ -41,6 +46,16 @@ clusters:
         bee-config: default
         config: default
         count: 3
+        endpoints:
+          - name: bee-0
+            api-url: https://bee-0.beekeeper.testnet.internal
+            debug-api-url: https://bee-0-debug.beekeeper.testnet.internal
+          - name: bee-1
+            api-url: https://bee-1.beekeeper.testnet.internal
+            debug-api-url: https://bee-1-debug.beekeeper.testnet.internal
+          - name: bee-2
+            api-url: https://bee-2.beekeeper.testnet.internal
+            debug-api-url: https://bee-2-debug.beekeeper.testnet.internal
         # nodes:
         # - clef:
         #     key: '{"address":"4558ab6d518bf60b813eeba3077eed986027c5da","crypto":{"cipher":"aes-128-ctr","ciphertext":"1bbeffa438a8b8fd592a46323fe0168d8d8e2625085ca8550023b5c0bd48a126","cipherparams":{"iv":"3f369a742a465aaf5e3025864639421a"},"kdf":"scrypt","kdfparams":{"dklen":32,"n":64,"p":1,"r":8,"salt":"4c2c1fde6491213ea3c6021c82a70327bc0a056569a6e7c2a3fda9e486c0f090"},"mac":"f733b77f675acf0539e7d3d60735408c6efd43893dc0d5b0f94124b0197f89dd"},"id":"1e526dc4-60bd-4c4d-897d-f284806abf2b","version":3}'
@@ -56,6 +71,16 @@ clusters:
         bee-config: light-node
         config: light-node
         count: 3
+        endpoints:
+          - name: light-0
+            api-url: https://light-0.beekeeper.testnet.internal
+            debug-api-url: https://light-0-debug.beekeeper.testnet.internal
+          - name: light-1
+            api-url: https://light-1.beekeeper.testnet.internal
+            debug-api-url: https://light-1-debug.beekeeper.testnet.internal
+          - name: light-2
+            api-url: https://light-2.beekeeper.testnet.internal
+            debug-api-url: https://light-2-debug.beekeeper.testnet.internal
 
 # node-groups defines node groups that can be registered in the cluster
 # node-groups may inherit it's configuration from already defined node-group and override specific fields from it
@@ -369,7 +394,6 @@ checks:
       data-size:
     type: redundancy
 
-
 # simulations defines simulations Beekeeper can execute against the cluster
 # type filed allows defining same simulation with different names and options
 simulations:

pkg/bee/client.go

@@ -24,10 +24,10 @@ const retryCount int = 5
 
 // Client manages communication with the Bee node
 type Client struct {
-	api    *api.Client
-	debug  *debugapi.Client
-	opts   ClientOptions
-	logger logging.Logger
+	api   *api.Client
+	debug *debugapi.Client
+	opts  ClientOptions
+	log   logging.Logger
 	// number of times to retry call
 	retry int
 }
@@ -43,11 +43,11 @@ type ClientOptions struct {
 }
 
 // NewClient returns Bee client
-func NewClient(opts ClientOptions, logger logging.Logger) (c *Client) {
+func NewClient(opts ClientOptions, log logging.Logger) (c *Client) {
 	c = &Client{
-		retry:  retryCount,
-		opts:   opts,
-		logger: logger,
+		retry: retryCount,
+		opts:  opts,
+		log:   log,
 	}
 
 	if opts.APIURL != nil {
@@ -384,7 +384,7 @@ func (c *Client) CreatePostageBatch(ctx context.Context, amount int64, depth uin
 		if err != nil {
 			return "", fmt.Errorf("print reserve state (before): %w", err)
 		}
-		c.logger.Infof("reserve state (prior to buying the batch):%s", rs.String())
+		c.log.Infof("reserve state (prior to buying the batch):%s", rs.String())
 	}
 	id, err := c.debug.Postage.CreatePostageBatch(ctx, amount, depth, label)
 	if err != nil {
@@ -420,8 +420,8 @@ func (c *Client) CreatePostageBatch(ctx context.Context, amount int64, depth uin
 		if err != nil {
 			return "", fmt.Errorf("print reserve state (after): %w", err)
 		}
-		c.logger.Infof("reserve state (after buying the batch):\n%s", rs.String())
-		c.logger.Infof("created batch id %s with depth %d and amount %d", id, depth, amount)
+		c.log.Infof("reserve state (after buying the batch):\n%s", rs.String())
+		c.log.Infof("created batch id %s with depth %d and amount %d", id, depth, amount)
 	}
 	return id, nil
 }

pkg/config/cluster.go

@@ -14,6 +14,7 @@ type Cluster struct {
 	Name                *string                      `yaml:"name"`
 	Namespace           *string                      `yaml:"namespace"`
 	DisableNamespace    *bool                        `yaml:"disable-namespace"`
+	UseStaticEndpoints  *bool                        `yaml:"use-static-endpoints"`
 	APIDomain           *string                      `yaml:"api-domain"`
 	APIInsecureTLS      *bool                        `yaml:"api-insecure-tls"`
 	APIScheme           *string                      `yaml:"api-scheme"`
@@ -27,11 +28,13 @@ type Cluster struct {
 
 // ClusterNodeGroup represents node group in the cluster
 type ClusterNodeGroup struct {
-	Mode      string        `yaml:"mode"`
-	BeeConfig string        `yaml:"bee-config"`
-	Config    string        `yaml:"config"`
-	Count     int           `yaml:"count"`
-	Nodes     []ClusterNode `yaml:"nodes"`
+	cluster       *Cluster
+	Mode          string         `yaml:"mode"`
+	BeeConfig     string         `yaml:"bee-config"`
+	Config        string         `yaml:"config"`
+	Count         int            `yaml:"count"`
+	Nodes         []ClusterNode  `yaml:"nodes"`
+	NodeEndpoints []NodeEndpoint `yaml:"endpoints"`
 }
 
 // ClusterNode represents node in the cluster
@@ -43,6 +46,12 @@ type ClusterNode struct {
 	SwarmKey  string `yaml:"swarm-key"`
 }
 
+type NodeEndpoint struct {
+	Name        string `yaml:"name"`
+	APIURL      string `yaml:"api-url"`
+	DebugAPIURL string `yaml:"debug-api-url"`
+}
+
 type Clef struct {
 	Key      string `yaml:"key"`
 	Password string `yaml:"password"`
@@ -91,5 +100,28 @@ func (c *Cluster) GetNodeGroups() map[string]ClusterNodeGroup {
 	if c.NodeGroups == nil {
 		return nil
 	}
-	return *c.NodeGroups
+
+	nodeGroups := *c.NodeGroups
+	for key, group := range nodeGroups {
+		group.cluster = c // Set the reference to the parent cluster
+		nodeGroups[key] = group
+	}
+
+	return nodeGroups
+}
+
+// IsUsingStaticEndpoints
+func (c *Cluster) IsUsingStaticEndpoints() bool {
+	if c.UseStaticEndpoints == nil {
+		return false
+	}
+	return *c.UseStaticEndpoints
+}
+
+func (ng *ClusterNodeGroup) GetEndpoints() map[string]NodeEndpoint {
+	endpoints := make(map[string]NodeEndpoint)
+	for _, endpoint := range ng.NodeEndpoints {
+		endpoints[endpoint.Name] = endpoint
+	}
+	return endpoints
 }