Feature/network config override (#316)

* run/gui: add network override option
* update substreams-endpoint cmd flag usage string
* show deprecation warning if endpoint cannot be resolved
* use hardcoded endpoints map, if possible, if user does not specify an endpoint
* improve log message for network-> endpoint
* updated changelog

---------

Co-authored-by: Stéphane Duchesneau <[email protected]>

cmd/substreams/gui.go

@@ -3,10 +3,6 @@ package main
 import (
 	"errors"
 	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-
 	tea "github.com/charmbracelet/bubbletea"
 	"github.com/spf13/cobra"
 	"github.com/streamingfast/cli"
@@ -15,11 +11,14 @@ import (
 	"github.com/streamingfast/substreams/tools"
 	"github.com/streamingfast/substreams/tui2"
 	"github.com/streamingfast/substreams/tui2/pages/request"
+	"os"
+	"path/filepath"
+	"strings"
 )
 
 func init() {
 	guiCmd.Flags().String("substreams-api-token-envvar", "SUBSTREAMS_API_TOKEN", "name of variable containing Substreams Authentication token")
-	guiCmd.Flags().StringP("substreams-endpoint", "e", "mainnet.eth.streamingfast.io:443", "Substreams gRPC endpoint")
+	guiCmd.Flags().StringP("substreams-endpoint", "e", "", "Substreams gRPC endpoint. If empty, will be replaced by the SUBSTREAMS_ENDPOINT_{network_name} environment variable, where `network_name` is determined from the substreams manifest. Some network names have default endpoints.")
 	guiCmd.Flags().Bool("insecure", false, "Skip certificate validation on GRPC connection")
 	guiCmd.Flags().Bool("plaintext", false, "Establish GRPC connection in plaintext")
 	guiCmd.Flags().StringSliceP("header", "H", nil, "Additional headers to be sent in the substreams request")
@@ -79,13 +78,6 @@ func runGui(cmd *cobra.Command, args []string) error {
 
 	outputModule := args[0]
 
-	substreamsClientConfig := client.NewSubstreamsClientConfig(
-		mustGetString(cmd, "substreams-endpoint"),
-		tools.ReadAPIToken(cmd, "substreams-api-token-envvar"),
-		mustGetBool(cmd, "insecure"),
-		mustGetBool(cmd, "plaintext"),
-	)
-
 	manifestReader, err := manifest.NewReader(manifestPath)
 	if err != nil {
 		return fmt.Errorf("manifest reader: %w", err)
@@ -94,6 +86,19 @@ func runGui(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("read manifest %q: %w", manifestPath, err)
 	}
+
+	endpoint, err := manifest.ExtractNetworkEndpoint(pkg.Network, mustGetString(cmd, "substreams-endpoint"), zlog)
+	if err != nil {
+		return fmt.Errorf("extracting endpoint: %w", err)
+	}
+
+	substreamsClientConfig := client.NewSubstreamsClientConfig(
+		endpoint,
+		tools.ReadAPIToken(cmd, "substreams-api-token-envvar"),
+		mustGetBool(cmd, "insecure"),
+		mustGetBool(cmd, "plaintext"),
+	)
+
 	params := mustGetStringArray(cmd, "params")
 	if err := manifest.ApplyParams(params, pkg); err != nil {
 		return err

cmd/substreams/run.go

@@ -17,7 +17,7 @@ import (
 )
 
 func init() {
-	runCmd.Flags().StringP("substreams-endpoint", "e", "mainnet.eth.streamingfast.io:443", "Substreams gRPC endpoint")
+	runCmd.Flags().StringP("substreams-endpoint", "e", "", "Substreams gRPC endpoint. If empty, will be replaced by the SUBSTREAMS_ENDPOINT_{network_name} environment variable, where `network_name` is determined from the substreams manifest. Some network names have default endpoints.")
 	runCmd.Flags().String("substreams-api-token-envvar", "SUBSTREAMS_API_TOKEN", "name of variable containing Substreams Authentication token")
 	runCmd.Flags().StringP("start-block", "s", "", "Start block to stream from. If empty, will be replaced by initialBlock of the first module you are streaming. If negative, will be resolved by the server relative to the chain head")
 	runCmd.Flags().StringP("cursor", "c", "", "Cursor to stream from. Leave blank for no cursor")
@@ -78,6 +78,11 @@ func runRun(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("read manifest %q: %w", manifestPath, err)
 	}
 
+	endpoint, err := manifest.ExtractNetworkEndpoint(pkg.Network, mustGetString(cmd, "substreams-endpoint"), zlog)
+	if err != nil {
+		return fmt.Errorf("extracting endpoint: %w", err)
+	}
+
 	if err := manifest.ApplyParams(mustGetStringArray(cmd, "params"), pkg); err != nil {
 		return err
 	}
@@ -124,7 +129,7 @@ func runRun(cmd *cobra.Command, args []string) error {
 	}
 
 	substreamsClientConfig := client.NewSubstreamsClientConfig(
-		mustGetString(cmd, "substreams-endpoint"),
+		endpoint,
 		tools.ReadAPIToken(cmd, "substreams-api-token-envvar"),
 		mustGetBool(cmd, "insecure"),
 		mustGetBool(cmd, "plaintext"),

docs/release-notes/change-log.md

@@ -50,6 +50,10 @@ params:
   module_name_1: "override_paramete_value_here"
 ```
 
+* The `substreams run` and `substreams gui` commands now determine the endpoint from the 'network' field in the manifest if no value is passed in the `--substreams-endpoint` flag.
+* The endpoint for each network can be set by using an environment variable `SUBSTREAMS_ENDPOINTS_CONFIG_<network_name>`, ex: `SUBSTREAMS_ENDPOINTS_CONFIG_MAINNET=my-endpoint:443`
+
+
 ## v1.1.14
 
 ### Bug fixes

manifest/utils.go

@@ -2,6 +2,10 @@ package manifest
 
 import (
 	"fmt"
+	"os"
+	"strings"
+
+	"go.uber.org/zap"
 
 	"gopkg.in/yaml.v3"
 )
@@ -27,3 +31,51 @@ func (s *mapSlice) UnmarshalYAML(n *yaml.Node) error {
 
 	return nil
 }
+
+func ExtractNetworkEndpoint(networkFromManifest, fromFlag string, logger *zap.Logger) (string, error) {
+	if fromFlag != "" {
+		return fromFlag, nil
+	}
+
+	if networkFromManifest == "" {
+		logger.Warn("DEPRECATION WARNING: This substreams does not define a 'network' field. To allow endpoint inference, define a 'network' field in your Substreams manifest. See --help for more information. Assuming 'mainnet' as network")
+		networkFromManifest = "mainnet"
+	}
+
+	if endpoint := getNetworkEndpointFromEnvironment(networkFromManifest); endpoint != "" {
+		logger.Info("using endpoint from environment", zap.String("manifest_network", networkFromManifest), zap.String("endpoint", endpoint))
+		return endpoint, nil
+	}
+
+	if ep, ok := HardcodedEndpoints[networkFromManifest]; ok {
+		logger.Info("using endpoint from hardcoded list", zap.String("manifest_network", networkFromManifest), zap.String("endpoint", ep))
+		return ep, nil
+	}
+
+	return "", fmt.Errorf("cannot determine endpoint for network %q: make sure that you set SUBSTREAMS_ENDPOINTS_CONFIG_%s environment variable to a valid endpoint, or use the endpoint flag", networkFromManifest, strings.ToUpper(networkFromManifest))
+}
+
+func getNetworkEndpointFromEnvironment(networkName string) string {
+	networkEndpoint := os.Getenv(fmt.Sprintf("SUBSTREAMS_ENDPOINTS_CONFIG_%s", strings.ToUpper(networkName)))
+	return networkEndpoint
+}
+
+// TODO: replace by the blockchain-based discovery when available
+var HardcodedEndpoints = map[string]string{
+	"mainnet":      "mainnet.eth.streamingfast.io:443",
+	"matic":        "polygon.streamingfast.io:443",
+	"polygon":      "polygon.streamingfast.io:443",
+	"mumbais":      "mumbai.streamingfast.io:443",
+	"bnb":          "bnb.streamingfast.io:443",
+	"bsc":          "bnb.streamingfast.io:443",
+	"sepolia":      "sepolia.eth.streamingfast.io:443",
+	"holesky":      "holesky.eth.streamingfast.io:443",
+	"near":         "mainnet.near.streamingfast.io:443",
+	"near-mainnet": "mainnet.near.streamingfast.io:443",
+	"near-testnet": "testnet.near.streamingfast.io:443",
+	"arbitrum":     "arb-one.streamingfast.io:443",
+	"arb":          "arb-one.streamingfast.io:443",
+	"arb-one":      "arb-one.streamingfast.io:443",
+
+	//"antelope": "",
+}