From 391f24ae4ece3844ba016ac81959a62339ca072a Mon Sep 17 00:00:00 2001 From: Colin Dickson Date: Wed, 27 Sep 2023 15:36:46 -0400 Subject: [PATCH] Feature/network config override (#316) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * run/gui: add network override option * update substreams-endpoint cmd flag usage string * show deprecation warning if endpoint cannot be resolved * use hardcoded endpoints map, if possible, if user does not specify an endpoint * improve log message for network-> endpoint * updated changelog --------- Co-authored-by: Stéphane Duchesneau --- cmd/substreams/gui.go | 29 ++++++++++-------- cmd/substreams/run.go | 9 ++++-- docs/release-notes/change-log.md | 4 +++ manifest/utils.go | 52 ++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 14 deletions(-) diff --git a/cmd/substreams/gui.go b/cmd/substreams/gui.go index 6f35b2b86..00543bab0 100644 --- a/cmd/substreams/gui.go +++ b/cmd/substreams/gui.go @@ -3,10 +3,6 @@ package main import ( "errors" "fmt" - "os" - "path/filepath" - "strings" - tea "github.com/charmbracelet/bubbletea" "github.com/spf13/cobra" "github.com/streamingfast/cli" @@ -15,11 +11,14 @@ import ( "github.com/streamingfast/substreams/tools" "github.com/streamingfast/substreams/tui2" "github.com/streamingfast/substreams/tui2/pages/request" + "os" + "path/filepath" + "strings" ) func init() { guiCmd.Flags().String("substreams-api-token-envvar", "SUBSTREAMS_API_TOKEN", "name of variable containing Substreams Authentication token") - guiCmd.Flags().StringP("substreams-endpoint", "e", "mainnet.eth.streamingfast.io:443", "Substreams gRPC endpoint") + guiCmd.Flags().StringP("substreams-endpoint", "e", "", "Substreams gRPC endpoint. If empty, will be replaced by the SUBSTREAMS_ENDPOINT_{network_name} environment variable, where `network_name` is determined from the substreams manifest. Some network names have default endpoints.") guiCmd.Flags().Bool("insecure", false, "Skip certificate validation on GRPC connection") guiCmd.Flags().Bool("plaintext", false, "Establish GRPC connection in plaintext") guiCmd.Flags().StringSliceP("header", "H", nil, "Additional headers to be sent in the substreams request") @@ -79,13 +78,6 @@ func runGui(cmd *cobra.Command, args []string) error { outputModule := args[0] - substreamsClientConfig := client.NewSubstreamsClientConfig( - mustGetString(cmd, "substreams-endpoint"), - tools.ReadAPIToken(cmd, "substreams-api-token-envvar"), - mustGetBool(cmd, "insecure"), - mustGetBool(cmd, "plaintext"), - ) - manifestReader, err := manifest.NewReader(manifestPath) if err != nil { return fmt.Errorf("manifest reader: %w", err) @@ -94,6 +86,19 @@ func runGui(cmd *cobra.Command, args []string) error { if err != nil { return fmt.Errorf("read manifest %q: %w", manifestPath, err) } + + endpoint, err := manifest.ExtractNetworkEndpoint(pkg.Network, mustGetString(cmd, "substreams-endpoint"), zlog) + if err != nil { + return fmt.Errorf("extracting endpoint: %w", err) + } + + substreamsClientConfig := client.NewSubstreamsClientConfig( + endpoint, + tools.ReadAPIToken(cmd, "substreams-api-token-envvar"), + mustGetBool(cmd, "insecure"), + mustGetBool(cmd, "plaintext"), + ) + params := mustGetStringArray(cmd, "params") if err := manifest.ApplyParams(params, pkg); err != nil { return err diff --git a/cmd/substreams/run.go b/cmd/substreams/run.go index 4a9ff0314..195bf9315 100644 --- a/cmd/substreams/run.go +++ b/cmd/substreams/run.go @@ -17,7 +17,7 @@ import ( ) func init() { - runCmd.Flags().StringP("substreams-endpoint", "e", "mainnet.eth.streamingfast.io:443", "Substreams gRPC endpoint") + runCmd.Flags().StringP("substreams-endpoint", "e", "", "Substreams gRPC endpoint. If empty, will be replaced by the SUBSTREAMS_ENDPOINT_{network_name} environment variable, where `network_name` is determined from the substreams manifest. Some network names have default endpoints.") runCmd.Flags().String("substreams-api-token-envvar", "SUBSTREAMS_API_TOKEN", "name of variable containing Substreams Authentication token") runCmd.Flags().StringP("start-block", "s", "", "Start block to stream from. If empty, will be replaced by initialBlock of the first module you are streaming. If negative, will be resolved by the server relative to the chain head") runCmd.Flags().StringP("cursor", "c", "", "Cursor to stream from. Leave blank for no cursor") @@ -78,6 +78,11 @@ func runRun(cmd *cobra.Command, args []string) error { return fmt.Errorf("read manifest %q: %w", manifestPath, err) } + endpoint, err := manifest.ExtractNetworkEndpoint(pkg.Network, mustGetString(cmd, "substreams-endpoint"), zlog) + if err != nil { + return fmt.Errorf("extracting endpoint: %w", err) + } + if err := manifest.ApplyParams(mustGetStringArray(cmd, "params"), pkg); err != nil { return err } @@ -124,7 +129,7 @@ func runRun(cmd *cobra.Command, args []string) error { } substreamsClientConfig := client.NewSubstreamsClientConfig( - mustGetString(cmd, "substreams-endpoint"), + endpoint, tools.ReadAPIToken(cmd, "substreams-api-token-envvar"), mustGetBool(cmd, "insecure"), mustGetBool(cmd, "plaintext"), diff --git a/docs/release-notes/change-log.md b/docs/release-notes/change-log.md index 3ae721fa1..9e388cfd5 100644 --- a/docs/release-notes/change-log.md +++ b/docs/release-notes/change-log.md @@ -50,6 +50,10 @@ params: module_name_1: "override_paramete_value_here" ``` +* The `substreams run` and `substreams gui` commands now determine the endpoint from the 'network' field in the manifest if no value is passed in the `--substreams-endpoint` flag. +* The endpoint for each network can be set by using an environment variable `SUBSTREAMS_ENDPOINTS_CONFIG_`, ex: `SUBSTREAMS_ENDPOINTS_CONFIG_MAINNET=my-endpoint:443` + + ## v1.1.14 ### Bug fixes diff --git a/manifest/utils.go b/manifest/utils.go index 532cb908c..acbc31a96 100644 --- a/manifest/utils.go +++ b/manifest/utils.go @@ -2,6 +2,10 @@ package manifest import ( "fmt" + "os" + "strings" + + "go.uber.org/zap" "gopkg.in/yaml.v3" ) @@ -27,3 +31,51 @@ func (s *mapSlice) UnmarshalYAML(n *yaml.Node) error { return nil } + +func ExtractNetworkEndpoint(networkFromManifest, fromFlag string, logger *zap.Logger) (string, error) { + if fromFlag != "" { + return fromFlag, nil + } + + if networkFromManifest == "" { + logger.Warn("DEPRECATION WARNING: This substreams does not define a 'network' field. To allow endpoint inference, define a 'network' field in your Substreams manifest. See --help for more information. Assuming 'mainnet' as network") + networkFromManifest = "mainnet" + } + + if endpoint := getNetworkEndpointFromEnvironment(networkFromManifest); endpoint != "" { + logger.Info("using endpoint from environment", zap.String("manifest_network", networkFromManifest), zap.String("endpoint", endpoint)) + return endpoint, nil + } + + if ep, ok := HardcodedEndpoints[networkFromManifest]; ok { + logger.Info("using endpoint from hardcoded list", zap.String("manifest_network", networkFromManifest), zap.String("endpoint", ep)) + return ep, nil + } + + return "", fmt.Errorf("cannot determine endpoint for network %q: make sure that you set SUBSTREAMS_ENDPOINTS_CONFIG_%s environment variable to a valid endpoint, or use the endpoint flag", networkFromManifest, strings.ToUpper(networkFromManifest)) +} + +func getNetworkEndpointFromEnvironment(networkName string) string { + networkEndpoint := os.Getenv(fmt.Sprintf("SUBSTREAMS_ENDPOINTS_CONFIG_%s", strings.ToUpper(networkName))) + return networkEndpoint +} + +// TODO: replace by the blockchain-based discovery when available +var HardcodedEndpoints = map[string]string{ + "mainnet": "mainnet.eth.streamingfast.io:443", + "matic": "polygon.streamingfast.io:443", + "polygon": "polygon.streamingfast.io:443", + "mumbais": "mumbai.streamingfast.io:443", + "bnb": "bnb.streamingfast.io:443", + "bsc": "bnb.streamingfast.io:443", + "sepolia": "sepolia.eth.streamingfast.io:443", + "holesky": "holesky.eth.streamingfast.io:443", + "near": "mainnet.near.streamingfast.io:443", + "near-mainnet": "mainnet.near.streamingfast.io:443", + "near-testnet": "testnet.near.streamingfast.io:443", + "arbitrum": "arb-one.streamingfast.io:443", + "arb": "arb-one.streamingfast.io:443", + "arb-one": "arb-one.streamingfast.io:443", + + //"antelope": "", +}