refactor: move s3 flags to their own package cli.go

e2e/setup.go

@@ -72,14 +72,14 @@ func createS3Config(eigendaCfg server.Config) server.CLIConfig {
 	createS3Bucket(bucketName)
 
 	eigendaCfg.S3Config = s3.Config{
-		Profiling:        true,
-		Bucket:           bucketName,
-		Path:             "",
-		Endpoint:         "localhost:4566",
-		AccessKeySecret:  "minioadmin",
-		AccessKeyID:      "minioadmin",
-		S3CredentialType: s3.CredentialTypeStatic,
-		Backup:           false,
+		Profiling:       true,
+		Bucket:          bucketName,
+		Path:            "",
+		Endpoint:        "localhost:4566",
+		AccessKeySecret: "minioadmin",
+		AccessKeyID:     "minioadmin",
+		CredentialType:  s3.CredentialTypeStatic,
+		Backup:          false,
 	}
 	return server.CLIConfig{
 		EigenDAConfig: eigendaCfg,

flags/flags.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/redis"
+	"github.com/Layr-Labs/eigenda-proxy/store/precomputed_key/s3"
 	"github.com/urfave/cli/v2"
 
 	opservice "github.com/ethereum-optimism/optimism/op-service"
@@ -14,6 +15,7 @@ import (
 const (
 	MemstoreFlagsCategory = "Memstore"
 	RedisCategory         = "Redis"
+	S3Category            = "S3"
 )
 
 const (
@@ -50,16 +52,6 @@ const (
 	MemstorePutLatencyFlagName = "memstore.put-latency"
 	MemstoreGetLatencyFlagName = "memstore.get-latency"
 
-	// S3 client flags
-	S3CredentialTypeFlagName  = "s3.credential-type" // #nosec G101
-	S3BucketFlagName          = "s3.bucket"          // #nosec G101
-	S3PathFlagName            = "s3.path"
-	S3EndpointFlagName        = "s3.endpoint"
-	S3AccessKeyIDFlagName     = "s3.access-key-id"     // #nosec G101
-	S3AccessKeySecretFlagName = "s3.access-key-secret" // #nosec G101
-	S3BackupFlagName          = "s3.backup"
-	S3TimeoutFlagName         = "s3.timeout"
-
 	// routing flags
 	FallbackTargetsFlagName = "routing.fallback-targets"
 	CacheTargetsFlagName    = "routing.cache-targets"
@@ -71,76 +63,21 @@ func prefixEnvVars(name string) []string {
 	return opservice.PrefixEnvVar(EnvVarPrefix, name)
 }
 
-// Flags contains the list of configuration options available to the binary.
-var Flags = []cli.Flag{
-	&cli.StringFlag{
-		Name:    ListenAddrFlagName,
-		Usage:   "server listening address",
-		Value:   "0.0.0.0",
-		EnvVars: prefixEnvVars("ADDR"),
-	},
-	&cli.IntFlag{
-		Name:    PortFlagName,
-		Usage:   "server listening port",
-		Value:   3100,
-		EnvVars: prefixEnvVars("PORT"),
-	},
-}
-
-// s3Flags ... used for S3 backend configuration
-func s3Flags() []cli.Flag {
-	return []cli.Flag{
-		&cli.StringFlag{
-			Name:    S3CredentialTypeFlagName,
-			Usage:   "The way to authenticate to S3, options are [iam, static]",
-			EnvVars: prefixEnvVars("S3_CREDENTIAL_TYPE"),
-		},
-		&cli.StringFlag{
-			Name:    S3BucketFlagName,
-			Usage:   "bucket name for S3 storage",
-			EnvVars: prefixEnvVars("S3_BUCKET"),
-		},
-		&cli.StringFlag{
-			Name:    S3PathFlagName,
-			Usage:   "path for S3 storage",
-			EnvVars: prefixEnvVars("S3_PATH"),
-		},
-		&cli.StringFlag{
-			Name:    S3EndpointFlagName,
-			Usage:   "endpoint for S3 storage",
-			Value:   "",
-			EnvVars: prefixEnvVars("S3_ENDPOINT"),
-		},
-		&cli.StringFlag{
-			Name:    S3AccessKeyIDFlagName,
-			Usage:   "access key id for S3 storage",
-			Value:   "",
-			EnvVars: prefixEnvVars("S3_ACCESS_KEY_ID"),
-		},
-		&cli.StringFlag{
-			Name:    S3AccessKeySecretFlagName,
-			Usage:   "access key secret for S3 storage",
-			Value:   "",
-			EnvVars: prefixEnvVars("S3_ACCESS_KEY_SECRET"),
-		},
-		&cli.BoolFlag{
-			Name:    S3BackupFlagName,
-			Usage:   "whether to use S3 as a backup store to ensure resiliency in case of EigenDA read failure",
-			Value:   false,
-			EnvVars: prefixEnvVars("S3_BACKUP"),
-		},
-		&cli.DurationFlag{
-			Name:    S3TimeoutFlagName,
-			Usage:   "timeout for S3 storage operations (e.g. get, put)",
-			Value:   5 * time.Second,
-			EnvVars: prefixEnvVars("S3_TIMEOUT"),
-		},
-	}
-}
-
 func CLIFlags() []cli.Flag {
 	// TODO: Decompose all flags into constituent parts based on their respective category / usage
 	flags := []cli.Flag{
+		&cli.StringFlag{
+			Name:    ListenAddrFlagName,
+			Usage:   "server listening address",
+			Value:   "0.0.0.0",
+			EnvVars: prefixEnvVars("ADDR"),
+		},
+		&cli.IntFlag{
+			Name:    PortFlagName,
+			Usage:   "server listening port",
+			Value:   3100,
+			EnvVars: prefixEnvVars("PORT"),
+		},
 		&cli.StringFlag{
 			Name:    EigenDADisperserRPCFlagName,
 			Usage:   "RPC endpoint of the EigenDA disperser.",
@@ -281,13 +218,16 @@ func CLIFlags() []cli.Flag {
 		},
 	}
 
-	flags = append(flags, s3Flags()...)
 	return flags
 }
 
+// Flags contains the list of configuration options available to the binary.
+var Flags = []cli.Flag{}
+
 func init() {
 	Flags = CLIFlags()
 	Flags = append(Flags, oplog.CLIFlags(EnvVarPrefix)...)
 	Flags = append(Flags, opmetrics.CLIFlags(EnvVarPrefix)...)
 	Flags = append(Flags, redis.CLIFlags(EnvVarPrefix, RedisCategory)...)
+	Flags = append(Flags, s3.CLIFlags(EnvVarPrefix, S3Category)...)
 }

server/config.go

@@ -117,16 +117,7 @@ func (cfg *Config) VerificationCfg() *verify.Config {
 func ReadConfig(ctx *cli.Context) Config {
 	cfg := Config{
 		RedisConfig: redis.ReadConfig(ctx),
-		S3Config: s3.Config{
-			S3CredentialType: s3.StringToCredentialType(ctx.String(flags.S3CredentialTypeFlagName)),
-			Bucket:           ctx.String(flags.S3BucketFlagName),
-			Path:             ctx.String(flags.S3PathFlagName),
-			Endpoint:         ctx.String(flags.S3EndpointFlagName),
-			AccessKeyID:      ctx.String(flags.S3AccessKeyIDFlagName),
-			AccessKeySecret:  ctx.String(flags.S3AccessKeySecretFlagName),
-			Backup:           ctx.Bool(flags.S3BackupFlagName),
-			Timeout:          ctx.Duration(flags.S3TimeoutFlagName),
-		},
+		S3Config:    s3.ReadConfig(ctx),
 		ClientConfig: clients.EigenDAClientConfig{
 			RPC:                          ctx.String(flags.EigenDADisperserRPCFlagName),
 			StatusQueryRetryInterval:     ctx.Duration(flags.StatusQueryRetryIntervalFlagName),
@@ -214,10 +205,10 @@ func (cfg *Config) Check() error {
 		}
 	}
 
-	if cfg.S3Config.S3CredentialType == s3.CredentialTypeUnknown && cfg.S3Config.Endpoint != "" {
+	if cfg.S3Config.CredentialType == s3.CredentialTypeUnknown && cfg.S3Config.Endpoint != "" {
 		return fmt.Errorf("s3 credential type must be set")
 	}
-	if cfg.S3Config.S3CredentialType == s3.CredentialTypeStatic {
+	if cfg.S3Config.CredentialType == s3.CredentialTypeStatic {
 		if cfg.S3Config.Endpoint != "" && (cfg.S3Config.AccessKeyID == "" || cfg.S3Config.AccessKeySecret == "") {
 			return fmt.Errorf("s3 endpoint is set, but access key id or access key secret is not set")
 		}

server/config_test.go

@@ -111,7 +111,7 @@ func TestConfigVerification(t *testing.T) {
 	t.Run("MissingS3AccessKeys", func(t *testing.T) {
 		cfg := validCfg()
 
-		cfg.S3Config.S3CredentialType = s3.CredentialTypeStatic
+		cfg.S3Config.CredentialType = s3.CredentialTypeStatic
 		cfg.S3Config.Endpoint = "http://localhost:9000"
 		cfg.S3Config.AccessKeyID = ""
 
@@ -122,7 +122,7 @@ func TestConfigVerification(t *testing.T) {
 	t.Run("MissingS3Credential", func(t *testing.T) {
 		cfg := validCfg()
 
-		cfg.S3Config.S3CredentialType = s3.CredentialTypeUnknown
+		cfg.S3Config.CredentialType = s3.CredentialTypeUnknown
 
 		err := cfg.Check()
 		require.Error(t, err)

store/precomputed_key/s3/cli.go

@@ -0,0 +1,92 @@
+package s3
+
+import (
+	"time"
+
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	EndpointFlagName        = withFlagPrefix("endpoint")
+	CredentialTypeFlagName  = withFlagPrefix("credential-type")
+	AccessKeyIDFlagName     = withFlagPrefix("access-key-id")     // #nosec G101
+	AccessKeySecretFlagName = withFlagPrefix("access-key-secret") // #nosec G101
+	BucketFlagName          = withFlagPrefix("bucket")
+	PathFlagName            = withFlagPrefix("path")
+	BackupFlagName          = withFlagPrefix("backup")
+	TimeoutFlagName         = withFlagPrefix("timeout")
+)
+
+func withFlagPrefix(s string) string {
+	return "s3." + s
+}
+
+func withEnvPrefix(envPrefix, s string) []string {
+	return []string{envPrefix + "_S3_" + s}
+}
+
+// CLIFlags ... used for S3 backend configuration
+// category is used to group the flags in the help output (see https://cli.urfave.org/v2/examples/flags/#grouping)
+func CLIFlags(envPrefix, category string) []cli.Flag {
+	return []cli.Flag{
+		&cli.StringFlag{
+			Name:    EndpointFlagName,
+			Usage:   "endpoint for S3 storage",
+			Value:   "",
+			EnvVars: withEnvPrefix(envPrefix, "S3_ENDPOINT"),
+		},
+		&cli.StringFlag{
+			Name:     CredentialTypeFlagName,
+			Usage:    "The way to authenticate to S3, options are [iam, static]",
+			EnvVars:  withEnvPrefix(envPrefix, "CREDENTIAL_TYPE"),
+			Category: category,
+		},
+		&cli.StringFlag{
+			Name:    AccessKeyIDFlagName,
+			Usage:   "access key id for S3 storage",
+			Value:   "",
+			EnvVars: withEnvPrefix(envPrefix, "ACCESS_KEY_ID"),
+		},
+		&cli.StringFlag{
+			Name:    AccessKeySecretFlagName,
+			Usage:   "access key secret for S3 storage",
+			Value:   "",
+			EnvVars: withEnvPrefix(envPrefix, "ACCESS_KEY_SECRET"),
+		},
+		&cli.StringFlag{
+			Name:    BucketFlagName,
+			Usage:   "bucket name for S3 storage",
+			EnvVars: withEnvPrefix(envPrefix, "BUCKET"),
+		},
+		&cli.StringFlag{
+			Name:    PathFlagName,
+			Usage:   "path for S3 storage",
+			EnvVars: withEnvPrefix(envPrefix, "PATH"),
+		},
+		&cli.BoolFlag{
+			Name:    BackupFlagName,
+			Usage:   "whether to use S3 as a backup store to ensure resiliency in case of EigenDA read failure",
+			Value:   false,
+			EnvVars: withEnvPrefix(envPrefix, "BACKUP"),
+		},
+		&cli.DurationFlag{
+			Name:    TimeoutFlagName,
+			Usage:   "timeout for S3 storage operations (e.g. get, put)",
+			Value:   5 * time.Second,
+			EnvVars: withEnvPrefix(envPrefix, "TIMEOUT"),
+		},
+	}
+}
+
+func ReadConfig(ctx *cli.Context) Config {
+	return Config{
+		CredentialType:  StringToCredentialType(ctx.String(CredentialTypeFlagName)),
+		Endpoint:        ctx.String(EndpointFlagName),
+		AccessKeyID:     ctx.String(AccessKeyIDFlagName),
+		AccessKeySecret: ctx.String(AccessKeySecretFlagName),
+		Bucket:          ctx.String(BucketFlagName),
+		Path:            ctx.String(PathFlagName),
+		Backup:          ctx.Bool(BackupFlagName),
+		Timeout:         ctx.Duration(TimeoutFlagName),
+	}
+}

store/precomputed_key/s3/s3.go

@@ -37,15 +37,15 @@ var _ store.PrecomputedKeyStore = (*Store)(nil)
 
 type CredentialType string
 type Config struct {
-	S3CredentialType CredentialType
-	Bucket           string
-	Path             string
-	Endpoint         string
-	AccessKeyID      string
-	AccessKeySecret  string
-	Profiling        bool
-	Backup           bool
-	Timeout          time.Duration
+	CredentialType  CredentialType
+	Endpoint        string
+	AccessKeyID     string
+	AccessKeySecret string
+	Bucket          string
+	Path            string
+	Backup          bool
+	Timeout         time.Duration
+	Profiling       bool
 }
 
 type Store struct {
@@ -126,7 +126,7 @@ func (s *Store) BackendType() store.BackendType {
 }
 
 func creds(cfg Config) *credentials.Credentials {
-	if cfg.S3CredentialType == CredentialTypeIAM {
+	if cfg.CredentialType == CredentialTypeIAM {
 		return credentials.NewIAM("")
 	}
 	return credentials.NewStaticV4(cfg.AccessKeyID, cfg.AccessKeySecret, "")