diff --git a/Makefile b/Makefile index 6cbd8bc..53a3699 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ build: .PHONY: build build-docker: - docker-compose build + docker compose build .PHONY: build-docker clean: @@ -21,8 +21,8 @@ test: .PHONY: test integration: - docker-compose down - docker-compose up -d minio create-buckets + docker compose down + docker compose up -d minio create-buckets RUN_INTEGRATION_TESTS=true go test -v ./... .PHONY: integration diff --git a/README.md b/README.md index fe41cc3..d879f65 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ To run the project locally, you should first copy `.env.template` to `.env` and to your beacon client and storage backend of choice. Then you can run the project with: ```sh -docker-compose up +docker compose up ``` You can see a full list of configuration options by running: diff --git a/archiver/flags/config.go b/archiver/flags/config.go index d680e6e..e0d2f32 100644 --- a/archiver/flags/config.go +++ b/archiver/flags/config.go @@ -2,6 +2,7 @@ package flags import ( "fmt" + "strings" "time" common "github.com/base-org/blob-archiver/common/flags" @@ -35,7 +36,7 @@ func (c ArchiverConfig) Check() error { } if c.OriginBlock == (geth.Hash{}) { - return fmt.Errorf("invalid origin block") + return fmt.Errorf("invalid origin block %s", c.OriginBlock) } if c.ListenAddr == "" { @@ -53,7 +54,7 @@ func ReadConfig(cliCtx *cli.Context) ArchiverConfig { BeaconConfig: common.NewBeaconConfig(cliCtx), StorageConfig: common.NewStorageConfig(cliCtx), PollInterval: pollInterval, - OriginBlock: geth.HexToHash(cliCtx.String(ArchiverOriginBlock.Name)), + OriginBlock: geth.HexToHash(strings.Trim(cliCtx.String(ArchiverOriginBlock.Name), "\"")), ListenAddr: cliCtx.String(ArchiverListenAddrFlag.Name), } } diff --git a/common/flags/config.go b/common/flags/config.go index 369f3a3..906379c 100644 --- a/common/flags/config.go +++ b/common/flags/config.go @@ -24,6 +24,7 @@ type S3Config struct { Endpoint string UseHttps bool Bucket string + Path string S3CredentialType S3CredentialType AccessKey string @@ -106,6 +107,7 @@ func readS3Config(ctx *cli.Context) S3Config { SecretAccessKey: ctx.String(S3SecretAccessKeyFlagName), UseHttps: ctx.Bool(S3EndpointHttpsFlagName), Bucket: ctx.String(S3BucketFlagName), + Path: ctx.String(S3PathFlagName), S3CredentialType: toS3CredentialType(ctx.String(S3CredentialTypeFlagName)), Compress: ctx.Bool(S3CompressFlagName), } diff --git a/common/flags/flags.go b/common/flags/flags.go index a1f2c48..d41eb90 100644 --- a/common/flags/flags.go +++ b/common/flags/flags.go @@ -17,6 +17,7 @@ const ( S3AccessKeyFlagName = "s3-access-key" S3SecretAccessKeyFlagName = "s3-secret-access-key" S3BucketFlagName = "s3-bucket" + S3PathFlagName = "s3-path" FileStorageDirectoryFlagName = "file-directory" ) @@ -77,6 +78,13 @@ func CLIFlags(envPrefix string) []cli.Flag { Hidden: true, EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_BUCKET"), }, + &cli.StringFlag{ + Name: S3PathFlagName, + Usage: "The path to append to file", + Hidden: true, + EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_PATH"), + Value: "", + }, // File Data Store Flags &cli.StringFlag{ Name: FileStorageDirectoryFlagName, diff --git a/common/storage/s3.go b/common/storage/s3.go index 6be01fc..bf68e92 100644 --- a/common/storage/s3.go +++ b/common/storage/s3.go @@ -6,6 +6,7 @@ import ( "context" "encoding/json" "io" + "path" "github.com/base-org/blob-archiver/common/flags" "github.com/ethereum/go-ethereum/common" @@ -17,6 +18,7 @@ import ( type S3Storage struct { s3 *minio.Client bucket string + path string log log.Logger compress bool } @@ -41,13 +43,14 @@ func NewS3Storage(cfg flags.S3Config, l log.Logger) (*S3Storage, error) { return &S3Storage{ s3: client, bucket: cfg.Bucket, + path: cfg.Path, log: l, compress: cfg.Compress, }, nil } func (s *S3Storage) Exists(ctx context.Context, hash common.Hash) (bool, error) { - _, err := s.s3.StatObject(ctx, s.bucket, hash.String(), minio.StatObjectOptions{}) + _, err := s.s3.StatObject(ctx, s.bucket, path.Join(s.path, hash.String()), minio.StatObjectOptions{}) if err != nil { errResponse := minio.ToErrorResponse(err) if errResponse.Code == "NoSuchKey" { @@ -61,7 +64,7 @@ func (s *S3Storage) Exists(ctx context.Context, hash common.Hash) (bool, error) } func (s *S3Storage) Read(ctx context.Context, hash common.Hash) (BlobData, error) { - res, err := s.s3.GetObject(ctx, s.bucket, hash.String(), minio.GetObjectOptions{}) + res, err := s.s3.GetObject(ctx, s.bucket, path.Join(s.path, hash.String()), minio.GetObjectOptions{}) if err != nil { s.log.Info("unexpected error fetching blob", "hash", hash.String(), "err", err) return BlobData{}, ErrStorage @@ -122,7 +125,7 @@ func (s *S3Storage) Write(ctx context.Context, data BlobData) error { reader := bytes.NewReader(b) - _, err = s.s3.PutObject(ctx, s.bucket, data.Header.BeaconBlockHash.String(), reader, int64(len(b)), options) + _, err = s.s3.PutObject(ctx, s.bucket, path.Join(s.path, data.Header.BeaconBlockHash.String()), reader, int64(len(b)), options) if err != nil { s.log.Warn("error writing blob", "err", err) diff --git a/common/storage/s3_test.go b/common/storage/s3_test.go index 3a233a2..e1f6753 100644 --- a/common/storage/s3_test.go +++ b/common/storage/s3_test.go @@ -14,8 +14,8 @@ import ( // Prior to running these tests, a local Minio server must be running. // You can accomplish this with: -// docker-compose down # shut down any running services -// docker-compose up minio create-buckets # start the minio service +// docker compose down # shut down any running services +// docker compose up minio create-buckets # start the minio service func setupS3(t *testing.T) *S3Storage { if os.Getenv("RUN_INTEGRATION_TESTS") == "" { t.Skip("skipping integration tests: set RUN_INTEGRATION_TESTS environment variable")