diff --git a/.env.example.holesky b/.env.example.holesky
index 210ab4b..05a241b 100644
--- a/.env.example.holesky
+++ b/.env.example.holesky
@@ -74,3 +74,15 @@ EIGENDA_PROXY_SERVICE_MANAGER_ADDR=0xD4A7E1Bd8015057293f0D0A557088c286942e84b
# Metrics listening port
# EIGENDA_PROXY_METRICS_PORT=7300
+
+# access key id for S3 storage
+# EIGENDA_PROXY_S3_ACCESS_KEY_ID=
+
+# access key secret for S3 storage
+# EIGENDA_PROXY_S3_ACCESS_KEY_SECRET=
+
+# bucket name for S3 storage
+# EIGENDA_PROXY_S3_BUCKET=
+
+# endpoint for S3 storage
+# EIGENDA_PROXY_S3_ENDPOINT
\ No newline at end of file
diff --git a/.env.example.mainnet b/.env.example.mainnet
index 53b016c..741a3d7 100644
--- a/.env.example.mainnet
+++ b/.env.example.mainnet
@@ -74,3 +74,15 @@ EIGENDA_PROXY_SERVICE_MANAGER_ADDR=0x870679E138bCdf293b7Ff14dD44b70FC97e12fc0
# Metrics listening port
# EIGENDA_PROXY_METRICS_PORT=7300
+
+# access key id for S3 storage
+# EIGENDA_PROXY_S3_ACCESS_KEY_ID=
+
+# access key secret for S3 storage
+# EIGENDA_PROXY_S3_ACCESS_KEY_SECRET=
+
+# bucket name for S3 storage
+# EIGENDA_PROXY_S3_BUCKET=
+
+# endpoint for S3 storage
+# EIGENDA_PROXY_S3_ENDPOINT
\ No newline at end of file
diff --git a/README.md b/README.md
index 0a7d26e..aaff3a8 100644
--- a/README.md
+++ b/README.md
@@ -12,40 +12,47 @@ Features:
* Performs KZG verification during dispersal to ensure that DA certificates returned from the EigenDA disperser have correct KZG commitments.
* Performs DA certificate verification during dispersal to ensure that DA certificates have been properly bridged to Ethereum by the disperser.
* Performs DA certificate verification during retrieval to ensure that data represented by bad DA certificates do not become part of the canonical chain.
+* Compatibility with Optimism's keccak-256 commitment type with S3 storage.
In order to disperse to the EigenDA network in production, or at high throughput on testnet, please register your authentication ethereum address through [this form](https://forms.gle/3QRNTYhSMacVFNcU8). Your EigenDA authentication keypair address should not be associated with any funds anywhere.
## Configuration Options
+| Option | Default Value | Environment Variable | Description |
+|--------|---------------|----------------------|-------------|
+| `--addr` | `"127.0.0.1"` | `$EIGENDA_PROXY_ADDR` | Server listening address |
+| `--eigenda-cache-path` | `"resources/SRSTables/"` | `$EIGENDA_PROXY_TARGET_CACHE_PATH` | Directory path to SRS tables for caching. |
+| `--eigenda-custom-quorum-ids` | | `$EIGENDA_PROXY_CUSTOM_QUORUM_IDS` | Custom quorum IDs for writing blobs. Should not include default quorums 0 or 1. |
+| `--eigenda-disable-point-verification-mode` | `false` | `$EIGENDA_PROXY_DISABLE_POINT_VERIFICATION_MODE` | Disable point verification mode. This mode performs IFFT on data before writing and FFT on data after reading. Disabling requires supplying the entire blob for verification against the KZG commitment. |
+| `--eigenda-disable-tls` | `false` | `$EIGENDA_PROXY_GRPC_DISABLE_TLS` | Disable TLS for gRPC communication with the EigenDA disperser. Default is false. |
+| `--eigenda-disperser-rpc` | | `$EIGENDA_PROXY_EIGENDA_DISPERSER_RPC` | RPC endpoint of the EigenDA disperser. |
+| `--eigenda-eth-confirmation-depth` | `6` | `$EIGENDA_PROXY_ETH_CONFIRMATION_DEPTH` | The number of Ethereum blocks of confirmation that the DA bridging transaction must have before it is assumed by the proxy to be final. The value of `0` indicates that the proxy should wait for weak-subjectivity finalization (12-14 minutes). |
+| `--eigenda-eth-rpc` | | `$EIGENDA_PROXY_ETH_RPC` | JSON RPC node endpoint for the Ethereum network used for finalizing DA blobs. See available list here: https://docs.eigenlayer.xyz/eigenda/networks/ |
+| `--eigenda-g1-path` | `"resources/g1.point"` | `$EIGENDA_PROXY_TARGET_KZG_G1_PATH` | Directory path to g1.point file. |
+| `--eigenda-g2-tau-path` | `"resources/g2.point.powerOf2"` | `$EIGENDA_PROXY_TARGET_G2_TAU_PATH` | Directory path to g2.point.powerOf2 file. |
+| `--eigenda-max-blob-length` | `"2MiB"` | `$EIGENDA_PROXY_MAX_BLOB_LENGTH` | Maximum blob length to be written or read from EigenDA. Determines the number of SRS points loaded into memory for KZG commitments. Example units: '30MiB', '4Kb', '30MB'. Maximum size slightly exceeds 1GB. |
+| `--eigenda-put-blob-encoding-version` | `0` | `$EIGENDA_PROXY_PUT_BLOB_ENCODING_VERSION` | Blob encoding version to use when writing blobs from the high-level interface. |
+| `--eigenda-response-timeout` | `10s` | `$EIGENDA_PROXY_RESPONSE_TIMEOUT` | Total time to wait for a response from the EigenDA disperser. Default is 10 seconds. |
+| `--eigenda-signer-private-key-hex` | | `$EIGENDA_PROXY_SIGNER_PRIVATE_KEY_HEX` | Hex-encoded signer private key. This key should not be associated with an Ethereum address holding any funds. |
+| `--eigenda-status-query-retry-interval` | `5s` | `$EIGENDA_PROXY_STATUS_QUERY_INTERVAL` | Interval between retries when awaiting network blob finalization. Default is 5 seconds. |
+| `--eigenda-status-query-timeout` | `30m0s` | `$EIGENDA_PROXY_STATUS_QUERY_TIMEOUT` | Duration to wait for a blob to finalize after being sent for dispersal. Default is 30 minutes. |
+| `--eigenda-svc-manager-addr` | | `$EIGENDA_PROXY_SERVICE_MANAGER_ADDR` | The deployed EigenDA service manager address. The list can be found here: https://github.com/Layr-Labs/eigenlayer-middleware/?tab=readme-ov-file#current-mainnet-deployment |
+| `--log.color` | `false` | `$EIGENDA_PROXY_LOG_COLOR` | Color the log output if in terminal mode. |
+| `--log.format` | `text` | `$EIGENDA_PROXY_LOG_FORMAT` | Format the log output. Supported formats: 'text', 'terminal', 'logfmt', 'json', 'json-pretty'. |
+| `--log.level` | `INFO` | `$EIGENDA_PROXY_LOG_LEVEL` | The lowest log level that will be output. |
+| `--log.pid` | `false` | `$EIGENDA_PROXY_LOG_PID` | Show pid in the log. |
+| `--memstore.enabled` | `false` | `$MEMSTORE_ENABLED` | Whether to use mem-store for DA logic. |
+| `--memstore.expiration` | `25m0s` | `$MEMSTORE_EXPIRATION` | Duration that a mem-store blob/commitment pair are allowed to live. |
+| `--metrics.addr` | `"0.0.0.0"` | `$EIGENDA_PROXY_METRICS_ADDR` | Metrics listening address. |
+| `--metrics.enabled` | `false` | `$EIGENDA_PROXY_METRICS_ENABLED` | Enable the metrics server. |
+| `--metrics.port` | `7300` | `$EIGENDA_PROXY_METRICS_PORT` | Metrics listening port. |
+| `--port` | `3100` | `$EIGENDA_PROXY_PORT` | Server listening port. |
+| `--s3.access-key-id` | | `$EIGENDA_PROXY_S3_ACCESS_KEY_ID` | Access key id for S3 storage. |
+| `--s3.access-key-secret` | | `$EIGENDA_PROXY_S3_ACCESS_KEY_SECRET` | Access key secret for S3 storage. |
+| `--s3.bucket` | | `$EIGENDA_PROXY_S3_BUCKET` | Bucket name for S3 storage. |
+| `--s3.endpoint` | | `$EIGENDA_PROXY_S3_ENDPOINT` | Endpoint for S3 storage. |
+| `--help, -h` | `false` | | Show help. |
+| `--version, -v` | `false` | | Print the version. |
-| CLI Flag Name | Env Var Flag Name | Input Type | Default Value | Required | Description |
-|----------------------------------------------|-------------------------------------------------|------------|---------------------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `--addr` | None | string | None | Yes | Server listening address. |
-| `--port` | None | int | None | Yes | Server listening port. |
-| `--eigenda-disperser-rpc` | `EIGENDA_PROXY_EIGENDA_DISPERSER_RPC` | string | None | Yes | RPC host of the EigenDA disperser service (e.g., on Holesky this is `disperser-holesky.eigenda.xyz:443`). Full network list available in the documentation. |
-| `--eigenda-signer-private-key-hex` | `EIGENDA_PROXY_SIGNER_PRIVATE_KEY_HEX` | string | None | Yes | Hex-encoded signer private key. This key should not be associated with an Ethereum address holding any funds. |
-| `--eigenda-eth-rpc` | `EIGENDA_PROXY_ETH_RPC` | string | None | Yes | JSON RPC node endpoint for the Ethereum network used for finalizing DA blobs. See available list here: |
-| `--eigenda-svc-manager-addr` | `EIGENDA_PROXY_SERVICE_MANAGER_ADDR` | string | None | Yes | The deployed EigenDA service manager address. The list can be found here: |
-| `--eigenda-g1-path` | `EIGENDA_PROXY_TARGET_KZG_G1_PATH` | string | "resources/g1.point" | No | Directory path to g1.point file. |
-| `--eigenda-g2-tau-path` | `EIGENDA_PROXY_TARGET_G2_TAU_PATH` | string | "resources/g2.point.powerOf2" | No | Directory path to g2.point.powerOf2 file. |
-| `--eigenda-cache-path` | `EIGENDA_PROXY_TARGET_CACHE_PATH` | string | "resources/SRSTables/" | No | Directory path to SRS tables for caching. |
-| `--eigenda-eth-confirmation-depth` | `EIGENDA_PROXY_ETH_CONFIRMATION_DEPTH` | int | 6 | No | The number of Ethereum blocks of confirmation that the DA briging transaction must have before it is assumed by the proxy to be final. The value of `0` indicates that the proxy should wait for weak-subjectivity finalization (12-14 minutes). |
-| `--eigenda-disable-tls` | `EIGENDA_PROXY_GRPC_DISABLE_TLS` | bool | false | No | Disable TLS for gRPC communication with the EigenDA disperser. |
-| `--eigenda-custom-quorum-ids` | `EIGENDA_PROXY_CUSTOM_QUORUM_IDS` | string | None | No | Custom quorum IDs for writing blobs. Should not include default quorums 0 or 1. |
-| `--eigenda-disable-point-verification-mode` | `EIGENDA_PROXY_DISABLE_POINT_VERIFICATION_MODE` | bool | false | No | Disable point verification mode. This mode performs IFFT on data before writing and FFT on data after reading. Disabling requires supplying the entire blob for verification against the KZG commitment. |
-| `--eigenda-max-blob-length` | `EIGENDA_PROXY_MAX_BLOB_LENGTH` | string | "2MiB" | No | Maximum blob length to be written or read from EigenDA. Determines the number of SRS points loaded into memory for KZG commitments. Example units: '30MiB', '4Kb', '30MB'. Maximum size slightly exceeds 1GB. |
-| `--eigenda-put-blob-encoding-version` | `EIGENDA_PROXY_PUT_BLOB_ENCODING_VERSION` | int | 0 | No | Blob encoding version to use when writing blobs from the high-level interface. |
-| `--eigenda-status-query-retry-interval` | `EIGENDA_PROXY_STATUS_QUERY_INTERVAL` | duration | 5s | No | Interval between retries when awaiting network blob finalization. |
-| `--eigenda-status-query-timeout` | `EIGENDA_PROXY_STATUS_QUERY_TIMEOUT` | duration | 30m0s | No | Duration to wait for a blob to finalize after being sent for dispersal. |
-| `--eigenda-response-timeout` | `EIGENDA_PROXY_RESPONSE_TIMEOUT` | duration | 10s | No | Total time to wait for a response from the EigenDA disperser. |
-| `--memstore.enabled` | `MEMSTORE_ENABLED` | bool | false | No | Whether to use mem-store for DA logic. |
-| `--memstore.expiration` | `MEMSTORE_EXPIRATION` | duration | 25m0s | No | Duration that a blob/commitment pair are allowed to live. |
-| `--metrics.addr` | `EIGENDA_PROXY_METRICS_ADDR` | string | "0.0.0.0" | No | Metrics listening address. |
-| `--metrics.enabled` | `EIGENDA_PROXY_METRICS_ENABLED` | bool | false | No | Enable the metrics server. |
-| `--metrics.port` | `EIGENDA_PROXY_METRICS_PORT` | int | 7300 | No | Metrics listening port. |
-| `--log.color` | `EIGENDA_PROXY_LOG_COLOR` | bool | false | No | Color the log output if in terminal mode. |
-| `--log.format` | `EIGENDA_PROXY_LOG_FORMAT` | string | text | No | Format the log output. Supported formats: 'text', 'terminal', 'logfmt', 'json', 'json-pretty'. |
-| `--log.level` | `EIGENDA_PROXY_LOG_LEVEL` | string | INFO | No | The lowest log level that will be output. |
-| `--log.pid` | `EIGENDA_PROXY_LOG_PID` | bool | false | No | Show pid in the log. |
### Certificate verification
diff --git a/e2e/server_test.go b/e2e/server_test.go
index 8214050..3e25a46 100644
--- a/e2e/server_test.go
+++ b/e2e/server_test.go
@@ -41,6 +41,8 @@ func TestOptimismClientWithS3Backend(t *testing.T) {
}
func TestOptimismClientWithEigenDABackend(t *testing.T) {
+ // this test asserts that the data can be posted/read to EigenDA with a concurrent S3 backend configured
+
if !runIntegrationTests && !runTestnetIntegrationTests {
t.Skip("Skipping test as INTEGRATION or TESTNET env var not set")
}
diff --git a/e2e/setup.go b/e2e/setup.go
index 5911fbd..15c0139 100644
--- a/e2e/setup.go
+++ b/e2e/setup.go
@@ -149,7 +149,6 @@ func (ts *TestSuite) Address() string {
func createS3Bucket(bucketName string) {
- println("Creating S3 bucket: ", bucketName)
// Initialize minio client object.
endpoint := "localhost:4566"
accessKeyID := "minioadmin"
diff --git a/server/server.go b/server/server.go
index 7488e9a..2dbab07 100644
--- a/server/server.go
+++ b/server/server.go
@@ -70,6 +70,7 @@ func WithMetrics(handleFn func(http.ResponseWriter, *http.Request) error, m metr
}
}
+// WithLogging is a middleware that logs the request method and URL.
func WithLogging(handleFn func(http.ResponseWriter, *http.Request) error, log log.Logger) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
log.Info("request", "method", r.Method, "url", r.URL)
@@ -188,7 +189,7 @@ func (svr *Server) HandlePut(w http.ResponseWriter, r *http.Request) error {
key := path.Base(r.URL.Path)
var commitment []byte
- if len(key) > 0 && key != "put" { // commitment key pre-provided (keccak256)
+ if len(key) > 0 && key != "put" { // commitment key already provided (keccak256)
comm, err := commitments.StringToDecodedCommitment(key, ct)
if err != nil {
svr.log.Info("failed to decode commitment", "err", err, "key", key)
@@ -203,14 +204,12 @@ func (svr *Server) HandlePut(w http.ResponseWriter, r *http.Request) error {
}
} else { // without
- svr.log.Info("Put without key")
commitment, err = svr.router.PutWithoutKey(context.Background(), input)
if err != nil {
return err
}
}
- println("Encoding commitment: ", hexutil.Encode(commitment))
comm, err := commitments.EncodeCommitment(commitment, ct)
if err != nil {
svr.log.Info("failed to encode commitment", "err", err)
@@ -263,11 +262,8 @@ func ReadCommitmentMode(r *http.Request) (commitments.CommitmentMode, error) {
return commitments.SimpleCommitmentMode, err
}
- println("byte", decodedCommit[0])
-
switch decodedCommit[0] {
case byte(commitments.GenericCommitmentType):
- println("Returning generic commitment")
return commitments.OptimismAltDA, nil
case byte(commitments.Keccak256CommitmentType):
@@ -277,7 +273,6 @@ func ReadCommitmentMode(r *http.Request) (commitments.CommitmentMode, error) {
return commitments.SimpleCommitmentMode, fmt.Errorf("unknown commit byte prefix")
}
-
}
return commitments.OptimismAltDA, nil
diff --git a/store/router.go b/store/router.go
index f6f5c8a..7fabe61 100644
--- a/store/router.go
+++ b/store/router.go
@@ -28,9 +28,6 @@ func NewRouter(eigenda *EigenDAStore, mem *MemStore, s3 *S3Store, l log.Logger)
}, nil
}
-// The general pseudo-opinionated way of processing commitments is as follows:
-// 1. generic --> EigenDA
-// 2. Optimism --> EigenDA || S3 (if not EigenDA)
func (r *Router) Get(ctx context.Context, key []byte, cm commitments.CommitmentMode) ([]byte, error) {
switch cm {
@@ -40,14 +37,25 @@ func (r *Router) Get(ctx context.Context, key []byte, cm commitments.CommitmentM
return nil, errors.New("expected S3 backend for OP keccak256 commitment type, but none configured")
}
- r.log.Debug("Fetching data from S3 backend")
- return r.s3.Get(ctx, key)
+ r.log.Debug("Retrieving data from S3 backend")
+ value, err := r.s3.Get(ctx, key)
+ if err != nil {
+ return nil, err
+ }
+
+ if actualHash := crypto.Keccak256(value); !utils.EqualSlices(actualHash, key) {
+ return nil, fmt.Errorf("expected key %s to be the hash of value %s, but got %s", hexutil.Encode(key), hexutil.Encode(value), hexutil.Encode(actualHash))
+ }
+
+ return value, nil
case commitments.SimpleCommitmentMode, commitments.OptimismAltDA:
if r.mem != nil {
+ r.log.Debug("Retrieving data from memstore")
return r.mem.Get(ctx, key)
}
+ r.log.Debug("Retrieving data from eigenda")
return r.eigenda.Get(ctx, key)
default:
@@ -101,7 +109,6 @@ func (r *Router) PutWithoutKey(ctx context.Context, value []byte) (key []byte, e
// PutWithKey is only supported for S3 storage backends using OP's alt-da keccak256 commitment type
func (r *Router) PutWithKey(ctx context.Context, key []byte, value []byte) ([]byte, error) {
- println("Storing to s3 with key")
if r.s3 == nil {
return nil, errors.New("S3 is disabled but is only supported for posting known commitment keys")
}