Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update internal data structures #732

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion api/clients/disperser_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ func (c *disperserClient) DisperseBlobAuthenticated(ctx context.Context, data []
return nil, nil, errors.New("expected challenge")
}

authHeader := core.BlobAuthHeader{
authHeader := core.BlobHeader{
BlobCommitments: encoding.BlobCommitments{},
AccountID: "",
Nonce: authHeaderReply.BlobAuthHeader.ChallengeParameter,
Expand Down
4 changes: 2 additions & 2 deletions api/clients/mock/node_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func NewNodeClient() *MockNodeClient {
return &MockNodeClient{}
}

func (c *MockNodeClient) GetBlobHeader(ctx context.Context, socket string, batchHeaderHash [32]byte, blobIndex uint32) (*core.BlobHeader, *merkletree.Proof, error) {
func (c *MockNodeClient) GetBlobHeader(ctx context.Context, socket string, batchHeaderHash [32]byte, blobIndex uint32) (*core.BlobCertificate, *merkletree.Proof, error) {
mooselumph marked this conversation as resolved.
Show resolved Hide resolved
args := c.Called(socket, batchHeaderHash, blobIndex)
var hashes [][]byte
if args.Get(1) != nil {
Expand All @@ -40,7 +40,7 @@ func (c *MockNodeClient) GetBlobHeader(ctx context.Context, socket string, batch
Hashes: hashes,
Index: index,
}
return (args.Get(0)).(*core.BlobHeader), proof, err
return (args.Get(0)).(*core.BlobCertificate), proof, err
}

func (c *MockNodeClient) GetChunks(
Expand Down
6 changes: 3 additions & 3 deletions api/clients/node_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ type RetrievedChunks struct {
}

type NodeClient interface {
GetBlobHeader(ctx context.Context, socket string, batchHeaderHash [32]byte, blobIndex uint32) (*core.BlobHeader, *merkletree.Proof, error)
GetBlobHeader(ctx context.Context, socket string, batchHeaderHash [32]byte, blobIndex uint32) (*core.BlobCertificate, *merkletree.Proof, error)
mooselumph marked this conversation as resolved.
Show resolved Hide resolved
GetChunks(ctx context.Context, opID core.OperatorID, opInfo *core.IndexedOperatorInfo, batchHeaderHash [32]byte, blobIndex uint32, quorumID core.QuorumID, chunksChan chan RetrievedChunks)
}

Expand All @@ -40,7 +40,7 @@ func (c client) GetBlobHeader(
socket string,
batchHeaderHash [32]byte,
blobIndex uint32,
) (*core.BlobHeader, *merkletree.Proof, error) {
) (*core.BlobCertificate, *merkletree.Proof, error) {
conn, err := grpc.Dial(
core.OperatorSocket(socket).GetRetrievalSocket(),
grpc.WithTransportCredentials(insecure.NewCredentials()),
Expand All @@ -64,7 +64,7 @@ func (c client) GetBlobHeader(
return nil, nil, err
}

blobHeader, err := node.GetBlobHeaderFromProto(reply.GetBlobHeader())
blobHeader, err := node.GetBlobCertFromProto(reply.GetBlobHeader())
mooselumph marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return nil, nil, err
}
Expand Down
20 changes: 10 additions & 10 deletions api/clients/retrieval_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,19 +71,19 @@ func (r *retrievalClient) RetrieveBlob(
}

// Get blob header from any operator
var blobHeader *core.BlobHeader
var blobCert *core.BlobCertificate
var proof *merkletree.Proof
var proofVerified bool
for opID := range operators {
opInfo := indexedOperatorState.IndexedOperators[opID]
blobHeader, proof, err = r.nodeClient.GetBlobHeader(ctx, opInfo.Socket, batchHeaderHash, blobIndex)
blobCert, proof, err = r.nodeClient.GetBlobHeader(ctx, opInfo.Socket, batchHeaderHash, blobIndex)
if err != nil {
// try another operator
r.logger.Warn("failed to dial operator while fetching BlobHeader, trying different operator", "operator", opInfo.Socket, "err", err)
continue
}

blobHeaderHash, err := blobHeader.GetBlobHeaderHash()
blobHeaderHash, err := blobCert.GetHash()
if err != nil {
r.logger.Warn("got invalid blob header, trying different operator", "operator", opInfo.Socket, "err", err)
continue
Expand All @@ -100,12 +100,12 @@ func (r *retrievalClient) RetrieveBlob(

break
}
if blobHeader == nil || proof == nil || !proofVerified {
if blobCert == nil || proof == nil || !proofVerified {
return nil, fmt.Errorf("failed to get blob header from all operators (header hash: %s, index: %d)", batchHeaderHash, blobIndex)
}

var quorumHeader *core.BlobQuorumInfo
for _, header := range blobHeader.QuorumInfos {
for _, header := range blobCert.QuorumInfos {
if header.QuorumID == quorumID {
quorumHeader = header
break
Expand All @@ -117,19 +117,19 @@ func (r *retrievalClient) RetrieveBlob(
}

// Validate the blob length
err = r.verifier.VerifyBlobLength(blobHeader.BlobCommitments)
err = r.verifier.VerifyBlobLength(blobCert.BlobCommitments)
if err != nil {
return nil, err
}

// Validate the commitments are equivalent
commitmentBatch := []encoding.BlobCommitments{blobHeader.BlobCommitments}
commitmentBatch := []encoding.BlobCommitments{blobCert.BlobCommitments}
err = r.verifier.VerifyCommitEquivalenceBatch(commitmentBatch)
if err != nil {
return nil, err
}

assignments, info, err := r.assignmentCoordinator.GetAssignments(indexedOperatorState.OperatorState, blobHeader.Length, quorumHeader)
assignments, info, err := r.assignmentCoordinator.GetAssignments(indexedOperatorState.OperatorState, blobCert.Length, quorumHeader)
if err != nil {
return nil, errors.New("failed to get assignments")
}
Expand Down Expand Up @@ -161,7 +161,7 @@ func (r *retrievalClient) RetrieveBlob(
return nil, fmt.Errorf("no assignment to operator %s", reply.OperatorID.Hex())
}

err = r.verifier.VerifyFrames(reply.Chunks, assignment.GetIndices(), blobHeader.BlobCommitments, encodingParams)
err = r.verifier.VerifyFrames(reply.Chunks, assignment.GetIndices(), blobCert.BlobCommitments, encodingParams)
if err != nil {
r.logger.Error("failed to verify chunks from operator", "operator", reply.OperatorID.Hex(), "err", err)
continue
Expand All @@ -173,5 +173,5 @@ func (r *retrievalClient) RetrieveBlob(
indices = append(indices, assignment.GetIndices()...)
}

return r.verifier.Decode(chunks, indices, encodingParams, uint64(blobHeader.Length)*encoding.BYTES_PER_SYMBOL)
return r.verifier.Decode(chunks, indices, encodingParams, uint64(blobCert.Length)*encoding.BYTES_PER_SYMBOL)
}
28 changes: 15 additions & 13 deletions api/clients/retrieval_client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,9 @@ var (
nodeClient *clientsmock.MockNodeClient
coordinator *core.StdAssignmentCoordinator
retrievalClient clients.RetrievalClient
blobHeader *core.BlobHeader
blobCert *core.BlobCertificate
encodedBlob core.EncodedBlob = core.EncodedBlob{
BlobHeader: nil,
BlobCert: nil,
EncodedBundlesByOperator: make(map[core.OperatorID]core.EncodedBundles),
}
batchHeaderHash [32]byte
Expand Down Expand Up @@ -165,17 +165,19 @@ func setup(t *testing.T) {
t.Fatal(err)
}

blobHeader = &core.BlobHeader{
BlobCommitments: encoding.BlobCommitments{
Commitment: commitments.Commitment,
LengthCommitment: commitments.LengthCommitment,
LengthProof: commitments.LengthProof,
Length: commitments.Length,
blobCert = &core.BlobCertificate{
BlobHeader: core.BlobHeader{
BlobCommitments: encoding.BlobCommitments{
Commitment: commitments.Commitment,
LengthCommitment: commitments.LengthCommitment,
LengthProof: commitments.LengthProof,
Length: commitments.Length,
},
},
QuorumInfos: []*core.BlobQuorumInfo{quorumHeader},
}

blobHeaderHash, err := blobHeader.GetBlobHeaderHash()
blobHeaderHash, err := blobCert.GetHash()
if err != nil {
t.Fatal(err)
}
Expand All @@ -195,9 +197,9 @@ func setup(t *testing.T) {
}

for id, assignment := range assignments {
bundles := make(map[core.QuorumID]core.Bundle, len(blobHeader.QuorumInfos))
bundles := make(map[core.QuorumID]core.Bundle, len(blobCert.QuorumInfos))
bundles[quorumID] = chunks[assignment.StartIndex : assignment.StartIndex+assignment.NumChunks]
encodedBlob.BlobHeader = blobHeader
encodedBlob.BlobCert = blobCert
eb, err := core.Bundles(bundles).ToEncodedBundles()
if err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -244,7 +246,7 @@ func TestInvalidBlobHeader(t *testing.T) {
setup(t)

// TODO: add the blob proof to the response
nodeClient.On("GetBlobHeader", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(blobHeader, [][]byte{{1}}, uint64(0), nil).Times(numOperators)
nodeClient.On("GetBlobHeader", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(blobCert, [][]byte{{1}}, uint64(0), nil).Times(numOperators)
nodeClient.
On("GetChunks", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(encodedBlob)
Expand All @@ -265,7 +267,7 @@ func TestValidBlobHeader(t *testing.T) {
setup(t)

// TODO: add the blob proof to the response
nodeClient.On("GetBlobHeader", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(blobHeader, [][]byte{}, uint64(0), nil).Once()
nodeClient.On("GetBlobHeader", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(blobCert, [][]byte{}, uint64(0), nil).Once()
nodeClient.
On("GetChunks", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).
Return(encodedBlob)
Expand Down
4 changes: 2 additions & 2 deletions core/assignment.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ type AssignmentCoordinator interface {
GetAssignments(state *OperatorState, blobLength uint, info *BlobQuorumInfo) (map[OperatorID]Assignment, AssignmentInfo, error)

// GetOperatorAssignment calculates the assignment for a specific DA node
GetOperatorAssignment(state *OperatorState, header *BlobHeader, quorum QuorumID, id OperatorID) (Assignment, AssignmentInfo, error)
GetOperatorAssignment(state *OperatorState, header *BlobCertificate, quorum QuorumID, id OperatorID) (Assignment, AssignmentInfo, error)

// ValidateChunkLength validates that the chunk length for the given quorum satisfies all protocol constraints
ValidateChunkLength(state *OperatorState, blobLength uint, info *BlobQuorumInfo) (bool, error)
Expand Down Expand Up @@ -162,7 +162,7 @@ func (c *StdAssignmentCoordinator) GetAssignments(state *OperatorState, blobLeng

}

func (c *StdAssignmentCoordinator) GetOperatorAssignment(state *OperatorState, header *BlobHeader, quorum QuorumID, id OperatorID) (Assignment, AssignmentInfo, error) {
func (c *StdAssignmentCoordinator) GetOperatorAssignment(state *OperatorState, header *BlobCertificate, quorum QuorumID, id OperatorID) (Assignment, AssignmentInfo, error) {
mooselumph marked this conversation as resolved.
Show resolved Hide resolved

quorumInfo := header.GetQuorumInfo(quorum)
if quorumInfo == nil {
Expand Down
8 changes: 5 additions & 3 deletions core/assignment_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,11 @@ func TestOperatorAssignments(t *testing.T) {
for operatorID, assignment := range assignments {
assert.Equal(t, assignment, expectedAssignments[operatorID])

header := &core.BlobHeader{
BlobCommitments: encoding.BlobCommitments{
Length: blobLength,
header := &core.BlobCertificate{
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

maybe cert :=?

BlobHeader: core.BlobHeader{
BlobCommitments: encoding.BlobCommitments{
Length: blobLength,
},
},
QuorumInfos: []*core.BlobQuorumInfo{quorumInfo},
}
Expand Down
4 changes: 2 additions & 2 deletions core/auth.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package core

type BlobRequestAuthenticator interface {
AuthenticateBlobRequest(header BlobAuthHeader) error
AuthenticateBlobRequest(header BlobHeader) error
}

type BlobRequestSigner interface {
SignBlobRequest(header BlobAuthHeader) ([]byte, error)
SignBlobRequest(header BlobHeader) ([]byte, error)
GetAccountID() string
}
4 changes: 2 additions & 2 deletions core/auth/auth_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ func TestAuthentication(t *testing.T) {
privateKeyHex := "0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
signer := auth.NewLocalBlobRequestSigner(privateKeyHex)

testHeader := core.BlobAuthHeader{
testHeader := core.BlobHeader{
BlobCommitments: encoding.BlobCommitments{},
AccountID: signer.GetAccountID(),
Nonce: rand.Uint32(),
Expand All @@ -46,7 +46,7 @@ func TestAuthenticationFail(t *testing.T) {
privateKeyHex := "0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
signer := auth.NewLocalBlobRequestSigner(privateKeyHex)

testHeader := core.BlobAuthHeader{
testHeader := core.BlobHeader{
BlobCommitments: encoding.BlobCommitments{},
AccountID: signer.GetAccountID(),
Nonce: rand.Uint32(),
Expand Down
2 changes: 1 addition & 1 deletion core/auth/authenticator.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func NewAuthenticator(config AuthConfig) core.BlobRequestAuthenticator {
}
}

func (*authenticator) AuthenticateBlobRequest(header core.BlobAuthHeader) error {
func (*authenticator) AuthenticateBlobRequest(header core.BlobHeader) error {
sig := header.AuthenticationData

// Ensure the signature is 65 bytes (Recovery ID is the last byte)
Expand Down
2 changes: 1 addition & 1 deletion core/auth/signer.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func NewLocalBlobRequestSigner(privateKeyHex string) *LocalBlobRequestSigner {
}
}

func (s *LocalBlobRequestSigner) SignBlobRequest(header core.BlobAuthHeader) ([]byte, error) {
func (s *LocalBlobRequestSigner) SignBlobRequest(header core.BlobHeader) ([]byte, error) {

// Message you want to sign
buf := make([]byte, 4)
Expand Down
28 changes: 13 additions & 15 deletions core/data.go
Original file line number Diff line number Diff line change
Expand Up @@ -220,11 +220,11 @@ type Blob struct {
Data []byte
}

// BlobAuthHeader contains the data that a user must sign to authenticate a blob request.
// BlobHeader contains the data that a user must sign to authenticate a blob request.
// Signing the combination of the Nonce and the BlobCommitments prohibits the disperser from
// using the signature to charge the user for a different blob or for dispersing the same blob
// multiple times (Replay attack).
type BlobAuthHeader struct {
type BlobHeader struct {
// Commitments
encoding.BlobCommitments `json:"commitments"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For the purpose of auth, is the kzg commitment itself sufficient (i.e. the 3 length related fields are not needed)?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the length is needed for payment purposes, but the length proof can probably be ommitted. I've made some updates to the document: https://www.notion.so/eigen-labs/Unified-blob-addressal-in-EigenDA-ae426c610f9e4e61b2e46eec76f7ca4d?pvs=4#ed6c05cfb2e34e3e9289a998cb25dc4d

// AccountID is the account that is paying for the blob to be stored. AccountID is hexadecimal representation of the ECDSA public key
Expand All @@ -238,7 +238,7 @@ type BlobAuthHeader struct {
// BlobRequestHeader contains the original data size of a blob and the security required
type BlobRequestHeader struct {
// BlobAuthHeader
BlobAuthHeader `json:"blob_auth_header"`
BlobHeader `json:"blob_auth_header"`
// For a blob to be accepted by EigenDA, it satisfy the AdversaryThreshold of each quorum contained in SecurityParams
SecurityParams []*SecurityParam `json:"security_params"`
}
Expand Down Expand Up @@ -267,17 +267,15 @@ type BlobQuorumInfo struct {
ChunkLength uint
}

// BlobHeader contains all metadata related to a blob including commitments and parameters for encoding
type BlobHeader struct {
encoding.BlobCommitments
// BlobCertificate contains all information about the blob to which the DA nodes must attest.
// Signed blob certificates are consumed by DA end-users to verify the availability of the blob.
type BlobCertificate struct {
BlobHeader
// QuorumInfos contains the quorum specific parameters for the blob
QuorumInfos []*BlobQuorumInfo

// AccountID is the account that is paying for the blob to be stored
AccountID AccountID
}

func (b *BlobHeader) GetQuorumInfo(quorum QuorumID) *BlobQuorumInfo {
func (b *BlobCertificate) GetQuorumInfo(quorum QuorumID) *BlobQuorumInfo {
for _, quorumInfo := range b.QuorumInfos {
if quorumInfo.QuorumID == quorum {
return quorumInfo
Expand All @@ -287,7 +285,7 @@ func (b *BlobHeader) GetQuorumInfo(quorum QuorumID) *BlobQuorumInfo {
}

// Returns the total encoded size in bytes of the blob across all quorums.
func (b *BlobHeader) EncodedSizeAllQuorums() int64 {
func (b *BlobCertificate) EncodedSizeAllQuorums() int64 {
size := int64(0)
for _, quorum := range b.QuorumInfos {

Expand All @@ -309,7 +307,7 @@ type BatchHeader struct {

// EncodedBlob contains the messages to be sent to a group of DA nodes corresponding to a single blob
type EncodedBlob struct {
BlobHeader *BlobHeader
BlobCert *BlobCertificate
BundlesByOperator map[OperatorID]Bundles
// EncodedBundlesByOperator is bundles in encoded format (not deserialized)
EncodedBundlesByOperator map[OperatorID]EncodedBundles
Expand All @@ -326,15 +324,15 @@ type EncodedBundles map[QuorumID]*ChunksData

// BlobMessage is the message that is sent to DA nodes. It contains the blob header and the associated chunk bundles.
type BlobMessage struct {
BlobHeader *BlobHeader
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need the concept of certificate internally? It looked nicer/more intuitive to have blob header + body (chunks/bundle).

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's pretty useful to maintain the distinction between the two since they have different purposes. Here it makes sense that it is the certificate because we are asking the DA node to sign the certificate.

That being said, I'm open to other naming suggestions!

Bundles Bundles
BlobCert *BlobCertificate
Bundles Bundles
}

// This is similar to BlobMessage, but keep the commitments and chunks in encoded format
// (i.e. not deserialized)
type EncodedBlobMessage struct {
// TODO(jianoaix): Change the commitments to encoded format.
BlobHeader *BlobHeader
BlobCert *BlobCertificate
EncodedBundles map[QuorumID]*ChunksData
}

Expand Down
Loading
Loading