[4/N] Chunk encoding optimization: Add bundle encoding (#632)

core/data.go

@@ -1,11 +1,13 @@
 package core
 
 import (
+	"encoding/binary"
 	"errors"
 	"fmt"
 
 	"github.com/Layr-Labs/eigenda/common"
 	"github.com/Layr-Labs/eigenda/encoding"
+	"github.com/consensys/gnark-crypto/ecc/bn254"
 )
 
 type AccountID = string
@@ -33,6 +35,8 @@ const (
 	// which means the max ID can not be larger than 254 (from 0 to 254, there are 255
 	// different IDs).
 	MaxQuorumID = 254
+
+	GnarkBundleEncodingFormat = 1
 )
 
 func (s *SecurityParam) String() string {
@@ -165,6 +169,83 @@ func (b Bundle) Size() uint64 {
 	return size
 }
 
+// Serialize returns the serialized bytes of the bundle.
+//
+// The bytes are packed in this format:
+// <8 bytes header><chunk 1 bytes>chunk 2 bytes>...
+//
+// The header format:
+//   - First byte: describes the encoding format. Currently, only GnarkBundleEncodingFormat (1)
+//     is supported.
+//   - Remaining 7 bytes: describes the information about chunks.
+//
+// The chunk format will depend on the encoding format. With the GnarkBundleEncodingFormat,
+// each chunk is formated as <32 bytes proof><32 bytes coeff>...<32 bytes coefff>, where the
+// proof and coeffs are all encoded with Gnark.
+func (b Bundle) Serialize() ([]byte, error) {
+	if len(b) == 0 {
+		return []byte{}, nil
+	}
+	if len(b[0].Coeffs) == 0 {
+		return nil, errors.New("invalid bundle: the coeffs length is zero")
+	}
+	size := 0
+	for _, f := range b {
+		if len(f.Coeffs) != len(b[0].Coeffs) {
+			return nil, errors.New("invalid bundle: all chunks should have the same length")
+		}
+		size += bn254.SizeOfG1AffineCompressed + encoding.BYTES_PER_SYMBOL*len(f.Coeffs)
+	}
+	result := make([]byte, size+8)
+	buf := result
+	metadata := uint64(GnarkBundleEncodingFormat) | (uint64(len(b[0].Coeffs)) << 8)
+	binary.LittleEndian.PutUint64(buf, metadata)
+	buf = buf[8:]
+	for _, f := range b {
+		chunk, err := f.SerializeGnark()
+		if err != nil {
+			return nil, err
+		}
+		copy(buf, chunk)
+		buf = buf[len(chunk):]
+	}
+	return result, nil
+}
+
+func (b Bundle) Deserialize(data []byte) (Bundle, error) {
+	if len(data) < 8 {
+		return nil, errors.New("bundle data must have at least 8 bytes")
+	}
+	// Parse metadata
+	meta := binary.LittleEndian.Uint64(data)
+	if (meta & 0xFF) != GnarkBundleEncodingFormat {
+		return nil, errors.New("invalid bundle data encoding format")
+	}
+	chunkLen := meta >> 8
+	if chunkLen == 0 {
+		return nil, errors.New("chunk length must be greater than zero")
+	}
+	chunkSize := bn254.SizeOfG1AffineCompressed + encoding.BYTES_PER_SYMBOL*int(chunkLen)
+	if (len(data)-8)%chunkSize != 0 {
+		return nil, errors.New("bundle data is invalid")
+	}
+	// Decode
+	bundle := make([]*encoding.Frame, 0, (len(data)-8)/chunkSize)
+	buf := data[8:]
+	for len(buf) > 0 {
+		if len(buf) < chunkSize {
+			return nil, errors.New("bundle data is invalid")
+		}
+		f, err := new(encoding.Frame).DeserializeGnark(buf[:chunkSize])
+		if err != nil {
+			return nil, err
+		}
+		bundle = append(bundle, f)
+		buf = buf[chunkSize:]
+	}
+	return bundle, nil
+}
+
 // Serialize encodes a batch of chunks into a byte array
 func (cb Bundles) Serialize() (map[uint32][][]byte, error) {
 	data := make(map[uint32][][]byte, len(cb))

core/data_test.go

@@ -0,0 +1,97 @@
+package core_test
+
+import (
+	"math/rand"
+	"testing"
+
+	"github.com/Layr-Labs/eigenda/core"
+	"github.com/Layr-Labs/eigenda/encoding"
+	"github.com/consensys/gnark-crypto/ecc/bn254/fp"
+	"github.com/consensys/gnark-crypto/ecc/bn254/fr"
+	"github.com/stretchr/testify/assert"
+)
+
+func createBundle(t *testing.T, numFrames, numCoeffs, seed int) core.Bundle {
+	var XCoord, YCoord fp.Element
+	_, err := XCoord.SetString("21661178944771197726808973281966770251114553549453983978976194544185382599016")
+	assert.NoError(t, err)
+	_, err = YCoord.SetString("9207254729396071334325696286939045899948985698134704137261649190717970615186")
+	assert.NoError(t, err)
+	r := rand.New(rand.NewSource(int64(seed)))
+	frames := make([]*encoding.Frame, numFrames)
+	for n := 0; n < numFrames; n++ {
+		frames[n] = new(encoding.Frame)
+		frames[n].Proof = encoding.Proof{
+			X: XCoord,
+			Y: YCoord,
+		}
+		for i := 0; i < numCoeffs; i++ {
+			frames[n].Coeffs = append(frames[n].Coeffs, fr.NewElement(r.Uint64()))
+		}
+	}
+	return frames
+}
+
+func TestInvalidBundleSer(t *testing.T) {
+	b1 := createBundle(t, 1, 0, 0)
+	_, err := b1.Serialize()
+	assert.EqualError(t, err, "invalid bundle: the coeffs length is zero")
+
+	b2 := createBundle(t, 1, 1, 0)
+	b3 := createBundle(t, 1, 2, 0)
+	b3 = append(b3, b2...)
+	_, err = b3.Serialize()
+	assert.EqualError(t, err, "invalid bundle: all chunks should have the same length")
+}
+
+func TestInvalidBundleDeser(t *testing.T) {
+	tooSmallBytes := []byte{byte(0b01000000)}
+	_, err := new(core.Bundle).Deserialize(tooSmallBytes)
+	assert.EqualError(t, err, "bundle data must have at least 8 bytes")
+
+	invalidFormat := make([]byte, 0, 8)
+	for i := 0; i < 7; i++ {
+		invalidFormat = append(invalidFormat, byte(0))
+	}
+	invalidFormat = append(invalidFormat, byte(0b01000000))
+	_, err = new(core.Bundle).Deserialize(invalidFormat)
+	assert.EqualError(t, err, "invalid bundle data encoding format")
+
+	invliadChunkLen := make([]byte, 0, 8)
+	invliadChunkLen = append(invliadChunkLen, byte(1))
+	for i := 0; i < 7; i++ {
+		invliadChunkLen = append(invliadChunkLen, byte(0))
+	}
+	_, err = new(core.Bundle).Deserialize(invliadChunkLen)
+	assert.EqualError(t, err, "chunk length must be greater than zero")
+
+	data := make([]byte, 0, 9)
+	data = append(data, byte(1))
+	for i := 0; i < 6; i++ {
+		data = append(data, byte(0))
+	}
+	data = append(data, byte(0b00100000))
+	data = append(data, byte(5))
+	data = append(data, byte(0b01000000))
+	_, err = new(core.Bundle).Deserialize(data)
+	assert.EqualError(t, err, "bundle data is invalid")
+}
+
+func TestBundleEncoding(t *testing.T) {
+	numTrials := 16
+	for i := 0; i < numTrials; i++ {
+		bundle := createBundle(t, 64, 64, i)
+		bytes, err := bundle.Serialize()
+		assert.Nil(t, err)
+		decoded, err := new(core.Bundle).Deserialize(bytes)
+		assert.Nil(t, err)
+		assert.Equal(t, len(bundle), len(decoded))
+		for i := 0; i < len(bundle); i++ {
+			assert.True(t, bundle[i].Proof.Equal(&decoded[i].Proof))
+			assert.Equal(t, len(bundle[i].Coeffs), len(decoded[i].Coeffs))
+			for j := 0; j < len(bundle[i].Coeffs); j++ {
+				assert.True(t, bundle[i].Coeffs[j].Equal(&decoded[i].Coeffs[j]))
+			}
+		}
+	}
+}