From 8674a5615f53a2fefe0515835c637ca90c77fb78 Mon Sep 17 00:00:00 2001 From: Jian Xiao <99709935+jianoaix@users.noreply.github.com> Date: Tue, 9 Jul 2024 20:52:41 -0700 Subject: [PATCH] [4/N] Chunk encoding optimization: Add bundle encoding (#632) --- core/data.go | 81 +++++++++++++++++++++++++++++++++++++++ core/data_test.go | 97 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+) create mode 100644 core/data_test.go diff --git a/core/data.go b/core/data.go index e71f54482..ca1ad1397 100644 --- a/core/data.go +++ b/core/data.go @@ -1,11 +1,13 @@ package core import ( + "encoding/binary" "errors" "fmt" "github.com/Layr-Labs/eigenda/common" "github.com/Layr-Labs/eigenda/encoding" + "github.com/consensys/gnark-crypto/ecc/bn254" ) type AccountID = string @@ -33,6 +35,8 @@ const ( // which means the max ID can not be larger than 254 (from 0 to 254, there are 255 // different IDs). MaxQuorumID = 254 + + GnarkBundleEncodingFormat = 1 ) func (s *SecurityParam) String() string { @@ -165,6 +169,83 @@ func (b Bundle) Size() uint64 { return size } +// Serialize returns the serialized bytes of the bundle. +// +// The bytes are packed in this format: +// <8 bytes header>chunk 2 bytes>... +// +// The header format: +// - First byte: describes the encoding format. Currently, only GnarkBundleEncodingFormat (1) +// is supported. +// - Remaining 7 bytes: describes the information about chunks. +// +// The chunk format will depend on the encoding format. With the GnarkBundleEncodingFormat, +// each chunk is formated as <32 bytes proof><32 bytes coeff>...<32 bytes coefff>, where the +// proof and coeffs are all encoded with Gnark. +func (b Bundle) Serialize() ([]byte, error) { + if len(b) == 0 { + return []byte{}, nil + } + if len(b[0].Coeffs) == 0 { + return nil, errors.New("invalid bundle: the coeffs length is zero") + } + size := 0 + for _, f := range b { + if len(f.Coeffs) != len(b[0].Coeffs) { + return nil, errors.New("invalid bundle: all chunks should have the same length") + } + size += bn254.SizeOfG1AffineCompressed + encoding.BYTES_PER_SYMBOL*len(f.Coeffs) + } + result := make([]byte, size+8) + buf := result + metadata := uint64(GnarkBundleEncodingFormat) | (uint64(len(b[0].Coeffs)) << 8) + binary.LittleEndian.PutUint64(buf, metadata) + buf = buf[8:] + for _, f := range b { + chunk, err := f.SerializeGnark() + if err != nil { + return nil, err + } + copy(buf, chunk) + buf = buf[len(chunk):] + } + return result, nil +} + +func (b Bundle) Deserialize(data []byte) (Bundle, error) { + if len(data) < 8 { + return nil, errors.New("bundle data must have at least 8 bytes") + } + // Parse metadata + meta := binary.LittleEndian.Uint64(data) + if (meta & 0xFF) != GnarkBundleEncodingFormat { + return nil, errors.New("invalid bundle data encoding format") + } + chunkLen := meta >> 8 + if chunkLen == 0 { + return nil, errors.New("chunk length must be greater than zero") + } + chunkSize := bn254.SizeOfG1AffineCompressed + encoding.BYTES_PER_SYMBOL*int(chunkLen) + if (len(data)-8)%chunkSize != 0 { + return nil, errors.New("bundle data is invalid") + } + // Decode + bundle := make([]*encoding.Frame, 0, (len(data)-8)/chunkSize) + buf := data[8:] + for len(buf) > 0 { + if len(buf) < chunkSize { + return nil, errors.New("bundle data is invalid") + } + f, err := new(encoding.Frame).DeserializeGnark(buf[:chunkSize]) + if err != nil { + return nil, err + } + bundle = append(bundle, f) + buf = buf[chunkSize:] + } + return bundle, nil +} + // Serialize encodes a batch of chunks into a byte array func (cb Bundles) Serialize() (map[uint32][][]byte, error) { data := make(map[uint32][][]byte, len(cb)) diff --git a/core/data_test.go b/core/data_test.go new file mode 100644 index 000000000..184866b6e --- /dev/null +++ b/core/data_test.go @@ -0,0 +1,97 @@ +package core_test + +import ( + "math/rand" + "testing" + + "github.com/Layr-Labs/eigenda/core" + "github.com/Layr-Labs/eigenda/encoding" + "github.com/consensys/gnark-crypto/ecc/bn254/fp" + "github.com/consensys/gnark-crypto/ecc/bn254/fr" + "github.com/stretchr/testify/assert" +) + +func createBundle(t *testing.T, numFrames, numCoeffs, seed int) core.Bundle { + var XCoord, YCoord fp.Element + _, err := XCoord.SetString("21661178944771197726808973281966770251114553549453983978976194544185382599016") + assert.NoError(t, err) + _, err = YCoord.SetString("9207254729396071334325696286939045899948985698134704137261649190717970615186") + assert.NoError(t, err) + r := rand.New(rand.NewSource(int64(seed))) + frames := make([]*encoding.Frame, numFrames) + for n := 0; n < numFrames; n++ { + frames[n] = new(encoding.Frame) + frames[n].Proof = encoding.Proof{ + X: XCoord, + Y: YCoord, + } + for i := 0; i < numCoeffs; i++ { + frames[n].Coeffs = append(frames[n].Coeffs, fr.NewElement(r.Uint64())) + } + } + return frames +} + +func TestInvalidBundleSer(t *testing.T) { + b1 := createBundle(t, 1, 0, 0) + _, err := b1.Serialize() + assert.EqualError(t, err, "invalid bundle: the coeffs length is zero") + + b2 := createBundle(t, 1, 1, 0) + b3 := createBundle(t, 1, 2, 0) + b3 = append(b3, b2...) + _, err = b3.Serialize() + assert.EqualError(t, err, "invalid bundle: all chunks should have the same length") +} + +func TestInvalidBundleDeser(t *testing.T) { + tooSmallBytes := []byte{byte(0b01000000)} + _, err := new(core.Bundle).Deserialize(tooSmallBytes) + assert.EqualError(t, err, "bundle data must have at least 8 bytes") + + invalidFormat := make([]byte, 0, 8) + for i := 0; i < 7; i++ { + invalidFormat = append(invalidFormat, byte(0)) + } + invalidFormat = append(invalidFormat, byte(0b01000000)) + _, err = new(core.Bundle).Deserialize(invalidFormat) + assert.EqualError(t, err, "invalid bundle data encoding format") + + invliadChunkLen := make([]byte, 0, 8) + invliadChunkLen = append(invliadChunkLen, byte(1)) + for i := 0; i < 7; i++ { + invliadChunkLen = append(invliadChunkLen, byte(0)) + } + _, err = new(core.Bundle).Deserialize(invliadChunkLen) + assert.EqualError(t, err, "chunk length must be greater than zero") + + data := make([]byte, 0, 9) + data = append(data, byte(1)) + for i := 0; i < 6; i++ { + data = append(data, byte(0)) + } + data = append(data, byte(0b00100000)) + data = append(data, byte(5)) + data = append(data, byte(0b01000000)) + _, err = new(core.Bundle).Deserialize(data) + assert.EqualError(t, err, "bundle data is invalid") +} + +func TestBundleEncoding(t *testing.T) { + numTrials := 16 + for i := 0; i < numTrials; i++ { + bundle := createBundle(t, 64, 64, i) + bytes, err := bundle.Serialize() + assert.Nil(t, err) + decoded, err := new(core.Bundle).Deserialize(bytes) + assert.Nil(t, err) + assert.Equal(t, len(bundle), len(decoded)) + for i := 0; i < len(bundle); i++ { + assert.True(t, bundle[i].Proof.Equal(&decoded[i].Proof)) + assert.Equal(t, len(bundle[i].Coeffs), len(decoded[i].Coeffs)) + for j := 0; j < len(bundle[i].Coeffs); j++ { + assert.True(t, bundle[i].Coeffs[j].Equal(&decoded[i].Coeffs[j])) + } + } + } +}