Optimize the bytes encoding at node (#606)

node/store.go

@@ -267,7 +267,7 @@ func (s *Store) StoreBatch(ctx context.Context, header *core.BatchHeader, blobs
 			for i := 0; i < len(bundle); i++ {
 				bundleRaw[i] = rawChunks[quorumID][i]
 			}
-			chunkBytes, err := encodeChunks(bundleRaw)
+			chunkBytes, err := EncodeChunks(bundleRaw)
 			if err != nil {
 				return nil, err
 			}
@@ -358,18 +358,21 @@ func (s *Store) DeleteKeys(ctx context.Context, keys *[][]byte) error {
 
 // Flattens an array of byte arrays (chunks) into a single byte array
 //
-// encodeChunks(chunks) = (len(chunks[0]), chunks[0], len(chunks[1]), chunks[1], ...)
-func encodeChunks(chunks [][]byte) ([]byte, error) {
-	buf := bytes.NewBuffer(make([]byte, 0))
+// EncodeChunks(chunks) = (len(chunks[0]), chunks[0], len(chunks[1]), chunks[1], ...)
+func EncodeChunks(chunks [][]byte) ([]byte, error) {
+	totalSize := 0
 	for _, chunk := range chunks {
-		if err := binary.Write(buf, binary.LittleEndian, uint64(len(chunk))); err != nil {
-			return nil, err
-		}
-		if _, err := buf.Write(chunk); err != nil {
-			return nil, err
-		}
+		totalSize += len(chunk) + 8 // Add size of uint64 for length
+	}
+	result := make([]byte, totalSize)
+	buf := result
+	for _, chunk := range chunks {
+		binary.LittleEndian.PutUint64(buf, uint64(len(chunk)))
+		buf = buf[8:]
+		copy(buf, chunk)
+		buf = buf[len(chunk):]
 	}
-	return buf.Bytes(), nil
+	return result, nil
 }
 
 // Converts a flattened array of chunks into an array of its constituent chunks,

node/store_test.go

@@ -3,6 +3,7 @@ package node_test
 import (
 	"bytes"
 	"context"
+	cryptorand "crypto/rand"
 	"testing"
 	"time"
 
@@ -264,3 +265,24 @@ func TestStoringBlob(t *testing.T) {
 	assert.False(t, s.HasKey(ctx, blobKey1))
 	assert.False(t, s.HasKey(ctx, blobKey2))
 }
+
+func BenchmarkEncodeChunks(b *testing.B) {
+	numSamples := 32
+	numChunks := 10
+	chunkSize := 2 * 1024
+	sampleChunks := make([][][]byte, numSamples)
+	for n := 0; n < numSamples; n++ {
+		chunks := make([][]byte, numChunks)
+		for i := 0; i < numChunks; i++ {
+			chunk := make([]byte, chunkSize)
+			_, _ = cryptorand.Read(chunk)
+			chunks[i] = chunk
+		}
+		sampleChunks[n] = chunks
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, _ = node.EncodeChunks(sampleChunks[i%numSamples])
+	}
+}