compute block length accounting for compression

Taken from PR #1698 and commit da5f789
dgraph-io · Feb 18, 2023 · 1375846 · 1375846
1 parent c569be1
commit 1375846
Show file tree

Hide file tree

Showing 3 changed files with 14 additions and 2 deletions.
diff --git a/db.go b/db.go
@@ -755,6 +755,9 @@ var requestPool = sync.Pool{
 }
 
 func (db *DB) writeToLSM(b *request) error {
+	db.lock.RLock()
+	defer db.lock.RUnlock()
+
 	// We should check the length of b.Prts and b.Entries only when badger is not
 	// running in InMemory mode. In InMemory mode, we don't write anything to the
 	// value log and that's why the length of b.Ptrs will always be zero.

diff --git a/go.sum b/go.sum
@@ -5,7 +5,6 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=

diff --git a/table/builder.go b/table/builder.go
@@ -156,6 +156,16 @@ func NewTableBuilder(opts Options) *Builder {
 	return b
 }
 
+func maxEncodedLen(ctype options.CompressionType, sz int) int {
+	switch ctype {
+	case options.Snappy:
+		return snappy.MaxEncodedLen(sz)
+	case options.ZSTD:
+		return y.ZSTDCompressBound(sz)
+	}
+	return sz
+}
+
 func (b *Builder) handleBlock() {
 	defer b.wg.Done()
 
@@ -178,7 +188,7 @@ func (b *Builder) handleBlock() {
 		// BlockBuf should always less than or equal to allocated space. If the blockBuf is greater
 		// than allocated space that means the data from this block cannot be stored in its
 		// existing location.
-		allocatedSpace := (item.end) + padding + 1
+		allocatedSpace := maxEncodedLen(b.opts.Compression, (item.end)) + padding + 1
 		y.AssertTrue(len(blockBuf) <= allocatedSpace)
 
 		// blockBuf was allocated on allocator. So, we don't need to copy it over.