perf: optimize listpack removeNext cost

internal/hash/bench_test.go

@@ -37,6 +37,14 @@ func benchMapI(name string, newf func() MapI, b *testing.B) {
 			m.Set(k, []byte(k))
 		}
 	})
+	b.Run(name+"/remove", func(b *testing.B) {
+		m := genMap(newf(), b.N)
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			k := genKey(i)
+			m.Remove(k)
+		}
+	})
 	b.Run(name+"/scan", func(b *testing.B) {
 		m := genMap(newf(), N)
 		b.ResetTimer()

internal/hash/benchmark/main.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"runtime"
+	"runtime/debug"
+	"time"
+
+	"github.com/xgzlucario/rotom/internal/hash"
+)
+
+var previousPause time.Duration
+
+func gcPause() time.Duration {
+	runtime.GC()
+	var stats debug.GCStats
+	debug.ReadGCStats(&stats)
+	pause := stats.PauseTotal - previousPause
+	previousPause = stats.PauseTotal
+	return pause
+}
+
+func genKV(id int) (string, []byte) {
+	k := fmt.Sprintf("%08x", id)
+	return k, []byte(k)
+}
+
+func main() {
+	c := ""
+	flag.StringVar(&c, "obj", "map", "object to bench.")
+	flag.Parse()
+	fmt.Println(c)
+
+	start := time.Now()
+
+	switch c {
+	case "map":
+		m := map[int]any{}
+		for i := 0; i < 10000; i++ {
+			hm := hash.NewMap()
+			for i := 0; i < 512; i++ {
+				k, v := genKV(i)
+				hm.Set(k, v)
+			}
+			m[i] = hm
+		}
+
+	case "zipmap":
+		m := map[int]any{}
+		for i := 0; i < 10000; i++ {
+			hm := hash.NewZipMap()
+			for i := 0; i < 512; i++ {
+				k, v := genKV(i)
+				hm.Set(k, v)
+			}
+			m[i] = hm
+		}
+
+	case "zipmap-compressed":
+		m := map[int]any{}
+		for i := 0; i < 10000; i++ {
+			hm := hash.NewZipMap()
+			for i := 0; i < 512; i++ {
+				k, v := genKV(i)
+				hm.Set(k, v)
+			}
+			hm.Compress()
+			m[i] = hm
+		}
+	}
+	cost := time.Since(start)
+
+	var mem runtime.MemStats
+	var stat debug.GCStats
+
+	runtime.ReadMemStats(&mem)
+	debug.ReadGCStats(&stat)
+
+	fmt.Println("gcsys:", mem.GCSys/1024/1024, "mb")
+	fmt.Println("heap inuse:", mem.HeapInuse/1024/1024, "mb")
+	fmt.Println("heap object:", mem.HeapObjects/1024, "k")
+	fmt.Println("gc:", stat.NumGC)
+	fmt.Println("pause:", gcPause())
+	fmt.Println("cost:", cost)
+}

internal/hash/set_test.go

@@ -11,13 +11,25 @@ func TestSet(t *testing.T) {
 	testSetI(NewZipSet(), t)
 }
 
+func TestZipSet2Set(t *testing.T) {
+	assert := assert.New(t)
+
+	m := NewZipSet()
+	m.Add("key1")
+	m.Add("key2")
+	m.Add("key3")
+
+	assert.ElementsMatch(m.ToSet().ToSlice(), []string{"key1", "key2", "key3"})
+}
+
 func testSetI(m SetI, t *testing.T) {
 	assert := assert.New(t)
 
 	// add
 	assert.True(m.Add("key1"))
 	assert.True(m.Add("key2"))
 	assert.True(m.Add("key3"))
+	assert.False(m.Add("key1"))
 
 	// len
 	assert.Equal(m.Len(), 3)

internal/hash/zipmap.go

@@ -51,8 +51,7 @@ func (zm *ZipMap) Remove(key string) bool {
 		it.Prev()
 		keyBytes := it.Prev()
 		if key == b2s(keyBytes) {
-			it.RemoveNext()
-			it.RemoveNext()
+			it.RemoveNexts(2)
 			return true
 		}
 	}
@@ -80,6 +79,10 @@ func (zm *ZipMap) ToMap() *Map {
 	return m
 }
 
+func (zm *ZipMap) Compress() { zm.m.Compress() }
+
+func (zm *ZipMap) Decompress() { zm.m.Decompress() }
+
 func b2s(b []byte) string {
 	return *(*string)(unsafe.Pointer(&b))
 }

internal/hash/zipset.go

@@ -51,17 +51,17 @@ func (zs *ZipSet) Scan(fn func(string)) {
 	}
 }
 
-func (zs *ZipSet) Pop() (string, bool) {
-	return zs.m.RPop()
-}
+func (zs *ZipSet) Pop() (string, bool) { return zs.m.RPop() }
 
-func (zs *ZipSet) Len() int {
-	return zs.m.Size()
-}
+func (zs *ZipSet) Len() int { return zs.m.Size() }
+
+func (zs *ZipSet) Compress() { zs.m.Compress() }
+
+func (zs *ZipSet) Decompress() { zs.m.Decompress() }
 
 func (zs *ZipSet) ToSet() *Set {
 	s := NewSet()
-	s.Scan(func(key string) {
+	zs.Scan(func(key string) {
 		s.Add(key)
 	})
 	return s

internal/list/listpack.go

@@ -40,8 +40,8 @@ type ListPack struct {
 	data     []byte
 }
 
-func NewListPack(val ...string) *ListPack {
-	return &ListPack{data: bpool.Get(32)[:0]}
+func NewListPack() *ListPack {
+	return &ListPack{data: make([]byte, 0, 32)}
 }
 
 func (lp *ListPack) Size() int {
@@ -70,17 +70,15 @@ func (lp *ListPack) Compress() {
 	if lp.compress {
 		return
 	}
-	dst := encoder.EncodeAll(lp.data, bpool.Get(len(lp.data))[:0])
-	bpool.Put(lp.data)
-	lp.data = slices.Clip(dst)
+	lp.data = encoder.EncodeAll(lp.data, make([]byte, 0, len(lp.data)/3))
 	lp.compress = true
 }
 
 func (lp *ListPack) Decompress() {
 	if !lp.compress {
 		return
 	}
-	lp.data, _ = decoder.DecodeAll(lp.data, bpool.Get(maxListPackSize)[:0])
+	lp.data, _ = decoder.DecodeAll(lp.data, nil)
 	lp.compress = false
 }
 
@@ -175,18 +173,32 @@ func (it *lpIterator) Insert(datas ...string) {
 }
 
 func (it *lpIterator) RemoveNext() (string, bool) {
-	if it.IsLast() {
+	res := it.RemoveNexts(1)
+	if len(res) == 0 {
 		return "", false
 	}
+	return res[0], true
+}
+
+func (it *lpIterator) RemoveNexts(num int) (res []string) {
+	res = make([]string, 0, num)
 	before := it.index
-	next := string(it.Next())
-	after := it.index
+	var after int
+
+	for i := 0; i < num; i++ {
+		if it.IsLast() {
+			goto BREAK
+		}
+		res = append(res, string(it.Next()))
+		it.size--
+	}
 
+BREAK:
+	after = it.index
 	it.data = slices.Delete(it.data, before, after)
 	it.index = before
-	it.size--
 
-	return next, true
+	return
 }
 
 func (it *lpIterator) ReplaceNext(key string) {

internal/list/listpack_test.go

@@ -43,6 +43,8 @@ func TestListpack(t *testing.T) {
 		lp.LPush("A", "B", "C")
 
 		it := lp.Iterator()
+		// bound check
+		it.Prev()
 
 		val, ok := it.RemoveNext()
 		assert.Equal(val, "A")
@@ -67,6 +69,8 @@ func TestListpack(t *testing.T) {
 		lp.LPush("A", "B", "C")
 
 		it := lp.Iterator().SeekLast()
+		// bound check
+		it.Next()
 
 		it.Prev()
 		val, ok := it.RemoveNext()
@@ -90,6 +94,25 @@ func TestListpack(t *testing.T) {
 		assert.False(ok)
 	})
 
+	t.Run("removeNexts", func(t *testing.T) {
+		lp := NewListPack()
+		lp.LPush("aa", "bb", "cc", "dd", "ee")
+
+		str, ok := lp.Iterator().RemoveNext()
+		assert.Equal(str, "aa")
+		assert.True(ok)
+
+		res := lp.Iterator().RemoveNexts(2)
+		assert.Equal(res, []string{"bb", "cc"})
+
+		res = lp.Iterator().RemoveNexts(3)
+		assert.Equal(res, []string{"dd", "ee"})
+
+		str, ok = lp.Iterator().RemoveNext()
+		assert.Equal(str, "")
+		assert.False(ok)
+	})
+
 	t.Run("replaceNext", func(t *testing.T) {
 		lp := NewListPack()
 		lp.LPush("TEST1", "TEST2", "TEST3")
@@ -103,12 +126,17 @@ func TestListpack(t *testing.T) {
 
 		it.ReplaceNext("TTTTTT")
 		assert.Equal(lp2list(lp), []string{"TTTTTT", "TEST2", "TEST3"})
+
+		it.SeekLast().ReplaceNext("a")
+		assert.Equal(lp2list(lp), []string{"TTTTTT", "TEST2", "TEST3"})
 	})
 
 	t.Run("compress", func(t *testing.T) {
 		lp := NewListPack()
 		lp.LPush("A", "B", "C", "D", "E")
 		lp.Compress()
+		lp.Compress()
+		lp.Decompress()
 		lp.Decompress()
 		assert.Equal(lp2list(lp), []string{"A", "B", "C", "D", "E"})
 	})