feat: add zipmap, zipset based on listpack (#40)

* opt: refact dict data structure

* opt: refact dict with swissmap

* opt: init fes slice cap in aewait

* feat: docker build support buildTime

* perf: optimise listpack perf

* perf: optimize listpack compress memalloc

* feat: add zipmap, zipset based on listpack

---------

Co-authored-by: guangzhixu <[email protected]>

.gitignore

@@ -1,2 +1,3 @@
 coverage.*
-*.aof
+*.aof
+rotom

Dockerfile

@@ -6,11 +6,13 @@ LABEL stage=gobuilder \
 ENV CGO_ENABLED 0
 ENV GOPROXY https://goproxy.cn,direct
 
+ARG BUILD_TIME
+
 WORKDIR /build
 
 COPY . .
 
-RUN go build -ldflags="-s -w" -o rotom .
+RUN go build -ldflags="-s -w -X main.buildTime=${BUILD_TIME}" -o rotom .
 
 FROM alpine:latest
 

Makefile

@@ -16,10 +16,14 @@ pprof:
 heap:
 	go tool pprof http://192.168.1.6:6060/debug/pprof/heap
 
-build-docker:
-	docker build -t rotom .
-
 bench:
 	go test -bench . -benchmem
 
+build:
+	go build -o rotom -ldflags "-s -w -X main.buildTime=$(shell date +%y%m%d_%H%M%S%z)"
+
+build-docker:
+	docker build --build-arg BUILD_TIME=$(shell date +%y%m%d_%H%M%S%z) -t rotom .
+
+# tmp command
 # rsync -av --exclude='.git' rotom/ 2:~/xgz/rotom

README.md

@@ -4,17 +4,15 @@
 
 ## 介绍
 
-这里是 rotom，一个使用 Go 编写的 tiny Redis Server。基于 IO 多路复用技术，还原了 Redis 中的 AeLoop 核心事件循环机制。
-
-rotom 基于 [godis](https://github.com/archeryue/godis) 项目
+这里是 rotom，一个使用 Go 编写的 tiny Redis Server。基于 IO 多路复用还原了 Redis 中的 AeLoop 核心事件循环机制。
 
 ### 实现特性
 
 1. 基于 epoll 网络模型，还原了 Redis 中的 AeLoop 单线程事件循环
 2. 兼容 Redis RESP 协议，你可以使用任何 redis 客户端连接 rotom
-3. 实现了 dict, quicklist, hash, set, zset 数据结构
+3. 实现了 dict, quicklist(listpack), hash(map, zipmap), set(mapset, zipset), zset 数据结构
 4. AOF 支持
-5. 支持 17 种常用命令
+5. 支持 18 种常用命令
 
 ### 原理介绍
 
@@ -64,7 +62,7 @@ $ go run .
 
 ```
 REPOSITORY       TAG           IMAGE ID       CREATED         SIZE
-rotom            latest        22f42ce9ae0e   8 seconds ago   18.6MB
+rotom            latest        22f42ce9ae0e   8 seconds ago   18.8MB
 ```
 
 然后启动容器：

ae.go

@@ -172,6 +172,7 @@ retry:
 	}
 
 	// collect file events
+	fes = make([]*AeFileEvent, 0, n)
 	for _, ev := range events[:n] {
 		if ev.Events&unix.EPOLLIN != 0 {
 			fe := loop.FileEvents[int(ev.Fd)]

command.go

@@ -13,7 +13,7 @@ type Command struct {
 	name string
 
 	// handler is this command real database handler function.
-	handler func(respWriter *RESPWriter, args []RESP)
+	handler func(writer *RESPWriter, args []RESP)
 
 	// arity represents the minimal number of arguments that command accepts.
 	arity int
@@ -37,6 +37,7 @@ var cmdTable []*Command = []*Command{
 	{"rpop", rpopCommand, 1, true},
 	{"lpop", lpopCommand, 1, true},
 	{"sadd", saddCommand, 2, true},
+	{"srem", sremCommand, 2, true},
 	{"spop", spopCommand, 1, true},
 	{"zadd", zaddCommand, 3, true},
 	{"ping", pingCommand, 0, false},
@@ -60,8 +61,8 @@ func equalCommand(str, lowerText string) bool {
 		return false
 	}
 	const s = 'a' - 'A'
-	for i, lo := range lowerText {
-		delta := lo - rune(str[i])
+	for i, lt := range lowerText {
+		delta := lt - rune(str[i])
 		if delta != 0 && delta != s {
 			return false
 		}
@@ -83,8 +84,8 @@ func pingCommand(writer *RESPWriter, _ []RESP) {
 
 func setCommand(writer *RESPWriter, args []RESP) {
 	key := args[0].ToString()
-	value := args[1].ToBytes()
-	db.strs.Set(key, value)
+	value := args[1].Clone()
+	db.dict.Set(key, value)
 	writer.WriteString("OK")
 }
 
@@ -96,48 +97,53 @@ func msetCommand(writer *RESPWriter, args []RESP) {
 	}
 	for i := 0; i < len(args); i += 2 {
 		key := args[i].ToString()
-		value := args[i+1].ToBytes()
-		db.strs.Set(key, value)
+		value := args[i+1].Clone()
+		db.dict.Set(key, value)
 	}
 	writer.WriteString("OK")
 }
 
 func incrCommand(writer *RESPWriter, args []RESP) {
 	key := args[0].ToString()
 
-	val, ok := db.strs.Get(key)
+	val, ok := db.dict.Get(key)
 	if !ok {
-		db.strs.Set(key, []byte("1"))
+		db.dict.Set(key, []byte("1"))
 		writer.WriteInteger(1)
 		return
 	}
 
-	num, err := RESP(val).ToInt()
+	valBytes, ok := val.([]byte)
+	if !ok {
+		writer.WriteError(ErrWrongType)
+		return
+	}
+
+	num, err := RESP(valBytes).ToInt()
 	if err != nil {
 		writer.WriteError(ErrParseInteger)
 		return
 	}
 	num++
 
-	db.strs.Set(key, []byte(strconv.Itoa(num)))
+	db.dict.Set(key, []byte(strconv.Itoa(num)))
 	writer.WriteInteger(num)
 }
 
 func getCommand(writer *RESPWriter, args []RESP) {
 	key := args[0].ToStringUnsafe()
 
-	value, ok := db.strs.Get(key)
-	if ok {
-		writer.WriteBulk(value)
+	val, ok := db.dict.Get(key)
+	if !ok {
+		writer.WriteNull()
 		return
 	}
 
-	// check extra maps
-	_, ok = db.extras.Get(key)
+	valBytes, ok := val.([]byte)
 	if ok {
-		writer.WriteError(ErrWrongType)
+		writer.WriteBulk(valBytes)
 	} else {
-		writer.WriteNull()
+		writer.WriteError(ErrWrongType)
 	}
 }
 
@@ -313,9 +319,8 @@ func lrangeCommand(writer *RESPWriter, args []RESP) {
 	}
 
 	writer.WriteArrayHead(size)
-	ls.Range(start, end, func(data []byte) (stop bool) {
+	ls.Range(start, end, func(data []byte) {
 		writer.WriteBulk(data)
-		return false
 	})
 }
 
@@ -338,6 +343,24 @@ func saddCommand(writer *RESPWriter, args []RESP) {
 	writer.WriteInteger(newItems)
 }
 
+func sremCommand(writer *RESPWriter, args []RESP) {
+	key := args[0].ToString()
+
+	set, err := fetchSet(key)
+	if err != nil {
+		writer.WriteError(err)
+		return
+	}
+
+	var count int
+	for _, arg := range args[1:] {
+		if set.Remove(arg.ToStringUnsafe()) {
+			count++
+		}
+	}
+	writer.WriteInteger(count)
+}
+
 func spopCommand(writer *RESPWriter, args []RESP) {
 	key := args[0].ToString()
 
@@ -403,7 +426,7 @@ func fetchZSet(key string, setnx ...bool) (ZSet, error) {
 }
 
 func fetch[T any](key string, new func() T, setnx ...bool) (v T, err error) {
-	item, ok := db.extras.Get(key)
+	item, ok := db.dict.Get(key)
 	if ok {
 		v, ok := item.(T)
 		if ok {
@@ -413,7 +436,7 @@ func fetch[T any](key string, new func() T, setnx ...bool) (v T, err error) {
 	}
 	v = new()
 	if len(setnx) > 0 && setnx[0] {
-		db.extras.Put(key, v)
+		db.dict.Set(key, v)
 	}
 	return v, nil
 }

command_test.go

@@ -97,10 +97,8 @@ func TestCommand(t *testing.T) {
 		assert.Equal(resm, map[string]string{"k1": "v1", "k2": "v2", "k3": "v3", "k4": "v4", "k5": "v5"})
 
 		// hdel
-		{
-			res, _ := rdb.HDel(ctx, "map", "k1", "k2", "k3", "k99").Result()
-			assert.Equal(res, int64(3))
-		}
+		res, _ = rdb.HDel(ctx, "map", "k1", "k2", "k3", "k99").Result()
+		assert.Equal(res, int64(3))
 
 		// error
 		_, err := rdb.HSet(ctx, "map").Result()
@@ -123,6 +121,9 @@ func TestCommand(t *testing.T) {
 		res, _ := rdb.LRange(ctx, "list", 0, -1).Result()
 		assert.Equal(res, []string{"c", "b", "a", "d", "e", "f"})
 
+		res, _ = rdb.LRange(ctx, "list", 1, 3).Result()
+		assert.Equal(res, []string{"b", "a"})
+
 		// lpop
 		val, _ := rdb.LPop(ctx, "list").Result()
 		assert.Equal(val, "c")
@@ -136,13 +137,19 @@ func TestCommand(t *testing.T) {
 		n, _ := rdb.SAdd(ctx, "set", "k1", "k2", "k3").Result()
 		assert.Equal(n, int64(3))
 
+		// spop
 		for i := 0; i < 3; i++ {
 			val, _ := rdb.SPop(ctx, "set").Result()
 			assert.NotEqual(val, "")
 		}
 
 		_, err := rdb.SPop(ctx, "set").Result()
 		assert.Equal(err, redis.Nil)
+
+		// srem
+		rdb.SAdd(ctx, "set", "k1", "k2", "k3").Result()
+		res, _ := rdb.SRem(ctx, "set", "k1", "k2", "k999").Result()
+		assert.Equal(res, int64(2))
 	})
 
 	t.Run("zset", func(t *testing.T) {

go.mod

@@ -4,12 +4,14 @@ go 1.22
 
 require (
 	github.com/cockroachdb/swiss v0.0.0-20240612210725-f4de07ae6964
+	github.com/deckarep/golang-set/v2 v2.6.0
+	github.com/klauspost/compress v1.17.9
 	github.com/redis/go-redis/v9 v9.5.2
 	github.com/rs/zerolog v1.33.0
 	github.com/sakeven/RbTree v1.1.1
 	github.com/stretchr/testify v1.9.0
 	github.com/tidwall/mmap v0.3.0
-	golang.org/x/sys v0.21.0
+	golang.org/x/sys v0.22.0
 )
 
 require (
@@ -22,7 +24,7 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
-	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect
+	golang.org/x/exp v0.0.0-20240707233637-46b078467d37 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -12,11 +12,15 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM=
+github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ=
 github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
+github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@@ -47,14 +51,14 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tidwall/mmap v0.3.0 h1:XXt1YsiXCF5/UAu3pLbu6g7iulJ9jsbs6vt7UpiV0sY=
 github.com/tidwall/mmap v0.3.0/go.mod h1:2/dNzF5zA+te/JVHfrqNLcRkb8LjdH3c80vYHFQEZRk=
-golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY=
-golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI=
+golang.org/x/exp v0.0.0-20240707233637-46b078467d37 h1:uLDX+AfeFCct3a2C7uIWBKMJIR3CJMhcgfrUAqjRK6w=
+golang.org/x/exp v0.0.0-20240707233637-46b078467d37/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
-golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
+golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=