hmap增加性能测试

ellyn_ast/processor.go

@@ -3,16 +3,16 @@ package ellyn_ast
 import (
 	"github.com/lvyahui8/ellyn/ellyn_common/log"
 	"github.com/lvyahui8/ellyn/ellyn_common/utils"
-	"io/fs"
 	"os"
 	"strings"
 )
 
 type Processor interface {
 	FilterPackage(progCtx *ProgramContext, pkg Package) bool
-	FilterFile(progCtx *ProgramContext, pkg Package, file fs.FileInfo) bool
-	HandleFile(progCtx *ProgramContext, pkg Package, file fs.FileInfo, content []byte)
-	HandleFunc(progCtx *ProgramContext, pkg Package, file fs.FileInfo, goFunc *GoFunc)
+	FilterFile(progCtx *ProgramContext, pkg Package, file os.DirEntry) bool
+	HandleFile(progCtx *ProgramContext, pkg Package, file os.DirEntry, content []byte)
+	HandlePackage(progCtx *ProgramContext, pkg Package)
+	HandleFunc(progCtx *ProgramContext, pkg Package, file os.DirEntry, goFunc *GoFunc)
 }
 
 type DefaultProcessor struct {
@@ -22,7 +22,7 @@ func (d DefaultProcessor) FilterPackage(progCtx *ProgramContext, pkg Package) bo
 	return strings.HasPrefix(pkg.Path, progCtx.RootPkgPath())
 }
 
-func (d DefaultProcessor) FilterFile(progCtx *ProgramContext, pkg Package, file fs.FileInfo) bool {
+func (d DefaultProcessor) FilterFile(progCtx *ProgramContext, pkg Package, file os.DirEntry) bool {
 	if file.IsDir() {
 		return false
 	}
@@ -38,10 +38,14 @@ func (d DefaultProcessor) FilterFile(progCtx *ProgramContext, pkg Package, file
 	return true
 }
 
-func (d DefaultProcessor) HandleFile(progCtx *ProgramContext, pkg Package, file fs.FileInfo, content []byte) {
+func (d DefaultProcessor) HandlePackage(progCtx *ProgramContext, pkg Package) {
+
+}
+
+func (d DefaultProcessor) HandleFile(progCtx *ProgramContext, pkg Package, file os.DirEntry, content []byte) {
 	log.Infof("dir %s,file %s", pkg.Dir, file.Name())
 }
 
-func (d DefaultProcessor) HandleFunc(progCtx *ProgramContext, pkg Package, file fs.FileInfo, goFunc *GoFunc) {
+func (d DefaultProcessor) HandleFunc(progCtx *ProgramContext, pkg Package, file os.DirEntry, goFunc *GoFunc) {
 
 }

ellyn_ast/program.go

@@ -6,7 +6,6 @@ import (
 	"go/ast"
 	"go/parser"
 	"go/token"
-	"io/ioutil"
 	"os"
 	"path"
 	"strings"
@@ -23,6 +22,7 @@ func (p *ProgramContext) RootPkgPath() string {
 	return p.rootPkgPath
 }
 
+// Program 封装程序信息，解析遍历程序的所有包、函数、代码块
 type Program struct {
 	mainPkg      Package
 	rootPkg      Package
@@ -57,13 +57,15 @@ func (p *Program) Visit() {
 	p.handleFiles()
 }
 
+// _init 初始化基础信息，为文件迭代做准备
 func (p *Program) _init() {
 	p.initOnce.Do(func() {
 		packages := utils.Go.AllPackages(p.mainPkg.Dir)
 		for pkgPath, pkgDir := range packages {
 			p.pkgMap[pkgDir] = NewPackage(pkgDir, pkgPath)
 		}
 		p.mainPkg.Name = p.pkgMap[p.mainPkg.Dir].Name
+
 		p.modFile = utils.Go.GetModFile(p.mainPkg.Dir)
 		rootPkgPath := utils.Go.GetProjectRootPkgPath(p.modFile)
 		p.rootPkg = NewPackage(path.Dir(p.modFile), rootPkgPath)
@@ -76,7 +78,7 @@ func (p *Program) handleFiles() {
 		if !p.processor.FilterPackage(p.progCtx, pkg) {
 			continue
 		}
-		files, err := ioutil.ReadDir(pkgDir)
+		files, err := os.ReadDir(pkgDir)
 		asserts.IsNil(err)
 		for _, file := range files {
 			if !strings.HasSuffix(file.Name(), ".go") {
@@ -87,7 +89,7 @@ func (p *Program) handleFiles() {
 			}
 			// 加入遍历集合
 			fileAbsPath := pkg.Dir + string(os.PathSeparator) + file.Name()
-			content, err := ioutil.ReadFile(fileAbsPath)
+			content, err := os.ReadFile(fileAbsPath)
 			asserts.IsNil(err)
 			p.processor.HandleFile(p.progCtx, pkg, file, content)
 			visitor := &FileVisitor{content: content}

ellyn_common/collections/hmap.go

@@ -5,16 +5,17 @@ import "sync"
 type keyHasher func(key interface{}) int
 
 type mapApi interface {
-	Put(key, val interface{})
-	Get(key interface{}) interface{}
-	Del(key interface{})
+	Store(key, val interface{})
+	Load(key interface{}) (interface{}, bool)
+	Delete(key interface{})
 }
 
 type concurrentMap struct {
-	segMask  int
-	segments []*mapSegment
-	hasher   keyHasher
-	size     uint64
+	segMask  int           // 8
+	segments []*mapSegment // 24字节，底层是一个slice struct，里面有array指针+两个int字段，3 * 8
+	hasher   keyHasher     // 8
+	size     uint64        // 8 字节
+	_padding [16]byte
 }
 
 func NewConcurrentMap(segSize int, hasher keyHasher) *concurrentMap {
@@ -32,16 +33,16 @@ func NewConcurrentMap(segSize int, hasher keyHasher) *concurrentMap {
 	return m
 }
 
-func (m *concurrentMap) Put(key, val interface{}) {
-	m.getSegment(key).Put(key, val)
+func (m *concurrentMap) Store(key, val interface{}) {
+	m.getSegment(key).Store(key, val)
 }
 
-func (m *concurrentMap) Get(key interface{}) interface{} {
-	return m.getSegment(key).Get(key)
+func (m *concurrentMap) Load(key interface{}) (interface{}, bool) {
+	return m.getSegment(key).Load(key)
 }
 
-func (m *concurrentMap) Del(key interface{}) {
-	m.getSegment(key).Del(key)
+func (m *concurrentMap) Delete(key interface{}) {
+	m.getSegment(key).Delete(key)
 }
 
 func (m *concurrentMap) getSegment(key interface{}) *mapSegment {
@@ -57,25 +58,27 @@ func (m *concurrentMap) Size() int {
 }
 
 type mapSegment struct {
-	sync.RWMutex
-	entries map[interface{}]interface{}
-	size    int
+	sync.RWMutex                             // 24
+	entries      map[interface{}]interface{} // 8
+	size         int                         // 8
+	_padding     [24]byte                    //
 }
 
-func (s *mapSegment) Put(key, val interface{}) {
+func (s *mapSegment) Store(key, val interface{}) {
 	s.Lock()
 	defer s.Unlock()
 	s.entries[key] = val
 	s.size = len(s.entries)
 }
 
-func (s *mapSegment) Get(key interface{}) interface{} {
+func (s *mapSegment) Load(key interface{}) (res interface{}, ok bool) {
 	s.RLock()
 	defer s.RUnlock()
-	return s.entries[key]
+	res, ok = s.entries[key]
+	return
 }
 
-func (s *mapSegment) Del(key interface{}) {
+func (s *mapSegment) Delete(key interface{}) {
 	s.Lock()
 	defer s.Unlock()
 	delete(s.entries, key)

ellyn_common/collections/hmap.md

@@ -0,0 +1,35 @@
+## map 性能测试结果
+
+测试方法：随机生成1000w个完全不相同的数字k，k<1亿 ，打乱顺序让多协程并发写入、读取、删除，每个协程处理其中的一部分号段
+
+> 结论: concurrentMap 在分段为2048时，性能和内存分配明显好于sync.Map。 但在资源竞争率高的时候，性能差异不明显，concurrentMap略好于syncMap。
+
+```shell
+go test -v -run ^$  -bench BenchmarkMap -benchtime=5s -benchmem
+```
+
+```text
+goos: windows
+goarch: amd64
+pkg: github.com/lvyahui8/ellyn/ellyn_common/collections
+cpu: AMD Ryzen 7 4800U with Radeon Graphics
+BenchmarkMap
+BenchmarkMap/concurrentMap_put
+BenchmarkMap/concurrentMap_put-16                     18         342347494 ns/op        80002544 B/op    9999784 allocs/op
+BenchmarkMap/concurrentMap_read
+BenchmarkMap/concurrentMap_read-16                    18         339383994 ns/op        80001507 B/op    9999781 allocs/op
+BenchmarkMap/concurrentMap_delete
+BenchmarkMap/concurrentMap_delete-16                  61          99936639 ns/op        80000277 B/op    9999779 allocs/op
+BenchmarkMap/concurrentMap_R&W
+BenchmarkMap/concurrentMap_R&W-16                      9         723391478 ns/op        160000385 B/op  19999569 allocs/op
+BenchmarkMap/syncMap_put
+BenchmarkMap/syncMap_put-16                            1        8409727400 ns/op        1470761288 B/op 40306906 allocs/op
+BenchmarkMap/syncMap_read
+BenchmarkMap/syncMap_read-16                           1        6106719400 ns/op        80012880 B/op    9999918 allocs/op
+BenchmarkMap/syncMap_delete
+BenchmarkMap/syncMap_delete-16                        15         369030780 ns/op        79999968 B/op    9999778 allocs/op
+BenchmarkMap/syncMap_R&W
+BenchmarkMap/syncMap_R&W-16                            7        1029901486 ns/op        319999952 B/op  29999555 allocs/op
+PASS
+ok      github.com/lvyahui8/ellyn/ellyn_common/collections      70.109s
+```

ellyn_common/collections/hmap_test.go

@@ -0,0 +1,141 @@
+package collections
+
+import (
+	"github.com/stretchr/testify/require"
+	"math/rand"
+	"runtime"
+	"sync"
+	"testing"
+	"time"
+	"unsafe"
+)
+
+const maxVal = 10000 * 10000
+const cnt = 1000 * 10000
+
+func shuffle(nums []int) (res []int) {
+	res = make([]int, len(nums))
+	for i, v := range nums {
+		res[i] = v
+	}
+	rand.Seed(time.Now().UnixMilli())
+	rand.Shuffle(len(nums), func(i, j int) {
+		res[i], res[j] = res[j], res[i]
+	})
+	return
+}
+
+// 生成100w个小于1000w不重复的数字
+func randomSeq() (res []int) {
+	raw := make([]int, maxVal)
+	for i := 0; i < maxVal; i++ {
+		raw[i] = i
+	}
+	res = shuffle(raw[0:cnt])
+	return
+}
+
+func testReadWrite(b *testing.B, name string, m mapApi, insertSeq, readSeq, deleteSeq []int) {
+	// 测试纯粹写入的并发性能
+	routineSize := runtime.NumCPU()
+	handleSize := len(insertSeq) / routineSize
+	b.Run(name+"_put", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			w := sync.WaitGroup{}
+			w.Add(routineSize)
+			for j := 0; j < routineSize; j++ {
+				go func(offset int) {
+					defer w.Done()
+					for i := offset; i < handleSize+offset; i++ {
+						m.Store(insertSeq[i], struct{}{})
+					}
+				}(handleSize * j)
+			}
+			w.Wait()
+		}
+	})
+	// 测试纯粹读取的并发性能
+	b.Run(name+"_read", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			w := sync.WaitGroup{}
+			w.Add(routineSize)
+			for j := 0; j < routineSize; j++ {
+				go func(offset int) {
+					defer w.Done()
+					for i := offset; i < handleSize+offset; i++ {
+						_, _ = m.Load(readSeq[i])
+					}
+				}(handleSize * j)
+			}
+			w.Wait()
+		}
+	})
+	// 测试纯粹删除的并发性能
+	b.Run(name+"_delete", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			w := sync.WaitGroup{}
+			w.Add(routineSize)
+			for j := 0; j < routineSize; j++ {
+				go func(offset int) {
+					defer w.Done()
+					for i := offset; i < handleSize+offset; i++ {
+						m.Delete(deleteSeq[i])
+					}
+				}(handleSize * j)
+			}
+			w.Wait()
+		}
+	})
+	// 测试同时写入、读取的并发性能
+	b.Run(name+"_R&W", func(b *testing.B) {
+		for i := 0; i < b.N; i++ {
+			w := sync.WaitGroup{}
+			w.Add(routineSize * 2)
+			for j := 0; j < routineSize; j++ {
+				go func(offset int) {
+					defer w.Done()
+					for i := offset; i < handleSize+offset; i++ {
+						m.Store(insertSeq[i], struct{}{})
+					}
+				}(handleSize * j)
+			}
+			for j := 0; j < routineSize; j++ {
+				go func(offset int) {
+					defer w.Done()
+					for i := offset; i < handleSize+offset; i++ {
+						_, _ = m.Load(readSeq[i])
+					}
+				}(handleSize * j)
+			}
+			w.Wait()
+		}
+	})
+}
+
+func BenchmarkMap(b *testing.B) {
+	insertSeq := randomSeq()
+	readSeq := shuffle(insertSeq)
+	deleteSeq := shuffle(insertSeq)
+	testReadWrite(b, "concurrentMap", NewConcurrentMap(2048, func(key interface{}) int {
+		return key.(int)
+	}), insertSeq, readSeq, deleteSeq)
+	sMap := &sync.Map{}
+	testReadWrite(b, "syncMap", sMap, insertSeq, readSeq, deleteSeq)
+}
+
+func TestMapPadding(t *testing.T) {
+	m := NewConcurrentMap(2048, func(key interface{}) int {
+		return key.(int)
+	})
+	require.Equal(t, 64, int(unsafe.Sizeof(*m)))
+	t.Log(unsafe.Sizeof(m.size))
+	t.Log(unsafe.Sizeof(m.segMask))
+	t.Log(unsafe.Sizeof(m.hasher))
+	t.Log(unsafe.Sizeof(m.segments))
+	t.Log("=======")
+	ms := mapSegment{}
+	require.Equal(t, 64, int(unsafe.Sizeof(ms)))
+	t.Log(unsafe.Sizeof(ms.RWMutex))
+	t.Log(unsafe.Sizeof(ms.entries))
+	t.Log(unsafe.Sizeof(ms.size))
+}

ellyn_common/collections/ringbuffer.go

@@ -1,6 +1,8 @@
 package collections
 
-import "sync/atomic"
+import (
+	"sync/atomic"
+)
 
 type ringBuffer struct {
 	// dequeuePos 指向下一个可消费点位，一直累加然后对capacity取模（&mask）取值

ellyn_common/collections/ringbuffer_test.go

@@ -3,6 +3,7 @@ package collections
 import (
 	"fmt"
 	"github.com/stretchr/testify/require"
+	"math"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -92,6 +93,12 @@ func queueReadWrite(queue Queue, cnt int) bool {
 	return produceCnt == consumeCnt
 }
 
+func TestMaxIdx(t *testing.T) {
+	s := math.MaxInt64 / int64(100*10000)
+	year := s / int64(60*60*24*365)
+	t.Log(year)
+}
+
 func TestRingBufferBasic(t *testing.T) {
 	buffer := NewRingBuffer(100000)
 	n := 4

ellyn_common/gls/goid.go

@@ -0,0 +1,7 @@
+package gls
+
+import "runtime"
+
+func GetGoId() uint64 {
+	return runtime.EllynGetGoid()
+}

ellyn_common/gls/routine_local.go

@@ -0,0 +1 @@
+package gls