diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 3efa3caeb5..e9e40fc468 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -30,7 +30,7 @@ jobs: max_enqueued: "20000000" is_london_fork_active: true is_bridge_active: false - gossip_msg_size: "16777216" + gossip_msg_size: "33554432" notification: false secrets: AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }} @@ -168,7 +168,7 @@ jobs: max_enqueued: "20000000" is_london_fork_active: true is_bridge_active: false - gossip_msg_size: "16777216" + gossip_msg_size: "33554432" logs: true build_blade_output: ${{ needs.ci.outputs.build_blade }} lint_output: ${{ needs.ci.outputs.lint }} diff --git a/go.mod b/go.mod index 1fab441695..af2b36e7ab 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/golang/protobuf v1.5.4 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 + github.com/hashicorp/go-bexpr v0.1.14 github.com/hashicorp/go-hclog v1.6.3 github.com/hashicorp/go-immutable-radix v1.3.1 github.com/hashicorp/go-multierror v1.1.1 @@ -72,11 +73,9 @@ require ( github.com/alibabacloud-go/tea-xml v1.1.3 // indirect github.com/clbanning/mxj/v2 v2.5.5 // indirect github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d // indirect + github.com/mitchellh/pointerstructure v1.2.1 // indirect github.com/tjfoc/gmsm v1.3.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect -) - -require ( cloud.google.com/go/auth v0.7.2 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.3 // indirect cloud.google.com/go/compute/metadata v0.5.0 // indirect diff --git a/go.sum b/go.sum index 6873fd196e..971ccdc839 100644 --- a/go.sum +++ b/go.sum @@ -344,6 +344,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-bexpr v0.1.14 h1:uKDeyuOhWhT1r5CiMTjdVY4Aoxdxs6EtwgTGnlosyp4= +github.com/hashicorp/go-bexpr v0.1.14/go.mod h1:gN7hRKB3s7yT+YvTdnhZVLTENejvhlkZ8UE4YVBS+Q8= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= @@ -512,6 +514,8 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQhoHZAz4x7TIw= +github.com/mitchellh/pointerstructure v1.2.1/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= diff --git a/jsonrpc/debug_endpoint.go b/jsonrpc/debug_endpoint.go index f66d5d0b25..b1752d8fe3 100644 --- a/jsonrpc/debug_endpoint.go +++ b/jsonrpc/debug_endpoint.go @@ -5,6 +5,9 @@ import ( "errors" "fmt" "os" + "path/filepath" + "runtime" + "runtime/debug" "time" "github.com/0xPolygon/polygon-edge/helper/hex" @@ -14,7 +17,12 @@ import ( "github.com/0xPolygon/polygon-edge/types" ) -const callTracerName = "callTracer" +const ( + callTracerName = "callTracer" + blockString = "block" + mutexString = "mutex" + heapString = "heap" +) var ( defaultTraceTimeout = 5 * time.Second @@ -77,6 +85,7 @@ type debugStore interface { type Debug struct { store debugStore throttling *Throttling + handler *DebugHandler ReadFileFunc func(filename string) ([]byte, error) } @@ -84,10 +93,282 @@ func NewDebug(store debugStore, requestsPerSecond uint64) *Debug { return &Debug{ store: store, throttling: NewThrottling(requestsPerSecond, time.Second), + handler: new(DebugHandler), ReadFileFunc: os.ReadFile, } } +// CpuProfile turns on CPU profiling for nsec seconds and writes +// profile data to file. +// +//nolint:stylecheck +func (d *Debug) CpuProfile(file string, nsec int64) (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + if err := d.handler.StartCPUProfile(file); err != nil { + return nil, err + } + + time.Sleep(time.Duration(nsec) * time.Second) + + if err := d.handler.StopCPUProfile(); err != nil { + return nil, err + } + + absPath, err := filepath.Abs(file) + if err != nil { + absPath = file + } + + return absPath, nil + }, + ) +} + +// FreeOSMemory forces a garbage collection. +func (d *Debug) FreeOSMemory() (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + debug.FreeOSMemory() + + return nil, nil + }, + ) +} + +// GcStats returns GC statistics. +func (d *Debug) GcStats() (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + s := new(debug.GCStats) + debug.ReadGCStats(s) + + return s, nil + }, + ) +} + +// MemStats returns detailed runtime memory statistics. +func (d *Debug) MemStats() (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + s := new(runtime.MemStats) + runtime.ReadMemStats(s) + + return s, nil + }, + ) +} + +// MutexProfile turns on mutex profiling for nsec seconds and writes profile data to file. +// It uses a profile rate of 1 for most accurate information. If a different rate is +// desired, set the rate and write the profile manually. +func (d *Debug) MutexProfile(file string, nsec int64) (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + runtime.SetMutexProfileFraction(1) + time.Sleep(time.Duration(nsec) * time.Second) + defer runtime.SetMutexProfileFraction(0) + + absPath, err := filepath.Abs(file) + if err != nil { + absPath = file + } + + return absPath, writeProfile(mutexString, file) + }, + ) +} + +// BlockProfile turns on goroutine profiling for nsec seconds and writes profile data to +// file. It uses a profile rate of 1 for most accurate information. If a different rate is +// desired, set the rate and write the profile manually. +func (d *Debug) BlockProfile(file string, nsec int64) (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + runtime.SetBlockProfileRate(1) + time.Sleep(time.Duration(nsec) * time.Second) + + defer runtime.SetBlockProfileRate(0) + + absPath, err := filepath.Abs(file) + if err != nil { + absPath = file + } + + return absPath, writeProfile(blockString, file) + }, + ) +} + +// SetBlockProfileRate sets the rate of goroutine block profile data collection. +// rate 0 disables block profiling. +func (d *Debug) SetBlockProfileRate(rate int) (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + runtime.SetBlockProfileRate(rate) + + return nil, nil + }, + ) +} + +// SetGCPercent sets the garbage collection target percentage. It returns the previous +// setting. A negative value disables GC. +func (d *Debug) SetGCPercent(v int) (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + return debug.SetGCPercent(v), nil + }, + ) +} + +// SetMutexProfileFraction sets the rate of mutex profiling. +func (d *Debug) SetMutexProfileFraction(rate int) (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + runtime.SetMutexProfileFraction(rate) + + return nil, nil + }, + ) +} + +// StartCPUProfile turns on CPU profiling, writing to the given file. +func (d *Debug) StartCPUProfile(file string) (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + if err := d.handler.StartCPUProfile(file); err != nil { + return nil, err + } + + absPath, err := filepath.Abs(file) + if err != nil { + absPath = file + } + + return absPath, nil + }, + ) +} + +// StartGoTrace turns on tracing, writing to the given file. +func (d *Debug) StartGoTrace(file string) (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + if err := d.handler.StartGoTrace(file); err != nil { + return nil, err + } + + absPath, err := filepath.Abs(file) + if err != nil { + absPath = file + } + + return absPath, nil + }, + ) +} + +// StopCPUProfile stops an ongoing CPU profile. +func (d *Debug) StopCPUProfile() (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + if err := d.handler.StopCPUProfile(); err != nil { + return nil, err + } + + return nil, nil + }, + ) +} + +// StopGoTrace stops an ongoing trace. +func (d *Debug) StopGoTrace() (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + if err := d.handler.StopGoTrace(); err != nil { + return nil, err + } + + return nil, nil + }, + ) +} + +// WriteBlockProfile writes a goroutine blocking profile to the given file. +func (d *Debug) WriteBlockProfile(file string) (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + absPath, err := filepath.Abs(file) + if err != nil { + absPath = file + } + + return absPath, writeProfile(blockString, file) + }, + ) +} + +// WriteMemProfile writes an allocation profile to the given file. +// Note that the profiling rate cannot be set through the API, +// it must be set on the command line. +// WriteBlockProfile writes a goroutine blocking profile to the given file. +func (d *Debug) WriteMemProfile(file string) (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + absPath, err := filepath.Abs(file) + if err != nil { + absPath = file + } + + return absPath, writeProfile(heapString, file) + }, + ) +} + +// WriteMutexProfile writes a goroutine blocking profile to the given file. +func (d *Debug) WriteMutexProfile(file string) (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + absPath, err := filepath.Abs(file) + if err != nil { + absPath = file + } + + return absPath, writeProfile(mutexString, file) + }, + ) +} + +// Stacks returns a printed representation of the stacks of all goroutines. It +// also permits the following optional filters to be used: +// - filter: boolean expression of packages to filter for +func (d *Debug) Stacks(filter *string) (interface{}, error) { + return d.throttling.AttemptRequest( + context.Background(), + func() (interface{}, error) { + return d.handler.Stacks(filter) + }, + ) +} + type TraceConfig struct { EnableMemory bool `json:"enableMemory"` DisableStack bool `json:"disableStack"` diff --git a/jsonrpc/debug_handler.go b/jsonrpc/debug_handler.go new file mode 100644 index 0000000000..c4f3092632 --- /dev/null +++ b/jsonrpc/debug_handler.go @@ -0,0 +1,182 @@ +package jsonrpc + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "regexp" + "runtime/pprof" + "runtime/trace" + "strings" + "sync" + + "github.com/hashicorp/go-bexpr" +) + +var ( + errCPUProfilingInProgress = errors.New("CPU profiling already in progress") + errCPUProfilingNotInProgress = errors.New("CPU profiling not in progress") + errTraceAlreadyInProgress = errors.New("trace already in progress") + errTraceNotInProgress = errors.New("trace not in progress") + expressionRegex = regexp.MustCompile(`[:/\.A-Za-z0-9_-]+`) + notExpressionRegex = regexp.MustCompile(`!([:/\.A-Za-z0-9_-]+)`) +) + +// DebugHandler implements the debugging API. +// Do not create values of this type, use the one +// in the Handler variable instead. +type DebugHandler struct { + mux sync.Mutex + cpuW io.WriteCloser + cpuProfileFile string + traceW io.WriteCloser + traceFile string +} + +// StartCPUProfile turns on CPU profiling, writing to the given file. +func (debug *DebugHandler) StartCPUProfile(file string) error { + debug.mux.Lock() + defer debug.mux.Unlock() + + if debug.cpuW != nil { + return errCPUProfilingInProgress + } + + f, err := os.Create(expandHomeDirectory(file)) + if err != nil { + return err + } + + if err := pprof.StartCPUProfile(f); err != nil { + f.Close() + + return err + } + + debug.cpuW = f + debug.cpuProfileFile = file + + return nil +} + +// StopCPUProfile stops an ongoing CPU profile. +func (debug *DebugHandler) StopCPUProfile() error { + debug.mux.Lock() + defer debug.mux.Unlock() + + if debug.cpuW == nil { + return errCPUProfilingNotInProgress + } + + pprof.StopCPUProfile() + + if err := debug.cpuW.Close(); err != nil { + return err + } + + debug.cpuW = nil + debug.cpuProfileFile = "" + + return nil +} + +// StartGoTrace turns on tracing, writing to the given file. +func (debug *DebugHandler) StartGoTrace(file string) error { + debug.mux.Lock() + defer debug.mux.Unlock() + + if debug.traceW != nil { + return errTraceAlreadyInProgress + } + + f, err := os.Create(expandHomeDirectory(file)) + + if err != nil { + return err + } + + if err := trace.Start(f); err != nil { + f.Close() + + return err + } + + debug.traceW = f + debug.traceFile = file + + return nil +} + +// StopTrace stops an ongoing trace. +func (debug *DebugHandler) StopGoTrace() error { + debug.mux.Lock() + defer debug.mux.Unlock() + + if debug.traceW == nil { + return errTraceNotInProgress + } + + trace.Stop() + + debug.traceW.Close() + debug.traceW = nil + debug.traceFile = "" + + return nil +} + +// Stacks returns a printed representation of the stacks of all goroutines. It +// also permits the following optional filters to be used: +// - filter: boolean expression of packages to filter for +func (*DebugHandler) Stacks(filter *string) (string, error) { + buf := new(bytes.Buffer) + if err := pprof.Lookup("goroutine").WriteTo(buf, 2); err != nil { + return "", err + } + + // Apply filtering if a filter is provided + if filter != nil && len(*filter) > 0 { + expanded, err := expandFilter(*filter) + if err != nil { + return "", fmt.Errorf("failed to parse filter expression: expanded=%v, err=%w", expanded, err) + } + + // Filter the goroutine stack trace + if err := filterStackTrace(buf, expanded); err != nil { + return "", err + } + } + + return buf.String(), nil +} + +func expandFilter(filter string) (string, error) { + expanded := expressionRegex.ReplaceAllString(filter, "$0 in Value") + expanded = notExpressionRegex.ReplaceAllString(expanded, "$1 not") + + expanded = strings.ReplaceAll(expanded, "||", "or") + expanded = strings.ReplaceAll(expanded, "&&", "and") + + return expanded, nil +} + +func filterStackTrace(buf *bytes.Buffer, expanded string) error { + expr, err := bexpr.CreateEvaluator(expanded) + if err != nil { + return err + } + + dump := buf.String() + buf.Reset() + + for _, trace := range strings.Split(dump, "\n\n") { + if ok, _ := expr.Evaluate(map[string]string{"Value": trace}); ok { + buf.WriteString(trace) + buf.WriteString("\n\n") + } + } + + return nil +} diff --git a/jsonrpc/dispatcher.go b/jsonrpc/dispatcher.go index 1fd222a4a3..a428768114 100644 --- a/jsonrpc/dispatcher.go +++ b/jsonrpc/dispatcher.go @@ -497,7 +497,7 @@ func (d *Dispatcher) registerService(serviceName string, service interface{}) er continue } - name := lowerCaseFirst(mv.Name) + name := lowerCaseFirstRune(mv.Name) funcName := serviceName + "_" + name fd := &funcData{ fv: mv.Func, @@ -587,10 +587,11 @@ func getError(v reflect.Value) error { return extractedErr } -func lowerCaseFirst(str string) string { - for i, v := range str { - return string(unicode.ToLower(v)) + str[i+1:] +// lowerCaseFirstRune converts the first character of the string to lowercase. +func lowerCaseFirstRune(str string) string { + if len(str) == 0 { + return "" } - return "" + return string(unicode.ToLower(rune(str[0]))) + str[1:] } diff --git a/jsonrpc/dispatcher_test.go b/jsonrpc/dispatcher_test.go index fb0cfec923..0743d0da70 100644 --- a/jsonrpc/dispatcher_test.go +++ b/jsonrpc/dispatcher_test.go @@ -559,6 +559,32 @@ func TestDispatcher_WebsocketConnection_Unsubscribe(t *testing.T) { assert.Equal(t, "true", string(resp.Result)) } +func TestLowerCaseFirstRune(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"Debug_CPUProfile", "debug_CPUProfile"}, + {"BlockNumber", "blockNumber"}, + {"CPUProfileNew", "cPUProfileNew"}, + {"", ""}, + {"A", "a"}, + {"a", "a"}, + {"AB", "aB"}, + {"aB", "aB"}, + {"1234", "1234"}, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + result := lowerCaseFirstRune(test.input) + if result != test.expected { + t.Errorf("lowerCaseFirst(%q) = %q; want %q", test.input, result, test.expected) + } + }) + } +} + func newTestDispatcher(tb testing.TB, logger hclog.Logger, store JSONRPCStore, params *dispatcherParams) *Dispatcher { tb.Helper() diff --git a/jsonrpc/helper.go b/jsonrpc/helper.go index d255650366..306e43a7a1 100644 --- a/jsonrpc/helper.go +++ b/jsonrpc/helper.go @@ -4,6 +4,11 @@ import ( "errors" "fmt" "math/big" + "os" + "os/user" + "path/filepath" + "runtime/pprof" + "strings" "github.com/0xPolygon/polygon-edge/types" ) @@ -270,3 +275,34 @@ func DecodeTxn(arg *txnArgs, store nonceGetter, forceSetNonce bool) (*types.Tran return txn, nil } + +// expandHomeDirectory expands home directory in file paths and sanitizes it. +// For example ~someuser/tmp will not be expanded. +func expandHomeDirectory(p string) string { + if strings.HasPrefix(p, "~/") || strings.HasPrefix(p, "~\\") { + home := os.Getenv("HOME") + if home == "" { + if usr, err := user.Current(); err == nil { + home = usr.HomeDir + } + } + + if home != "" { + p = home + p[1:] + } + } + + return filepath.Clean(p) +} + +func writeProfile(name, file string) error { + p := pprof.Lookup(name) + + f, err := os.Create(expandHomeDirectory(file)) + if err != nil { + return err + } + defer f.Close() + + return p.WriteTo(f, 0) +}