From a569bb66e49ff87ae5f3868fe14ca684b8e85464 Mon Sep 17 00:00:00 2001 From: Petri-Johan Last Date: Thu, 28 Sep 2023 10:14:40 +0200 Subject: [PATCH] Add code host rate limiter details to code host connection page (#56638) --- .../ExternalServiceInformation.tsx | 36 +++++- .../externalServices/ExternalServicePage.tsx | 1 + .../components/externalServices/backend.ts | 22 ++++ client/web/src/site-admin/fixtures.ts | 9 ++ cmd/frontend/graphqlbackend/BUILD.bazel | 3 + .../graphqlbackend/external_service.go | 15 +++ .../graphqlbackend/external_services_test.go | 108 ++++++++++++++++-- cmd/frontend/graphqlbackend/ratelimiter.go | 36 ++++++ cmd/frontend/graphqlbackend/schema.graphql | 40 +++++++ 9 files changed, 262 insertions(+), 8 deletions(-) create mode 100644 cmd/frontend/graphqlbackend/ratelimiter.go diff --git a/client/web/src/components/externalServices/ExternalServiceInformation.tsx b/client/web/src/components/externalServices/ExternalServiceInformation.tsx index 51708961c8149..e2938d0467b79 100644 --- a/client/web/src/components/externalServices/ExternalServiceInformation.tsx +++ b/client/web/src/components/externalServices/ExternalServiceInformation.tsx @@ -5,6 +5,8 @@ import classNames from 'classnames' import type { ExternalServiceKind } from '@sourcegraph/shared/src/graphql-operations' import { Icon, Link, LoadingSpinner, Tooltip } from '@sourcegraph/wildcard' +import type { RateLimiterState } from './backend' + import styles from '../../site-admin/WebhookInformation.module.scss' interface ExternalServiceInformationProps { @@ -14,6 +16,7 @@ interface ExternalServiceInformationProps { icon: React.ComponentType> kind: ExternalServiceKind displayName: string + rateLimiterState?: RateLimiterState | null codeHostID: string reposNumber: number syncInProgress: boolean @@ -23,8 +26,38 @@ interface ExternalServiceInformationProps { } | null } +export const RateLimiterStateInfo: FC<{ rateLimiterState: RateLimiterState }> = props => { + const { rateLimiterState } = props + const rateLimiterDebug = Object.entries(rateLimiterState).map(([key, value]) => ( +
+ {key}: {value.toString()} +
+ )) + + return ( + + Rate limit + {rateLimiterState.infinite ? ( + + + No rate limit + + + ) : ( + + + + {(rateLimiterState.limit / rateLimiterState.interval).toFixed(2)} requests per second + + + + )} + + ) +} + export const ExternalServiceInformation: FC = props => { - const { icon, kind, displayName, codeHostID, reposNumber, syncInProgress, gitHubApp } = props + const { icon, kind, displayName, codeHostID, reposNumber, syncInProgress, gitHubApp, rateLimiterState } = props return ( @@ -66,6 +99,7 @@ export const ExternalServiceInformation: FC = p )} + {rateLimiterState && }
) diff --git a/client/web/src/components/externalServices/ExternalServicePage.tsx b/client/web/src/components/externalServices/ExternalServicePage.tsx index f90ae233ee96b..535c0c01360fb 100644 --- a/client/web/src/components/externalServices/ExternalServicePage.tsx +++ b/client/web/src/components/externalServices/ExternalServicePage.tsx @@ -250,6 +250,7 @@ export const ExternalServicePage: FC = props => { diff --git a/client/web/src/site-admin/fixtures.ts b/client/web/src/site-admin/fixtures.ts index 1a6eeaee3bacd..4dbb54e700965 100644 --- a/client/web/src/site-admin/fixtures.ts +++ b/client/web/src/site-admin/fixtures.ts @@ -20,6 +20,15 @@ export function createExternalService(kind: ExternalServiceKind, url: string): L nextSyncAt: null, updatedAt: '2021-03-15T19:39:11Z', createdAt: '2021-03-15T19:39:11Z', + rateLimiterState: { + __typename: 'RateLimiterState', + currentCapacity: 10, + burst: 10, + limit: 5000, + interval: 1, + lastReplenishment: '2021-03-15T19:39:11Z', + infinite: false, + }, webhookURL: null, hasConnectionCheck: true, syncJobs: { diff --git a/cmd/frontend/graphqlbackend/BUILD.bazel b/cmd/frontend/graphqlbackend/BUILD.bazel index 5e586964f0d22..af4975cd73cec 100644 --- a/cmd/frontend/graphqlbackend/BUILD.bazel +++ b/cmd/frontend/graphqlbackend/BUILD.bazel @@ -126,6 +126,7 @@ go_library( "product_license_info.go", "product_subscription_status.go", "rate_limit.go", + "ratelimiter.go", "rbac.go", "recorded_commands.go", "repositories.go", @@ -303,6 +304,7 @@ go_library( "//internal/observation", "//internal/oobmigration", "//internal/perforce", + "//internal/ratelimit", "//internal/rbac", "//internal/rcache", "//internal/redispool", @@ -503,6 +505,7 @@ go_test( "//internal/highlight", "//internal/inventory", "//internal/oobmigration", + "//internal/ratelimit", "//internal/rbac", "//internal/rbac/types", "//internal/rcache", diff --git a/cmd/frontend/graphqlbackend/external_service.go b/cmd/frontend/graphqlbackend/external_service.go index 36baeed9082d9..9f1fe2d131380 100644 --- a/cmd/frontend/graphqlbackend/external_service.go +++ b/cmd/frontend/graphqlbackend/external_service.go @@ -19,6 +19,7 @@ import ( "github.com/sourcegraph/sourcegraph/internal/extsvc" "github.com/sourcegraph/sourcegraph/internal/gqlutil" "github.com/sourcegraph/sourcegraph/internal/httpcli" + "github.com/sourcegraph/sourcegraph/internal/ratelimit" "github.com/sourcegraph/sourcegraph/internal/repos" "github.com/sourcegraph/sourcegraph/internal/repoupdater/protocol" "github.com/sourcegraph/sourcegraph/internal/types" @@ -128,6 +129,20 @@ func (r *externalServiceResolver) DisplayName() string { return r.externalService.DisplayName } +func (r *externalServiceResolver) RateLimiterState(ctx context.Context) (*rateLimiterStateResolver, error) { + info, err := ratelimit.GetGlobalLimiterState(ctx) + if err != nil { + return nil, errors.Wrap(err, "getting rate limiter state") + } + + state, ok := info[r.externalService.URN()] + if !ok { + return nil, nil + } + + return &rateLimiterStateResolver{state: state}, nil +} + func (r *externalServiceResolver) Config(ctx context.Context) (JSONCString, error) { redacted, err := r.externalService.RedactedConfig(ctx) if err != nil { diff --git a/cmd/frontend/graphqlbackend/external_services_test.go b/cmd/frontend/graphqlbackend/external_services_test.go index d0eb26663ce87..2626eb2a22061 100644 --- a/cmd/frontend/graphqlbackend/external_services_test.go +++ b/cmd/frontend/graphqlbackend/external_services_test.go @@ -19,6 +19,7 @@ import ( "github.com/sourcegraph/sourcegraph/internal/api" "github.com/sourcegraph/sourcegraph/internal/database/dbmocks" "github.com/sourcegraph/sourcegraph/internal/extsvc/github" + "github.com/sourcegraph/sourcegraph/internal/ratelimit" "github.com/sourcegraph/sourcegraph/internal/repoupdater/protocol" "github.com/sourcegraph/sourcegraph/lib/errors" @@ -622,6 +623,13 @@ func TestExternalServices(t *testing.T) { users.GetByCurrentAuthUserFunc.SetDefaultReturn(&types.User{SiteAdmin: true}, nil) externalServices := dbmocks.NewMockExternalServiceStore() + ess := []*types.ExternalService{ + {ID: 1, Config: extsvc.NewEmptyConfig()}, + {ID: 2, Config: extsvc.NewEmptyConfig(), Kind: extsvc.KindGitHub}, + {ID: 3, Config: extsvc.NewEmptyConfig(), Kind: extsvc.KindGitHub}, + {ID: 4, Config: extsvc.NewEmptyConfig(), Kind: extsvc.KindAWSCodeCommit}, + {ID: 5, Config: extsvc.NewEmptyConfig(), Kind: extsvc.KindGerrit}, + } externalServices.ListFunc.SetDefaultHook(func(_ context.Context, opt database.ExternalServicesListOptions) ([]*types.ExternalService, error) { if opt.AfterID > 0 || opt.RepoID == 42 { return []*types.ExternalService{ @@ -630,18 +638,20 @@ func TestExternalServices(t *testing.T) { }, nil } - ess := []*types.ExternalService{ - {ID: 1, Config: extsvc.NewEmptyConfig()}, - {ID: 2, Config: extsvc.NewEmptyConfig(), Kind: extsvc.KindGitHub}, - {ID: 3, Config: extsvc.NewEmptyConfig(), Kind: extsvc.KindGitHub}, - {ID: 4, Config: extsvc.NewEmptyConfig(), Kind: extsvc.KindAWSCodeCommit}, - {ID: 5, Config: extsvc.NewEmptyConfig(), Kind: extsvc.KindGerrit}, - } if opt.LimitOffset != nil { return ess[:opt.LimitOffset.Limit], nil } return ess, nil }) + + // Set up rate limits + ctx := context.Background() + ratelimit.SetupForTest(t) + for _, es := range ess { + rl := ratelimit.NewGlobalRateLimiter(logtest.NoOp(t), es.URN()) + rl.SetTokenBucketConfig(ctx, 10, time.Hour) + } + externalServices.CountFunc.SetDefaultHook(func(ctx context.Context, opt database.ExternalServicesListOptions) (int, error) { if opt.AfterID > 0 { return 1, nil @@ -698,6 +708,90 @@ func TestExternalServices(t *testing.T) { } `, }, + { + Schema: mustParseGraphQLSchema(t, db), + Label: "Read with rate limiter state", + Query: ` + { + externalServices { + nodes { + id + rateLimiterState { + burst + currentCapacity + infinite + interval + lastReplenishment + limit + } + } + } + } + `, + ExpectedResult: ` + { + "externalServices": { + "nodes": [ + { + "id":"RXh0ZXJuYWxTZXJ2aWNlOjE=", + "rateLimiterState": { + "burst": 10, + "currentCapacity": 0, + "infinite": false, + "interval": 3600, + "lastReplenishment": "1970-01-01T00:00:00Z", + "limit": 10 + } + }, + { + "id":"RXh0ZXJuYWxTZXJ2aWNlOjI=", + "rateLimiterState": { + "burst": 10, + "currentCapacity": 0, + "infinite": false, + "interval": 3600, + "lastReplenishment": "1970-01-01T00:00:00Z", + "limit": 10 + } + }, + { + "id":"RXh0ZXJuYWxTZXJ2aWNlOjM=", + "rateLimiterState": { + "burst": 10, + "currentCapacity": 0, + "infinite": false, + "interval": 3600, + "lastReplenishment": "1970-01-01T00:00:00Z", + "limit": 10 + } + }, + { + "id":"RXh0ZXJuYWxTZXJ2aWNlOjQ=", + "rateLimiterState": { + "burst": 10, + "currentCapacity": 0, + "infinite": false, + "interval": 3600, + "lastReplenishment": "1970-01-01T00:00:00Z", + "limit": 10 + } + }, + { + "id":"RXh0ZXJuYWxTZXJ2aWNlOjU=", + "rateLimiterState": { + "burst": 10, + "currentCapacity": 0, + "infinite": false, + "interval": 3600, + "lastReplenishment": "1970-01-01T00:00:00Z", + "limit": 10 + } + } + ] + } + } + `, + }, { Schema: mustParseGraphQLSchema(t, db), Label: "Read all external services for a given repo", diff --git a/cmd/frontend/graphqlbackend/ratelimiter.go b/cmd/frontend/graphqlbackend/ratelimiter.go new file mode 100644 index 0000000000000..e92c31eda67ff --- /dev/null +++ b/cmd/frontend/graphqlbackend/ratelimiter.go @@ -0,0 +1,36 @@ +package graphqlbackend + +import ( + "time" + + "github.com/sourcegraph/sourcegraph/internal/gqlutil" + "github.com/sourcegraph/sourcegraph/internal/ratelimit" +) + +type rateLimiterStateResolver struct { + state ratelimit.GlobalLimiterInfo +} + +func (rl *rateLimiterStateResolver) Burst() int32 { + return int32(rl.state.Burst) +} + +func (rl *rateLimiterStateResolver) CurrentCapacity() int32 { + return int32(rl.state.CurrentCapacity) +} + +func (rl *rateLimiterStateResolver) Infinite() bool { + return rl.state.Infinite +} + +func (rl *rateLimiterStateResolver) Interval() int32 { + return int32(rl.state.Interval / time.Second) +} + +func (rl *rateLimiterStateResolver) LastReplenishment() gqlutil.DateTime { + return gqlutil.DateTime{Time: rl.state.LastReplenishment} +} + +func (rl *rateLimiterStateResolver) Limit() int32 { + return int32(rl.state.Limit) +} diff --git a/cmd/frontend/graphqlbackend/schema.graphql b/cmd/frontend/graphqlbackend/schema.graphql index a2daa50328585..ee6df1a59e343 100755 --- a/cmd/frontend/graphqlbackend/schema.graphql +++ b/cmd/frontend/graphqlbackend/schema.graphql @@ -3260,6 +3260,41 @@ enum ExternalServiceKind { RUBYPACKAGES } +""" +The current state of a rate limiter. +""" +type RateLimiterState { + """ + The number of tokens in the bucket. + """ + burst: Int! + + """ + The current capacity of the rate limiter. + """ + currentCapacity: Int! + + """ + If infinite is true, the rate limiter is functionally disabled. + """ + infinite: Boolean! + + """ + The amount of time in seconds it takes for the rate limiter to refill the number of tokens specified by limit. + """ + interval: Int! + + """ + The last time at which the rate limiter refilled tokens. + """ + lastReplenishment: DateTime! + + """ + The number of tokens the rate limiter refills in the specified amount of time. + """ + limit: Int! +} + """ A configured external service. """ @@ -3279,6 +3314,11 @@ type ExternalService implements Node { """ displayName: String! + """ + The current rate limiter state for the external service. + """ + rateLimiterState: RateLimiterState + """ The JSON configuration of the external service. """