From c6048faba8208fb606cbe85bc0df0d77b84e0ec3 Mon Sep 17 00:00:00 2001 From: Daniel Blando Date: Tue, 3 Sep 2024 07:02:24 -0700 Subject: [PATCH] Add users stats http api to ingester (#6178) * Add users stats http api to ingester Signed-off-by: Daniel Deluiggi * Changelog Signed-off-by: Daniel Deluiggi * Change name for loadedBlocks Signed-off-by: Daniel Deluiggi --------- Signed-off-by: Daniel Deluiggi Signed-off-by: Daniel Blando --- CHANGELOG.md | 1 + docs/api/_index.md | 12 +- pkg/api/api.go | 4 + pkg/distributor/distributor.go | 34 ++-- pkg/distributor/http_server.go | 9 - pkg/ingester/client/ingester.pb.go | 209 ++++++++++++-------- pkg/ingester/client/ingester.proto | 1 + pkg/{distributor => ingester}/http_admin.go | 48 +++-- pkg/ingester/ingester.go | 64 ++++-- pkg/ingester/ingester_test.go | 80 ++++++++ 10 files changed, 326 insertions(+), 136 deletions(-) rename pkg/{distributor => ingester}/http_admin.go (61%) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf33519981..fc857f5e71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * [ENHANCEMENT] Ruler: Add new ruler metric `cortex_ruler_rule_groups_in_store` that is the total rule groups per tenant in store, which can be used to compare with `cortex_prometheus_rule_group_rules` to count the number of rule groups that are not loaded by a ruler. #5869 * [ENHANCEMENT] Ingester/Ring: New `READONLY` status on ring to be used by Ingester. New ingester API to change mode of ingester #6163 * [ENHANCEMENT] Ruler: Add query statistics metrics when --ruler.query-stats-enabled=true. #6173 +* [ENHANCEMENT] Ingester: Add new API `/ingester/all_user_stats` which shows loaded blocks, active timeseries and ingestion rate for a specific ingester. #6178 * [ENHANCEMENT] Distributor: Add new `cortex_reduced_resolution_histogram_samples_total` metric to to track the number of histogram samples which resolution was reduced. #6182 ## 1.18.0 in progress diff --git a/docs/api/_index.md b/docs/api/_index.md index 1505f9fbb8..1ab231ea83 100644 --- a/docs/api/_index.md +++ b/docs/api/_index.md @@ -32,6 +32,7 @@ For the sake of clarity, in this document we have grouped API endpoints by servi | [Flush blocks](#flush-blocks) | Ingester || `GET,POST /ingester/flush` | | [Shutdown](#shutdown) | Ingester || `GET,POST /ingester/shutdown` | | [Ingesters ring status](#ingesters-ring-status) | Ingester || `GET /ingester/ring` | +| [Ingester tenants stats](#ingester-tenants-stats) | Ingester || `GET /ingester/all_user_stats` | | [Ingester mode](#ingester-mode) | Ingester || `GET,POST /ingester/mode` | | [Instant query](#instant-query) | Querier, Query-frontend || `GET,POST /api/v1/query` | | [Range query](#range-query) | Querier, Query-frontend || `GET,POST /api/v1/query_range` | @@ -242,7 +243,7 @@ GET /distributor/all_user_stats GET /all_user_stats ``` -Displays a web page with per-tenant statistics updated in realtime, including the total number of active series across all ingesters and the current ingestion rate (samples / sec). +Displays a web page with per-tenant statistics updated in realtime, including the total number of loaded blocks and active series across all ingesters as well as the current ingestion rate (samples / sec). ### HA tracker status @@ -297,6 +298,15 @@ GET /ring Displays a web page with the ingesters hash ring status, including the state, healthy and last heartbeat time of each ingester. +### Ingester tenants stats + +``` +GET /ingester/all_user_stats + +``` + +Displays a web page with per-tenant statistics updated in realtime, including the total number of loaded blocks and active series from a specific ingester as well as the current ingestion rate (samples / sec). + ### Ingester mode ``` diff --git a/pkg/api/api.go b/pkg/api/api.go index efca613ea2..5eb13cbb24 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -284,6 +284,7 @@ type Ingester interface { FlushHandler(http.ResponseWriter, *http.Request) ShutdownHandler(http.ResponseWriter, *http.Request) RenewTokenHandler(http.ResponseWriter, *http.Request) + AllUserStatsHandler(http.ResponseWriter, *http.Request) ModeHandler(http.ResponseWriter, *http.Request) Push(context.Context, *cortexpb.WriteRequest) (*cortexpb.WriteResponse, error) } @@ -292,6 +293,8 @@ type Ingester interface { func (a *API) RegisterIngester(i Ingester, pushConfig distributor.Config) { client.RegisterIngesterServer(a.server.GRPC, i) + a.indexPage.AddLink(SectionAdminEndpoints, "/ingester/all_user_stats", "Usage Statistics") + a.indexPage.AddLink(SectionDangerous, "/ingester/flush", "Trigger a Flush of data from Ingester to storage") a.indexPage.AddLink(SectionDangerous, "/ingester/shutdown", "Trigger Ingester Shutdown (Dangerous)") a.indexPage.AddLink(SectionDangerous, "/ingester/renewTokens", "Renew Ingester Tokens (10%)") @@ -300,6 +303,7 @@ func (a *API) RegisterIngester(i Ingester, pushConfig distributor.Config) { a.RegisterRoute("/ingester/flush", http.HandlerFunc(i.FlushHandler), false, "GET", "POST") a.RegisterRoute("/ingester/shutdown", http.HandlerFunc(i.ShutdownHandler), false, "GET", "POST") a.RegisterRoute("/ingester/renewTokens", http.HandlerFunc(i.RenewTokenHandler), false, "GET", "POST") + a.RegisterRoute("/ingester/all_user_stats", http.HandlerFunc(i.AllUserStatsHandler), false, "GET") a.RegisterRoute("/ingester/mode", http.HandlerFunc(i.ModeHandler), false, "GET", "POST") a.RegisterRoute("/ingester/push", push.Handler(pushConfig.MaxRecvMsgSize, a.sourceIPs, i.Push), true, "POST") // For testing and debugging. diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 82b49a5631..351187ba83 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -29,6 +29,7 @@ import ( "github.com/cortexproject/cortex/pkg/cortexpb" "github.com/cortexproject/cortex/pkg/ha" + "github.com/cortexproject/cortex/pkg/ingester" ingester_client "github.com/cortexproject/cortex/pkg/ingester/client" "github.com/cortexproject/cortex/pkg/ring" ring_client "github.com/cortexproject/cortex/pkg/ring/client" @@ -1319,7 +1320,7 @@ func (d *Distributor) MetricsMetadata(ctx context.Context) ([]scrape.MetricMetad } // UserStats returns statistics about the current user. -func (d *Distributor) UserStats(ctx context.Context) (*UserStats, error) { +func (d *Distributor) UserStats(ctx context.Context) (*ingester.UserStats, error) { replicationSet, err := d.GetIngestersForMetadata(ctx) if err != nil { return nil, err @@ -1336,7 +1337,7 @@ func (d *Distributor) UserStats(ctx context.Context) (*UserStats, error) { return nil, err } - totalStats := &UserStats{} + totalStats := &ingester.UserStats{} for _, resp := range resps { r := resp.(*ingester_client.UserStatsResponse) totalStats.IngestionRate += r.IngestionRate @@ -1354,17 +1355,11 @@ func (d *Distributor) UserStats(ctx context.Context) (*UserStats, error) { return totalStats, nil } -// UserIDStats models ingestion statistics for one user, including the user ID -type UserIDStats struct { - UserID string `json:"userID"` - UserStats -} - // AllUserStats returns statistics about all users. // Note it does not divide by the ReplicationFactor like UserStats() -func (d *Distributor) AllUserStats(ctx context.Context) ([]UserIDStats, error) { +func (d *Distributor) AllUserStats(ctx context.Context) ([]ingester.UserIDStats, error) { // Add up by user, across all responses from ingesters - perUserTotals := make(map[string]UserStats) + perUserTotals := make(map[string]ingester.UserStats) req := &ingester_client.UserStatsRequest{} ctx = user.InjectOrgID(ctx, "1") // fake: ingester insists on having an org ID @@ -1389,21 +1384,23 @@ func (d *Distributor) AllUserStats(ctx context.Context) ([]UserIDStats, error) { s.RuleIngestionRate += u.Data.RuleIngestionRate s.NumSeries += u.Data.NumSeries s.ActiveSeries += u.Data.ActiveSeries + s.LoadedBlocks += u.Data.LoadedBlocks perUserTotals[u.UserId] = s } } // Turn aggregated map into a slice for return - response := make([]UserIDStats, 0, len(perUserTotals)) + response := make([]ingester.UserIDStats, 0, len(perUserTotals)) for id, stats := range perUserTotals { - response = append(response, UserIDStats{ + response = append(response, ingester.UserIDStats{ UserID: id, - UserStats: UserStats{ + UserStats: ingester.UserStats{ IngestionRate: stats.IngestionRate, APIIngestionRate: stats.APIIngestionRate, RuleIngestionRate: stats.RuleIngestionRate, NumSeries: stats.NumSeries, ActiveSeries: stats.ActiveSeries, + LoadedBlocks: stats.LoadedBlocks, }, }) } @@ -1411,6 +1408,17 @@ func (d *Distributor) AllUserStats(ctx context.Context) ([]UserIDStats, error) { return response, nil } +// AllUserStatsHandler shows stats for all users. +func (d *Distributor) AllUserStatsHandler(w http.ResponseWriter, r *http.Request) { + stats, err := d.AllUserStats(r.Context()) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + ingester.AllUserStatsRender(w, r, stats, d.ingestersRing.ReplicationFactor()) +} + func (d *Distributor) ServeHTTP(w http.ResponseWriter, req *http.Request) { if d.distributorsRing != nil { d.distributorsRing.ServeHTTP(w, req) diff --git a/pkg/distributor/http_server.go b/pkg/distributor/http_server.go index 679e5879c7..8934f3718e 100644 --- a/pkg/distributor/http_server.go +++ b/pkg/distributor/http_server.go @@ -6,15 +6,6 @@ import ( "github.com/cortexproject/cortex/pkg/util" ) -// UserStats models ingestion statistics for one user. -type UserStats struct { - IngestionRate float64 `json:"ingestionRate"` - NumSeries uint64 `json:"numSeries"` - APIIngestionRate float64 `json:"APIIngestionRate"` - RuleIngestionRate float64 `json:"RuleIngestionRate"` - ActiveSeries uint64 `json:"activeSeries"` -} - // UserStatsHandler handles user stats to the Distributor. func (d *Distributor) UserStatsHandler(w http.ResponseWriter, r *http.Request) { stats, err := d.UserStats(r.Context()) diff --git a/pkg/ingester/client/ingester.pb.go b/pkg/ingester/client/ingester.pb.go index 594292429f..23f1fbdf5b 100644 --- a/pkg/ingester/client/ingester.pb.go +++ b/pkg/ingester/client/ingester.pb.go @@ -744,6 +744,7 @@ type UserStatsResponse struct { ApiIngestionRate float64 `protobuf:"fixed64,3,opt,name=api_ingestion_rate,json=apiIngestionRate,proto3" json:"api_ingestion_rate,omitempty"` RuleIngestionRate float64 `protobuf:"fixed64,4,opt,name=rule_ingestion_rate,json=ruleIngestionRate,proto3" json:"rule_ingestion_rate,omitempty"` ActiveSeries uint64 `protobuf:"varint,5,opt,name=active_series,json=activeSeries,proto3" json:"active_series,omitempty"` + LoadedBlocks uint64 `protobuf:"varint,6,opt,name=loaded_blocks,json=loadedBlocks,proto3" json:"loaded_blocks,omitempty"` } func (m *UserStatsResponse) Reset() { *m = UserStatsResponse{} } @@ -813,6 +814,13 @@ func (m *UserStatsResponse) GetActiveSeries() uint64 { return 0 } +func (m *UserStatsResponse) GetLoadedBlocks() uint64 { + if m != nil { + return m.LoadedBlocks + } + return 0 +} + type UserIDStatsResponse struct { UserId string `protobuf:"bytes,1,opt,name=user_id,json=userId,proto3" json:"user_id,omitempty"` Data *UserStatsResponse `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"` @@ -1468,89 +1476,90 @@ func init() { func init() { proto.RegisterFile("ingester.proto", fileDescriptor_60f6df4f3586b478) } var fileDescriptor_60f6df4f3586b478 = []byte{ - // 1309 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0x4b, 0x6f, 0xd4, 0xd6, - 0x17, 0xb7, 0x33, 0x8f, 0x64, 0xce, 0x4c, 0x86, 0xc9, 0x4d, 0x20, 0x83, 0xf9, 0xe3, 0x80, 0xff, - 0xa2, 0x8d, 0xda, 0x92, 0x40, 0xda, 0x4a, 0xd0, 0x17, 0x4a, 0x20, 0x40, 0x80, 0x10, 0x70, 0x02, - 0xad, 0xaa, 0x56, 0x96, 0x33, 0x73, 0x49, 0x5c, 0xfc, 0xc2, 0xbe, 0x46, 0xd0, 0x55, 0xab, 0x7e, - 0x80, 0x76, 0xd9, 0x6d, 0x77, 0xfd, 0x28, 0x2c, 0x59, 0x74, 0x81, 0xaa, 0x0a, 0x95, 0x41, 0xaa, - 0xba, 0xa4, 0xdf, 0xa0, 0xf2, 0x7d, 0xf8, 0x15, 0x27, 0x19, 0x24, 0xd2, 0x9d, 0xef, 0x39, 0xbf, - 0x73, 0xee, 0x79, 0xde, 0x73, 0x0c, 0x6d, 0xcb, 0xdd, 0xc2, 0x21, 0xc1, 0xc1, 0x9c, 0x1f, 0x78, - 0xc4, 0x43, 0xf5, 0x9e, 0x17, 0x10, 0xfc, 0x48, 0x99, 0xda, 0xf2, 0xb6, 0x3c, 0x4a, 0x9a, 0x8f, - 0xbf, 0x18, 0x57, 0x39, 0xbf, 0x65, 0x91, 0xed, 0x68, 0x73, 0xae, 0xe7, 0x39, 0xf3, 0x0c, 0xe8, - 0x07, 0xde, 0x37, 0xb8, 0x47, 0xf8, 0x69, 0xde, 0xbf, 0xbf, 0x25, 0x18, 0x9b, 0xfc, 0x83, 0x89, - 0x6a, 0x9f, 0x42, 0x53, 0xc7, 0x66, 0x5f, 0xc7, 0x0f, 0x22, 0x1c, 0x12, 0x34, 0x07, 0xa3, 0x0f, - 0x22, 0x1c, 0x58, 0x38, 0xec, 0xca, 0x27, 0x2a, 0xb3, 0xcd, 0x85, 0xa9, 0x39, 0x0e, 0xbf, 0x1d, - 0xe1, 0xe0, 0x31, 0x87, 0xe9, 0x02, 0xa4, 0x5d, 0x80, 0x16, 0x13, 0x0f, 0x7d, 0xcf, 0x0d, 0x31, - 0x9a, 0x87, 0xd1, 0x00, 0x87, 0x91, 0x4d, 0x84, 0xfc, 0xe1, 0x82, 0x3c, 0xc3, 0xe9, 0x02, 0xa5, - 0x5d, 0x87, 0xf1, 0x1c, 0x07, 0x7d, 0x04, 0x40, 0x2c, 0x07, 0x87, 0x65, 0x46, 0xf8, 0x9b, 0x73, - 0x1b, 0x96, 0x83, 0xd7, 0x29, 0x6f, 0xa9, 0xfa, 0xe4, 0xf9, 0x8c, 0xa4, 0x67, 0xd0, 0xda, 0xcf, - 0x32, 0xb4, 0xb2, 0x76, 0xa2, 0xf7, 0x00, 0x85, 0xc4, 0x0c, 0x88, 0x41, 0x41, 0xc4, 0x74, 0x7c, - 0xc3, 0x89, 0x95, 0xca, 0xb3, 0x15, 0xbd, 0x43, 0x39, 0x1b, 0x82, 0xb1, 0x1a, 0xa2, 0x59, 0xe8, - 0x60, 0xb7, 0x9f, 0xc7, 0x8e, 0x50, 0x6c, 0x1b, 0xbb, 0xfd, 0x2c, 0xf2, 0x0c, 0x8c, 0x39, 0x26, - 0xe9, 0x6d, 0xe3, 0x20, 0xec, 0x56, 0xf2, 0x71, 0xba, 0x61, 0x6e, 0x62, 0x7b, 0x95, 0x31, 0xf5, - 0x04, 0xa5, 0xfd, 0x22, 0xc3, 0xd4, 0xf2, 0x23, 0xec, 0xf8, 0xb6, 0x19, 0xfc, 0x27, 0x26, 0x9e, - 0xdd, 0x61, 0xe2, 0xe1, 0x32, 0x13, 0xc3, 0x8c, 0x8d, 0x5f, 0xc1, 0x24, 0x35, 0x6d, 0x9d, 0x04, - 0xd8, 0x74, 0x92, 0x8c, 0x5c, 0x80, 0x66, 0x6f, 0x3b, 0x72, 0xef, 0xe7, 0x52, 0x32, 0x2d, 0x94, - 0xa5, 0x09, 0xb9, 0x18, 0x83, 0x78, 0x56, 0xb2, 0x12, 0xd7, 0xaa, 0x63, 0x23, 0x9d, 0x8a, 0xb6, - 0x0e, 0x87, 0x0b, 0x01, 0x78, 0x03, 0x19, 0xff, 0x4d, 0x06, 0x44, 0xdd, 0xb9, 0x6b, 0xda, 0x11, - 0x0e, 0x45, 0x50, 0x8f, 0x03, 0xd8, 0x31, 0xd5, 0x70, 0x4d, 0x07, 0xd3, 0x60, 0x36, 0xf4, 0x06, - 0xa5, 0xdc, 0x34, 0x1d, 0xbc, 0x4b, 0xcc, 0x47, 0x5e, 0x23, 0xe6, 0x95, 0x7d, 0x63, 0x5e, 0x3d, - 0x21, 0x0f, 0x11, 0x73, 0x34, 0x05, 0x35, 0xdb, 0x72, 0x2c, 0xd2, 0xad, 0x51, 0x8d, 0xec, 0xa0, - 0x9d, 0x83, 0xc9, 0x9c, 0x57, 0x3c, 0x52, 0x27, 0xa1, 0xc5, 0xdc, 0x7a, 0x48, 0xe9, 0x34, 0x56, - 0x0d, 0xbd, 0x69, 0xa7, 0x50, 0xed, 0x33, 0x38, 0x9a, 0x91, 0x2c, 0x64, 0x72, 0x08, 0xf9, 0xef, - 0x65, 0x98, 0xb8, 0x21, 0x02, 0x15, 0x1e, 0x74, 0x91, 0x26, 0xde, 0x57, 0xb2, 0xde, 0x7f, 0xc8, - 0x73, 0xca, 0x4d, 0xe0, 0xc6, 0xcf, 0x40, 0x33, 0xcd, 0xa9, 0xb0, 0x1d, 0x92, 0xa4, 0x86, 0xda, - 0xc7, 0xd0, 0x4d, 0xc5, 0x0a, 0x9e, 0xef, 0x2b, 0x8c, 0xa0, 0x73, 0x27, 0xc4, 0xc1, 0x3a, 0x31, - 0x89, 0xf0, 0x5a, 0xfb, 0x43, 0x86, 0x89, 0x0c, 0x91, 0xab, 0x3a, 0x25, 0x1e, 0x67, 0xcb, 0x73, - 0x8d, 0xc0, 0x24, 0xac, 0xbe, 0x64, 0x7d, 0x3c, 0xa1, 0xea, 0x26, 0xc1, 0x71, 0x09, 0xba, 0x91, - 0x63, 0xf0, 0xaa, 0x8e, 0xdd, 0xaf, 0xea, 0x0d, 0x37, 0x72, 0x58, 0x29, 0xc7, 0x11, 0x35, 0x7d, - 0xcb, 0x28, 0x68, 0xaa, 0x50, 0x4d, 0x1d, 0xd3, 0xb7, 0x56, 0x72, 0xca, 0xe6, 0x60, 0x32, 0x88, - 0x6c, 0x5c, 0x84, 0x57, 0x29, 0x7c, 0x22, 0x66, 0xe5, 0xf1, 0xff, 0x87, 0x71, 0xb3, 0x47, 0xac, - 0x87, 0x58, 0xdc, 0x5f, 0xa3, 0xf7, 0xb7, 0x18, 0x91, 0x99, 0xa0, 0x7d, 0x0d, 0x93, 0xb1, 0x77, - 0x2b, 0x97, 0xf2, 0xfe, 0x4d, 0xc3, 0x68, 0x14, 0xe2, 0xc0, 0xb0, 0xfa, 0xbc, 0x71, 0xea, 0xf1, - 0x71, 0xa5, 0x8f, 0x4e, 0x43, 0xb5, 0x6f, 0x12, 0x93, 0xfa, 0xd2, 0x5c, 0x38, 0x2a, 0x2a, 0x7b, - 0x47, 0x84, 0x74, 0x0a, 0xd3, 0xae, 0x00, 0x8a, 0x59, 0x61, 0x5e, 0xfb, 0x59, 0xa8, 0x85, 0x31, - 0x81, 0xf7, 0xf9, 0xb1, 0xac, 0x96, 0x82, 0x25, 0x3a, 0x43, 0x6a, 0x4f, 0x64, 0x50, 0x57, 0x31, - 0x09, 0xac, 0x5e, 0x78, 0xd9, 0x0b, 0xf2, 0x8d, 0x74, 0xc0, 0xf5, 0x79, 0x0e, 0x5a, 0xa2, 0x53, - 0x8d, 0x10, 0x93, 0xbd, 0x1f, 0xd2, 0xa6, 0x80, 0xae, 0x63, 0x92, 0x56, 0x76, 0x35, 0x5b, 0xd9, - 0xd7, 0x61, 0x66, 0x57, 0x4f, 0x78, 0x80, 0x66, 0xa1, 0xee, 0x50, 0x08, 0x8f, 0x50, 0x27, 0x7d, - 0x09, 0x99, 0xa8, 0xce, 0xf9, 0xda, 0x6d, 0x38, 0xb5, 0x8b, 0xb2, 0x42, 0xf1, 0x0f, 0xaf, 0xb2, - 0x0b, 0x47, 0xb8, 0xca, 0x55, 0x4c, 0xcc, 0x38, 0x8d, 0xa2, 0x17, 0xd6, 0x60, 0x7a, 0x07, 0x87, - 0xab, 0xff, 0x00, 0xc6, 0x1c, 0x4e, 0xe3, 0x17, 0x74, 0x8b, 0x17, 0x24, 0x32, 0x09, 0x52, 0xfb, - 0x47, 0x86, 0x43, 0x85, 0xd9, 0x11, 0x27, 0xe6, 0x5e, 0xe0, 0x39, 0x86, 0x58, 0x7e, 0xd2, 0x1a, - 0x6c, 0xc7, 0xf4, 0x15, 0x4e, 0x5e, 0xe9, 0x67, 0x8b, 0x74, 0x24, 0x57, 0xa4, 0x2e, 0xd4, 0x69, - 0x57, 0x8b, 0xa1, 0x37, 0x99, 0x9a, 0x42, 0x43, 0x74, 0xcb, 0xb4, 0x82, 0xa5, 0xc5, 0x78, 0x8e, - 0xfc, 0xfe, 0x7c, 0xe6, 0xb5, 0xf6, 0x26, 0x26, 0xbf, 0xd8, 0x37, 0x7d, 0x82, 0x03, 0x9d, 0xdf, - 0x82, 0xde, 0x85, 0x3a, 0x1b, 0x75, 0xdd, 0x2a, 0xbd, 0x6f, 0x5c, 0xd4, 0x46, 0x76, 0x1a, 0x72, - 0x88, 0xf6, 0xa3, 0x0c, 0x35, 0xe6, 0xe9, 0x41, 0x15, 0xac, 0x02, 0x63, 0xd8, 0xed, 0x79, 0x7d, - 0xcb, 0xdd, 0xa2, 0x8f, 0x49, 0x4d, 0x4f, 0xce, 0x08, 0xf1, 0xfe, 0x8d, 0x2b, 0xb2, 0xc5, 0x9b, - 0x74, 0x11, 0xc6, 0x73, 0x95, 0x93, 0xdb, 0x6c, 0xe4, 0xa1, 0x36, 0x1b, 0x03, 0x5a, 0x59, 0x0e, - 0x3a, 0x05, 0x55, 0xf2, 0xd8, 0x67, 0xaf, 0x62, 0x7b, 0x61, 0x42, 0x48, 0x53, 0xf6, 0xc6, 0x63, - 0x1f, 0xeb, 0x94, 0x1d, 0x5b, 0x43, 0x87, 0x33, 0x4b, 0x1f, 0xfd, 0x8e, 0x9b, 0x86, 0x4e, 0x26, - 0x6a, 0x7a, 0x43, 0x67, 0x07, 0xed, 0x07, 0x19, 0xda, 0x69, 0xa5, 0x5c, 0xb6, 0x6c, 0xfc, 0x26, - 0x0a, 0x45, 0x81, 0xb1, 0x7b, 0x96, 0x8d, 0xa9, 0x0d, 0xec, 0xba, 0xe4, 0x5c, 0x16, 0xa9, 0x77, - 0xae, 0x41, 0x23, 0x71, 0x01, 0x35, 0xa0, 0xb6, 0x7c, 0xfb, 0xce, 0xe2, 0x8d, 0x8e, 0x84, 0xc6, - 0xa1, 0x71, 0x73, 0x6d, 0xc3, 0x60, 0x47, 0x19, 0x1d, 0x82, 0xa6, 0xbe, 0x7c, 0x65, 0xf9, 0x0b, - 0x63, 0x75, 0x71, 0xe3, 0xe2, 0xd5, 0xce, 0x08, 0x42, 0xd0, 0x66, 0x84, 0x9b, 0x6b, 0x9c, 0x56, - 0x59, 0xf8, 0x6b, 0x14, 0xc6, 0x84, 0x8d, 0xe8, 0x3c, 0x54, 0x6f, 0x45, 0xe1, 0x36, 0x3a, 0x92, - 0x56, 0xea, 0xe7, 0x81, 0x45, 0x30, 0xef, 0x3c, 0x65, 0x7a, 0x07, 0x9d, 0xf5, 0x9d, 0x26, 0xa1, - 0x4b, 0xd0, 0xcc, 0x2c, 0x6c, 0xa8, 0x74, 0x57, 0x57, 0x8e, 0xe5, 0xa8, 0xf9, 0xa7, 0x41, 0x93, - 0xce, 0xc8, 0x68, 0x0d, 0xda, 0x94, 0x25, 0xb6, 0xb3, 0x10, 0xfd, 0x4f, 0x88, 0x94, 0x6d, 0xac, - 0xca, 0xf1, 0x5d, 0xb8, 0x89, 0x59, 0x57, 0xa1, 0x99, 0xd9, 0x41, 0x90, 0x92, 0x2b, 0xa0, 0xdc, - 0xa2, 0x96, 0x1a, 0x57, 0xb2, 0xee, 0x68, 0x12, 0xba, 0xcb, 0x97, 0x91, 0xec, 0x36, 0xb3, 0xa7, - 0xbe, 0x93, 0x25, 0xbc, 0x12, 0x97, 0x97, 0x01, 0xd2, 0x55, 0x01, 0x1d, 0xcd, 0x09, 0x65, 0x17, - 0x1f, 0x45, 0x29, 0x63, 0x25, 0xe6, 0xad, 0x43, 0xa7, 0xb8, 0x71, 0xec, 0xa5, 0xec, 0xc4, 0x4e, - 0x56, 0x89, 0x6d, 0x4b, 0xd0, 0x48, 0x46, 0x2a, 0xea, 0x96, 0x4c, 0x59, 0xa6, 0x6c, 0xf7, 0xf9, - 0xab, 0x49, 0xe8, 0x32, 0xb4, 0x16, 0x6d, 0x7b, 0x18, 0x35, 0x4a, 0x96, 0x13, 0x16, 0xf5, 0xd8, - 0xc9, 0xab, 0x5f, 0x1c, 0x31, 0xe8, 0xad, 0xa4, 0xb1, 0xf7, 0x1c, 0xcd, 0xca, 0xdb, 0xfb, 0xe2, - 0x92, 0xdb, 0xbe, 0x85, 0xe3, 0x7b, 0x0e, 0xb4, 0xa1, 0xef, 0x3c, 0xbd, 0x0f, 0xae, 0x24, 0xea, - 0x1b, 0x70, 0xa8, 0x30, 0xdf, 0x90, 0x5a, 0xd0, 0x52, 0x18, 0x89, 0xca, 0xcc, 0xae, 0x7c, 0xa1, - 0x77, 0xe9, 0x93, 0xa7, 0x2f, 0x54, 0xe9, 0xd9, 0x0b, 0x55, 0x7a, 0xf5, 0x42, 0x95, 0xbf, 0x1b, - 0xa8, 0xf2, 0xaf, 0x03, 0x55, 0x7e, 0x32, 0x50, 0xe5, 0xa7, 0x03, 0x55, 0xfe, 0x73, 0xa0, 0xca, - 0x7f, 0x0f, 0x54, 0xe9, 0xd5, 0x40, 0x95, 0x7f, 0x7a, 0xa9, 0x4a, 0x4f, 0x5f, 0xaa, 0xd2, 0xb3, - 0x97, 0xaa, 0xf4, 0x65, 0xbd, 0x67, 0x5b, 0xd8, 0x25, 0x9b, 0x75, 0xfa, 0x8b, 0xfe, 0xfe, 0xbf, - 0x01, 0x00, 0x00, 0xff, 0xff, 0xdd, 0xd4, 0xb0, 0x94, 0x0d, 0x10, 0x00, 0x00, + // 1328 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x57, 0x4b, 0x6f, 0xd4, 0x56, + 0x14, 0x1e, 0x67, 0x1e, 0xc9, 0x9c, 0x79, 0x30, 0xb9, 0x09, 0x64, 0x30, 0xc5, 0x01, 0x23, 0xda, + 0xa8, 0x2d, 0x09, 0xa4, 0xad, 0x04, 0x7d, 0xa1, 0x04, 0x02, 0x04, 0x08, 0x01, 0x27, 0xd0, 0xaa, + 0x6a, 0x65, 0x39, 0x33, 0x97, 0xc4, 0xc5, 0x8f, 0xc1, 0xf7, 0x1a, 0x41, 0x57, 0x45, 0xfd, 0x01, + 0xed, 0xb2, 0xdb, 0xee, 0xfa, 0x53, 0x58, 0xb2, 0xe8, 0x02, 0x75, 0x81, 0xca, 0x20, 0x55, 0x5d, + 0xd2, 0x7f, 0x50, 0xf9, 0x3e, 0xfc, 0x8a, 0xf3, 0x40, 0x82, 0xee, 0x7c, 0xcf, 0xf9, 0xce, 0xb9, + 0xe7, 0x79, 0xcf, 0x31, 0xb4, 0x6d, 0x6f, 0x13, 0x13, 0x8a, 0x83, 0xd9, 0x41, 0xe0, 0x53, 0x1f, + 0xd5, 0x7a, 0x7e, 0x40, 0xf1, 0x43, 0x75, 0x72, 0xd3, 0xdf, 0xf4, 0x19, 0x69, 0x2e, 0xfa, 0xe2, + 0x5c, 0xf5, 0xdc, 0xa6, 0x4d, 0xb7, 0xc2, 0x8d, 0xd9, 0x9e, 0xef, 0xce, 0x71, 0xe0, 0x20, 0xf0, + 0xbf, 0xc7, 0x3d, 0x2a, 0x4e, 0x73, 0x83, 0x7b, 0x9b, 0x92, 0xb1, 0x21, 0x3e, 0xb8, 0xa8, 0xfe, + 0x05, 0x34, 0x0c, 0x6c, 0xf5, 0x0d, 0x7c, 0x3f, 0xc4, 0x84, 0xa2, 0x59, 0x18, 0xbd, 0x1f, 0xe2, + 0xc0, 0xc6, 0xa4, 0xab, 0x1c, 0x2b, 0xcf, 0x34, 0xe6, 0x27, 0x67, 0x05, 0xfc, 0x56, 0x88, 0x83, + 0x47, 0x02, 0x66, 0x48, 0x90, 0x7e, 0x1e, 0x9a, 0x5c, 0x9c, 0x0c, 0x7c, 0x8f, 0x60, 0x34, 0x07, + 0xa3, 0x01, 0x26, 0xa1, 0x43, 0xa5, 0xfc, 0xc1, 0x9c, 0x3c, 0xc7, 0x19, 0x12, 0xa5, 0x5f, 0x83, + 0x56, 0x86, 0x83, 0x3e, 0x05, 0xa0, 0xb6, 0x8b, 0x49, 0x91, 0x11, 0x83, 0x8d, 0xd9, 0x75, 0xdb, + 0xc5, 0x6b, 0x8c, 0xb7, 0x58, 0x79, 0xf2, 0x7c, 0xba, 0x64, 0xa4, 0xd0, 0xfa, 0xaf, 0x0a, 0x34, + 0xd3, 0x76, 0xa2, 0x0f, 0x01, 0x11, 0x6a, 0x05, 0xd4, 0x64, 0x20, 0x6a, 0xb9, 0x03, 0xd3, 0x8d, + 0x94, 0x2a, 0x33, 0x65, 0xa3, 0xc3, 0x38, 0xeb, 0x92, 0xb1, 0x42, 0xd0, 0x0c, 0x74, 0xb0, 0xd7, + 0xcf, 0x62, 0x47, 0x18, 0xb6, 0x8d, 0xbd, 0x7e, 0x1a, 0x79, 0x1a, 0xc6, 0x5c, 0x8b, 0xf6, 0xb6, + 0x70, 0x40, 0xba, 0xe5, 0x6c, 0x9c, 0xae, 0x5b, 0x1b, 0xd8, 0x59, 0xe1, 0x4c, 0x23, 0x46, 0xe9, + 0xbf, 0x29, 0x30, 0xb9, 0xf4, 0x10, 0xbb, 0x03, 0xc7, 0x0a, 0xfe, 0x17, 0x13, 0xcf, 0x6c, 0x33, + 0xf1, 0x60, 0x91, 0x89, 0x24, 0x65, 0xe3, 0xb7, 0x30, 0xc1, 0x4c, 0x5b, 0xa3, 0x01, 0xb6, 0xdc, + 0x38, 0x23, 0xe7, 0xa1, 0xd1, 0xdb, 0x0a, 0xbd, 0x7b, 0x99, 0x94, 0x4c, 0x49, 0x65, 0x49, 0x42, + 0x2e, 0x44, 0x20, 0x91, 0x95, 0xb4, 0xc4, 0xd5, 0xca, 0xd8, 0x48, 0xa7, 0xac, 0xaf, 0xc1, 0xc1, + 0x5c, 0x00, 0xde, 0x40, 0xc6, 0xff, 0x50, 0x00, 0x31, 0x77, 0xee, 0x58, 0x4e, 0x88, 0x89, 0x0c, + 0xea, 0x51, 0x00, 0x27, 0xa2, 0x9a, 0x9e, 0xe5, 0x62, 0x16, 0xcc, 0xba, 0x51, 0x67, 0x94, 0x1b, + 0x96, 0x8b, 0x77, 0x88, 0xf9, 0xc8, 0x6b, 0xc4, 0xbc, 0xbc, 0x67, 0xcc, 0x2b, 0xc7, 0x94, 0x7d, + 0xc4, 0x1c, 0x4d, 0x42, 0xd5, 0xb1, 0x5d, 0x9b, 0x76, 0xab, 0x4c, 0x23, 0x3f, 0xe8, 0x67, 0x61, + 0x22, 0xe3, 0x95, 0x88, 0xd4, 0x71, 0x68, 0x72, 0xb7, 0x1e, 0x30, 0x3a, 0x8b, 0x55, 0xdd, 0x68, + 0x38, 0x09, 0x54, 0xff, 0x12, 0x0e, 0xa7, 0x24, 0x73, 0x99, 0xdc, 0x87, 0xfc, 0x63, 0x05, 0xc6, + 0xaf, 0xcb, 0x40, 0x91, 0xb7, 0x5d, 0xa4, 0xb1, 0xf7, 0xe5, 0xb4, 0xf7, 0x9f, 0x88, 0x9c, 0x0a, + 0x13, 0x84, 0xf1, 0xd3, 0xd0, 0x48, 0x72, 0x2a, 0x6d, 0x87, 0x38, 0xa9, 0x44, 0xff, 0x0c, 0xba, + 0x89, 0x58, 0xce, 0xf3, 0x3d, 0x85, 0x11, 0x74, 0x6e, 0x13, 0x1c, 0xac, 0x51, 0x8b, 0x4a, 0xaf, + 0xf5, 0xc7, 0x23, 0x30, 0x9e, 0x22, 0x0a, 0x55, 0x27, 0xe5, 0xe3, 0x6c, 0xfb, 0x9e, 0x19, 0x58, + 0x94, 0xd7, 0x97, 0x62, 0xb4, 0x62, 0xaa, 0x61, 0x51, 0x1c, 0x95, 0xa0, 0x17, 0xba, 0xa6, 0xa8, + 0xea, 0xc8, 0xfd, 0x8a, 0x51, 0xf7, 0x42, 0x97, 0x97, 0x72, 0x14, 0x51, 0x6b, 0x60, 0x9b, 0x39, + 0x4d, 0x65, 0xa6, 0xa9, 0x63, 0x0d, 0xec, 0xe5, 0x8c, 0xb2, 0x59, 0x98, 0x08, 0x42, 0x07, 0xe7, + 0xe1, 0x15, 0x06, 0x1f, 0x8f, 0x58, 0x59, 0xfc, 0x09, 0x68, 0x59, 0x3d, 0x6a, 0x3f, 0xc0, 0xf2, + 0xfe, 0x2a, 0xbb, 0xbf, 0xc9, 0x89, 0xc2, 0x84, 0x13, 0xd0, 0x72, 0x7c, 0xab, 0x8f, 0xfb, 0xe6, + 0x86, 0xe3, 0xf7, 0xee, 0x91, 0x6e, 0x8d, 0x83, 0x38, 0x71, 0x91, 0xd1, 0xf4, 0xef, 0x60, 0x22, + 0x0a, 0xc1, 0xf2, 0xc5, 0x6c, 0x10, 0xa6, 0x60, 0x34, 0x24, 0x38, 0x30, 0xed, 0xbe, 0xe8, 0xae, + 0x5a, 0x74, 0x5c, 0xee, 0xa3, 0x53, 0x50, 0xe9, 0x5b, 0xd4, 0x62, 0x0e, 0x37, 0xe6, 0x0f, 0xcb, + 0xf2, 0xdf, 0x16, 0x46, 0x83, 0xc1, 0xf4, 0xcb, 0x80, 0x22, 0x16, 0xc9, 0x6a, 0x3f, 0x03, 0x55, + 0x12, 0x11, 0xc4, 0x63, 0x70, 0x24, 0xad, 0x25, 0x67, 0x89, 0xc1, 0x91, 0xfa, 0x13, 0x05, 0xb4, + 0x15, 0x4c, 0x03, 0xbb, 0x47, 0x2e, 0xf9, 0x41, 0xb6, 0xdb, 0xde, 0x72, 0x11, 0x9f, 0x85, 0xa6, + 0x6c, 0x67, 0x93, 0x60, 0xba, 0xfb, 0x6b, 0xdb, 0x90, 0xd0, 0x35, 0x4c, 0x93, 0xf2, 0xaf, 0xa4, + 0xcb, 0xff, 0x1a, 0x4c, 0xef, 0xe8, 0x89, 0x08, 0xd0, 0x0c, 0xd4, 0x5c, 0x06, 0x11, 0x11, 0xea, + 0x24, 0xcf, 0x25, 0x17, 0x35, 0x04, 0x5f, 0xbf, 0x05, 0x27, 0x77, 0x50, 0x96, 0xeb, 0x90, 0xfd, + 0xab, 0xec, 0xc2, 0x21, 0xa1, 0x72, 0x05, 0x53, 0x2b, 0x4a, 0xa3, 0x6c, 0x98, 0x55, 0x98, 0xda, + 0xc6, 0x11, 0xea, 0x3f, 0x86, 0x31, 0x57, 0xd0, 0xc4, 0x05, 0xdd, 0xfc, 0x05, 0xb1, 0x4c, 0x8c, + 0xd4, 0xff, 0x55, 0xe0, 0x40, 0x6e, 0xc0, 0x44, 0x89, 0xb9, 0x1b, 0xf8, 0xae, 0x29, 0x37, 0xa4, + 0xa4, 0x06, 0xdb, 0x11, 0x7d, 0x59, 0x90, 0x97, 0xfb, 0xe9, 0x22, 0x1d, 0xc9, 0x14, 0xa9, 0x07, + 0x35, 0xd6, 0xfa, 0x72, 0x32, 0x4e, 0x24, 0xa6, 0xb0, 0x10, 0xdd, 0xb4, 0xec, 0x60, 0x71, 0x21, + 0x1a, 0x36, 0x7f, 0x3e, 0x9f, 0x7e, 0xad, 0xe5, 0x8a, 0xcb, 0x2f, 0xf4, 0xad, 0x01, 0xc5, 0x81, + 0x21, 0x6e, 0x41, 0x1f, 0x40, 0x8d, 0xcf, 0xc3, 0x6e, 0x85, 0xdd, 0xd7, 0x92, 0xb5, 0x91, 0x1e, + 0x99, 0x02, 0xa2, 0xff, 0xac, 0x40, 0x95, 0x7b, 0xfa, 0xb6, 0x0a, 0x56, 0x85, 0x31, 0xec, 0xf5, + 0xfc, 0xbe, 0xed, 0x6d, 0xb2, 0x17, 0xa7, 0x6a, 0xc4, 0x67, 0x84, 0x44, 0xff, 0x46, 0x15, 0xd9, + 0x14, 0x4d, 0xba, 0x00, 0xad, 0x4c, 0xe5, 0x64, 0xd6, 0x1f, 0x65, 0x5f, 0xeb, 0x8f, 0x09, 0xcd, + 0x34, 0x07, 0x9d, 0x84, 0x0a, 0x7d, 0x34, 0xe0, 0x4f, 0x67, 0x7b, 0x7e, 0x5c, 0x4a, 0x33, 0xf6, + 0xfa, 0xa3, 0x01, 0x36, 0x18, 0x3b, 0xb2, 0x86, 0x4d, 0x70, 0x9e, 0x3e, 0xf6, 0x1d, 0x35, 0x0d, + 0x1b, 0x5f, 0xcc, 0xf4, 0xba, 0xc1, 0x0f, 0xfa, 0x4f, 0x0a, 0xb4, 0x93, 0x4a, 0xb9, 0x64, 0x3b, + 0xf8, 0x4d, 0x14, 0x8a, 0x0a, 0x63, 0x77, 0x6d, 0x07, 0x33, 0x1b, 0xf8, 0x75, 0xf1, 0xb9, 0x28, + 0x52, 0xef, 0x5f, 0x85, 0x7a, 0xec, 0x02, 0xaa, 0x43, 0x75, 0xe9, 0xd6, 0xed, 0x85, 0xeb, 0x9d, + 0x12, 0x6a, 0x41, 0xfd, 0xc6, 0xea, 0xba, 0xc9, 0x8f, 0x0a, 0x3a, 0x00, 0x0d, 0x63, 0xe9, 0xf2, + 0xd2, 0xd7, 0xe6, 0xca, 0xc2, 0xfa, 0x85, 0x2b, 0x9d, 0x11, 0x84, 0xa0, 0xcd, 0x09, 0x37, 0x56, + 0x05, 0xad, 0x3c, 0xff, 0xf7, 0x28, 0x8c, 0x49, 0x1b, 0xd1, 0x39, 0xa8, 0xdc, 0x0c, 0xc9, 0x16, + 0x3a, 0x94, 0x54, 0xea, 0x57, 0x81, 0x4d, 0xb1, 0xe8, 0x3c, 0x75, 0x6a, 0x1b, 0x9d, 0xf7, 0x9d, + 0x5e, 0x42, 0x17, 0xa1, 0x91, 0xda, 0xea, 0x50, 0xe1, 0x42, 0xaf, 0x1e, 0xc9, 0x50, 0xb3, 0x4f, + 0x83, 0x5e, 0x3a, 0xad, 0xa0, 0x55, 0x68, 0x33, 0x96, 0x5c, 0xe1, 0x08, 0x7a, 0x47, 0x8a, 0x14, + 0xad, 0xb5, 0xea, 0xd1, 0x1d, 0xb8, 0xb1, 0x59, 0x57, 0xa0, 0x91, 0x5a, 0x54, 0x90, 0x9a, 0x29, + 0xa0, 0xcc, 0x36, 0x97, 0x18, 0x57, 0xb0, 0x13, 0xe9, 0x25, 0x74, 0x47, 0x6c, 0x2c, 0xe9, 0x95, + 0x67, 0x57, 0x7d, 0xc7, 0x0b, 0x78, 0x05, 0x2e, 0x2f, 0x01, 0x24, 0xfb, 0x04, 0x3a, 0x9c, 0x11, + 0x4a, 0x6f, 0x47, 0xaa, 0x5a, 0xc4, 0x8a, 0xcd, 0x5b, 0x83, 0x4e, 0x7e, 0x2d, 0xd9, 0x4d, 0xd9, + 0xb1, 0xed, 0xac, 0x02, 0xdb, 0x16, 0xa1, 0x1e, 0x8f, 0x54, 0xd4, 0x2d, 0x98, 0xb2, 0x5c, 0xd9, + 0xce, 0xf3, 0x57, 0x2f, 0xa1, 0x4b, 0xd0, 0x5c, 0x70, 0x9c, 0xfd, 0xa8, 0x51, 0xd3, 0x1c, 0x92, + 0xd7, 0xe3, 0xc4, 0xaf, 0x7e, 0x7e, 0xc4, 0xa0, 0x77, 0xe3, 0xc6, 0xde, 0x75, 0x34, 0xab, 0xef, + 0xed, 0x89, 0x8b, 0x6f, 0xfb, 0x01, 0x8e, 0xee, 0x3a, 0xd0, 0xf6, 0x7d, 0xe7, 0xa9, 0x3d, 0x70, + 0x05, 0x51, 0x5f, 0x87, 0x03, 0xb9, 0xf9, 0x86, 0xb4, 0x9c, 0x96, 0xdc, 0x48, 0x54, 0xa7, 0x77, + 0xe4, 0x4b, 0xbd, 0x8b, 0x9f, 0x3f, 0x7d, 0xa1, 0x95, 0x9e, 0xbd, 0xd0, 0x4a, 0xaf, 0x5e, 0x68, + 0xca, 0x8f, 0x43, 0x4d, 0xf9, 0x7d, 0xa8, 0x29, 0x4f, 0x86, 0x9a, 0xf2, 0x74, 0xa8, 0x29, 0x7f, + 0x0d, 0x35, 0xe5, 0x9f, 0xa1, 0x56, 0x7a, 0x35, 0xd4, 0x94, 0x5f, 0x5e, 0x6a, 0xa5, 0xa7, 0x2f, + 0xb5, 0xd2, 0xb3, 0x97, 0x5a, 0xe9, 0x9b, 0x5a, 0xcf, 0xb1, 0xb1, 0x47, 0x37, 0x6a, 0xec, 0x3f, + 0xfe, 0xa3, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0xc5, 0xfc, 0x4c, 0xf5, 0x32, 0x10, 0x00, 0x00, } func (x MatchType) String() string { @@ -2012,6 +2021,9 @@ func (this *UserStatsResponse) Equal(that interface{}) bool { if this.ActiveSeries != that1.ActiveSeries { return false } + if this.LoadedBlocks != that1.LoadedBlocks { + return false + } return true } func (this *UserIDStatsResponse) Equal(that interface{}) bool { @@ -2565,13 +2577,14 @@ func (this *UserStatsResponse) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 9) + s := make([]string, 0, 10) s = append(s, "&client.UserStatsResponse{") s = append(s, "IngestionRate: "+fmt.Sprintf("%#v", this.IngestionRate)+",\n") s = append(s, "NumSeries: "+fmt.Sprintf("%#v", this.NumSeries)+",\n") s = append(s, "ApiIngestionRate: "+fmt.Sprintf("%#v", this.ApiIngestionRate)+",\n") s = append(s, "RuleIngestionRate: "+fmt.Sprintf("%#v", this.RuleIngestionRate)+",\n") s = append(s, "ActiveSeries: "+fmt.Sprintf("%#v", this.ActiveSeries)+",\n") + s = append(s, "LoadedBlocks: "+fmt.Sprintf("%#v", this.LoadedBlocks)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -3868,6 +3881,11 @@ func (m *UserStatsResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.LoadedBlocks != 0 { + i = encodeVarintIngester(dAtA, i, uint64(m.LoadedBlocks)) + i-- + dAtA[i] = 0x30 + } if m.ActiveSeries != 0 { i = encodeVarintIngester(dAtA, i, uint64(m.ActiveSeries)) i-- @@ -4666,6 +4684,9 @@ func (m *UserStatsResponse) Size() (n int) { if m.ActiveSeries != 0 { n += 1 + sovIngester(uint64(m.ActiveSeries)) } + if m.LoadedBlocks != 0 { + n += 1 + sovIngester(uint64(m.LoadedBlocks)) + } return n } @@ -5090,6 +5111,7 @@ func (this *UserStatsResponse) String() string { `ApiIngestionRate:` + fmt.Sprintf("%v", this.ApiIngestionRate) + `,`, `RuleIngestionRate:` + fmt.Sprintf("%v", this.RuleIngestionRate) + `,`, `ActiveSeries:` + fmt.Sprintf("%v", this.ActiveSeries) + `,`, + `LoadedBlocks:` + fmt.Sprintf("%v", this.LoadedBlocks) + `,`, `}`, }, "") return s @@ -6737,6 +6759,25 @@ func (m *UserStatsResponse) Unmarshal(dAtA []byte) error { break } } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LoadedBlocks", wireType) + } + m.LoadedBlocks = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIngester + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LoadedBlocks |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipIngester(dAtA[iNdEx:]) diff --git a/pkg/ingester/client/ingester.proto b/pkg/ingester/client/ingester.proto index e9db9685c1..34b9662264 100644 --- a/pkg/ingester/client/ingester.proto +++ b/pkg/ingester/client/ingester.proto @@ -100,6 +100,7 @@ message UserStatsResponse { double api_ingestion_rate = 3; double rule_ingestion_rate = 4; uint64 active_series = 5; + uint64 loaded_blocks = 6; } message UserIDStatsResponse { diff --git a/pkg/distributor/http_admin.go b/pkg/ingester/http_admin.go similarity index 61% rename from pkg/distributor/http_admin.go rename to pkg/ingester/http_admin.go index 1aa13baff2..a0d2ca98c5 100644 --- a/pkg/distributor/http_admin.go +++ b/pkg/ingester/http_admin.go @@ -1,4 +1,4 @@ -package distributor +package ingester import ( "encoding/json" @@ -22,13 +22,16 @@ const tpl = `

Cortex Ingester Stats

Current time: {{ .Now }}

+ {{if (gt .ReplicationFactor 0)}}

NB stats do not account for replication factor, which is currently set to {{ .ReplicationFactor }}

+ {{end}}
+ @@ -40,6 +43,7 @@ const tpl = ` {{ range .Stats }} + @@ -53,31 +57,41 @@ const tpl = ` ` -var tmpl *template.Template +var UserStatsTmpl *template.Template func init() { - tmpl = template.Must(template.New("webpage").Parse(tpl)) + UserStatsTmpl = template.Must(template.New("webpage").Parse(tpl)) } -type userStatsByTimeseries []UserIDStats +type UserStatsByTimeseries []UserIDStats -func (s userStatsByTimeseries) Len() int { return len(s) } -func (s userStatsByTimeseries) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s UserStatsByTimeseries) Len() int { return len(s) } +func (s UserStatsByTimeseries) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s userStatsByTimeseries) Less(i, j int) bool { +func (s UserStatsByTimeseries) Less(i, j int) bool { return s[i].NumSeries > s[j].NumSeries || (s[i].NumSeries == s[j].NumSeries && s[i].UserID < s[j].UserID) } -// AllUserStatsHandler shows stats for all users. -func (d *Distributor) AllUserStatsHandler(w http.ResponseWriter, r *http.Request) { - stats, err := d.AllUserStats(r.Context()) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } +// UserIDStats models ingestion statistics for one user, including the user ID +type UserIDStats struct { + UserID string `json:"userID"` + UserStats +} + +// UserStats models ingestion statistics for one user. +type UserStats struct { + IngestionRate float64 `json:"ingestionRate"` + NumSeries uint64 `json:"numSeries"` + APIIngestionRate float64 `json:"APIIngestionRate"` + RuleIngestionRate float64 `json:"RuleIngestionRate"` + ActiveSeries uint64 `json:"activeSeries"` + LoadedBlocks uint64 `json:"loadedBlocks"` +} - sort.Sort(userStatsByTimeseries(stats)) +// AllUserStatsRender render data for all users or return in json format. +func AllUserStatsRender(w http.ResponseWriter, r *http.Request, stats []UserIDStats, rf int) { + sort.Sort(UserStatsByTimeseries(stats)) if encodings, found := r.Header["Accept"]; found && len(encodings) > 0 && strings.Contains(encodings[0], "json") { @@ -94,6 +108,6 @@ func (d *Distributor) AllUserStatsHandler(w http.ResponseWriter, r *http.Request }{ Now: time.Now(), Stats: stats, - ReplicationFactor: d.ingestersRing.ReplicationFactor(), - }, tmpl, r) + ReplicationFactor: rf, + }, UserStatsTmpl, r) } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 4269f04d91..82e4bff755 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1805,7 +1805,42 @@ func (i *Ingester) UserStats(ctx context.Context, req *client.UserStatsRequest) return &client.UserStatsResponse{}, nil } - return createUserStats(db, i.cfg.ActiveSeriesMetricsEnabled), nil + userStat := createUserStats(db, i.cfg.ActiveSeriesMetricsEnabled) + + return &client.UserStatsResponse{ + IngestionRate: userStat.IngestionRate, + NumSeries: userStat.NumSeries, + ApiIngestionRate: userStat.APIIngestionRate, + RuleIngestionRate: userStat.RuleIngestionRate, + ActiveSeries: userStat.ActiveSeries, + LoadedBlocks: userStat.LoadedBlocks, + }, nil +} + +func (i *Ingester) userStats() []UserIDStats { + i.stoppedMtx.RLock() + defer i.stoppedMtx.RUnlock() + + perUserTotals := make(map[string]UserStats) + + users := i.TSDBState.dbs + + response := make([]UserIDStats, 0, len(perUserTotals)) + for id, db := range users { + response = append(response, UserIDStats{ + UserID: id, + UserStats: createUserStats(db, i.cfg.ActiveSeriesMetricsEnabled), + }) + } + + return response +} + +// AllUserStatsHandler shows stats for all users. +func (i *Ingester) AllUserStatsHandler(w http.ResponseWriter, r *http.Request) { + stats := i.userStats() + + AllUserStatsRender(w, r, stats, 0) } // AllUserStats returns ingestion statistics for all users known to this ingester. @@ -1814,24 +1849,28 @@ func (i *Ingester) AllUserStats(_ context.Context, _ *client.UserStatsRequest) ( return nil, err } - i.stoppedMtx.RLock() - defer i.stoppedMtx.RUnlock() - - users := i.TSDBState.dbs + userStats := i.userStats() response := &client.UsersStatsResponse{ - Stats: make([]*client.UserIDStatsResponse, 0, len(users)), + Stats: make([]*client.UserIDStatsResponse, 0, len(userStats)), } - for userID, db := range users { + for _, userStat := range userStats { response.Stats = append(response.Stats, &client.UserIDStatsResponse{ - UserId: userID, - Data: createUserStats(db, i.cfg.ActiveSeriesMetricsEnabled), + UserId: userStat.UserID, + Data: &client.UserStatsResponse{ + IngestionRate: userStat.IngestionRate, + NumSeries: userStat.NumSeries, + ApiIngestionRate: userStat.APIIngestionRate, + RuleIngestionRate: userStat.RuleIngestionRate, + ActiveSeries: userStat.ActiveSeries, + LoadedBlocks: userStat.LoadedBlocks, + }, }) } return response, nil } -func createUserStats(db *userTSDB, activeSeriesMetricsEnabled bool) *client.UserStatsResponse { +func createUserStats(db *userTSDB, activeSeriesMetricsEnabled bool) UserStats { apiRate := db.ingestedAPISamples.Rate() ruleRate := db.ingestedRuleSamples.Rate() @@ -1840,12 +1879,13 @@ func createUserStats(db *userTSDB, activeSeriesMetricsEnabled bool) *client.User activeSeries = uint64(db.activeSeries.Active()) } - return &client.UserStatsResponse{ + return UserStats{ IngestionRate: apiRate + ruleRate, - ApiIngestionRate: apiRate, + APIIngestionRate: apiRate, RuleIngestionRate: ruleRate, NumSeries: db.Head().NumSeries(), ActiveSeries: activeSeries, + LoadedBlocks: uint64(len(db.Blocks())), } } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 96253d274a..e574a0d81a 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -4248,6 +4248,7 @@ func Test_Ingester_AllUserStats(t *testing.T) { ApiIngestionRate: 0.2, RuleIngestionRate: 0, ActiveSeries: 3, + LoadedBlocks: 0, }, }, { @@ -4258,12 +4259,91 @@ func Test_Ingester_AllUserStats(t *testing.T) { ApiIngestionRate: 0.13333333333333333, RuleIngestionRate: 0, ActiveSeries: 2, + LoadedBlocks: 0, }, }, } assert.ElementsMatch(t, expect, res.Stats) } +func Test_Ingester_AllUserStatsHandler(t *testing.T) { + series := []struct { + user string + lbls labels.Labels + value float64 + timestamp int64 + }{ + {"user-1", labels.Labels{{Name: labels.MetricName, Value: "test_1_1"}, {Name: "status", Value: "200"}, {Name: "route", Value: "get_user"}}, 1, 100000}, + {"user-1", labels.Labels{{Name: labels.MetricName, Value: "test_1_1"}, {Name: "status", Value: "500"}, {Name: "route", Value: "get_user"}}, 1, 110000}, + {"user-1", labels.Labels{{Name: labels.MetricName, Value: "test_1_2"}}, 2, 200000}, + {"user-2", labels.Labels{{Name: labels.MetricName, Value: "test_2_1"}}, 2, 200000}, + {"user-2", labels.Labels{{Name: labels.MetricName, Value: "test_2_2"}}, 2, 200000}, + } + + // Create ingester + i, err := prepareIngesterWithBlocksStorage(t, defaultIngesterTestConfig(t), prometheus.NewRegistry()) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) + defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck + + // Wait until it's ACTIVE + test.Poll(t, 1*time.Second, ring.ACTIVE, func() interface{} { + return i.lifecycler.GetState() + }) + for _, series := range series { + ctx := user.InjectOrgID(context.Background(), series.user) + req, _ := mockWriteRequest(t, series.lbls, series.value, series.timestamp) + _, err := i.Push(ctx, req) + require.NoError(t, err) + } + + // Force compaction to test loaded blocks + compactionCallbackCh := make(chan struct{}) + i.TSDBState.forceCompactTrigger <- requestWithUsersAndCallback{users: nil, callback: compactionCallbackCh} + <-compactionCallbackCh + + // force update statistics + for _, db := range i.TSDBState.dbs { + db.ingestedAPISamples.Tick() + db.ingestedRuleSamples.Tick() + } + + // Get label names + response := httptest.NewRecorder() + request := httptest.NewRequest("GET", "/all_user_stats", nil) + request.Header.Add("Accept", "application/json") + i.AllUserStatsHandler(response, request) + var resp UserStatsByTimeseries + err = json.Unmarshal(response.Body.Bytes(), &resp) + require.NoError(t, err) + + expect := UserStatsByTimeseries{ + { + UserID: "user-1", + UserStats: UserStats{ + IngestionRate: 0.2, + NumSeries: 0, + APIIngestionRate: 0.2, + RuleIngestionRate: 0, + ActiveSeries: 3, + LoadedBlocks: 1, + }, + }, + { + UserID: "user-2", + UserStats: UserStats{ + IngestionRate: 0.13333333333333333, + NumSeries: 0, + APIIngestionRate: 0.13333333333333333, + RuleIngestionRate: 0, + ActiveSeries: 2, + LoadedBlocks: 1, + }, + }, + } + assert.ElementsMatch(t, expect, resp) +} + func TestIngesterCompactIdleBlock(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.LifecyclerConfig.JoinAfter = 0
UserLoaded Blocks # Series # Active Series Total Ingest Rate
{{ .UserID }}{{ .UserStats.LoadBlocks }} {{ .UserStats.NumSeries }} {{ .UserStats.ActiveSeries }} {{ printf "%.2f" .UserStats.IngestionRate }}