From 423688c8597931b2af62d48e259475db9ab36aa8 Mon Sep 17 00:00:00 2001 From: LiZhenCheng9527 Date: Sat, 31 Aug 2024 16:36:10 +0800 Subject: [PATCH] batch update kmesh metrics to prometheus Signed-off-by: LiZhenCheng9527 --- pkg/controller/telemetry/metric.go | 209 ++++++++++++++-- pkg/controller/telemetry/metric_test.go | 310 ++++++++++++++++++++++-- 2 files changed, 471 insertions(+), 48 deletions(-) diff --git a/pkg/controller/telemetry/metric.go b/pkg/controller/telemetry/metric.go index 124519a60..661ddb8db 100644 --- a/pkg/controller/telemetry/metric.go +++ b/pkg/controller/telemetry/metric.go @@ -48,6 +48,20 @@ var osStartTime time.Time type MetricController struct { workloadCache cache.WorkloadCache + metricCache metricInfoCache +} + +type metricInfoCache struct { + WorkloadConnOpened map[workloadMetricLabels]float64 + WorkloadConnClosed map[workloadMetricLabels]float64 + WorkloadConnSentBytes map[workloadMetricLabels]float64 + WorkloadConnReceivedBytes map[workloadMetricLabels]float64 + WorkloadConnFailed map[workloadMetricLabels]float64 + ServiceConnOpened map[serviceMetricLabels]float64 + ServiceConnClosed map[serviceMetricLabels]float64 + ServiceConnSentBytes map[serviceMetricLabels]float64 + ServiceConnReceivedBytes map[serviceMetricLabels]float64 + ServiceConnFailed map[serviceMetricLabels]float64 } type connectionDataV4 struct { @@ -154,6 +168,22 @@ type serviceMetricLabels struct { func NewMetric(workloadCache cache.WorkloadCache) *MetricController { return &MetricController{ workloadCache: workloadCache, + metricCache: newMetricCache(), + } +} + +func newMetricCache() metricInfoCache { + return metricInfoCache{ + WorkloadConnOpened: map[workloadMetricLabels]float64{}, + WorkloadConnClosed: map[workloadMetricLabels]float64{}, + WorkloadConnSentBytes: map[workloadMetricLabels]float64{}, + WorkloadConnReceivedBytes: map[workloadMetricLabels]float64{}, + WorkloadConnFailed: map[workloadMetricLabels]float64{}, + ServiceConnOpened: map[serviceMetricLabels]float64{}, + ServiceConnClosed: map[serviceMetricLabels]float64{}, + ServiceConnSentBytes: map[serviceMetricLabels]float64{}, + ServiceConnReceivedBytes: map[serviceMetricLabels]float64{}, + ServiceConnFailed: map[serviceMetricLabels]float64{}, } } @@ -181,6 +211,21 @@ func (m *MetricController) Run(ctx context.Context, mapOfTcpInfo *ebpf.Map) { // Register metrics to Prometheus and start Prometheus server go RunPrometheusClient(ctx) + go func() { + for { + select { + case <-ctx.Done(): + return + default: + // Metrics updated every 3 seconds + time.Sleep(3 * time.Second) + err := m.updatePrometheusMetric() + if err != nil { + log.Errorf("update Kmesh metrics failed: %v", err) + } + } + } + }() for { select { @@ -231,8 +276,8 @@ func (m *MetricController) Run(ctx context.Context, mapOfTcpInfo *ebpf.Map) { if data.state == TCP_CLOSTED { OutputAccesslog(data, accesslog) } - buildWorkloadMetricsToPrometheus(data, workloadLabels) - buildServiceMetricsToPrometheus(data, serviceLabels) + m.buildWorkloadMetricsToPrometheus(data, workloadLabels) + m.buildServiceMetricsToPrometheus(data, serviceLabels) } } } @@ -429,36 +474,150 @@ func buildPrincipal(workload *workloadapi.Workload) string { return "-" } -func buildWorkloadMetricsToPrometheus(data requestMetric, labels workloadMetricLabels) { - commonLabels := struct2map(labels) - - if data.state == TCP_ESTABLISHED { - tcpConnectionOpenedInWorkload.With(commonLabels).Add(float64(1)) - } - if data.state == TCP_CLOSTED { - tcpConnectionClosedInWorkload.With(commonLabels).Add(float64(1)) +func (m *MetricController) buildWorkloadMetricsToPrometheus(data requestMetric, labels workloadMetricLabels) { + // commonLabels := struct2map(labels) + + // if data.state == TCP_ESTABLISHED { + // tcpConnectionOpenedInWorkload.With(commonLabels).Add(float64(1)) + // } + // if data.state == TCP_CLOSTED { + // tcpConnectionClosedInWorkload.With(commonLabels).Add(float64(1)) + // } + // if data.success != connection_success { + // tcpConnectionFailedInWorkload.With(commonLabels).Add(float64(1)) + // } + // tcpReceivedBytesInWorkload.With(commonLabels).Add(float64(data.receivedBytes)) + // tcpSentBytesInWorkload.With(commonLabels).Add(float64(data.sentBytes)) + _, ok := m.metricCache.WorkloadConnReceivedBytes[labels] + if ok { + if data.state == TCP_ESTABLISHED { + m.metricCache.WorkloadConnOpened[labels] = m.metricCache.WorkloadConnOpened[labels] + 1 + } + if data.state == TCP_CLOSTED { + m.metricCache.WorkloadConnClosed[labels] = m.metricCache.WorkloadConnClosed[labels] + 1 + } + if data.success != connection_success { + m.metricCache.WorkloadConnFailed[labels] = m.metricCache.WorkloadConnFailed[labels] + 1 + } + m.metricCache.WorkloadConnReceivedBytes[labels] = m.metricCache.WorkloadConnReceivedBytes[labels] + float64(data.receivedBytes) + m.metricCache.WorkloadConnSentBytes[labels] = m.metricCache.WorkloadConnSentBytes[labels] + float64(data.sentBytes) + } else { + if data.state == TCP_ESTABLISHED { + m.metricCache.WorkloadConnOpened[labels] = 1 + } + if data.state == TCP_CLOSTED { + m.metricCache.WorkloadConnClosed[labels] = 1 + } + if data.success != connection_success { + m.metricCache.WorkloadConnFailed[labels] = 1 + } + m.metricCache.WorkloadConnReceivedBytes[labels] = float64(data.receivedBytes) + m.metricCache.WorkloadConnSentBytes[labels] = float64(data.sentBytes) } - if data.success != connection_success { - tcpConnectionFailedInWorkload.With(commonLabels).Add(float64(1)) +} + +func (m *MetricController) buildServiceMetricsToPrometheus(data requestMetric, labels serviceMetricLabels) { + // commonLabels := struct2map(labels) + + // if data.state == TCP_ESTABLISHED { + // tcpConnectionOpenedInService.With(commonLabels).Add(float64(1)) + // } + // if data.state == TCP_CLOSTED { + // tcpConnectionClosedInService.With(commonLabels).Add(float64(1)) + // } + // if data.success != uint32(1) { + // tcpConnectionFailedInService.With(commonLabels).Add(float64(1)) + // } + // tcpReceivedBytesInService.With(commonLabels).Add(float64(data.receivedBytes)) + // tcpSentBytesInService.With(commonLabels).Add(float64(data.sentBytes)) + _, ok := m.metricCache.ServiceConnReceivedBytes[labels] + if ok { + if data.state == TCP_ESTABLISHED { + m.metricCache.ServiceConnOpened[labels] = m.metricCache.ServiceConnOpened[labels] + 1 + } + if data.state == TCP_CLOSTED { + m.metricCache.ServiceConnClosed[labels] = m.metricCache.ServiceConnClosed[labels] + 1 + } + if data.success != connection_success { + m.metricCache.ServiceConnFailed[labels] = m.metricCache.ServiceConnFailed[labels] + 1 + } + m.metricCache.ServiceConnReceivedBytes[labels] = m.metricCache.ServiceConnReceivedBytes[labels] + float64(data.receivedBytes) + m.metricCache.ServiceConnSentBytes[labels] = m.metricCache.ServiceConnSentBytes[labels] + float64(data.sentBytes) + } else { + if data.state == TCP_ESTABLISHED { + m.metricCache.ServiceConnOpened[labels] = 1 + } + if data.state == TCP_CLOSTED { + m.metricCache.ServiceConnClosed[labels] = 1 + } + if data.success != connection_success { + m.metricCache.ServiceConnFailed[labels] = 1 + } + m.metricCache.ServiceConnReceivedBytes[labels] = float64(data.receivedBytes) + m.metricCache.ServiceConnSentBytes[labels] = float64(data.sentBytes) } - tcpReceivedBytesInWorkload.With(commonLabels).Add(float64(data.receivedBytes)) - tcpSentBytesInWorkload.With(commonLabels).Add(float64(data.sentBytes)) } -func buildServiceMetricsToPrometheus(data requestMetric, labels serviceMetricLabels) { - commonLabels := struct2map(labels) +func (m *MetricController) updatePrometheusMetric() error { + val := reflect.ValueOf(m.metricCache) + typ := reflect.TypeOf(m.metricCache) + + // check if has pointer in struct + if typ.Kind() == reflect.Ptr { + val = val.Elem() + typ = typ.Elem() - if data.state == TCP_ESTABLISHED { - tcpConnectionOpenedInService.With(commonLabels).Add(float64(1)) - } - if data.state == TCP_CLOSTED { - tcpConnectionClosedInService.With(commonLabels).Add(float64(1)) } - if data.success != uint32(1) { - tcpConnectionFailedInService.With(commonLabels).Add(float64(1)) + num := val.NumField() + for i := 0; i < num; i++ { + sType := typ.Field(i) + sVal := val.Field(i).Interface() + workloadMap, isWorkload := sVal.(map[workloadMetricLabels]float64) + // fmt.Printf("\n ------- %v, \n%v -------- \n", workloadMap, isWorkload) + if isWorkload { + for k, v := range workloadMap { + name := sType.Name + commonLabels := struct2map(k) + // fmt.Printf("name is %v, value is: %v", name, v) + switch name { + case "WorkloadConnOpened": + tcpConnectionOpenedInWorkload.With(commonLabels).Set(float64(v)) + case "WorkloadConnClosed": + tcpConnectionClosedInWorkload.With(commonLabels).Set(float64(v)) + case "WorkloadConnSentBytes": + tcpSentBytesInWorkload.With(commonLabels).Set(float64(v)) + case "WorkloadConnReceivedBytes": + tcpReceivedBytesInWorkload.With(commonLabels).Set(float64(v)) + case "WorkloadConnFailed": + tcpConnectionFailedInWorkload.With(commonLabels).Set(float64(v)) + } + } + } + serviceMap, isService := sVal.(map[serviceMetricLabels]float64) + // fmt.Printf("\n ------- %v, \n%v -------- \n", serviceMap, isService) + if isService { + for k, v := range serviceMap { + name := sType.Name + commonLabels := struct2map(k) + switch name { + case "ServiceConnOpened": + tcpConnectionOpenedInService.With(commonLabels).Set(float64(v)) + case "ServiceConnClosed": + tcpConnectionClosedInService.With(commonLabels).Set(float64(v)) + case "ServiceConnSentBytes": + tcpSentBytesInService.With(commonLabels).Set(float64(v)) + case "ServiceConnReceivedBytes": + tcpReceivedBytesInService.With(commonLabels).Set(float64(v)) + case "ServiceConnFailed": + tcpConnectionFailedInService.With(commonLabels).Set(float64(v)) + } + } + } + if !isWorkload && !isService { + return fmt.Errorf("get metricCahce data failed") + } } - tcpReceivedBytesInService.With(commonLabels).Add(float64(data.receivedBytes)) - tcpSentBytesInService.With(commonLabels).Add(float64(data.sentBytes)) + return nil } func struct2map(labels interface{}) map[string]string { diff --git a/pkg/controller/telemetry/metric_test.go b/pkg/controller/telemetry/metric_test.go index 2a97a0853..0f04d2691 100644 --- a/pkg/controller/telemetry/metric_test.go +++ b/pkg/controller/telemetry/metric_test.go @@ -22,7 +22,7 @@ import ( "testing" "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "kmesh.net/kmesh/api/v2/workloadapi" @@ -172,13 +172,6 @@ func TestCommonTrafficLabels2map(t *testing.T) { } func TestBuildMetricsToPrometheus(t *testing.T) { - metrics := []*prometheus.GaugeVec{ - tcpConnectionClosedInWorkload, - tcpConnectionOpenedInWorkload, - tcpReceivedBytesInWorkload, - tcpSentBytesInWorkload, - } - type args struct { data requestMetric labels workloadMetricLabels @@ -189,7 +182,7 @@ func TestBuildMetricsToPrometheus(t *testing.T) { want []float64 }{ { - name: "test build metrisc to Prometheus", + name: "test build workload metrisc to metricCache", args: args{ data: requestMetric{ src: [4]uint32{183763210, 0, 0, 0}, @@ -234,21 +227,83 @@ func TestBuildMetricsToPrometheus(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - go RunPrometheusClient(ctx) - buildWorkloadMetricsToPrometheus(tt.args.data, tt.args.labels) - commonLabels := struct2map(tt.args.labels) - for index, metric := range metrics { - if gauge, err := metric.GetMetricWith(commonLabels); err != nil { - t.Errorf("use labels to get %v failed", metric) - } else { - var m dto.Metric - gauge.Write(&m) - value := m.Gauge.Value - assert.Equal(t, tt.want[index], *value) - } + m := MetricController{ + workloadCache: cache.NewWorkloadCache(), + metricCache: newMetricCache(), } - cancel() + m.buildWorkloadMetricsToPrometheus(tt.args.data, tt.args.labels) + assert.Equal(t, m.metricCache.WorkloadConnClosed[tt.args.labels], tt.want[0]) + assert.Equal(t, m.metricCache.WorkloadConnOpened[tt.args.labels], tt.want[1]) + assert.Equal(t, m.metricCache.WorkloadConnReceivedBytes[tt.args.labels], tt.want[2]) + assert.Equal(t, m.metricCache.WorkloadConnSentBytes[tt.args.labels], tt.want[3]) + }) + } +} + +func TestBuildServiceMetricsToPrometheus(t *testing.T) { + type args struct { + data requestMetric + labels serviceMetricLabels + } + tests := []struct { + name string + args args + want []float64 + }{ + { + name: "build service metrics in metricCache", + args: args{ + data: requestMetric{ + src: [4]uint32{183763210, 0, 0, 0}, + dst: [4]uint32{183762951, 0, 0, 0}, + sentBytes: 0x0000009, + receivedBytes: 0x0000008, + state: TCP_ESTABLISHED, + }, + labels: serviceMetricLabels{ + sourceWorkload: "kmesh-daemon", + sourceCanonicalService: "srcCanonical", + sourceCanonicalRevision: "srcVersion", + sourceWorkloadNamespace: "kmesh-system", + sourcePrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + sourceApp: "srcCanonical", + sourceVersion: "srcVersion", + sourceCluster: "Kubernetes", + destinationService: "kmesh.kmesh-system.svc.cluster.local", + destinationServiceNamespace: "kmesh-system", + destinationServiceName: "kmesh.kmesh-system.svc.cluster.local", + destinationWorkload: "kmesh-daemon", + destinationCanonicalService: "dstCanonical", + destinationCanonicalRevision: "dstVersion", + destinationWorkloadNamespace: "kmesh-system", + destinationPrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + destinationApp: "dstCanonical", + destinationVersion: "dstVersion", + destinationCluster: "Kubernetes", + requestProtocol: "tcp", + responseFlags: "-", + connectionSecurityPolicy: "mutual_tls", + }, + }, + want: []float64{ + 0, + 1, + 8, + 9, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m := MetricController{ + workloadCache: cache.NewWorkloadCache(), + metricCache: newMetricCache(), + } + m.buildServiceMetricsToPrometheus(tt.args.data, tt.args.labels) + assert.Equal(t, m.metricCache.ServiceConnClosed[tt.args.labels], tt.want[0]) + assert.Equal(t, m.metricCache.ServiceConnOpened[tt.args.labels], tt.want[1]) + assert.Equal(t, m.metricCache.ServiceConnReceivedBytes[tt.args.labels], tt.want[2]) + assert.Equal(t, m.metricCache.ServiceConnSentBytes[tt.args.labels], tt.want[3]) }) } } @@ -706,3 +761,212 @@ func Test_buildServiceMetric(t *testing.T) { }) } } + +func TestMetricController_updatePrometheusMetric(t *testing.T) { + testworkloadLabel1 := workloadMetricLabels{ + sourceWorkload: "kmesh-daemon", + sourceCanonicalService: "srcCanonical", + sourceCanonicalRevision: "srcVersion", + sourceWorkloadNamespace: "kmesh-system", + sourcePrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + sourceApp: "srcCanonical", + sourceVersion: "srcVersion", + sourceCluster: "Kubernetes", + destinationPodAddress: "192.168.224.22", + destinationPodNamespace: "kmesh-system", + destinationPodName: "kmesh", + destinationWorkload: "kmesh-daemon", + destinationCanonicalService: "dstCanonical", + destinationCanonicalRevision: "dstVersion", + destinationWorkloadNamespace: "kmesh-system", + destinationPrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + destinationApp: "dstCanonical", + destinationVersion: "dstVersion", + destinationCluster: "Kubernetes", + requestProtocol: "tcp", + responseFlags: "-", + connectionSecurityPolicy: "mutual_tls", + } + testworkloadLabel2 := workloadMetricLabels{ + sourceWorkload: "kmesh-daemon", + sourceCanonicalService: "srcCanonical", + sourceCanonicalRevision: "srcVersion", + sourceWorkloadNamespace: "kmesh-system", + sourcePrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + sourceApp: "srcCanonical", + sourceVersion: "srcVersion", + sourceCluster: "Kubernetes", + destinationPodAddress: "192.168.224.22", + destinationPodNamespace: "kmesh-system", + destinationPodName: "sleep", + destinationWorkload: "sleep", + destinationCanonicalService: "dstCanonical", + destinationCanonicalRevision: "dstVersion", + destinationWorkloadNamespace: "kmesh-system", + destinationPrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + destinationApp: "dstCanonical", + destinationVersion: "dstVersion", + destinationCluster: "Kubernetes", + requestProtocol: "tcp", + responseFlags: "-", + connectionSecurityPolicy: "mutual_tls", + } + + testServiceLabel1 := serviceMetricLabels{ + sourceWorkload: "kmesh-daemon", + sourceCanonicalService: "srcCanonical", + sourceCanonicalRevision: "srcVersion", + sourceWorkloadNamespace: "kmesh-system", + sourcePrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + sourceApp: "srcCanonical", + sourceVersion: "srcVersion", + sourceCluster: "Kubernetes", + destinationService: "kmesh.kmesh-system.svc.cluster.local", + destinationServiceNamespace: "kmesh-system", + destinationServiceName: "kmesh.kmesh-system.svc.cluster.local", + destinationWorkload: "kmesh-daemon", + destinationCanonicalService: "dstCanonical", + destinationCanonicalRevision: "dstVersion", + destinationWorkloadNamespace: "kmesh-system", + destinationPrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + destinationApp: "dstCanonical", + destinationVersion: "dstVersion", + destinationCluster: "Kubernetes", + requestProtocol: "tcp", + responseFlags: "-", + connectionSecurityPolicy: "mutual_tls", + } + testServiceLabel2 := serviceMetricLabels{ + sourceWorkload: "kmesh-daemon", + sourceCanonicalService: "srcCanonical", + sourceCanonicalRevision: "srcVersion", + sourceWorkloadNamespace: "kmesh-system", + sourcePrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + sourceApp: "srcCanonical", + sourceVersion: "srcVersion", + sourceCluster: "Kubernetes", + destinationService: "sleep.kmesh-system.svc.cluster.local", + destinationServiceNamespace: "kmesh-system", + destinationServiceName: "sleep.kmesh-system.svc.cluster.local", + destinationWorkload: "kmesh-daemon", + destinationCanonicalService: "dstCanonical", + destinationCanonicalRevision: "dstVersion", + destinationWorkloadNamespace: "kmesh-system", + destinationPrincipal: "spiffe://cluster.local/ns/kmesh-system/sa/default", + destinationApp: "dstCanonical", + destinationVersion: "dstVersion", + destinationCluster: "Kubernetes", + requestProtocol: "tcp", + responseFlags: "-", + connectionSecurityPolicy: "mutual_tls", + } + workloadPrometheusLabel1 := struct2map(testworkloadLabel1) + workloadPrometheusLabel2 := struct2map(testworkloadLabel2) + servicePrometheusLabel1 := struct2map(testServiceLabel1) + servicePrometheusLabel2 := struct2map(testServiceLabel2) + tests := []struct { + name string + metricCache metricInfoCache + exportMetrics []*prometheus.GaugeVec + labels []map[string]string + want []float64 + }{ + { + name: "update workload metric in Prometheus", + metricCache: metricInfoCache{ + WorkloadConnOpened: map[workloadMetricLabels]float64{ + testworkloadLabel1: float64(1), + testworkloadLabel2: float64(2), + }, + WorkloadConnClosed: map[workloadMetricLabels]float64{ + testworkloadLabel1: float64(3), + testworkloadLabel2: float64(4), + }, + WorkloadConnSentBytes: map[workloadMetricLabels]float64{ + testworkloadLabel1: float64(5), + testworkloadLabel2: float64(6), + }, + WorkloadConnReceivedBytes: map[workloadMetricLabels]float64{ + testworkloadLabel1: float64(7), + testworkloadLabel2: float64(8), + }, + WorkloadConnFailed: map[workloadMetricLabels]float64{ + testworkloadLabel1: float64(9), + testworkloadLabel2: float64(10), + }, + }, + exportMetrics: []*prometheus.GaugeVec{ + tcpConnectionOpenedInWorkload, + tcpConnectionClosedInWorkload, + tcpSentBytesInWorkload, + tcpReceivedBytesInWorkload, + tcpConnectionFailedInWorkload, + }, + labels: []map[string]string{ + workloadPrometheusLabel1, workloadPrometheusLabel2, + }, + want: []float64{ + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, + }, + }, + { + name: "update service metrics in Prometheus", + metricCache: metricInfoCache{ + ServiceConnOpened: map[serviceMetricLabels]float64{ + testServiceLabel1: float64(11), + testServiceLabel2: float64(12), + }, + ServiceConnClosed: map[serviceMetricLabels]float64{ + testServiceLabel1: float64(13), + testServiceLabel2: float64(14), + }, + ServiceConnSentBytes: map[serviceMetricLabels]float64{ + testServiceLabel1: float64(15), + testServiceLabel2: float64(16), + }, + ServiceConnReceivedBytes: map[serviceMetricLabels]float64{ + testServiceLabel1: float64(17), + testServiceLabel2: float64(18), + }, + ServiceConnFailed: map[serviceMetricLabels]float64{ + testServiceLabel1: float64(19), + testServiceLabel2: float64(20), + }, + }, + exportMetrics: []*prometheus.GaugeVec{ + tcpConnectionOpenedInService, + tcpConnectionClosedInService, + tcpSentBytesInService, + tcpReceivedBytesInService, + tcpConnectionFailedInService, + }, + labels: []map[string]string{ + servicePrometheusLabel1, servicePrometheusLabel2, + }, + want: []float64{ + 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + go RunPrometheusClient(ctx) + m := &MetricController{ + workloadCache: cache.NewWorkloadCache(), + metricCache: tt.metricCache, + } + err := m.updatePrometheusMetric() + assert.NoError(t, err) + index := 0 + for _, metric := range tt.exportMetrics { + v1 := testutil.ToFloat64(metric.With(tt.labels[0])) + assert.Equal(t, tt.want[index], v1) + v2 := testutil.ToFloat64(metric.With(tt.labels[1])) + assert.Equal(t, tt.want[index+1], v2) + index = index + 2 + } + cancel() + }) + } +}