Expose query duration as histogram

Signed-off-by: Wilfried Roset <[email protected]>

config.go

@@ -13,6 +13,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/jmoiron/sqlx"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/robfig/cron/v3"
 	"gopkg.in/yaml.v2"
 )
@@ -25,9 +26,10 @@ func getenv(key, defaultVal string) string {
 }
 
 var (
+	metricsPrefix = "sql_exporter"
 	failedScrapes = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "sql_exporter_last_scrape_failed",
+			Name: fmt.Sprintf("%s_last_scrape_failed", metricsPrefix),
 			Help: "Failed scrapes",
 		},
 		[]string{"driver", "host", "database", "user", "sql_job", "query"},
@@ -41,6 +43,18 @@ var (
 			regexp.QuoteMeta(tmplEnd),
 		),
 	)
+	QueryMetricsLabels = []string{"sql_job", "query"}
+	queryCounter       = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: fmt.Sprintf("%s_queries_total", metricsPrefix),
+	}, QueryMetricsLabels)
+	failedQueryCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: fmt.Sprintf("%s_query_failures_total", metricsPrefix),
+	}, QueryMetricsLabels)
+
+	// Those are the default buckets
+	DefaultQueryDurationHistogramBuckets = []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60, 120}
+	// To make the buckets configurable let's init it after loading the configuration.
+	queryDurationHistogram *prometheus.HistogramVec
 )
 
 func init() {
@@ -100,11 +114,16 @@ type CloudSQLConfig struct {
 
 // File is a collection of jobs
 type File struct {
+	Misc           Misc              `yaml:"misc,omitempty"`
 	Jobs           []*Job            `yaml:"jobs"`
 	Queries        map[string]string `yaml:"queries"`
 	CloudSQLConfig *CloudSQLConfig   `yaml:"cloudsql_config"`
 }
 
+type Misc struct {
+	HistogramBuckets []float64 `yaml:"histogram_buckets"`
+}
+
 type cronConfig struct {
 	definition string
 	schedule   cron.Schedule

exporter.go

@@ -1,14 +1,16 @@
 package main
 
 import (
+	"context"
+	"fmt"
+
 	"cloud.google.com/go/cloudsqlconn"
 	"cloud.google.com/go/cloudsqlconn/mysql/mysql"
 	"cloud.google.com/go/cloudsqlconn/postgres/pgxv4"
-	"context"
-	"fmt"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/robfig/cron/v3"
 	"google.golang.org/api/option"
 	sqladmin "google.golang.org/api/sqladmin/v1beta4"
@@ -34,6 +36,18 @@ func NewExporter(logger log.Logger, configFile string) (*Exporter, error) {
 		return nil, err
 	}
 
+	var queryDurationHistogramBuckets []float64
+	if len(cfg.Misc.HistogramBuckets) == 0 {
+		queryDurationHistogramBuckets = DefaultQueryDurationHistogramBuckets
+	} else {
+		queryDurationHistogramBuckets = cfg.Misc.HistogramBuckets
+	}
+	queryDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    fmt.Sprintf("%s_query_duration_seconds", metricsPrefix),
+		Help:    "Time spent by querying the database.",
+		Buckets: queryDurationHistogramBuckets,
+	}, QueryMetricsLabels)
+
 	exp := &Exporter{
 		jobs:          make([]*Job, 0, len(cfg.Jobs)),
 		logger:        logger,

job.go

@@ -329,6 +329,8 @@ func (j *Job) runOnceConnection(conn *connection, done chan int) {
 	if err := conn.connect(j); err != nil {
 		level.Warn(j.log).Log("msg", "Failed to connect", "err", err, "host", conn.host)
 		j.markFailed(conn)
+		// we don't have the query name yet.
+		failedQueryCounter.WithLabelValues(j.Name, "").Inc()
 		return
 	}
 

query.go

@@ -15,22 +15,30 @@ func (q *Query) Run(conn *connection) error {
 	if q.log == nil {
 		q.log = log.NewNopLogger()
 	}
+	queryCounter.WithLabelValues(q.jobName, q.Name).Inc()
 	if q.desc == nil {
+		failedQueryCounter.WithLabelValues(q.jobName, q.Name).Inc()
 		return fmt.Errorf("metrics descriptor is nil")
 	}
 	if q.Query == "" {
+		failedQueryCounter.WithLabelValues(q.jobName, q.Name).Inc()
 		return fmt.Errorf("query is empty")
 	}
 	if conn == nil || conn.conn == nil {
+		failedQueryCounter.WithLabelValues(q.jobName, q.Name).Inc()
 		return fmt.Errorf("db connection not initialized (should not happen)")
 	}
 	// execute query
+	now := time.Now()
 	rows, err := conn.conn.Queryx(q.Query)
 	if err != nil {
 		failedScrapes.WithLabelValues(conn.driver, conn.host, conn.database, conn.user, q.jobName, q.Name).Set(1.0)
+		failedQueryCounter.WithLabelValues(q.jobName, q.Name).Inc()
 		return err
 	}
 	defer rows.Close()
+	duration := time.Since(now)
+	queryDurationHistogram.WithLabelValues(q.jobName, q.Name).Observe(duration.Seconds())
 
 	updated := 0
 	metrics := make([]prometheus.Metric, 0, len(q.metrics))