Merge pull request justwatchcom#121 from wilfriedroset/histogram

Expose query duration via histogram

.github/workflows/go.yml

@@ -14,7 +14,7 @@ jobs:
     - uses: actions/checkout@v3
     - uses: actions/setup-go@v5
       with:
-        go-version: '>=1.21'
+        go-version-file: 'go.mod'
     - name: Build
       run: go build -v ./...
 
@@ -24,3 +24,5 @@ jobs:
     - uses: dominikh/[email protected]
       with:
         version: "2023.1"
+        # go is already installed by actions/setup-go with the correct version from go.mod
+        install-go: false

config.go

@@ -13,6 +13,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/jmoiron/sqlx"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/robfig/cron/v3"
 	"gopkg.in/yaml.v2"
 )
@@ -25,9 +26,10 @@ func getenv(key, defaultVal string) string {
 }
 
 var (
+	metricsPrefix = "sql_exporter"
 	failedScrapes = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "sql_exporter_last_scrape_failed",
+			Name: fmt.Sprintf("%s_last_scrape_failed", metricsPrefix),
 			Help: "Failed scrapes",
 		},
 		[]string{"driver", "host", "database", "user", "sql_job", "query"},
@@ -41,6 +43,18 @@ var (
 			regexp.QuoteMeta(tmplEnd),
 		),
 	)
+	QueryMetricsLabels = []string{"sql_job", "query"}
+	queryCounter       = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: fmt.Sprintf("%s_queries_total", metricsPrefix),
+	}, QueryMetricsLabels)
+	failedQueryCounter = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: fmt.Sprintf("%s_query_failures_total", metricsPrefix),
+	}, QueryMetricsLabels)
+
+	// Those are the default buckets
+	DefaultQueryDurationHistogramBuckets = prometheus.DefBuckets
+	// To make the buckets configurable let's init it after loading the configuration.
+	queryDurationHistogram *prometheus.HistogramVec
 )
 
 func init() {
@@ -100,11 +114,16 @@ type CloudSQLConfig struct {
 
 // File is a collection of jobs
 type File struct {
+	Configuration  Configuration     `yaml:"configuration,omitempty"`
 	Jobs           []*Job            `yaml:"jobs"`
 	Queries        map[string]string `yaml:"queries"`
 	CloudSQLConfig *CloudSQLConfig   `yaml:"cloudsql_config"`
 }
 
+type Configuration struct {
+	HistogramBuckets []float64 `yaml:"histogram_buckets"`
+}
+
 type cronConfig struct {
 	definition string
 	schedule   cron.Schedule

exporter.go

@@ -1,14 +1,16 @@
 package main
 
 import (
+	"context"
+	"fmt"
+
 	"cloud.google.com/go/cloudsqlconn"
 	"cloud.google.com/go/cloudsqlconn/mysql/mysql"
 	"cloud.google.com/go/cloudsqlconn/postgres/pgxv4"
-	"context"
-	"fmt"
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/robfig/cron/v3"
 	"google.golang.org/api/option"
 	sqladmin "google.golang.org/api/sqladmin/v1beta4"
@@ -34,6 +36,18 @@ func NewExporter(logger log.Logger, configFile string) (*Exporter, error) {
 		return nil, err
 	}
 
+	var queryDurationHistogramBuckets []float64
+	if len(cfg.Configuration.HistogramBuckets) == 0 {
+		queryDurationHistogramBuckets = DefaultQueryDurationHistogramBuckets
+	} else {
+		queryDurationHistogramBuckets = cfg.Configuration.HistogramBuckets
+	}
+	queryDurationHistogram = promauto.NewHistogramVec(prometheus.HistogramOpts{
+		Name:    fmt.Sprintf("%s_query_duration_seconds", metricsPrefix),
+		Help:    "Time spent by querying the database.",
+		Buckets: queryDurationHistogramBuckets,
+	}, QueryMetricsLabels)
+
 	exp := &Exporter{
 		jobs:          make([]*Job, 0, len(cfg.Jobs)),
 		logger:        logger,

go.mod

@@ -12,8 +12,8 @@ require (
 	github.com/gobwas/glob v0.2.3
 	github.com/jmoiron/sqlx v1.3.5
 	github.com/lib/pq v1.10.9
-	github.com/prometheus/client_golang v1.18.0
-	github.com/prometheus/common v0.46.0
+	github.com/prometheus/client_golang v1.19.0
+	github.com/prometheus/common v0.48.0
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/segmentio/go-athena v0.0.0-20230626212750-5fac08ed8dab
 	github.com/snowflakedb/gosnowflake v1.7.2

go.sum

@@ -330,11 +330,15 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
 github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
 github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
 github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y=
 github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
 github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=

job.go

@@ -329,6 +329,8 @@ func (j *Job) runOnceConnection(conn *connection, done chan int) {
 	if err := conn.connect(j); err != nil {
 		level.Warn(j.log).Log("msg", "Failed to connect", "err", err, "host", conn.host)
 		j.markFailed(conn)
+		// we don't have the query name yet.
+		failedQueryCounter.WithLabelValues(j.Name, "").Inc()
 		return
 	}
 

query.go

@@ -15,22 +15,30 @@ func (q *Query) Run(conn *connection) error {
 	if q.log == nil {
 		q.log = log.NewNopLogger()
 	}
+	queryCounter.WithLabelValues(q.jobName, q.Name).Inc()
 	if q.desc == nil {
+		failedQueryCounter.WithLabelValues(q.jobName, q.Name).Inc()
 		return fmt.Errorf("metrics descriptor is nil")
 	}
 	if q.Query == "" {
+		failedQueryCounter.WithLabelValues(q.jobName, q.Name).Inc()
 		return fmt.Errorf("query is empty")
 	}
 	if conn == nil || conn.conn == nil {
+		failedQueryCounter.WithLabelValues(q.jobName, q.Name).Inc()
 		return fmt.Errorf("db connection not initialized (should not happen)")
 	}
 	// execute query
+	now := time.Now()
 	rows, err := conn.conn.Queryx(q.Query)
 	if err != nil {
 		failedScrapes.WithLabelValues(conn.driver, conn.host, conn.database, conn.user, q.jobName, q.Name).Set(1.0)
+		failedQueryCounter.WithLabelValues(q.jobName, q.Name).Inc()
 		return err
 	}
 	defer rows.Close()
+	duration := time.Since(now)
+	queryDurationHistogram.WithLabelValues(q.jobName, q.Name).Observe(duration.Seconds())
 
 	updated := 0
 	metrics := make([]prometheus.Metric, 0, len(q.metrics))