feat: add scrape_error_drop_interval param to clear error metrics

cmd/sql_exporter/main.go

@@ -105,6 +105,20 @@ func main() {
 		http.HandleFunc("/reload", reloadHandler(exporter))
 	}
 
+	// Drop scrape error metrics if configured
+	scrapeErrorsDropInterval := exporter.Config().Globals.ScrapeErrorDropInterval
+	if scrapeErrorsDropInterval > 0 {
+		klog.Warning("Starting scrape error drop interval ticker")
+		ticker := time.NewTicker(time.Duration(scrapeErrorsDropInterval))
+		defer ticker.Stop()
+		go func() {
+			for range ticker.C {
+				sql_exporter.DropErrorMetrics()
+				klog.Warning("Dropped scrape_error_total metrics")
+			}
+		}()
+	}
+
 	// Handle SIGHUP for reloading the configuration
 	go func() {
 		c := make(chan os.Signal, 1)

config/global_config.go

@@ -9,12 +9,14 @@ import (
 
 // GlobalConfig contains globally applicable defaults.
 type GlobalConfig struct {
-	MinInterval     model.Duration `yaml:"min_interval" env:"MIN_INTERVAL"`                       // minimum interval between query executions, default is 0
-	ScrapeTimeout   model.Duration `yaml:"scrape_timeout" env:"SCRAPE_TIMEOUT"`                   // per-scrape timeout, global
-	TimeoutOffset   model.Duration `yaml:"scrape_timeout_offset" env:"SCRAPE_TIMEOUT_OFFSET"`     // offset to subtract from timeout in seconds
-	MaxConnLifetime time.Duration  `yaml:"max_connection_lifetime" env:"MAX_CONNECTION_LIFETIME"` // maximum amount of time a connection may be reused to any one target
-	MaxConns        int            `yaml:"max_connections" env:"MAX_CONNECTIONS"`                 // maximum number of open connections to any one target
-	MaxIdleConns    int            `yaml:"max_idle_connections" env:"MAX_IDLE_CONNECTIONS"`       // maximum number of idle connections to any one target
+	MinInterval             model.Duration `yaml:"min_interval" env:"MIN_INTERVAL"`                             // minimum interval between query executions, default is 0
+	ScrapeTimeout           model.Duration `yaml:"scrape_timeout" env:"SCRAPE_TIMEOUT"`                         // per-scrape timeout, global
+	TimeoutOffset           model.Duration `yaml:"scrape_timeout_offset" env:"SCRAPE_TIMEOUT_OFFSET"`           // offset to subtract from timeout in seconds
+	ScrapeErrorDropInterval model.Duration `yaml:"scrape_error_drop_interval" env:"SCRAPE_ERROR_DROP_INTERVAL"` // interval to drop scrape errors from the error counter, default is 0
+	MaxConnLifetime         time.Duration  `yaml:"max_connection_lifetime" env:"MAX_CONNECTION_LIFETIME"`       // maximum amount of time a connection may be reused to any one target
+
+	MaxConns     int `yaml:"max_connections" env:"MAX_CONNECTIONS"`           // maximum number of open connections to any one target
+	MaxIdleConns int `yaml:"max_idle_connections" env:"MAX_IDLE_CONNECTIONS"` // maximum number of idle connections to any one target
 
 	// Catches all undefined fields and must be empty after parsing.
 	XXX map[string]any `yaml:",inline" json:"-"`
@@ -26,6 +28,8 @@ func (g *GlobalConfig) UnmarshalYAML(unmarshal func(any) error) error {
 	g.MinInterval = model.Duration(0)
 	// Default to 10 seconds, since Prometheus has a 10 second scrape timeout default.
 	g.ScrapeTimeout = model.Duration(10 * time.Second)
+	// Default to 0 for scrape error drop interval.
+	g.ScrapeErrorDropInterval = model.Duration(0)
 	// Default to .5 seconds.
 	g.TimeoutOffset = model.Duration(500 * time.Millisecond)
 	g.MaxConns = 3

documentation/sql_exporter.yml

@@ -13,6 +13,8 @@ global:
   #
   # Must be strictly positive. The default is 500ms.
   scrape_timeout_offset: 500ms
+  # Interval between dropping scrape_errors_total metric: by default (0s) metrics are persistent.
+  scrape_error_drop_interval: 0s
   # Minimum interval between collector runs: by default (0s) collectors are executed on every scrape.
   min_interval: 0s
   # Maximum number of open connections to any one target. Metric queries will run concurrently on multiple connections,

exporter.go

@@ -238,3 +238,7 @@ func TrimMissingCtx(logContext string) string {
 	}
 	return logContext
 }
+
+func DropErrorMetrics() {
+	scrapeErrorsMetric.Reset()
+}