query-tee: Add an option to shift the query times for comparison

pkg/frontend/querymiddleware/codec.go

@@ -554,17 +554,17 @@ func (c prometheusCodec) EncodeMetricsQueryRequest(ctx context.Context, r Metric
 		u = &url.URL{
 			Path: r.GetPath(),
 			RawQuery: url.Values{
-				"start": []string{encodeTime(r.GetStart())},
-				"end":   []string{encodeTime(r.GetEnd())},
-				"step":  []string{encodeDurationMs(r.GetStep())},
+				"start": []string{EncodeTime(r.GetStart())},
+				"end":   []string{EncodeTime(r.GetEnd())},
+				"step":  []string{EncodeDurationMs(r.GetStep())},
 				"query": []string{r.GetQuery()},
 			}.Encode(),
 		}
 	case *PrometheusInstantQueryRequest:
 		u = &url.URL{
 			Path: r.GetPath(),
 			RawQuery: url.Values{
-				"time":  []string{encodeTime(r.GetTime())},
+				"time":  []string{EncodeTime(r.GetTime())},
 				"query": []string{r.GetQuery()},
 			}.Encode(),
 		}
@@ -617,10 +617,10 @@ func (c prometheusCodec) EncodeLabelsQueryRequest(ctx context.Context, req Label
 	case *PrometheusLabelNamesQueryRequest:
 		urlValues := url.Values{}
 		if req.GetStart() != 0 {
-			urlValues["start"] = []string{encodeTime(req.Start)}
+			urlValues["start"] = []string{EncodeTime(req.Start)}
 		}
 		if req.GetEnd() != 0 {
-			urlValues["end"] = []string{encodeTime(req.End)}
+			urlValues["end"] = []string{EncodeTime(req.End)}
 		}
 		if len(req.GetLabelMatcherSets()) > 0 {
 			urlValues["match[]"] = req.GetLabelMatcherSets()
@@ -637,10 +637,10 @@ func (c prometheusCodec) EncodeLabelsQueryRequest(ctx context.Context, req Label
 		// does not support accessing struct members on a typeA|typeB switch
 		urlValues := url.Values{}
 		if req.GetStart() != 0 {
-			urlValues["start"] = []string{encodeTime(req.Start)}
+			urlValues["start"] = []string{EncodeTime(req.Start)}
 		}
 		if req.GetEnd() != 0 {
-			urlValues["end"] = []string{encodeTime(req.End)}
+			urlValues["end"] = []string{EncodeTime(req.End)}
 		}
 		if len(req.GetLabelMatcherSets()) > 0 {
 			urlValues["match[]"] = req.GetLabelMatcherSets()
@@ -958,12 +958,12 @@ func parseDurationMs(s string) (int64, error) {
 	return 0, apierror.Newf(apierror.TypeBadData, "cannot parse %q to a valid duration", s)
 }
 
-func encodeTime(t int64) string {
+func EncodeTime(t int64) string {
 	f := float64(t) / 1.0e3
 	return strconv.FormatFloat(f, 'f', -1, 64)
 }
 
-func encodeDurationMs(d int64) string {
+func EncodeDurationMs(d int64) string {
 	return strconv.FormatFloat(float64(d)/float64(time.Second/time.Millisecond), 'f', -1, 64)
 }
 

pkg/frontend/querymiddleware/roundtrip.go

@@ -30,8 +30,8 @@ import (
 
 const (
 	day                                               = 24 * time.Hour
-	queryRangePathSuffix                              = "/api/v1/query_range"
-	instantQueryPathSuffix                            = "/api/v1/query"
+	QueryRangePathSuffix                              = "/api/v1/query_range"
+	InstantQueryPathSuffix                            = "/api/v1/query"
 	cardinalityLabelNamesPathSuffix                   = "/api/v1/cardinality/label_names"
 	cardinalityLabelValuesPathSuffix                  = "/api/v1/cardinality/label_values"
 	cardinalityActiveSeriesPathSuffix                 = "/api/v1/cardinality/active_series"
@@ -499,11 +499,11 @@ func newQueryCountTripperware(registerer prometheus.Registerer) Tripperware {
 }
 
 func IsRangeQuery(path string) bool {
-	return strings.HasSuffix(path, queryRangePathSuffix)
+	return strings.HasSuffix(path, QueryRangePathSuffix)
 }
 
 func IsInstantQuery(path string) bool {
-	return strings.HasSuffix(path, instantQueryPathSuffix)
+	return strings.HasSuffix(path, InstantQueryPathSuffix)
 }
 
 func IsCardinalityQuery(path string) bool {

pkg/frontend/querymiddleware/roundtrip_test.go

@@ -815,12 +815,12 @@ func TestTripperware_ShouldSupportReadConsistencyOffsetsInjection(t *testing.T)
 	}{
 		"range query": {
 			makeRequest: func() *http.Request {
-				return httptest.NewRequest("GET", queryRangePathSuffix+"?start=1536673680&end=1536716880&step=120&query=up", nil)
+				return httptest.NewRequest("GET", QueryRangePathSuffix+"?start=1536673680&end=1536716880&step=120&query=up", nil)
 			},
 		},
 		"instant query": {
 			makeRequest: func() *http.Request {
-				return httptest.NewRequest("GET", instantQueryPathSuffix+"?time=1536673680&query=up", nil)
+				return httptest.NewRequest("GET", InstantQueryPathSuffix+"?time=1536673680&query=up", nil)
 			},
 		},
 		"cardinality label names": {

tools/querytee/proxy.go

@@ -34,6 +34,7 @@ type ProxyConfig struct {
 	PreferredBackend                    string
 	BackendReadTimeout                  time.Duration
 	CompareResponses                    bool
+	ShiftComparisonQueriesBy            time.Duration
 	LogSlowQueryResponseThreshold       time.Duration
 	ValueComparisonTolerance            float64
 	UseRelativeError                    bool
@@ -43,6 +44,7 @@ type ProxyConfig struct {
 	BackendSkipTLSVerify                bool
 	AddMissingTimeParamToInstantQueries bool
 	SecondaryBackendsRequestProportion  float64
+	ShiftComparisonSamplingRatio        float64
 }
 
 func (cfg *ProxyConfig) RegisterFlags(f *flag.FlagSet) {
@@ -69,6 +71,9 @@ func (cfg *ProxyConfig) RegisterFlags(f *flag.FlagSet) {
 	f.BoolVar(&cfg.PassThroughNonRegisteredRoutes, "proxy.passthrough-non-registered-routes", false, "Passthrough requests for non-registered routes to preferred backend.")
 	f.BoolVar(&cfg.AddMissingTimeParamToInstantQueries, "proxy.add-missing-time-parameter-to-instant-queries", true, "Add a 'time' parameter to proxied instant query requests if they do not have one.")
 	f.Float64Var(&cfg.SecondaryBackendsRequestProportion, "proxy.secondary-backends-request-proportion", 1.0, "Proportion of requests to send to secondary backends. Must be between 0 and 1 (inclusive), and if not 1, then -backend.preferred must be set.")
+	f.DurationVar(&cfg.ShiftComparisonQueriesBy, "proxy.shift-comparison-queries-by", 0, "Shift the timestamps of the queries by the given duration before querying and comparing them. This will still do the query for the preferred backend with the original timestamps but do another query with shifted timestamps for comparison.")
+	// Defaulted to 0 to avoid mistakes of not setting this correctly and overloading the store-gateways with shifted queries.
+	f.Float64Var(&cfg.ShiftComparisonSamplingRatio, "proxy.shift-comparison-sampling-ratio", 0, "Ratio of queries for which query times are shifted based on -proxy.shift-comparison-queries-by config, sampled randomly. Must be between 0 and 1 (inclusive).")
 }
 
 type Route struct {
@@ -232,7 +237,7 @@ func (p *Proxy) Start() error {
 		if p.cfg.CompareResponses {
 			comparator = route.ResponseComparator
 		}
-		router.Path(route.Path).Methods(route.Methods...).Handler(NewProxyEndpoint(p.backends, route, p.metrics, p.logger, comparator, p.cfg.LogSlowQueryResponseThreshold, p.cfg.SecondaryBackendsRequestProportion))
+		router.Path(route.Path).Methods(route.Methods...).Handler(NewProxyEndpoint(p.backends, route, p.metrics, p.logger, comparator, p.cfg))
 	}
 
 	if p.cfg.PassThroughNonRegisteredRoutes {