diff --git a/expfmt/decode.go b/expfmt/decode.go index 1448439b..7274d16d 100644 --- a/expfmt/decode.go +++ b/expfmt/decode.go @@ -76,6 +76,8 @@ func NewDecoder(r io.Reader, format Format) Decoder { switch format.FormatType() { case TypeProtoDelim: return &protoDecoder{r: bufio.NewReader(r)} + case TypeOpenMetrics: + return &openMetricsDecoder{r: r} } return &textDecoder{r: r} } @@ -115,6 +117,36 @@ func (d *protoDecoder) Decode(v *dto.MetricFamily) error { return nil } +type openMetricsDecoder struct { + r io.Reader + fams map[string]*dto.MetricFamily + err error +} + +// Decode implements Decoder. +func (d *openMetricsDecoder) Decode(mf *dto.MetricFamily) error { + if d.err == nil { + // Read all metrics in one shot. + var p OpenMetricsParser + d.fams, d.err = p.OpenMetricsToMetricFamilies(d.r) + // If we don't get an error, store io.EOF for the end. + if d.err == nil { + d.err = io.EOF + } + } + // Pick off one MetricFamily per Decode until there's nothing left. + for key, fam := range d.fams { + mf.Name = fam.Name + mf.Help = fam.Help + mf.Type = fam.Type + mf.Unit = fam.Unit + mf.Metric = fam.Metric + delete(d.fams, key) + return nil + } + return d.err +} + // textDecoder implements the Decoder interface for the text protocol. type textDecoder struct { r io.Reader diff --git a/expfmt/decode_test.go b/expfmt/decode_test.go index 1b261401..7641da49 100644 --- a/expfmt/decode_test.go +++ b/expfmt/decode_test.go @@ -103,6 +103,78 @@ mf2 4 } } +func TestOpenMetricsDecoder(t *testing.T) { + var ( + ts = model.Now() + in = ` +# Only a quite simple scenario with two metric families. +# More complicated tests of the parser itself can be found in the openmetrics package. +# TYPE metric1 counter +metric1_total 3 +mf1{label="value1"} -3.14 123456 +mf1{label="value2"} 42 +metric1_total 4 +# EOF +` + out = model.Vector{ + &model.Sample{ + Metric: model.Metric{ + model.MetricNameLabel: "mf1", + "label": "value1", + }, + Value: -3.14, + Timestamp: 123456, + }, + &model.Sample{ + Metric: model.Metric{ + model.MetricNameLabel: "mf1", + "label": "value2", + }, + Value: 42, + Timestamp: ts, + }, + &model.Sample{ + Metric: model.Metric{ + model.MetricNameLabel: "metric1", + }, + Value: 3, + Timestamp: ts, + }, + &model.Sample{ + Metric: model.Metric{ + model.MetricNameLabel: "metric1", + }, + Value: 4, + Timestamp: ts, + }, + } + ) + + dec := &SampleDecoder{ + Dec: NewDecoder(strings.NewReader(in), FmtOpenMetrics_1_0_0), + Opts: &DecodeOptions{ + Timestamp: ts, + }, + } + var all model.Vector + for { + var smpls model.Vector + err := dec.Decode(&smpls) + if err != nil && errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatal(err) + } + all = append(all, smpls...) + } + sort.Sort(all) + sort.Sort(out) + if !reflect.DeepEqual(all, out) { + t.Fatalf("output does not match") + } +} + func TestProtoDecoder(t *testing.T) { testTime := model.Now() diff --git a/expfmt/openmetrics_parse.go b/expfmt/openmetrics_parse.go new file mode 100644 index 00000000..f5fd5485 --- /dev/null +++ b/expfmt/openmetrics_parse.go @@ -0,0 +1,932 @@ +// Copyright 2014 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expfmt + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "math" + "strconv" + "strings" + "time" + + dto "github.com/prometheus/client_model/go" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/prometheus/common/model" +) + +var UnsupportMetricType = map[string]struct{}{ + "INFO": {}, + "STATESET": {}, +} + +// OpenMetricsParser is used to parse the simple and flat openmetrics-based exchange format. Its +// zero value is ready to use. +type OpenMetricsParser struct { + metricFamiliesByName map[string]*dto.MetricFamily + buf *bufio.Reader // Where the parsed input is read through. + err error // Most recent error. + lineCount int // Tracks the line count for error messages. + currentByte byte // The most recent byte read. + currentToken bytes.Buffer // Re-used each time a token has to be gathered from multiple bytes. + currentMF *dto.MetricFamily + currentMetric *dto.Metric + currentLabelPair *dto.LabelPair + currentIsExemplar bool + currentExemplar *dto.Exemplar + + // The remaining member variables are only used for summaries/histograms. + currentLabels map[string]string // All labels including '__name__' but excluding 'quantile'/'le' + // Summary specific. + summaries map[uint64]*dto.Metric // Key is created with LabelsToSignature. + currentQuantile float64 + // Histogram specific. + histograms map[uint64]*dto.Metric // Key is created with LabelsToSignature. + currentBucketValue float64 + currentBucket *dto.Bucket + + // This tell us if the currently processed line ends on '_created', + // representing the created timestamp of the metric + currentIsMetricCreated bool + + // This tell us have read 'EOF' line, representing the end of the metrics + currentIsEOF bool + + // These tell us if the currently processed line ends on '_count' or + // '_sum' respectively and belong to a summary/histogram, representing the sample + // count and sum of that summary/histogram. + currentIsSummaryCount, currentIsSummarySum bool + currentIsHistogramCount, currentIsHistogramSum bool +} + +// OpenMetricsToMetricFamilies reads 'in' as the simple and flat openmetrics-based +// exchange format and creates MetricFamily proto messages. It returns the MetricFamily +// proto messages in a map where the metric names are the keys, along with any +// error encountered. +// +// If the input contains duplicate metrics (i.e. lines with the same metric name +// and exactly the same label set), the resulting MetricFamily will contain +// duplicate Metric proto messages. Similar is true for duplicate label +// names. Checks for duplicates have to be performed separately, if required. +// Also note that neither the metrics within each MetricFamily are sorted nor +// the label pairs within each Metric. Sorting is not required for the most +// frequent use of this method, which is sample ingestion in the Prometheus +// server. However, for presentation purposes, you might want to sort the +// metrics, and in some cases, you must sort the labels, e.g. for consumption by +// the metric family injection hook of the Prometheus registry. +// +// Summaries and histograms are rather special beasts. You would probably not +// use them in the simple openmetrics format anyway. This method can deal with +// summaries and histograms if they are presented in exactly the way the +// openmetrics.Create function creates them. +// +// This method must not be called concurrently. If you want to parse different +// input concurrently, instantiate a separate Parser for each goroutine. +func (p *OpenMetricsParser) OpenMetricsToMetricFamilies(in io.Reader) (map[string]*dto.MetricFamily, error) { + p.reset(in) + p.currentIsEOF = false + for nextState := p.startOfLine; nextState != nil; nextState = nextState() { + // Magic happens here... + } + // Get rid of empty metric families. + for k, mf := range p.metricFamiliesByName { + if len(mf.GetMetric()) == 0 { + delete(p.metricFamiliesByName, k) + } + } + // If p.err is io.EOF now, we have run into a premature end of the input + // stream. Turn this error into something nicer and more + // meaningful. (io.EOF is often used as a signal for the legitimate end + // of an input stream.) + if p.err != nil && errors.Is(p.err, io.EOF) { + p.parseError("unexpected end of input stream") + } + return p.metricFamiliesByName, p.err +} + +func (p *OpenMetricsParser) reset(in io.Reader) { + p.metricFamiliesByName = map[string]*dto.MetricFamily{} + if p.buf == nil { + p.buf = bufio.NewReader(in) + } else { + p.buf.Reset(in) + } + p.err = nil + p.lineCount = 0 + if p.summaries == nil || len(p.summaries) > 0 { + p.summaries = map[uint64]*dto.Metric{} + } + if p.histograms == nil || len(p.histograms) > 0 { + p.histograms = map[uint64]*dto.Metric{} + } + p.currentQuantile = math.NaN() + p.currentBucketValue = math.NaN() +} + +// startOfLine represents the state where the next byte read from p.buf is the +// start of a line (or whitespace leading up to it). +func (p *OpenMetricsParser) startOfLine() stateFn { + p.lineCount++ + p.currentByte, p.err = p.buf.ReadByte() + if p.err != nil { + // This is the only place that we expect to see io.EOF, + // which is not an error but the signal that we are done. + // Any other error that happens to align with the start of + // a line is still an error. + if errors.Is(p.err, io.EOF) { + if p.currentIsEOF { + p.err = nil + } else { + p.parseError("expected EOF keyword at the end") + } + } + return nil + } + if p.currentIsEOF { + p.parseError(fmt.Sprintf("unexpected line after EOF, got %q", p.currentByte)) + return nil + } + switch p.currentByte { + case '#': + return p.startComment + case '\n': + return p.startOfLine // Empty line, start the next one. + } + if !isValidMetricNameStart(p.currentByte) { + p.parseError(fmt.Sprintf("%q is not a valid start token", p.currentByte)) + return nil + } + return p.readingMetricName +} + +// startComment represents the state where the next byte read from p.buf is the +// start of a comment (or whitespace leading up to it). +func (p *OpenMetricsParser) startComment() stateFn { + if p.skipBlankTab(); p.err != nil { + return nil // Unexpected end of input. + } + if p.currentByte == '\n' { + return p.startOfLine + } + if p.readTokenUntilWhitespace(); p.err != nil { + return nil // Unexpected end of input. + } + // If we have hit the end of line already, there is nothing left + // to do. This is not considered a syntax error. + if p.currentByte == '\n' && p.currentToken.String() != "EOF" { + return p.startOfLine + } + keyword := p.currentToken.String() + if keyword != "HELP" && keyword != "TYPE" && keyword != "UNIT" && keyword != "EOF" { + // Generic comment, ignore by fast forwarding to end of line. + for p.currentByte != '\n' { + if p.currentByte, p.err = p.buf.ReadByte(); p.err != nil { + return nil // Unexpected end of input. + } + } + return p.startOfLine + } + if keyword == "EOF" { + p.currentIsEOF = true + return p.startOfLine + } + // There is something. Next has to be a metric name. + if p.skipBlankTab(); p.err != nil { + return nil // Unexpected end of input. + } + if p.readTokenAsMetricName(); p.err != nil { + return nil // Unexpected end of input. + } + if p.currentByte == '\n' { + // At the end of the line already. + // Again, this is not considered a syntax error. + return p.startOfLine + } + if !isBlankOrTab(p.currentByte) { + p.parseError("invalid metric name in comment") + return nil + } + p.setOrCreateCurrentMF() + if p.skipBlankTab(); p.err != nil { + return nil // Unexpected end of input. + } + if p.currentByte == '\n' { + // At the end of the line already. + // Again, this is not considered a syntax error. + return p.startOfLine + } + switch keyword { + case "HELP": + return p.readingHelp + case "TYPE": + return p.readingType + case "UNIT": + return p.readingUnit + } + panic(fmt.Sprintf("code error: unexpected keyword %q", keyword)) +} + +// readingMetricName represents the state where the last byte read (now in +// p.currentByte) is the first byte of a metric name. +func (p *OpenMetricsParser) readingMetricName() stateFn { + if p.readTokenAsMetricName(); p.err != nil { + return nil + } + if p.currentToken.Len() == 0 { + p.parseError("invalid metric name") + return nil + } + + p.setOrCreateCurrentMF() + // Now is the time to fix the type if it hasn't happened yet. + if p.currentMF.Type == nil { + p.currentMF.Type = dto.MetricType_UNTYPED.Enum() + } + if p.currentMF.GetType() == dto.MetricType_COUNTER { + if !strings.HasSuffix(p.currentToken.String(), "_total") && !strings.HasSuffix(p.currentToken.String(), "_created") { + p.parseError(fmt.Sprintf("expected '_total' or '_created' as counter metric name suffix, got metric name %q", p.currentToken.String())) + return nil + } + } + // metric is not new if the metric ends with "_created". + if !p.currentIsMetricCreated { + p.currentMetric = &dto.Metric{} + } + // Do not append the newly created currentMetric to + // currentMF.Metric right now. First wait if this is a summary, + // and the metric exists already, which we can only know after + // having read all the labels. + if p.skipBlankTabIfCurrentBlankTab(); p.err != nil { + return nil // Unexpected end of input. + } + return p.readingLabels +} + +// readingLabels represents the state where the last byte read (now in +// p.currentByte) is either the first byte of the label set (i.e. a '{'), or the +// first byte of the value (otherwise). +func (p *OpenMetricsParser) readingLabels() stateFn { + if p.currentIsExemplar { + if p.currentByte != '{' { + p.parseError(fmt.Sprintf("exemplar must contain labelset, got %q", p.currentByte)) + return nil + } + } else { + // Summaries/histograms are special. We have to reset the + // currentLabels map, currentQuantile and currentBucket before starting to + // read labels. + if p.currentMF.GetType() == dto.MetricType_SUMMARY || + p.currentMF.GetType() == dto.MetricType_HISTOGRAM || + p.currentMF.GetType() == dto.MetricType_GAUGE_HISTOGRAM { + p.currentLabels = map[string]string{} + p.currentLabels[string(model.MetricNameLabel)] = p.currentMF.GetName() + p.currentQuantile = math.NaN() + p.currentBucketValue = math.NaN() + } + if p.currentByte != '{' { + return p.readingValue + } + } + return p.startLabelName +} + +// startLabelName represents the state where the next byte read from p.buf is +// the start of a label name (or whitespace leading up to it). +func (p *OpenMetricsParser) startLabelName() stateFn { + if p.skipBlankTab(); p.err != nil { + return nil // Unexpected end of input. + } + if p.currentByte == '}' { + if p.skipBlankTab(); p.err != nil { + return nil // Unexpected end of input. + } + return p.readingValue + } + if p.readTokenAsLabelName(); p.err != nil { + return nil // Unexpected end of input. + } + if p.currentToken.Len() == 0 { + p.parseError(fmt.Sprintf("invalid label name for metric %q", p.currentMF.GetName())) + return nil + } + p.currentLabelPair = &dto.LabelPair{Name: proto.String(p.currentToken.String())} + if p.currentLabelPair.GetName() == string(model.MetricNameLabel) { + p.parseError(fmt.Sprintf("label name %q is reserved", model.MetricNameLabel)) + return nil + } + // Special summary/histogram treatment. Don't add 'quantile' and 'le' + // labels to 'real' labels. + if !(p.currentMF.GetType() == dto.MetricType_SUMMARY && p.currentLabelPair.GetName() == model.QuantileLabel) && + !(p.currentMF.GetType() == dto.MetricType_HISTOGRAM && p.currentLabelPair.GetName() == model.BucketLabel) && + !(p.currentMF.GetType() == dto.MetricType_GAUGE_HISTOGRAM && p.currentLabelPair.GetName() == model.BucketLabel) && + !p.currentIsMetricCreated { + if p.currentIsExemplar { + p.currentExemplar.Label = append(p.currentExemplar.Label, p.currentLabelPair) + } else { + p.currentMetric.Label = append(p.currentMetric.Label, p.currentLabelPair) + } + } + if p.skipBlankTabIfCurrentBlankTab(); p.err != nil { + return nil // Unexpected end of input. + } + if p.currentByte != '=' { + p.parseError(fmt.Sprintf("expected '=' after label name, found %q", p.currentByte)) + return nil + } + if !p.currentIsExemplar { + // Check for duplicate label names. + labels := make(map[string]struct{}) + for _, l := range p.currentMetric.Label { + lName := l.GetName() + if _, exists := labels[lName]; !exists { + labels[lName] = struct{}{} + } else { + if !p.currentIsMetricCreated { + p.parseError(fmt.Sprintf("duplicate label names for metric %q", p.currentMF.GetName())) + return nil + } + } + } + } + return p.startLabelValue +} + +// startLabelValue represents the state where the next byte read from p.buf is +// the start of a (quoted) label value (or whitespace leading up to it). +func (p *OpenMetricsParser) startLabelValue() stateFn { + if p.skipBlankTab(); p.err != nil { + return nil // Unexpected end of input. + } + if p.currentByte != '"' { + p.parseError(fmt.Sprintf("expected '\"' at start of label value, found %q", p.currentByte)) + return nil + } + if p.readTokenAsLabelValue(); p.err != nil { + return nil + } + if !model.LabelValue(p.currentToken.String()).IsValid() { + p.parseError(fmt.Sprintf("invalid label value %q", p.currentToken.String())) + return nil + } + p.currentLabelPair.Value = proto.String(p.currentToken.String()) + // Special treatment of summaries: + // - Quantile labels are special, will result in dto.Quantile later. + // - Other labels have to be added to currentLabels for signature calculation. + if p.currentMF.GetType() == dto.MetricType_SUMMARY { + if p.currentLabelPair.GetName() == model.QuantileLabel { + if p.currentQuantile, p.err = parseFloat(p.currentLabelPair.GetValue()); p.err != nil { + // Create a more helpful error message. + p.parseError(fmt.Sprintf("expected float as value for 'quantile' label, got %q", p.currentLabelPair.GetValue())) + return nil + } + } else { + p.currentLabels[p.currentLabelPair.GetName()] = p.currentLabelPair.GetValue() + } + } + // Similar special treatment of histograms. + if p.currentMF.GetType() == dto.MetricType_HISTOGRAM || p.currentMF.GetType() == dto.MetricType_GAUGE_HISTOGRAM { + if p.currentLabelPair.GetName() == model.BucketLabel { + if p.currentBucketValue, p.err = parseFloat(p.currentLabelPair.GetValue()); p.err != nil { + // Create a more helpful error message. + p.parseError(fmt.Sprintf("expected float as value for 'le' label, got %q", p.currentLabelPair.GetValue())) + return nil + } + } else { + p.currentLabels[p.currentLabelPair.GetName()] = p.currentLabelPair.GetValue() + } + } + if p.skipBlankTab(); p.err != nil { + return nil // Unexpected end of input. + } + switch p.currentByte { + case ',': + return p.startLabelName + + case '}': + if p.skipBlankTab(); p.err != nil { + return nil // Unexpected end of input. + } + return p.readingValue + default: + p.parseError(fmt.Sprintf("unexpected end of label value %q", p.currentLabelPair.GetValue())) + return nil + } +} + +// readingValue represents the state where the last byte read (now in +// p.currentByte) is the first byte of the sample value (i.e. a float). +func (p *OpenMetricsParser) readingValue() stateFn { + if !p.currentIsMetricCreated && !p.currentIsExemplar { + // When we are here, we have read all the labels, so for the + // special case of a summary/histogram, we can finally find out + // if the metric already exists. + if p.currentMF.GetType() == dto.MetricType_SUMMARY { + signature := model.LabelsToSignature(p.currentLabels) + if summary := p.summaries[signature]; summary != nil { + p.currentMetric = summary + } else { + p.summaries[signature] = p.currentMetric + p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric) + } + } else if p.currentMF.GetType() == dto.MetricType_HISTOGRAM || p.currentMF.GetType() == dto.MetricType_GAUGE_HISTOGRAM { + signature := model.LabelsToSignature(p.currentLabels) + if histogram := p.histograms[signature]; histogram != nil { + p.currentMetric = histogram + } else { + p.histograms[signature] = p.currentMetric + p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric) + } + } else { + p.currentMF.Metric = append(p.currentMF.Metric, p.currentMetric) + } + } + + if p.readTokenUntilWhitespace(); p.err != nil { + return nil // Unexpected end of input. + } + value, err := parseFloat(p.currentToken.String()) + if err != nil { + // Create a more helpful error message. + p.parseError(fmt.Sprintf("expected float as value, got %q", p.currentToken.String())) + return nil + } + switch p.currentMF.GetType() { + case dto.MetricType_COUNTER: + if p.currentIsExemplar { + p.currentExemplar.Value = proto.Float64(value) + p.currentMetric.Counter.Exemplar = p.currentExemplar + } else { + if p.currentMetric.Counter == nil { + p.currentMetric.Counter = &dto.Counter{} + } + if !p.currentIsMetricCreated { + p.currentMetric.Counter = &dto.Counter{ + Value: proto.Float64(value), + } + } else { + p.currentMetric.Counter.CreatedTimestamp = timestamppb.New(time.Unix(int64(value), 600000000)) + } + } + case dto.MetricType_GAUGE: + p.currentMetric.Gauge = &dto.Gauge{Value: proto.Float64(value)} + case dto.MetricType_UNTYPED: + p.currentMetric.Untyped = &dto.Untyped{Value: proto.Float64(value)} + case dto.MetricType_SUMMARY: + // *sigh* + if p.currentMetric.Summary == nil { + p.currentMetric.Summary = &dto.Summary{} + } + switch { + case p.currentIsSummaryCount: + p.currentMetric.Summary.SampleCount = proto.Uint64(uint64(value)) + case p.currentIsSummarySum: + p.currentMetric.Summary.SampleSum = proto.Float64(value) + case p.currentIsMetricCreated: + p.currentMetric.Summary.CreatedTimestamp = timestamppb.New(time.Unix(int64(value), 600000000)) + case !math.IsNaN(p.currentQuantile): + p.currentMetric.Summary.Quantile = append( + p.currentMetric.Summary.Quantile, + &dto.Quantile{ + Quantile: proto.Float64(p.currentQuantile), + Value: proto.Float64(value), + }, + ) + } + case dto.MetricType_HISTOGRAM, dto.MetricType_GAUGE_HISTOGRAM: + // *sigh* + if p.currentMetric.Histogram == nil { + p.currentMetric.Histogram = &dto.Histogram{} + } + switch { + case p.currentIsExemplar: + p.currentExemplar.Value = proto.Float64(value) + p.currentBucket.Exemplar = p.currentExemplar + case p.currentIsHistogramCount: + p.currentMetric.Histogram.SampleCount = proto.Uint64(uint64(value)) + case p.currentIsHistogramSum: + p.currentMetric.Histogram.SampleSum = proto.Float64(value) + case p.currentIsMetricCreated: + p.currentMetric.Histogram.CreatedTimestamp = timestamppb.New(time.Unix(int64(value), 600000000)) + case !math.IsNaN(p.currentBucketValue): + p.currentBucket = &dto.Bucket{ + UpperBound: proto.Float64(p.currentBucketValue), + CumulativeCount: proto.Uint64(uint64(value)), + } + p.currentMetric.Histogram.Bucket = append( + p.currentMetric.Histogram.Bucket, + p.currentBucket, + ) + } + + default: + p.err = fmt.Errorf("unexpected type for metric name %q", p.currentMF.GetName()) + } + if p.currentByte == '\n' { + return p.startOfLine + } else if p.currentByte == '#' { + return p.startExemplar + } + return p.startTimestamp +} + +// startExemplar represents the state where the next byte read from p.buf is +// the start of the exemplar (or whitespace leading up to it). +func (p *OpenMetricsParser) startExemplar() stateFn { + if p.currentMF.GetType() == dto.MetricType_GAUGE || p.currentMF.GetType() == dto.MetricType_SUMMARY { + p.parseError(fmt.Sprintf("unexpected exemplar for metric name %q type %s", p.currentMF.GetName(), strings.ToLower(p.currentMF.GetType().String()))) + return nil + } + if p.skipBlankTab(); p.err != nil { + return nil + } + p.currentIsExemplar = true + p.currentExemplar = &dto.Exemplar{} + return p.readingLabels +} + +// startTimestamp represents the state where the next byte read from p.buf is +// the start of the timestamp (or whitespace leading up to it). +func (p *OpenMetricsParser) startTimestamp() stateFn { + if p.skipBlankTab(); p.err != nil { + return nil // Unexpected end of input. + } + + if p.currentByte == '#' { + return p.startExemplar + } + + if p.readTokenUntilWhitespace(); p.err != nil { + return nil // Unexpected end of input. + } + timestamp, err := strconv.ParseInt(p.currentToken.String(), 10, 64) + if err != nil { + // Create a more helpful error message. + p.parseError(fmt.Sprintf("expected integer as timestamp, got %q", p.currentToken.String())) + return nil + } + if p.currentIsExemplar { + p.currentExemplar.Timestamp = timestamppb.New(time.Unix(timestamp, 600000000)) + } else { + p.currentMetric.TimestampMs = proto.Int64(timestamp) + } + if p.skipBlankTabUntilNewline(); p.err != nil { + return nil // Unexpected end of input. + } + if p.readTokenUntilWhitespace(); p.err != nil { + return nil // Unexpected end of input. + } + + if p.currentToken.Len() > 0 { + if p.currentToken.String() == "#" { + return p.startExemplar + } else { + p.parseError(fmt.Sprintf("spurious string after timestamp: %q", p.currentToken.String())) + return nil + } + } + return p.startOfLine +} + +// readingHelp represents the state where the last byte read (now in +// p.currentByte) is the first byte of the docstring after 'HELP'. +func (p *OpenMetricsParser) readingHelp() stateFn { + if p.currentMF.Help != nil { + p.parseError(fmt.Sprintf("second HELP line for metric name %q", p.currentMF.GetName())) + return nil + } + // Rest of line is the docstring. + if p.readTokenUntilNewline(true); p.err != nil { + return nil // Unexpected end of input. + } + p.currentMF.Help = proto.String(p.currentToken.String()) + return p.startOfLine +} + +// readingType represents the state where the last byte read (now in +// p.currentByte) is the first byte of the type hint after 'HELP'. +func (p *OpenMetricsParser) readingType() stateFn { + if p.currentMF.Type != nil { + p.parseError(fmt.Sprintf("second TYPE line for metric name %q, or TYPE reported after samples", p.currentMF.GetName())) + return nil + } + // Rest of line is the type. + if p.readTokenUntilNewline(false); p.err != nil { + return nil // Unexpected end of input. + } + + // if the type is unsupported type (now only info and stateset), + // use untyped to instead + if _, ok := UnsupportMetricType[strings.ToUpper(p.currentToken.String())]; ok { + p.currentMF.Type = dto.MetricType_UNTYPED.Enum() + } else { + if strings.ToUpper(p.currentToken.String()) == "GAUGEHISTOGRAM" { + p.currentMF.Type = dto.MetricType_GAUGE_HISTOGRAM.Enum() + return p.startOfLine + } + metricType, ok := dto.MetricType_value[strings.ToUpper(p.currentToken.String())] + if !ok { + p.parseError(fmt.Sprintf("unknown metric type %q", p.currentToken.String())) + return nil + } + p.currentMF.Type = dto.MetricType(metricType).Enum() + } + return p.startOfLine +} + +func (p *OpenMetricsParser) readingUnit() stateFn { + if p.currentMF.Unit != nil { + p.parseError(fmt.Sprintf("second UNIT line for metric name %q", p.currentMF.GetUnit())) + return nil + } + if p.readTokenUntilNewline(true); p.err != nil { + return nil + } + if !strings.HasSuffix(p.currentMF.GetName(), p.currentToken.String()) { + p.parseError(fmt.Sprintf("expected unit as metric name suffix, found metric %q", p.currentMF.GetName())) + return nil + } + p.currentMF.Unit = proto.String(p.currentToken.String()) + return p.startOfLine +} + +// parseError sets p.err to a ParseError at the current line with the given +// message. +func (p *OpenMetricsParser) parseError(msg string) { + p.err = ParseError{ + Line: p.lineCount, + Msg: msg, + Format: FormatOpenMetrics, + } +} + +// skipBlankTab reads (and discards) bytes from p.buf until it encounters a byte +// that is neither ' ' nor '\t'. That byte is left in p.currentByte. +func (p *OpenMetricsParser) skipBlankTab() { + for { + if p.currentByte, p.err = p.buf.ReadByte(); p.err != nil || !isBlankOrTab(p.currentByte) { + return + } + } +} + +// skipBlankTabUntilNewline reads (and discards) bytes from p.buf until it encounters a byte +// that is neither ' ' nor '\t'. That byte is left in p.currentByte. +func (p *OpenMetricsParser) skipBlankTabUntilNewline() { + if p.currentByte != '\n' { + for { + if p.currentByte, p.err = p.buf.ReadByte(); p.err != nil || !isBlankOrTab(p.currentByte) || p.currentByte != '\n' { + return + } + } + } +} + +// skipBlankTabIfCurrentBlankTab works exactly as skipBlankTab but doesn't do +// anything if p.currentByte is neither ' ' nor '\t'. +func (p *OpenMetricsParser) skipBlankTabIfCurrentBlankTab() { + if isBlankOrTab(p.currentByte) { + p.skipBlankTab() + } +} + +// readTokenUntilWhitespace copies bytes from p.buf into p.currentToken. The +// first byte considered is the byte already read (now in p.currentByte). The +// first whitespace byte encountered is still copied into p.currentByte, but not +// into p.currentToken. +func (p *OpenMetricsParser) readTokenUntilWhitespace() { + p.currentToken.Reset() + for p.err == nil && !isBlankOrTab(p.currentByte) && p.currentByte != '\n' { + p.currentToken.WriteByte(p.currentByte) + p.currentByte, p.err = p.buf.ReadByte() + } +} + +// readTokenUntilNewline copies bytes from p.buf into p.currentToken. The first +// byte considered is the byte already read (now in p.currentByte). The first +// newline byte encountered is still copied into p.currentByte, but not into +// p.currentToken. If recognizeEscapeSequence is true, two escape sequences are +// recognized: '\\' translates into '\', and '\n' into a line-feed character. +// All other escape sequences are invalid and cause an error. +func (p *OpenMetricsParser) readTokenUntilNewline(recognizeEscapeSequence bool) { + p.currentToken.Reset() + escaped := false + for p.err == nil { + if recognizeEscapeSequence && escaped { + switch p.currentByte { + case '\\': + p.currentToken.WriteByte(p.currentByte) + case 'n': + p.currentToken.WriteByte('\n') + default: + p.parseError(fmt.Sprintf("invalid escape sequence '\\%c'", p.currentByte)) + return + } + escaped = false + } else { + switch p.currentByte { + case '\n': + return + case '\\': + escaped = true + default: + p.currentToken.WriteByte(p.currentByte) + } + } + p.currentByte, p.err = p.buf.ReadByte() + } +} + +// readTokenAsMetricName copies a metric name from p.buf into p.currentToken. +// The first byte considered is the byte already read (now in p.currentByte). +// The first byte not part of a metric name is still copied into p.currentByte, +// but not into p.currentToken. +func (p *OpenMetricsParser) readTokenAsMetricName() { + p.currentToken.Reset() + if !isValidMetricNameStart(p.currentByte) { + return + } + for { + p.currentToken.WriteByte(p.currentByte) + p.currentByte, p.err = p.buf.ReadByte() + if p.err != nil || !isValidMetricNameContinuation(p.currentByte) { + return + } + } +} + +// readTokenAsLabelName copies a label name from p.buf into p.currentToken. +// The first byte considered is the byte already read (now in p.currentByte). +// The first byte not part of a label name is still copied into p.currentByte, +// but not into p.currentToken. +func (p *OpenMetricsParser) readTokenAsLabelName() { + p.currentToken.Reset() + if !isValidLabelNameStart(p.currentByte) { + return + } + for { + p.currentToken.WriteByte(p.currentByte) + p.currentByte, p.err = p.buf.ReadByte() + if p.err != nil || !isValidLabelNameContinuation(p.currentByte) { + return + } + } +} + +// readTokenAsLabelValue copies a label value from p.buf into p.currentToken. +// In contrast to the other 'readTokenAs...' functions, which start with the +// last read byte in p.currentByte, this method ignores p.currentByte and starts +// with reading a new byte from p.buf. The first byte not part of a label value +// is still copied into p.currentByte, but not into p.currentToken. +func (p *OpenMetricsParser) readTokenAsLabelValue() { + p.currentToken.Reset() + escaped := false + for { + if p.currentByte, p.err = p.buf.ReadByte(); p.err != nil { + return + } + if escaped { + switch p.currentByte { + case '"', '\\': + p.currentToken.WriteByte(p.currentByte) + case 'n': + p.currentToken.WriteByte('\n') + default: + p.parseError(fmt.Sprintf("invalid escape sequence '\\%c'", p.currentByte)) + return + } + escaped = false + continue + } + switch p.currentByte { + case '"': + return + case '\n': + p.parseError(fmt.Sprintf("label value %q contains unescaped new-line", p.currentToken.String())) + return + case '\\': + escaped = true + default: + p.currentToken.WriteByte(p.currentByte) + } + } +} + +func (p *OpenMetricsParser) setOrCreateCurrentMF() { + p.currentIsSummaryCount = false + p.currentIsSummarySum = false + p.currentIsHistogramCount = false + p.currentIsHistogramSum = false + p.currentIsMetricCreated = false + p.currentIsExemplar = false + p.currentIsEOF = false + + name := p.currentToken.String() + if isCreated(name) { + p.currentIsMetricCreated = true + name = strings.TrimSuffix(name, "_created") + } + if p.currentMF = p.metricFamiliesByName[name]; p.currentMF != nil { + return + } + // Try out if this is a _total for a counter + counterName := counterMetricName(name) + if p.currentMF = p.metricFamiliesByName[counterName]; p.currentMF != nil { + if p.currentMF.GetType() == dto.MetricType_COUNTER { + return + } + } + // Try out if this is a _sum or _count for a summary/histogram. + summaryName := summaryMetricName(name) + if p.currentMF = p.metricFamiliesByName[summaryName]; p.currentMF != nil { + if p.currentMF.GetType() == dto.MetricType_SUMMARY { + if isCount(name) { + p.currentIsSummaryCount = true + } + if isSum(name) { + p.currentIsSummarySum = true + } + return + } + } + histogramName := gaugehistogramMetricName(name) + if p.currentMF = p.metricFamiliesByName[histogramName]; p.currentMF != nil { + if p.currentMF.GetType() == dto.MetricType_HISTOGRAM { + if isCount(name) { + p.currentIsHistogramCount = true + } + if isSum(name) { + p.currentIsHistogramSum = true + } + return + } else if p.currentMF.GetType() == dto.MetricType_GAUGE_HISTOGRAM { + if isGCount(name) { + p.currentIsHistogramCount = true + } + if isGSum(name) { + p.currentIsHistogramSum = true + } + return + } + } + p.currentMF = &dto.MetricFamily{Name: proto.String(name)} + p.metricFamiliesByName[name] = p.currentMF +} + +func counterMetricName(name string) string { + switch { + case isTotal(name): + return name[:len(name)-6] + default: + return name + } +} + +func gaugehistogramMetricName(name string) string { + switch { + case isGCount(name): + return name[:len(name)-7] + case isGSum(name): + return name[:len(name)-5] + case isCount(name): + return name[:len(name)-6] + case isSum(name): + return name[:len(name)-4] + case isBucket(name): + return name[:len(name)-7] + default: + return name + } +} + +func isGCount(name string) bool { + return len(name) > 7 && name[len(name)-7:] == "_gcount" +} + +func isGSum(name string) bool { + return len(name) > 5 && name[len(name)-5:] == "_gsum" +} + +func isTotal(name string) bool { + return len(name) > 6 && name[len(name)-6:] == "_total" +} + +func isCreated(name string) bool { + return len(name) > 8 && name[len(name)-8:] == "_created" +} diff --git a/expfmt/openmetrics_parse_test.go b/expfmt/openmetrics_parse_test.go new file mode 100644 index 00000000..13ebd2c0 --- /dev/null +++ b/expfmt/openmetrics_parse_test.go @@ -0,0 +1,1664 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expfmt + +import ( + "math" + "strings" + "testing" + "time" + + dto "github.com/prometheus/client_model/go" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func testOpenMetricsParse(t testing.TB) { + var omParser OpenMetricsParser + metricTimestamp := timestamppb.New(time.Unix(123456, 600000000)) + scenarios := []struct { + in string + out []*dto.MetricFamily + }{ + // 1: EOF as input + { + in: `# EOF +`, + + out: []*dto.MetricFamily{}, + }, + + // 2: Counter with int64 value + { + in: `# TYPE foo counter +foo_total 12345678901234567890 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foo"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(12345678901234567890), + }, + }, + }, + }, + }, + }, + + // 3: Counter without unit. + { + in: `# HELP foos Number of foos. +# TYPE foos counter +foos_total 42.0 +foos_created 123456.7 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foos"), + Help: proto.String("Number of foos."), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(42), + CreatedTimestamp: metricTimestamp, + }, + }, + }, + }, + }, + }, + + // 4: Counter with unit + { + in: `# TYPE foos_seconds counter +# HELP foos_seconds help +# UNIT foos_seconds seconds +foos_seconds_total 1 +foos_seconds_created 123456.7 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foos_seconds"), + Help: proto.String("help"), + Type: dto.MetricType_COUNTER.Enum(), + Unit: proto.String("seconds"), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(1), + CreatedTimestamp: metricTimestamp, + }, + }, + }, + }, + }, + }, + + // 5: Counter with labels + { + in: `# TYPE foos_seconds counter +# HELP foos_seconds help +# UNIT foos_seconds seconds +foos_seconds_total{a="1", b="2"} 1 +foos_seconds_created{a="1", b="2"} 12345.6 +foos_seconds_total{a="2", b="3"} 2 +foos_seconds_created{a="2", b="3"} 123456.6 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foos_seconds"), + Help: proto.String("help"), + Type: dto.MetricType_COUNTER.Enum(), + Unit: proto.String("seconds"), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("a"), + Value: proto.String("1"), + }, + { + Name: proto.String("b"), + Value: proto.String("2"), + }, + }, + Counter: &dto.Counter{ + Value: proto.Float64(1), + CreatedTimestamp: timestamppb.New(time.Unix(12345, 600000000)), + }, + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("a"), + Value: proto.String("2"), + }, + { + Name: proto.String("b"), + Value: proto.String("3"), + }, + }, + Counter: &dto.Counter{ + Value: proto.Float64(2), + CreatedTimestamp: timestamppb.New(time.Unix(123456, 600000000)), + }, + }, + }, + }, + }, + }, + + // 6: Counter without timestamp and created + { + in: `# TYPE foo counter +foo_total 17.0 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foo"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(17), + }, + }, + }, + }, + }, + }, + + // 7: Counter with timestamp + { + in: `# TYPE foo counter +foo_total 17.0 123456 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foo"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(17), + }, + TimestampMs: proto.Int64(123456), + }, + }, + }, + }, + }, + + // 8: Counter with exemplar + { + in: `# TYPE foo counter +# HELP foo help +foo_total{b="c"} 0 123456 # {a="b"} 0.5 123456 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foo"), + Help: proto.String("help"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("b"), + Value: proto.String("c"), + }, + }, + Counter: &dto.Counter{ + Value: proto.Float64(0), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + { + Name: proto.String("a"), + Value: proto.String("b"), + }, + }, + Value: proto.Float64(0.5), + Timestamp: metricTimestamp, + }, + }, + TimestampMs: proto.Int64(123456), + }, + }, + }, + }, + }, + + // 9: Counter empty labelset + { + in: `# TYPE foo counter +# HELP foo help +foo_total{} 0 123456 # {a="b"} 0.5 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foo"), + Help: proto.String("help"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Counter: &dto.Counter{ + Value: proto.Float64(0), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + { + Name: proto.String("a"), + Value: proto.String("b"), + }, + }, + Value: proto.Float64(0.5), + }, + }, + TimestampMs: proto.Int64(123456), + }, + }, + }, + }, + }, + + // 10: Gauge with unit + { + in: `# TYPE foos_seconds gauge +# HELP foos_seconds help +# UNIT foos_seconds seconds +foos_seconds{b="c"} 0 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foos_seconds"), + Help: proto.String("help"), + Unit: proto.String("seconds"), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("b"), + Value: proto.String("c"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(0), + }, + }, + }, + }, + }, + }, + + // 11: Gauge with unit and timestamp + { + in: `# TYPE foos_seconds gauge +# HELP foos_seconds help +# UNIT foos_seconds seconds +foos_seconds{b="c"} 0 123456 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foos_seconds"), + Help: proto.String("help"), + Unit: proto.String("seconds"), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("b"), + Value: proto.String("c"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(0), + }, + TimestampMs: proto.Int64(123456), + }, + }, + }, + }, + }, + + // 12: Gauge with float value + { + in: `# TYPE foos_seconds gauge +# HELP foos_seconds help +# UNIT foos_seconds seconds +foos_seconds{b="c"} 0.12345678 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foos_seconds"), + Help: proto.String("help"), + Unit: proto.String("seconds"), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("b"), + Value: proto.String("c"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(0.12345678), + }, + }, + }, + }, + }, + }, + + // 13: Gauge empty labelset + { + in: `# TYPE foos_seconds gauge +# HELP foos_seconds help +# UNIT foos_seconds seconds +foos_seconds{} 0.12345678 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foos_seconds"), + Help: proto.String("help"), + Type: dto.MetricType_GAUGE.Enum(), + Unit: proto.String("seconds"), + Metric: []*dto.Metric{ + { + Gauge: &dto.Gauge{ + Value: proto.Float64(0.12345678), + }, + }, + }, + }, + }, + }, + + // 14: Untyped metric + { + in: `# TYPE foos_seconds untyped +# HELP foos_seconds help +# UNIT foos_seconds seconds +foos_seconds{a="v"} 0.12345678 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foos_seconds"), + Help: proto.String("help"), + Type: dto.MetricType_UNTYPED.Enum(), + Unit: proto.String("seconds"), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("a"), + Value: proto.String("v"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(0.12345678), + }, + }, + }, + }, + }, + }, + + // 15: Unsupported metric type(info, stateset) + { + in: `# TYPE foos_info info +# HELP foos_info help +foos_info{a="v"} 1 +# TYPE foos stateset +# HELP foos help +foos{foos="a"} 1 +foos{foos="b"} 0 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foos_info"), + Help: proto.String("help"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("a"), + Value: proto.String("v"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(1), + }, + }, + }, + }, + { + Name: proto.String("foos"), + Help: proto.String("help"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("foos"), + Value: proto.String("a"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(1), + }, + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("foos"), + Value: proto.String("b"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(0), + }, + }, + }, + }, + }, + }, + + // 16: Simple summary with quantile + { + in: `# TYPE a summary +# HELP a help +a_count 1 +a_sum 2 +a{quantile="0.5"} 0.7 +a{quantile="1"} 0.8 +a_created 123456 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("a"), + Help: proto.String("help"), + Type: dto.MetricType_SUMMARY.Enum(), + Metric: []*dto.Metric{ + { + Summary: &dto.Summary{ + SampleCount: proto.Uint64(1), + SampleSum: proto.Float64(2), + Quantile: []*dto.Quantile{ + { + Quantile: proto.Float64(0.5), + Value: proto.Float64(0.7), + }, + { + Quantile: proto.Float64(1), + Value: proto.Float64(0.8), + }, + }, + CreatedTimestamp: metricTimestamp, + }, + }, + }, + }, + }, + }, + + // 17: Simple summary with labels + { + in: `# TYPE a summary +# HELP a help +a_count{b="c1"} 1 +a_sum{b="c1"} 2 +a{b="c1", quantile="0.5"} 0.7 +a{b="c1", quantile="1"} 0.8 +a_created{b="c1"} 123456 +a_count{b="c2"} 2 +a_sum{b="c2"} 3 +a{b="c2", quantile="0.5"} 0.1 +a{b="c2", quantile="1"} 0.2 +a_created{b="c2"} 123456 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("a"), + Help: proto.String("help"), + Type: dto.MetricType_SUMMARY.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("b"), + Value: proto.String("c1"), + }, + }, + Summary: &dto.Summary{ + SampleCount: proto.Uint64(1), + SampleSum: proto.Float64(2), + Quantile: []*dto.Quantile{ + { + Quantile: proto.Float64(0.5), + Value: proto.Float64(0.7), + }, + { + Quantile: proto.Float64(1), + Value: proto.Float64(0.8), + }, + }, + CreatedTimestamp: metricTimestamp, + }, + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("b"), + Value: proto.String("c2"), + }, + }, + Summary: &dto.Summary{ + SampleCount: proto.Uint64(2), + SampleSum: proto.Float64(3), + Quantile: []*dto.Quantile{ + { + Quantile: proto.Float64(0.5), + Value: proto.Float64(0.1), + }, + { + Quantile: proto.Float64(1), + Value: proto.Float64(0.2), + }, + }, + CreatedTimestamp: metricTimestamp, + }, + }, + }, + }, + }, + }, + + // 18: Simple histogram with labels + { + in: `# TYPE foo histogram +# HELP foo help +foo_bucket{a="b", le="0.0"} 0 +foo_bucket{a="b", le="1e-05"} 0 +foo_bucket{a="b", le="0.0001"} 5 +foo_bucket{a="b", le="0.1"} 8 +foo_bucket{a="b", le="1.0"} 10 +foo_bucket{a="b", le="10.0"} 11 +foo_bucket{a="b", le="100000.0"} 11 +foo_bucket{a="b", le="1e+06"} 15 +foo_bucket{a="b", le="1e+23"} 16 +foo_bucket{a="b", le="1.1e+23"} 17 +foo_bucket{a="b", le="+Inf"} 17 +foo_count{a="b"} 17 +foo_sum{a="b"} 324789.3 +foo_created{a="b"} 123456 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foo"), + Help: proto.String("help"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("a"), + Value: proto.String("b"), + }, + }, + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(17), + SampleSum: proto.Float64(324789.3), + Bucket: []*dto.Bucket{ + { + UpperBound: proto.Float64(0.0), + CumulativeCount: proto.Uint64(0), + }, + { + UpperBound: proto.Float64(1e-05), + CumulativeCount: proto.Uint64(0), + }, + { + UpperBound: proto.Float64(0.0001), + CumulativeCount: proto.Uint64(5), + }, + { + UpperBound: proto.Float64(0.1), + CumulativeCount: proto.Uint64(8), + }, + { + UpperBound: proto.Float64(1), + CumulativeCount: proto.Uint64(10), + }, + { + UpperBound: proto.Float64(10.0), + CumulativeCount: proto.Uint64(11), + }, + { + UpperBound: proto.Float64(100000), + CumulativeCount: proto.Uint64(11), + }, + { + UpperBound: proto.Float64(1e+06), + CumulativeCount: proto.Uint64(15), + }, + { + UpperBound: proto.Float64(1e+23), + CumulativeCount: proto.Uint64(16), + }, + { + UpperBound: proto.Float64(1.1e+23), + CumulativeCount: proto.Uint64(17), + }, + { + UpperBound: proto.Float64(math.Inf(+1)), + CumulativeCount: proto.Uint64(17), + }, + }, + CreatedTimestamp: metricTimestamp, + }, + }, + }, + }, + }, + }, + + // 19: Simple histogram with exemplars + { + in: `# TYPE foo histogram +# HELP foo help +foo_bucket{a="b", le="0.0"} 0 # {l="1"} 0.5 +foo_bucket{a="b", le="1e-05"} 0 +foo_bucket{a="b", le="0.0001"} 5 +foo_bucket{a="b", le="0.1"} 8 +foo_bucket{a="b", le="1.0"} 10 +foo_bucket{a="b", le="10.0"} 11 +foo_bucket{a="b", le="100000.0"} 11 +foo_bucket{a="b", le="1e+06"} 15 # {l="2"} 1 +foo_bucket{a="b", le="1e+23"} 16 +foo_bucket{a="b", le="1.1e+23"} 17 +foo_bucket{a="b", le="+Inf"} 17 +foo_count{a="b"} 17 +foo_sum{a="b"} 324789.3 +foo_created{a="b"} 123456 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foo"), + Help: proto.String("help"), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("a"), + Value: proto.String("b"), + }, + }, + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(17), + SampleSum: proto.Float64(324789.3), + Bucket: []*dto.Bucket{ + { + UpperBound: proto.Float64(0.0), + CumulativeCount: proto.Uint64(0), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + { + Name: proto.String("l"), + Value: proto.String("1"), + }, + }, + Value: proto.Float64(0.5), + }, + }, + { + UpperBound: proto.Float64(1e-05), + CumulativeCount: proto.Uint64(0), + }, + { + UpperBound: proto.Float64(0.0001), + CumulativeCount: proto.Uint64(5), + }, + { + UpperBound: proto.Float64(0.1), + CumulativeCount: proto.Uint64(8), + }, + { + UpperBound: proto.Float64(1), + CumulativeCount: proto.Uint64(10), + }, + { + UpperBound: proto.Float64(10.0), + CumulativeCount: proto.Uint64(11), + }, + { + UpperBound: proto.Float64(100000), + CumulativeCount: proto.Uint64(11), + }, + { + UpperBound: proto.Float64(1e+06), + CumulativeCount: proto.Uint64(15), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + { + Name: proto.String("l"), + Value: proto.String("2"), + }, + }, + Value: proto.Float64(1), + }, + }, + { + UpperBound: proto.Float64(1e+23), + CumulativeCount: proto.Uint64(16), + }, + { + UpperBound: proto.Float64(1.1e+23), + CumulativeCount: proto.Uint64(17), + }, + { + UpperBound: proto.Float64(math.Inf(+1)), + CumulativeCount: proto.Uint64(17), + }, + }, + CreatedTimestamp: metricTimestamp, + }, + }, + }, + }, + }, + }, + + // 20: Simple gaugehistogram + { + in: `# TYPE foo gaugehistogram +foo_bucket{le="0.01"} 20.0 +foo_bucket{le="0.1"} 25.0 +foo_bucket{le="1"} 34.0 +foo_bucket{le="10"} 34.0 +foo_bucket{le="+Inf"} 42.0 +foo_gcount 42.0 +foo_gsum 3289.3 +foo_created 123456 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foo"), + Type: dto.MetricType_GAUGE_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(42), + SampleSum: proto.Float64(3289.3), + Bucket: []*dto.Bucket{ + { + UpperBound: proto.Float64(0.01), + CumulativeCount: proto.Uint64(20), + }, + { + UpperBound: proto.Float64(0.1), + CumulativeCount: proto.Uint64(25), + }, + { + UpperBound: proto.Float64(1), + CumulativeCount: proto.Uint64(34), + }, + { + UpperBound: proto.Float64(10), + CumulativeCount: proto.Uint64(34), + }, + { + UpperBound: proto.Float64(math.Inf(+1)), + CumulativeCount: proto.Uint64(42), + }, + }, + CreatedTimestamp: metricTimestamp, + }, + }, + }, + }, + }, + }, + + // 21: Simple gaugehistogram with labels and exemplars + { + in: `# TYPE foo gaugehistogram +foo_bucket{l="label", le="0.01"} 20.0 # {trace_id="a"} 0.5 123456 +foo_bucket{l="label", le="0.1"} 25.0 # {trace_id="b"} 0.6 +foo_bucket{l="label", le="1"} 34.0 +foo_bucket{l="label", le="10"} 34.0 +foo_bucket{l="label", le="+Inf"} 42.0 +foo_gcount{l="label"} 42.0 +foo_gsum{l="label"} 3289.3 +foo_created{l="label"} 123456 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("foo"), + Type: dto.MetricType_GAUGE_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("l"), + Value: proto.String("label"), + }, + }, + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(42), + SampleSum: proto.Float64(3289.3), + Bucket: []*dto.Bucket{ + { + UpperBound: proto.Float64(0.01), + CumulativeCount: proto.Uint64(20), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + { + Name: proto.String("trace_id"), + Value: proto.String("a"), + }, + }, + Value: proto.Float64(0.5), + Timestamp: metricTimestamp, + }, + }, + { + UpperBound: proto.Float64(0.1), + CumulativeCount: proto.Uint64(25), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + { + Name: proto.String("trace_id"), + Value: proto.String("b"), + }, + }, + Value: proto.Float64(0.6), + }, + }, + { + UpperBound: proto.Float64(1), + CumulativeCount: proto.Uint64(34), + }, + { + UpperBound: proto.Float64(10), + CumulativeCount: proto.Uint64(34), + }, + { + UpperBound: proto.Float64(math.Inf(+1)), + CumulativeCount: proto.Uint64(42), + }, + }, + CreatedTimestamp: metricTimestamp, + }, + }, + }, + }, + }, + }, + + // 22: Minimal case + { + in: `minimal_metric 1.234 +another_metric -3e3 103948 +# Even that: +no_labels{} 3 +# HELP line for non-existing metric will be ignored. +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("minimal_metric"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Untyped: &dto.Untyped{ + Value: proto.Float64(1.234), + }, + }, + }, + }, + { + Name: proto.String("another_metric"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Untyped: &dto.Untyped{ + Value: proto.Float64(-3e3), + }, + TimestampMs: proto.Int64(103948), + }, + }, + }, + { + Name: proto.String("no_labels"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Untyped: &dto.Untyped{ + Value: proto.Float64(3), + }, + }, + }, + }, + }, + }, + + // 23: Counters with exemplars and created timestamp & gauges, + // docstrings, various whitespace, escape sequences. + { + in: `# A normal comment. +# +# TYPE name_seconds counter +# UNIT name_seconds seconds +name_seconds_total{labelname="val1",basename="basevalue"} NaN # {a="b"} 0.5 +name_seconds_created{labelname="val1",basename="basevalue"} 123456789 +name_seconds_total{labelname="val2",basename="base\"v\\al\nue"} 0.23 1234567890 # {a="c"} 1 +# HELP name_seconds two-line\n doc str\\ing + +# HELP name2 doc str"ing 2 +# TYPE name2 gauge +name2{labelname="val2" ,basename = "basevalue2" } +Inf 54321 +name2{ labelname = "val1" , }-Inf +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("name_seconds"), + Unit: proto.String("seconds"), + Help: proto.String("two-line\n doc str\\ing"), + Type: dto.MetricType_COUNTER.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("labelname"), + Value: proto.String("val1"), + }, + { + Name: proto.String("basename"), + Value: proto.String("basevalue"), + }, + }, + Counter: &dto.Counter{ + Value: proto.Float64(math.NaN()), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + { + Name: proto.String("a"), + Value: proto.String("b"), + }, + }, + Value: proto.Float64(0.5), + }, + CreatedTimestamp: timestamppb.New(time.Unix(123456789, 600000000)), + }, + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("labelname"), + Value: proto.String("val2"), + }, + { + Name: proto.String("basename"), + Value: proto.String("base\"v\\al\nue"), + }, + }, + Counter: &dto.Counter{ + Value: proto.Float64(.23), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + { + Name: proto.String("a"), + Value: proto.String("c"), + }, + }, + Value: proto.Float64(1), + }, + }, + TimestampMs: proto.Int64(1234567890), + }, + }, + }, + { + Name: proto.String("name2"), + Help: proto.String("doc str\"ing 2"), + Type: dto.MetricType_GAUGE.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("labelname"), + Value: proto.String("val2"), + }, + { + Name: proto.String("basename"), + Value: proto.String("basevalue2"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(math.Inf(+1)), + }, + TimestampMs: proto.Int64(54321), + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("labelname"), + Value: proto.String("val1"), + }, + }, + Gauge: &dto.Gauge{ + Value: proto.Float64(math.Inf(-1)), + }, + }, + }, + }, + }, + }, + + // 24: The evil summary with created timestamp, + // mixed with other types and funny comments. + { + in: `# TYPE my_summary summary +my_summary{n1="val1",quantile="0.5"} 110 +decoy -1 -2 +my_summary{n1="val1",quantile="0.9"} 140 1 +my_summary_count{n1="val1"} 42 +# Latest timestamp wins in case of a summary. +my_summary_sum{n1="val1"} 4711 2 +my_summary_created{n1="val1"} 123456789 +fake_sum{n1="val1"} 2001 +# TYPE another_summary summary +another_summary_count{n2="val2",n1="val1"} 20 +my_summary_count{n2="val2",n1="val1"} 5 5 +another_summary{n1="val1",n2="val2",quantile=".3"} -1.2 +my_summary_sum{n1="val2"} 08 15 +my_summary{n1="val3", quantile="0.2"} 4711 +my_summary{n1="val1",n2="val2",quantile="-12.34",} NaN +# some +# funny comments +# HELP +# HELP +# HELP my_summary +# HELP my_summary +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("fake_sum"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("n1"), + Value: proto.String("val1"), + }, + }, + Untyped: &dto.Untyped{ + Value: proto.Float64(2001), + }, + }, + }, + }, + { + Name: proto.String("decoy"), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Untyped: &dto.Untyped{ + Value: proto.Float64(-1), + }, + TimestampMs: proto.Int64(-2), + }, + }, + }, + { + Name: proto.String("my_summary"), + Type: dto.MetricType_SUMMARY.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("n1"), + Value: proto.String("val1"), + }, + }, + Summary: &dto.Summary{ + SampleCount: proto.Uint64(42), + SampleSum: proto.Float64(4711), + Quantile: []*dto.Quantile{ + { + Quantile: proto.Float64(0.5), + Value: proto.Float64(110), + }, + { + Quantile: proto.Float64(0.9), + Value: proto.Float64(140), + }, + }, + CreatedTimestamp: timestamppb.New(time.Unix(123456789, 600000000)), + }, + TimestampMs: proto.Int64(2), + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("n2"), + Value: proto.String("val2"), + }, + { + Name: proto.String("n1"), + Value: proto.String("val1"), + }, + }, + Summary: &dto.Summary{ + SampleCount: proto.Uint64(5), + Quantile: []*dto.Quantile{ + { + Quantile: proto.Float64(-12.34), + Value: proto.Float64(math.NaN()), + }, + }, + }, + TimestampMs: proto.Int64(5), + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("n1"), + Value: proto.String("val2"), + }, + }, + Summary: &dto.Summary{ + SampleSum: proto.Float64(8), + }, + TimestampMs: proto.Int64(15), + }, + { + Label: []*dto.LabelPair{ + { + Name: proto.String("n1"), + Value: proto.String("val3"), + }, + }, + Summary: &dto.Summary{ + Quantile: []*dto.Quantile{ + { + Quantile: proto.Float64(0.2), + Value: proto.Float64(4711), + }, + }, + }, + }, + }, + }, + { + Name: proto.String("another_summary"), + Type: dto.MetricType_SUMMARY.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{ + { + Name: proto.String("n2"), + Value: proto.String("val2"), + }, + { + Name: proto.String("n1"), + Value: proto.String("val1"), + }, + }, + Summary: &dto.Summary{ + SampleCount: proto.Uint64(20), + Quantile: []*dto.Quantile{ + { + Quantile: proto.Float64(0.3), + Value: proto.Float64(-1.2), + }, + }, + }, + }, + }, + }, + }, + }, + + // 25: The histogram with created timestamp and exemplars. + { + in: `# HELP request_duration_microseconds The response latency. +# TYPE request_duration_microseconds histogram +request_duration_microseconds_bucket{le="100"} 123 # {trace_id="a"} 0.67 +request_duration_microseconds_bucket{le="120"} 412 # {trace_id="b"} 1 123456 +request_duration_microseconds_bucket{le="144"} 592 +request_duration_microseconds_bucket{le="172.8"} 1524 +request_duration_microseconds_bucket{le="+Inf"} 2693 # {} 2 +request_duration_microseconds_sum 1.7560473e+06 +request_duration_microseconds_count 2693 +request_duration_microseconds_created 123456789.123 +# EOF +`, + out: []*dto.MetricFamily{ + { + Name: proto.String("request_duration_microseconds"), + Help: proto.String("The response latency."), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(2693), + SampleSum: proto.Float64(1756047.3), + CreatedTimestamp: timestamppb.New(time.Unix(123456789, 600000000)), + Bucket: []*dto.Bucket{ + { + UpperBound: proto.Float64(100), + CumulativeCount: proto.Uint64(123), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + { + Name: proto.String("trace_id"), + Value: proto.String("a"), + }, + }, + Value: proto.Float64(0.67), + }, + }, + { + UpperBound: proto.Float64(120), + CumulativeCount: proto.Uint64(412), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{ + { + Name: proto.String("trace_id"), + Value: proto.String("b"), + }, + }, + Value: proto.Float64(1), + Timestamp: metricTimestamp, + }, + }, + { + UpperBound: proto.Float64(144), + CumulativeCount: proto.Uint64(592), + }, + { + UpperBound: proto.Float64(172.8), + CumulativeCount: proto.Uint64(1524), + }, + { + UpperBound: proto.Float64(math.Inf(+1)), + CumulativeCount: proto.Uint64(2693), + Exemplar: &dto.Exemplar{ + Label: []*dto.LabelPair{}, + Value: proto.Float64(2), + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for i, scenario := range scenarios { + out, err := omParser.OpenMetricsToMetricFamilies(strings.NewReader(scenario.in)) + if err != nil { + t.Errorf("%d. error: %s", i, err) + continue + } + if expected, got := len(scenario.out), len(out); expected != got { + t.Errorf( + "%d. expected %d MetricFamilies, got %d", + i, expected, got, + ) + } + for _, expected := range scenario.out { + got, ok := out[expected.GetName()] + if !ok { + t.Errorf( + "%d. expected MetricFamily %q, found none", + i, expected.GetName(), + ) + continue + } + if expected.String() != got.String() { + t.Errorf( + "%d. expected MetricFamily %s, got %s", + i, expected, got, + ) + } + } + } +} + +func testOpenMetricParseError(t testing.TB) { + scenarios := []struct { + in string + err string + }{ + // 0: No new-line at end of input. + { + in: `bla 3.14 +blubber 42`, + err: "openmetrics format parsing error in line 2: unexpected end of input stream", + }, + // 1: Invalid escape sequence in label value. + { + in: `metric{label="\t"} 3.14`, + err: "openmetrics format parsing error in line 1: invalid escape sequence", + }, + // 2: Newline in label value. + { + in: `metric{label="new +line"} 3.14 +`, + err: `openmetrics format parsing error in line 1: label value "new" contains unescaped new-line`, + }, + // 3: + { + in: `metric{@="bla"} 3.14`, + err: "openmetrics format parsing error in line 1: invalid label name for metric", + }, + // 4: + { + in: `metric{__name__="bla"} 3.14`, + err: `openmetrics format parsing error in line 1: label name "__name__" is reserved`, + }, + // 5: + { + in: `metric{label+="bla"} 3.14`, + err: "openmetrics format parsing error in line 1: expected '=' after label name", + }, + // 6: + { + in: `metric{label=bla} 3.14`, + err: "openmetrics format parsing error in line 1: expected '\"' at start of label value", + }, + // 7: + { + in: `# TYPE metric summary +metric{quantile="bla"} 3.14 +`, + err: "openmetrics format parsing error in line 2: expected float as value for 'quantile' label", + }, + // 8: + { + in: `metric{label="bla"+} 3.14`, + err: "openmetrics format parsing error in line 1: unexpected end of label value", + }, + // 9: + { + in: `metric{label="bla"} 3.14 2.72 + `, + err: "openmetrics format parsing error in line 1: expected integer as timestamp", + }, + // 10: + { + in: `metric{label="bla"} 3.14 2 3 + `, + err: "openmetrics format parsing error in line 1: spurious string after timestamp", + }, + // 11: + { + in: `metric{label="bla"} blubb + `, + err: "openmetrics format parsing error in line 1: expected float as value", + }, + // 12: + { + in: `# HELP metric one +# HELP metric two +`, + err: "openmetrics format parsing error in line 2: second HELP line for metric name", + }, + // 13: + { + in: `# TYPE metric counter +# TYPE metric untyped +`, + err: `openmetrics format parsing error in line 2: second TYPE line for metric name "metric", or TYPE reported after samples`, + }, + // 14: + { + in: `metric 4.12 +# TYPE metric counter +`, + err: `openmetrics format parsing error in line 2: second TYPE line for metric name "metric", or TYPE reported after samples`, + }, + // 15: + { + in: `# TYPE metric bla +`, + err: "openmetrics format parsing error in line 1: unknown metric type", + }, + // 16: + { + in: `# TYPE met-ric +`, + err: "openmetrics format parsing error in line 1: invalid metric name in comment", + }, + // 17: + { + in: `@invalidmetric{label="bla"} 3.14 2`, + err: "openmetrics format parsing error in line 1: '@' is not a valid start token", + }, + // 18: + { + in: `{label="bla"} 3.14 2`, + err: "openmetrics format parsing error in line 1: '{' is not a valid start token", + }, + // 19: + { + in: `# TYPE metric histogram +metric_bucket{le="bla"} 3.14 + `, + err: "openmetrics format parsing error in line 2: expected float as value for 'le' label", + }, + // 20: Invalid UTF-8 in label value. + { + in: "metric{l=\"\xbd\"} 3.14\n", + err: "openmetrics format parsing error in line 1: invalid label value \"\\xbd\"", + }, + // 21: Go 1.13 sometimes allows underscores in numbers. + { + in: "foo 1_2\n", + err: "openmetrics format parsing error in line 1: expected float as value", + }, + // 22: Go 1.13 supports hex floating point. + { + in: "foo 0x1p-3\n", + err: "openmetrics format parsing error in line 1: expected float as value", + }, + // 23: Check for various other literals variants, just in case. + { + in: "foo 0x1P-3\n", + err: "openmetrics format parsing error in line 1: expected float as value", + }, + // 24: + { + in: "foo 0B1\n", + err: "openmetrics format parsing error in line 1: expected float as value", + }, + // 25: + { + in: "foo 0O1\n", + err: "openmetrics format parsing error in line 1: expected float as value", + }, + // 26: + { + in: "foo 0X1\n", + err: "openmetrics format parsing error in line 1: expected float as value", + }, + // 27: + { + in: "foo 0x1\n", + err: "openmetrics format parsing error in line 1: expected float as value", + }, + // 28: + { + in: "foo 0b1\n", + err: "openmetrics format parsing error in line 1: expected float as value", + }, + // 29: + { + in: "foo 0o1\n", + err: "openmetrics format parsing error in line 1: expected float as value", + }, + // 30: + { + in: "foo 0x1\n", + err: "openmetrics format parsing error in line 1: expected float as value", + }, + // 31: + { + in: "foo 0x1\n", + err: "openmetrics format parsing error in line 1: expected float as value", + }, + // 32: Check histogram label. + { + in: `# TYPE metric histogram +metric_bucket{le="0x1p-3"} 3.14 +`, + err: "openmetrics format parsing error in line 2: expected float as value for 'le' label", + }, + // 33: Check quantile label. + { + in: `# TYPE metric summary +metric{quantile="0x1p-3"} 3.14 +`, + err: "openmetrics format parsing error in line 2: expected float as value for 'quantile' label", + }, + // 34: Check duplicate label. + { + in: `metric{label="bla",label="bla"} 3.14`, + err: "openmetrics format parsing error in line 1: duplicate label names for metric", + }, + // 35: Exemplars in gauge metric. + { + in: `# TYPE metric gauge +metric{le="0x1p-3"} 3.14 # {} 1 +`, + err: `openmetrics format parsing error in line 2: unexpected exemplar for metric name "metric" type gauge`, + }, + // 36: Exemplars in summary metric. + { + in: `# TYPE metric summary +metric{quantile="0.1"} 3.14 # {} 1 +`, + err: `openmetrics format parsing error in line 2: unexpected exemplar for metric name "metric" type summary`, + }, + // 37: Counter ends without '_total' + { + in: `# TYPE metric counter +metric{t="1"} 3.14 +`, + err: `openmetrics format parsing error in line 2: expected '_total' or '_created' as counter metric name suffix, got metric name "metric"`, + }, + // 38: metrics ends without unit + { + in: `# TYPE metric counter +# UNIT metric seconds +`, + err: `openmetrics format parsing error in line 2: expected unit as metric name suffix, found metric "metric"`, + }, + + // 39: metrics ends without EOF + { + in: `# TYPE metric_seconds counter +# UNIT metric_seconds seconds +`, + err: `openmetrics format parsing error in line 3: expected EOF keyword at the end`, + }, + + // 40: line after EOF + { + in: `# EOF +# TYPE metric counter +`, + err: `openmetrics format parsing error in line 2: unexpected line after EOF, got '#'`, + }, + + // 41: invalid start token + { + in: `# TYPE metric_seconds counter + # UNIT metric_seconds seconds +`, + err: `openmetrics format parsing error in line 2: '\t' is not a valid start token`, + }, + } + var omParser OpenMetricsParser + + for i, scenario := range scenarios { + _, err := omParser.OpenMetricsToMetricFamilies(strings.NewReader(scenario.in)) + if err == nil { + t.Errorf("%d. expected error, got nil", i) + continue + } + if expected, got := scenario.err, err.Error(); strings.Index(got, expected) != 0 { + t.Errorf( + "%d. expected error starting with %q, got %q", + i, expected, got, + ) + } + } +} + +func TestOpenMetricsParse(t *testing.T) { + testOpenMetricsParse(t) +} + +func BenchmarkOpenMetricParse(b *testing.B) { + for i := 0; i < b.N; i++ { + testOpenMetricsParse(b) + } +} + +func TestOpenMetricParseError(t *testing.T) { + testOpenMetricParseError(t) +} + +func BenchmarkOpenMetricParseError(b *testing.B) { + for i := 0; i < b.N; i++ { + testOpenMetricParseError(b) + } +} diff --git a/expfmt/text_parse.go b/expfmt/text_parse.go index 26490211..51f4da17 100644 --- a/expfmt/text_parse.go +++ b/expfmt/text_parse.go @@ -36,16 +36,23 @@ import ( // by nil. type stateFn func() stateFn +var ( + // The format values for the parse error. + FormatText = "text" + FormatOpenMetrics = "openmetrics" +) + // ParseError signals errors while parsing the simple and flat text-based // exchange format. type ParseError struct { - Line int - Msg string + Line int + Msg string + Format string } // Error implements the error interface. func (e ParseError) Error() string { - return fmt.Sprintf("text format parsing error in line %d: %s", e.Line, e.Msg) + return fmt.Sprintf("%s format parsing error in line %d: %s", e.Format, e.Line, e.Msg) } // TextParser is used to parse the simple and flat text-based exchange format. Its @@ -534,8 +541,9 @@ func (p *TextParser) readingType() stateFn { // message. func (p *TextParser) parseError(msg string) { p.err = ParseError{ - Line: p.lineCount, - Msg: msg, + Line: p.lineCount, + Msg: msg, + Format: FormatText, } }