Track errors when recording spans

server.go

@@ -158,6 +158,7 @@ func NewFromConfig(conf Config) (ret Server, err error) {
 	}
 	ret.Statsd.Namespace = "veneur."
 	ret.Statsd.Tags = append(ret.Tags, "veneurlocalonly")
+	trace.DefaultClient.SetErrorStats(ret.Statsd)
 
 	// nil is a valid sentry client that noops all methods, if there is no DSN
 	// we can just leave it as nil

trace/client.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"sync"
 	"time"
 
 	"github.com/stripe/veneur/protocol"
@@ -36,6 +37,9 @@ type Client struct {
 	backend
 	cancel context.CancelFunc
 	ops    chan op
+
+	stats    IncrClient
+	statsMtx sync.Mutex
 }
 
 // Close tears down the entire client. It waits until the backend has
@@ -61,6 +65,18 @@ func (c *Client) run(ctx context.Context) {
 	}
 }
 
+type IncrClient interface {
+	Incr(name string, tags []string, rate float64) error
+}
+
+// SetErrorStats sets an IncrClient used for reporting errors
+func (c *Client) SetErrorStats(statsClient IncrClient) {
+	c.statsMtx.Lock()
+	defer c.statsMtx.Unlock()
+
+	c.stats = statsClient
+}
+
 // ClientParam is an option for NewClient. Its implementation borrows
 // from Dave Cheney's functional options API
 // (https://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis).

trace/trace.go

@@ -8,6 +8,7 @@ package trace
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"math/rand"
 	"reflect"
@@ -194,7 +195,14 @@ func (t *Trace) ClientRecord(cl *Client, name string, tags map[string]string) er
 	span := t.SSFSpan()
 	span.Name = name
 
-	return Record(cl, span, t.Sent)
+	err := Record(cl, span, t.Sent)
+	if err != nil {
+		cl.statsMtx.Lock()
+		defer cl.statsMtx.Unlock()
+		tags := []string{fmt.Sprintf("error:%s", err.Error())}
+		cl.stats.Incr("veneur.trace.record.errors_total", tags, 0.1)
+	}
+	return err
 }
 
 func (t *Trace) Error(err error) {