Merge pull request #96 from librato/stddev_m2

added sample standard deviation

README.md

@@ -212,7 +212,8 @@ If you want to contribute:
 2. `yarn install`
 3. Hack away
 4. If you are adding new functionality, document it in the README
-5. Push the branch up to GitHub
-6. Send a pull request
+5. for tests, run `yarn test`
+6. Push the branch up to GitHub
+7. Send a pull request
 
 [statsd]: https://github.com/etsy/statsd

lib/librato.js

@@ -181,6 +181,21 @@ var timerGaugePct = function(timerName, values, pct, suffix) {
   var sum = values.slice(0, numInThreshold).reduce(function(s, current) {
     return s + current;
   }, 0);
+
+  // calculate sample standard deviation if count > 1
+  var stddev = 0;
+  if (numInThreshold > 1) {
+    var sampleAvg = sum / numInThreshold;
+    var sampleSquareDiff = values.slice(0, numInThreshold).map(function(v) {
+      var diff = v - sampleAvg;
+      return diff * diff;
+    });
+    var sampleSquareDiffSum = sampleSquareDiff.reduce(function(sum, v) {
+      return sum + v;
+    });
+    var sampleSquareDiffAvg = sampleSquareDiffSum / (numInThreshold - 1);
+    stddev = Math.sqrt(sampleSquareDiffAvg);
+  }
   var names = timerName.split('#');
   var name;
   if (names.length > 1) {
@@ -204,6 +219,7 @@ var timerGaugePct = function(timerName, values, pct, suffix) {
     count: numInThreshold,
     sum: sum,
     min: min,
+    stddev_m2: stddev,
     max: max,
   };
 };
@@ -354,9 +370,8 @@ var flushStats = function libratoFlush(ts, metrics) {
     if (excludeMetric(key)) {
       continue;
     }
-    var sortedVals = metrics.timers[key].sort(function(a, b) {
-      return a - b;
-    });
+    // already sorted by statsd
+    var sortedVals = metrics.timers[key];
     // First build the 100% percentile
     var gauge = timerGaugePct(key, sortedVals, 100, alwaysSuffixPercentile ? '.100' : null);
     if (gauge) {

test/librato_tests.js

@@ -155,6 +155,136 @@ module.exports.tags = {
     this.emitter.emit('flush', 123, metrics);
   },
 
+  testTimerWithOneMeasurement: function(test) {
+    test.expect(8);
+    let metrics = {
+      timers: {
+        'my_timer#tag=foo': [
+          41,
+        ],
+      },
+      timer_data: {'my_timer#tag=foo': null},
+    };
+    this.apiServer.post('/v1/measurements')
+      .reply(200, (uri, requestBody) => {
+        let measurement = requestBody.measurements[0];
+        test.ok(measurement);
+        test.equal(measurement.name, 'my_timer');
+        test.equal(measurement.value, undefined);
+        test.equal(measurement.stddev_m2, 0);
+        test.equal(measurement.min, 41);
+        test.equal(measurement.max, 41);
+        test.equal(measurement.sum, 41);
+        test.deepEqual(measurement.tags, {tag: 'foo'});
+        test.done();
+      });
+
+    this.emitter.emit('flush', 123, metrics);
+  },
+
+
+  testLargeTimersPercentiles: function(test) {
+    test.expect(18);
+    let metrics = {
+      timers: {
+        'my_timer#tag=foo': [
+          1,
+          4,
+          7,
+          10,
+          13,
+          16,
+          19,
+          22,
+          25,
+          29,
+          33,
+          37,
+          40,
+          45,
+          50,
+          56,
+          62,
+          69,
+        ],
+      },
+      timer_data: {'my_timer#tag=foo': null},
+      pctThreshold: {90: 90},
+    };
+    this.apiServer.post('/v1/measurements')
+      .reply(200, (uri, requestBody) => {
+        let hundredth = requestBody.measurements[0];
+        test.ok(hundredth);
+        test.equal(hundredth.name, 'my_timer');
+        test.equal(hundredth.value, undefined);
+        test.equal(hundredth.min, 1);
+        test.equal(hundredth.count, 18);
+        test.equal(hundredth.stddev_m2, 20.502191816511857);
+        test.equal(hundredth.max, 69);
+        test.equal(hundredth.sum, 538);
+        test.deepEqual(hundredth.tags, {tag: 'foo'});
+
+        let measurement = requestBody.measurements[1];
+        test.ok(measurement);
+        test.equal(measurement.name, 'my_timer.90');
+        test.equal(measurement.value, undefined);
+        test.equal(measurement.min, 1);
+        test.equal(measurement.count, 16);
+        test.equal(measurement.stddev_m2, 16.86799237214277);
+        test.equal(measurement.max, 56);
+        test.equal(measurement.sum, 407);
+        test.deepEqual(measurement.tags, {tag: 'foo'});
+        test.done();
+      });
+
+    this.emitter.emit('flush', 123, metrics);
+  },
+
+  testLargeTimers: function(test) {
+    test.expect(9);
+    let metrics = {
+      timers: {
+        'my_timer#tag=foo': [
+          1,
+          4,
+          7,
+          10,
+          13,
+          16,
+          19,
+          22,
+          25,
+          29,
+          33,
+          37,
+          40,
+          45,
+          50,
+          56,
+          62,
+          69,
+        ],
+      },
+      timer_data: {'my_timer#tag=foo': null},
+    };
+    this.apiServer.post('/v1/measurements')
+      .reply(200, (uri, requestBody) => {
+        let measurement = requestBody.measurements[0];
+        test.ok(measurement);
+        test.equal(measurement.name, 'my_timer');
+        test.equal(measurement.value, undefined);
+        test.equal(measurement.min, 1);
+        test.equal(measurement.count, 18);
+        test.equal(measurement.stddev_m2, 20.502191816511857);
+        test.equal(measurement.max, 69);
+        test.equal(measurement.sum, 538);
+        test.deepEqual(measurement.tags, {tag: 'foo'});
+        test.done();
+      });
+
+    this.emitter.emit('flush', 123, metrics);
+  },
+
   testTimers: function(test) {
     test.expect(7);
     let metrics = {