Skip to content

Commit

Permalink
Update the retry mechanism with the list of http errors and backoff p…
Browse files Browse the repository at this point in the history
…eriods and remove the sanitization (#18)

* update changelog with samir's changes

* add lint

* will it fix?

* add test and see if the script is fixed

* add eslint block

* experiment will it work without the changes?

* validate the yaml

* add script command

* run lint for index

* change the md

* Change retry mech with the backoff. Remove sanitazation. And update the list of the errors when to retry

* ;remove

* correct

* lints

* remove the unused line

* crate a new variable
  • Loading branch information
vilyapilya authored and smusali committed Nov 11, 2019
1 parent 9d887c9 commit f3cb7b2
Show file tree
Hide file tree
Showing 5 changed files with 272 additions and 24 deletions.
29 changes: 27 additions & 2 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,14 @@
version: 2
jobs:
eslint:
docker:
- image: circleci/node:10
steps:
- checkout
- run: cp .eslintrc ./logdna/.eslintrc
- run: cd ./logdna && npm install .
- run: cd ./logdna && npm install eslint
- run: cd ./logdna && npm run lint
build:
docker:
- image: circleci/node:10
Expand Down Expand Up @@ -43,17 +52,33 @@ workflows:
version: 2
update:
jobs:
- eslint:
filters:
tags:
only: /[0-9].[0-9].[0-9]/
branches:
ignore: /.*/
- build:
requires:
- eslint
filters:
branches:
ignore: /.*/
tags:
only: /[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?/
only: /[0-9].[0-9].[0-9]/
- publish:
requires:
- build
filters:
branches:
ignore: /.*/
tags:
only: /[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?/
only: /[0-9].[0-9].[0-9]/
eslint:
jobs:
- eslint:
filters:
branches:
only: /.*/
tags:
ignore: /.*/
208 changes: 208 additions & 0 deletions .eslintrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,208 @@
{
"env": {
"browser": true,
"es6": true,
"node": true
},
"globals": {
"debug": false
},
"rules": {
"accessor-pairs": 0,
"array-bracket-spacing": [2, "never"],
"array-callback-return": 2,
"arrow-parens": ["error", "as-needed", {
"requireForBlockBody": true
}],
"block-scoped-var": 2,
"block-spacing": [2, "always"],
"brace-style": [2, "1tbs", {
"allowSingleLine": true
}],
"class-methods-use-this": [0, {
"exceptMethods": []
}],
"comma-dangle": [2, "never"],
"comma-spacing": [2, {
"before": false,
"after": true
}],
"comma-style": [2, "first"],
"complexity": [0, 11],
"consistent-return": 0,
"consistent-this": [2, "that"],
"curly": [2, "multi-line"],
"default-case": 0,
"dot-location": [2, "property"],
"dot-notation": [1, {
"allowKeywords": true
}],
"eol-last": 2,
"eqeqeq": [2, "always", {
"null": "ignore"
}],
"guard-for-in": 2,
"indent": [1, 4, {
"FunctionDeclaration": {
"parameters": "first"
},
"FunctionExpression": {
"parameters": "first"
},
"SwitchCase": 1
}],
"key-spacing": [0, {
"align": "colon",
"afterColon": true
}],
"keyword-spacing": [2, {
"overrides": {
"else": {
"before": true
},
"while": {
"before": true
},
"catch": {
"before": true
}
}
}],
"max-len": [1, 160, 2, {
"ignoreComments": true
}],
"no-alert": 1,
"no-caller": 2,
"no-case-declarations": 2,
"no-cond-assign": [2, "except-parens"],
"no-const-assign": "error",
"no-div-regex": 0,
"no-else-return": 0,
"no-empty-function": [2, {
"allow": ["arrowFunctions", "functions", "methods"]
}],
"no-empty-pattern": 2,
"no-eq-null": 0,
"no-eval": 2,
"no-extend-native": 2,
"no-extra-bind": 2,
"no-extra-label": 2,
"no-fallthrough": 1,
"no-floating-decimal": 2,
"no-global-assign": [2, {
"exceptions": []
}],
"no-implicit-coercion": [0, {
"boolean": false,
"number": true,
"string": true,
"allow": []
}],
"no-implicit-globals": 0,
"no-implied-eval": 2,
"no-invalid-this": 0,
"no-iterator": 2,
"no-labels": [2, {
"allowLoop": false,
"allowSwitch": false
}],
"no-lone-blocks": 2,
"no-loop-func": 1,
"no-magic-numbers": [1, {
"ignore": [0, 1, -1, 2, 3, 4, 5, 6, 7, 8, 9,
10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
32, 50, 80, 100, 443, 8080, 8090,
1024, 2048, 4096, 8192,
200, 201, 202, 204, 301, 302, 400, 401, 403, 404, 405, 409, 410, 415, 500, 505,
30, 60, 120, 180, 240, 300, 900, 1800, 3600, 7200, 14400, 43200, 86400, 172800,
1000, 2000, 2500, 3000, 5000, 10000, 30000, 32000, 60000, 120000, 180000, 240000, 300000,
900000, 1800000, 3600000, 7200000, 14400000, 43200000, 86400000, 172800000],
"ignoreArrayIndexes": true,
"enforceConst": false,
"detectObjects": false
}],
"no-mixed-spaces-and-tabs": 2,
"no-multi-spaces": 2,
"no-multi-str": 2,
"no-native-reassign": 0,
"no-new": 0,
"no-new-func": 2,
"no-new-wrappers": 2,
"no-octal": 2,
"no-octal-escape": 2,
"no-param-reassign": 0,
"no-proto": 2,
"no-redeclare": 2,
"no-restricted-properties": [2, {
"object": "arguments",
"property": "callee",
"message": "arguments.callee is deprecated,"
}, {
"property": "__defineGetter__",
"message": "Please use Object.defineProperty instead."
}, {
"property": "__defineSetter__",
"message": "Please use Object.defineProperty instead."
}],
"no-return-assign": 2,
"no-script-url": 2,
"no-self-assign": 2,
"no-self-compare": 2,
"no-sequences": 2,
"no-spaced-func": 2,
"no-throw-literal": 2,
"no-trailing-spaces": 2,
"no-undef": 2,
"no-unmodified-loop-condition": 0,
"no-unused-expressions": [2, {
"allowShortCircuit": false,
"allowTernary": false
}],
"no-unused-labels": 2,
"no-unused-vars": [1, {
"vars": "local",
"args": "none"
}],
"no-use-before-define": [2, {
"functions": false
}],
"no-useless-call": 0,
"no-useless-concat": 2,
"no-useless-escape": 2,
"no-void": 2,
"no-warning-comments": [0, {
"terms": ["todo", "fixme", "xxx"],
"location": "start"
}],
"no-with": 2,
"one-var": [0, {
"uninitialized": "always",
"initialized": "never"
}],
"operator-linebreak": [2, "after"],
"padded-blocks": [0, "never"],
"quote-props": [1, "consistent-as-needed"],
"quotes": [1, "single"],
"radix": 0,
"semi": [2, "always"],
"semi-spacing": [2, {
"before": false,
"after": true
}],
"space-before-blocks": [2, "always"],
"space-before-function-paren": [2, "never"],
"space-in-parens": [2, "never"],
"space-infix-ops": 2,
"space-unary-ops": [2, {
"words": false,
"nonwords": false
}],
"spaced-comment": [2, "always"],
"vars-on-top": 0,
"wrap-iife": [2, "outside", {
"functionPrototypeMethods": false
}],
"yoda": 0
}
}
21 changes: 13 additions & 8 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,23 @@

This file documents all notable changes in `LogDNA CloudWatch Lambda Function`. The release numbering uses [semantic versioning](http://semver.org).

## v2.0.3 - Released on November 1, 2019
* Fix bugs: change constants values to milliseconds [commit](https://github.com/logdna/logdna-cloudwatch/commit/7a26f4730cbac052387c782ec86711a132ab7082)

## v2.0.2 - Released on September 11, 2019

## v2.0.1 - Released on September 5, 2019
* Added `user-agent` to `req.headers` to track the clients better
* Added `CHANGELOG` and `Semantic Versioning`
* Add `user-agent` to `req.headers` to track the clients better
* Add `CHANGELOG` and `Semantic Versioning`

## v2.0.0 - Released on June 5, 2019
* Released new `Lambda Function` written/implemented on `node.js`
* Added Retry on `ETIMEDOUT` or `ESOCKETTIMEDOUT`
* Moved `log.group` and `log.stream` info from `meta` to the main `line`
* Added the extra custom options configurable with environment variables
* Release new `Lambda Function` written/implemented on `node.js`
* Add Retry on `ETIMEDOUT` or `ESOCKETTIMEDOUT`
* Move `log.group` and `log.stream` info from `meta` to the main `line`
* Add the extra custom options configurable with environment variables

## v1.1.0 - Released on March 8, 2019
* Added Custom Ingestion URL Option with `LOGDNA_URL` environment variable
* Add Custom Ingestion URL Option with `LOGDNA_URL` environment variable

## v1.0.0 - Released on July 18, 2018
* Initial Release
* Initial Release
35 changes: 21 additions & 14 deletions logdna/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,15 @@ const FREE_SOCKET_TIMEOUT_MS = parseInt(process.env.LOGDNA_FREE_SOCKET_TIMEOUT)
const LOGDNA_URL = process.env.LOGDNA_URL || 'https://logs.logdna.com/logs/ingest';
const MAX_REQUEST_RETRIES = parseInt(process.env.LOGDNA_MAX_REQUEST_RETRIES) || 5;
const REQUEST_RETRY_INTERVAL_MS = parseInt(process.env.LOGDNA_REQUEST_RETRY_INTERVAL) || 100;
const DEFAULT_HTTP_ERRORS = [
'ECONNRESET'
, 'EHOSTUNREACH'
, 'ETIMEDOUT'
, 'ESOCKETTIMEDOUT'
, 'ECONNREFUSED'
, 'ENOTFOUND'];

const INTERNAL_SERVER_ERROR = 500;
// Get Configuration from Environment Variables
const getConfig = () => {
const pkg = require('./package.json');
Expand All @@ -22,20 +30,12 @@ const getConfig = () => {
if (process.env.LOGDNA_KEY) config.key = process.env.LOGDNA_KEY;
if (process.env.LOGDNA_HOSTNAME) config.hostname = process.env.LOGDNA_HOSTNAME;
if (process.env.LOGDNA_TAGS && process.env.LOGDNA_TAGS.length > 0) {
config.tags = process.env.LOGDNA_TAGS.split(',').map((tag) => tag.trim()).join(',');
config.tags = process.env.LOGDNA_TAGS.split(',').map(tag => tag.trim()).join(',');
}

return config;
};

// Sanity Check: Truncate Long Message
const sanitizeMessage = (message) => {
if (message.length > MAX_LINE_LENGTH) {
return message.substring(0, MAX_LINE_LENGTH) + ' (truncated)';
}
return message;
};

// Parse the GZipped Log Data
const parseEvent = (event) => {
return JSON.parse(zlib.unzipSync(Buffer.from(event.awslogs.data, 'base64')));
Expand All @@ -46,7 +46,7 @@ const prepareLogs = (eventData) => {
return eventData.logEvents.map((event) => {
return {
line: JSON.stringify({
message: sanitizeMessage(event.message)
message: event.message
, source: 'cloudwatch'
, event: {
type: eventData.messageType
Expand Down Expand Up @@ -106,13 +106,20 @@ const sendLine = (payload, config, callback) => {
// Flush the Log
asyncRetry({
times: MAX_REQUEST_RETRIES
, interval: REQUEST_RETRY_INTERVAL_MS
, errorFilter: (err) => {
return err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT';
, interval: (retryCount) => {
return REQUEST_RETRY_INTERVAL_MS * Math.pow(2, retryCount);
}
, errorFilter: (errCode) => {
return DEFAULT_HTTP_ERRORS.includes(errCode) || errCode === 'INTERNAL_SERVER_ERROR';
}
}, (reqCallback) => {
return request(options, (error, response, body) => {
if (error) return reqCallback(error);
if (error) {
return reqCallback(error.code);
}
if (response.statusCode >= INTERNAL_SERVER_ERROR) {
return reqCallback('INTERNAL_SERVER_ERROR');
}
return reqCallback(null, body);
});
}, (error, result) => {
Expand Down
3 changes: 3 additions & 0 deletions logdna/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@
"version": "2.0.3",
"description": "Lambda Functions to Stream Logs from AWS CloudWatch to LogDNA",
"main": "index.js",
"scripts": {
"lint": "./node_modules/.bin/eslint -c .eslintrc *.js"
},
"dependencies": {
"agentkeepalive": "^4.0.2",
"async": "^2.6.2",
Expand Down

0 comments on commit f3cb7b2

Please sign in to comment.