Skip to content

Commit

Permalink
add LOG_RAW_EVENT environment variable option to set line to raw …
Browse files Browse the repository at this point in the history
…`event.message` (#22)

no need for extra check

restructure prepareLogs

add explanation for RAW_EVENT_MESSAGE

fix linting

add explanation for the change

make value check stricter and rename the variable

add default for the variable in the docs

rename the environment variable

add `tap` unit testing coverage

add tap tests

newlines

remove source and fix test name

update README
  • Loading branch information
smusali authored Jun 9, 2020
1 parent c50cb93 commit 04b327f
Show file tree
Hide file tree
Showing 8 changed files with 213 additions and 36 deletions.
3 changes: 2 additions & 1 deletion .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@ jobs:
steps:
- checkout
- run: npm install
- run: npm run eslint
- run: npm run lint
- run: npm run test
build:
docker:
- image: circleci/node:10
Expand Down
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,5 @@ node_modules

# No Testing Materials
data
generateData.js
generateData.js
.nyc*
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

This file documents all notable changes in `LogDNA CloudWatch Lambda Function`. The release numbering uses [semantic versioning](http://semver.org).

## v2.2.0 - Released on June 9, 2020
* Add `LOG_RAW_EVENT` environment variable option to set `line` to raw `event.message`

## v2.1.0 - Released on November 14, 2019
* Update retry mechanism
* Remove message truncation
Expand Down
2 changes: 1 addition & 1 deletion LICENSE
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
MIT License

Copyright (c) 2019 LogDNA, Inc.
Copyright (c) 2020 LogDNA, Inc.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
Expand Down
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ The LogDNA AWS CloudWatch integration relies on [AWS Lambda](https://aws.amazon.
* `LOGDNA_HOSTNAME`: Alternative Host Name *(Optional)*
* `LOGDNA_TAGS`: Comma-separated Tags *(Optional)*
* `LOGDNA_URL`: Custom Ingestion URL *(Optional)*
* `LOG_RAW_EVENT`: Setting `line` to Raw `event.message` *(Optional, Default: false)*:
* It can be enabled by setting `LOG_RAW_EVENT` to `YES` or `TRUE`
* Enabling it moves the following `event`-related `meta` data from the `line` field to the `meta` field:
* `event.type`: `messageType` of `CloudWatch Log` encoded inside `awslogs.data` in `base64`
* `event.id`: `id` of each `CloudWatch Log` encoded inside `awslogs.data` in `base64`
* `log.group`: `LogGroup` where the log is coming from
* `log.stream`: `LogStream` where the log is coming from
4. For Execution role, assign an [IAM user with basic execution permissions](https://docs.aws.amazon.com/IAM/latest/UserGuide/getting-started_create-admin-group.html) by choosing an existing role and selecting a role that has permissions to upload logs to Amazon CloudWatch logs.

### Configure your AWS CloudWatch Log Group
Expand Down
77 changes: 47 additions & 30 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,27 @@ const request = require('request');
const zlib = require('zlib');

// Constants
const MAX_LINE_LENGTH = parseInt(process.env.LOGDNA_MAX_LINE_LENGTH) || 32000;
const MAX_REQUEST_TIMEOUT_MS = parseInt(process.env.LOGDNA_MAX_REQUEST_TIMEOUT) || 30000;
const FREE_SOCKET_TIMEOUT_MS = parseInt(process.env.LOGDNA_FREE_SOCKET_TIMEOUT) || 300000;
const LOGDNA_URL = process.env.LOGDNA_URL || 'https://logs.logdna.com/logs/ingest';
const MAX_REQUEST_RETRIES = parseInt(process.env.LOGDNA_MAX_REQUEST_RETRIES) || 5;
const REQUEST_RETRY_INTERVAL_MS = parseInt(process.env.LOGDNA_REQUEST_RETRY_INTERVAL) || 100;
const INTERNAL_SERVER_ERROR = 500;
const DEFAULT_HTTP_ERRORS = [
'ECONNRESET'
, 'EHOSTUNREACH'
, 'ETIMEDOUT'
, 'ESOCKETTIMEDOUT'
, 'ECONNREFUSED'
, 'ENOTFOUND'];
, 'ENOTFOUND'
];

const INTERNAL_SERVER_ERROR = 500;
// Get Configuration from Environment Variables
const getConfig = () => {
const pkg = require('./package.json');
let config = {
UserAgent: `${pkg.name}/${pkg.version}`
log_raw_event: false
, UserAgent: `${pkg.name}/${pkg.version}`
};

if (process.env.LOGDNA_KEY) config.key = process.env.LOGDNA_KEY;
Expand All @@ -33,6 +34,11 @@ const getConfig = () => {
config.tags = process.env.LOGDNA_TAGS.split(',').map(tag => tag.trim()).join(',');
}

if (process.env.LOG_RAW_EVENT) {
config.log_raw_event = process.env.LOG_RAW_EVENT.toLowerCase();
config.log_raw_event = config.log_raw_event === 'yes' || config.log_raw_event === 'true';
}

return config;
};

Expand All @@ -42,28 +48,35 @@ const parseEvent = (event) => {
};

// Prepare the Messages and Options
const prepareLogs = (eventData) => {
const prepareLogs = (eventData, log_raw_event) => {
return eventData.logEvents.map((event) => {
return {
line: JSON.stringify({
message: event.message
, source: 'cloudwatch'
, event: {
type: eventData.messageType
, id: event.id
}
, log: {
group: eventData.logGroup
, stream: eventData.logStream
}
})
, timestamp: event.timestamp
const eventMetadata = {
event: {
type: eventData.messageType
, id: event.id
}, log: {
group: eventData.logGroup
, stream: eventData.logStream
}
};

const eventLog = {
timestamp: event.timestamp
, file: eventData.logStream
, meta: {
owner: eventData.owner
, filters: eventData.subscriptionFilters
}
}, line: JSON.stringify(Object.assign({}, {
message: event.message
}, eventMetadata))
};

if (log_raw_event) {
eventLog.line = event.message;
eventLog.meta = Object.assign({}, eventLog.meta, eventMetadata);
}

return eventLog;
});
};

Expand All @@ -81,17 +94,13 @@ const sendLine = (payload, config, callback) => {
, qs: config.tags ? {
tags: config.tags
, hostname: hostname
} : {
hostname: hostname
}
} : { hostname: hostname }
, method: 'POST'
, body: JSON.stringify({
e: 'ls'
, ls: payload
})
, auth: {
username: config.key
}
, auth: { username: config.key }
, headers: {
'Content-Type': 'application/json; charset=UTF-8'
, 'user-agent': config.UserAgent
Expand All @@ -108,8 +117,7 @@ const sendLine = (payload, config, callback) => {
times: MAX_REQUEST_RETRIES
, interval: (retryCount) => {
return REQUEST_RETRY_INTERVAL_MS * Math.pow(2, retryCount);
}
, errorFilter: (errCode) => {
}, errorFilter: (errCode) => {
return DEFAULT_HTTP_ERRORS.includes(errCode) || errCode === 'INTERNAL_SERVER_ERROR';
}
}, (reqCallback) => {
Expand All @@ -129,6 +137,15 @@ const sendLine = (payload, config, callback) => {
};

// Main Handler
exports.handler = (event, context, callback) => {
return sendLine(prepareLogs(parseEvent(event)), getConfig(), callback);
const handler = (event, context, callback) => {
const config = getConfig();
return sendLine(prepareLogs(parseEvent(event), config.log_raw_event), config, callback);
};

module.exports = {
getConfig
, handler
, parseEvent
, prepareLogs
, sendLine
};
8 changes: 5 additions & 3 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
{
"name": "logdna-cloudwatch",
"version": "2.1.0",
"version": "2.2.0",
"description": "Lambda Functions to Stream Logs from AWS CloudWatch to LogDNA",
"main": "index.js",
"scripts": {
"eslint": "./node_modules/.bin/eslint -c .eslintrc *.js"
"lint": "./node_modules/.bin/eslint -c .eslintrc index.js",
"test": "tap"
},
"dependencies": {
"agentkeepalive": "^4.0.2",
"async": "^2.6.2",
"request": "^2.88.0"
},
"devDependencies": {
"eslint": "^6.7.2"
"eslint": "^6.7.2",
"tap": "^14.10.7"
},
"keywords": [
"lambda",
Expand Down
146 changes: 146 additions & 0 deletions test/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
// External Modules
const { test } = require('tap');

// Internal Modules
const index = require('../index');
const pkg = require('../package.json');

// Constants
const missingKey = 'Missing LogDNA Ingestion Key';
const hostname = 'sampleCloudWatchHostname';
const inputTags = ' cloudwatch, logging, test';
const outputTags = 'cloudwatch,logging,test';
const sampleKey = '0123456789';
const rawEvent = {
'awslogs': {
'data': 'H4sIAAAAAAAAEzWQQW+DMAyF/wrKmaEkJCbhhjbWCzuBtMNUVSmkNBIQRMKqqep/X6Cb5Ivfs58++45G7ZzqdfMza5Sjt6IpTh9lXReHEsXI3ia9BJnQlHHIhMSEBnmw/WGx6xwcp8Z50M9uN2q/aDUGx2vn/5oYufXs2sXM3tjp3QxeLw7lX6hS47lTz6lTO9i1uynfXkOMe5lsp9Fxzyy/9eS3hTsyXYhOGVCaEsBSgsyEYBkGzrDMAIMQlAq+gQIQSjFhBFgqJOUMAog34WAfoFFOOM8kA0Y5SSH+f0SIb67GRaHq/baosn1UmUlHF7tErxvk5wa56b2Z+iRJ0OP4+AWj9ITzSgEAAA=='
}
};

const eventData = {
messageType: 'DATA_MESSAGE'
, owner: '123456789012'
, logGroup: 'sampleGroup'
, logStream: 'testStream'
, subscriptionFilters: [ 'LambdaStream_cloudwatchlogs-node' ]
, logEvents: [{
id: '34622316099697884706540976068822859012661220141643892546'
, timestamp: 1557946425136
, message: 'This is Sample Log Line for CloudWatch Logging...'
}]
};

const eventMetaData = {
event: {
type: eventData.messageType
, id: eventData.logEvents[0].id
}, log: {
group: eventData.logGroup
, stream: eventData.logStream
}
};

// Test parseEvent
test('test parseEvent with the sample test data described in README', (t) => {
t.deepEqual(index.parseEvent(rawEvent), eventData);
t.end();
});

// Test getConfig
test('test getConfig', (t) => {
// Test getConfig without any environment variable set
let config = index.getConfig();
t.equal(config.key, undefined);
t.equal(config.log_raw_event, false);
t.equal(config.UserAgent, `${pkg.name}/${pkg.version}`);
t.equal(config.hostname, undefined);
t.equal(config.tags, undefined);

// Set Hostname, Key and Tags
process.env.LOGDNA_HOSTNAME = hostname;
process.env.LOGDNA_TAGS = inputTags;
process.env.LOGDNA_KEY = sampleKey;
config = index.getConfig();
t.equal(config.key, sampleKey);
t.equal(config.log_raw_event, false);
t.equal(config.UserAgent, `${pkg.name}/${pkg.version}`);
t.equal(config.hostname, hostname);
t.equal(config.tags, outputTags);

// Set LOG_RAW_EVENT to True
process.env.LOG_RAW_EVENT = 'True';
config = index.getConfig();
t.equal(config.key, sampleKey);
t.equal(config.log_raw_event, true);
t.equal(config.UserAgent, `${pkg.name}/${pkg.version}`);
t.equal(config.hostname, hostname);
t.equal(config.tags, outputTags);

// Unset some environment variables
process.env.LOG_RAW_EVENT = '';
process.env.LOGDNA_TAGS = '';
config = index.getConfig();
t.equal(config.key, sampleKey);
t.equal(config.log_raw_event, false);
t.equal(config.UserAgent, `${pkg.name}/${pkg.version}`);
t.equal(config.hostname, hostname);
t.equal(config.tags, undefined);

// Set LOG_RAW_EVENT to Yes
process.env.LOG_RAW_EVENT = 'yEs';
config = index.getConfig();
t.equal(config.key, sampleKey);
t.equal(config.log_raw_event, true);
t.equal(config.UserAgent, `${pkg.name}/${pkg.version}`);
t.equal(config.hostname, hostname);
t.equal(config.tags, undefined);

// Finish the test suite
t.end();
});

// Test prepareLogs
test('test prepareLogs', (t) => {
// Without log_raw_event set to true
let eventLog = index.prepareLogs(eventData, false)[0];
t.assert(eventLog.timestamp < Date.now());
t.equal(eventLog.file, eventData.logStream);
t.equal(eventLog.meta.owner, eventData.owner);
t.deepEqual(eventLog.meta.filters, eventData.subscriptionFilters);
t.deepEqual(JSON.parse(eventLog.line), Object.assign({
message: eventData.logEvents[0].message
}, eventMetaData));

// With log_raw_event set to true
eventLog = index.prepareLogs(eventData, true)[0];
t.assert(eventLog.timestamp < Date.now());
t.equal(eventLog.file, eventData.logStream);
t.equal(eventLog.line, eventData.logEvents[0].message);
t.deepEqual(eventLog.meta, Object.assign({
owner: eventData.owner
, filters: eventData.subscriptionFilters
}, eventMetaData));

// Finish the test suite
t.end();
});

// Test sendLine
test('test sendLine', (t) => {
index.sendLine({ line: eventData.logEvents[0].message }, {}, (error, response) => {
t.equal(error, missingKey);

// Finish the test suite
t.end();
});
});

// Test handler
test('test handler', (t) => {
index.sendLine(rawEvent, {}, (error, response) => {
t.equal(error, missingKey);

// Finish the test suite
t.end();
});
});

0 comments on commit 04b327f

Please sign in to comment.