diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0fccf9b..7c9faf2 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -41,19 +41,21 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build CloudFormation Lambda function + working-directory: ./cfn-lambda run: | - cd cfn-lambda cp -R ../common ./common + cp -R ../logger ./logger GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o bootstrap . zip -x "*_test.go" -r ../cfn-lambda.zip . cd .. - - name: Build and Zip EventBridge Lambda function with common dependencies + - name: Build and Zip LogGroups events Lambda function with common dependencies + working-directory: ./log-group-events-lambda run: | - cd eventbridge-lambda cp -R ../common ./common + cp -R ../logger ./logger GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o bootstrap . - zip -x "*_test.go" -r ../eventbridge-lambda.zip . + zip -x "*_test.go" -r ../log-group-events-lambda.zip . cd .. - name: Upload Lambdas ZIP as Artifact @@ -62,12 +64,14 @@ jobs: name: lambdas path: | cfn-lambda.zip - eventbridge-lambda.zip + log-group-events-lambda.zip - name: Cleanup common folders run: | rm -rf cfn-lambda/common - rm -rf eventbridge-lambda/common + rm -rf cfn-lambda/logger + rm -rf log-group-events-lambda/common + rm -rf log-group-events-lambda/logger # Upload built artifacts to S3 upload_to_buckets: @@ -127,9 +131,9 @@ jobs: run: | aws s3 cp ./cfn-lambda.zip s3://logzio-aws-integrations-${{ matrix.aws_region }}/firehose-logs/${{ github.event.release.tag_name }}/cfn-lambda.zip --acl public-read - - name: Upload EventBridge Lambda ZIP to S3 + - name: Upload LogGroups events Lambda ZIP to S3 run: | - aws s3 cp ./eventbridge-lambda.zip s3://logzio-aws-integrations-${{ matrix.aws_region }}/firehose-logs/${{ github.event.release.tag_name }}/eventbridge-lambda.zip --acl public-read + aws s3 cp ./log-group-events-lambda.zip s3://logzio-aws-integrations-${{ matrix.aws_region }}/firehose-logs/${{ github.event.release.tag_name }}/log-group-events-lambda.zip --acl public-read - name: Prepare SAM Template run: | diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 4cb07c9..d198b40 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -23,8 +23,8 @@ jobs: - name: Test CFN lambda working-directory: ./cfn-lambda/handler run: go test -v -race -covermode=atomic -coverprofile=coverage.out - - name: Test EventBridge lambda - working-directory: ./eventbridge-lambda/handler + - name: Test LogGroups events lambda lambda + working-directory: ./log-group-events-lambda/handler run: go test -v -race -covermode=atomic -coverprofile=coverage.out - name: Test common functions working-directory: ./common @@ -54,14 +54,16 @@ jobs: working-directory: ./cfn-lambda run: | cp -R ../common ./common + cp -R ../logger ./logger GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o bootstrap . zip -x "*_test.go" -r ../cfn-lambda.zip . - - name: Build and Zip EventBridge Lambda function - working-directory: ./eventbridge-lambda + - name: Build and Zip LogGroups events Lambda function + working-directory: ./log-group-events-lambda run: | cp -R ../common ./common + cp -R ../logger ./logger GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o bootstrap . - zip -x "*_test.go" -r ../eventbridge-lambda.zip . + zip -x "*_test.go" -r ../log-group-events-lambda.zip . # Setup and upload to AWS - name: Setup AWS @@ -73,7 +75,7 @@ jobs: - name: Upload ZIP to S3 run: | aws s3 cp ./cfn-lambda.zip s3://logzio-aws-integrations-${{ env.AWS_REGION }}/firehose-logs/${{ env.TEST_VERSION }}/cfn-lambda.zip --acl public-read - aws s3 cp ./eventbridge-lambda.zip s3://logzio-aws-integrations-${{ env.AWS_REGION }}/firehose-logs/${{ env.TEST_VERSION }}/eventbridge-lambda.zip --acl public-read + aws s3 cp ./log-group-events-lambda.zip s3://logzio-aws-integrations-${{ env.AWS_REGION }}/firehose-logs/${{ env.TEST_VERSION }}/log-group-events-lambda.zip --acl public-read # Generate test data - name: Create API Gateway service diff --git a/README.md b/README.md index 866e34d..ccf61a2 100644 --- a/README.md +++ b/README.md @@ -13,55 +13,55 @@ This project will use a Cloudformation template to create a Stack that deploys: To deploy this project, click the button that matches the region you wish to deploy your Stack to: -| Region | Deployment | -|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `us-east-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-east-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `us-east-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-east-2.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `us-west-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-west-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `us-west-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-west-2.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `eu-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-central-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `eu-central-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-central-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-central-2.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `eu-north-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-north-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-north-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `eu-west-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-west-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `eu-west-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-west-2.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `eu-west-3` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-3#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-west-3.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `eu-south-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-south-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-south-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `eu-south-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-south-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-south-2.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `sa-east-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=sa-east-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-sa-east-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `ap-northeast-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-northeast-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `ap-northeast-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-northeast-2.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `ap-northeast-3` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-3#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-northeast-3.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `ap-south-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-south-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-south-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `ap-south-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-south-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-south-2.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `ap-southeast-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `ap-southeast-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-2.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `ap-southeast-3` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-3#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-3.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `ap-southeast-4` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-4#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-4.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `ap-east-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-east-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-east-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `ca-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ca-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ca-central-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `ca-west-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ca-west-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ca-west-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `af-south-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=af-south-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-af-south-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `me-south-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=me-south-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-me-south-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `me-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=me-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-me-central-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | -| `il-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=il-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-il-central-1.s3.amazonaws.com/firehose-logs/0.2.1/sam-template.yaml&stackName=logzio-firehose) | +| Region | Deployment | +|-------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `us-east-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-east-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `us-east-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-east-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-east-2.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `us-west-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-west-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `us-west-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=us-west-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-us-west-2.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-central-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-central-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-central-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-central-2.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-north-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-north-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-north-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-west-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-west-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-west-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-west-2.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-west-3` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-west-3#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-west-3.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-south-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-south-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-south-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `eu-south-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=eu-south-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-eu-south-2.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `sa-east-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=sa-east-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-sa-east-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-northeast-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-northeast-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-northeast-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-northeast-2.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-northeast-3` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-northeast-3#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-northeast-3.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-south-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-south-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-south-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-south-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-south-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-south-2.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-southeast-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-southeast-2` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-2#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-2.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-southeast-3` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-3#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-3.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-southeast-4` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-southeast-4#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-southeast-4.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `ap-east-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ap-east-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ap-east-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `ca-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ca-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ca-central-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `ca-west-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=ca-west-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-ca-west-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `af-south-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=af-south-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-af-south-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `me-south-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=me-south-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-me-south-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `me-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=me-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-me-central-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | +| `il-central-1` | [![Deploy to AWS](https://dytvr9ot2sszz.cloudfront.net/logz-docs/lights/LightS-button.png)](https://console.aws.amazon.com/cloudformation/home?region=il-central-1#/stacks/create/review?templateURL=https://logzio-aws-integrations-il-central-1.s3.amazonaws.com/firehose-logs/0.3.0/sam-template.yaml&stackName=logzio-firehose) | ### 1. Specify stack details Specify the stack details as per the table below, check the checkboxes and select **Create stack**. -| Parameter | Description | Required/Default | -|--------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------| -| `logzioToken` | The [token](https://app.logz.io/#/dashboard/settings/general) of the account you want to ship logs to. | **Required** | -| `logzioListener` | Listener host. | **Required** | -| `logzioType` | The log type you'll use with this Lambda. This can be a [built-in log type](https://docs.logz.io/user-guide/log-shipping/built-in-log-types.html), or a custom log type. | `logzio_firehose` | -| `services` | A comma-seperated list of services you want to collect logs from. Supported options are: `apigateway`, `rds`, `cloudhsm`, `cloudtrail`, `codebuild`, `connect`, `elasticbeanstalk`, `ecs`, `eks`, `aws-glue`, `aws-iot`, `lambda`, `macie`, `amazon-mq`, `batch` | - | -| `customLogGroups` | A comma-separated list of custom log groups to collect logs from, or the ARN of the Secret parameter ([explanation below](#custom-log-group-list-exceeds-4096-characters-limit)) storing the log groups list if it exceeds 4096 characters. | - | -| `useCustomLogGroupsFromSecret` | If you want to provide list of `customLogGroups` which exceeds 4096 characters, set to `true` and configure your customLogGroups as [defined below](#custom-log-group-list-exceeds-4096-characters-limit). | `false` | -| `triggerLambdaTimeout` | The amount of seconds that Lambda allows a function to run before stopping it, for the trigger function. | `60` | -| `triggerLambdaMemory` | Trigger function's allocated CPU proportional to the memory configured, in MB. | `512` | -| `triggerLambdaLogLevel` | Log level for the Lambda function. Can be one of: `debug`, `info`, `warn`, `error`, `fatal`, `panic` | `info` | -| `httpEndpointDestinationIntervalInSeconds` | The length of time, in seconds, that Kinesis Data Firehose buffers incoming data before delivering it to the destination | `60` | -| `httpEndpointDestinationSizeInMBs` | The size of the buffer, in MBs, that Kinesis Data Firehose uses for incoming data before delivering it to the destination | `5` | +| Parameter | Description | Required/Default | +|--------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------| +| `logzioToken` | The [token](https://app.logz.io/#/dashboard/settings/general) of the account you want to ship logs to. | **Required** | +| `logzioListener` | Listener host. | **Required** | +| `logzioType` | The log type you'll use with this Lambda. This can be a [built-in log type](https://docs.logz.io/user-guide/log-shipping/built-in-log-types.html), or a custom log type. | `logzio_firehose` | +| `services` | A comma-seperated list of services you want to collect logs from. Supported options are: `apigateway`, `rds`, `cloudhsm`, `cloudtrail`, `codebuild`, `connect`, `elasticbeanstalk`, `ecs`, `eks`, `aws-glue`, `aws-iot`, `lambda`, `macie`, `amazon-mq`, `batch` | - | +| `customLogGroups` | A comma-separated list of custom log groups to collect logs from, or the ARN of the Secret parameter ([explanation below](#custom-log-group-list-exceeds-4096-characters-limit)) storing the log groups list if it exceeds 4096 characters. **Note**: You can also specify a prefix of the log group names by using a wildcard at the end (e.g., `prefix*`). This will match all log groups that start with the specified prefix | - | +| `useCustomLogGroupsFromSecret` | If you want to provide list of `customLogGroups` which exceeds 4096 characters, set to `true` and configure your customLogGroups as [defined below](#custom-log-group-list-exceeds-4096-characters-limit). | `false` | +| `triggerLambdaTimeout` | The amount of seconds that Lambda allows a function to run before stopping it, for the trigger function. | `60` | +| `triggerLambdaMemory` | Trigger function's allocated CPU proportional to the memory configured, in MB. | `512` | +| `triggerLambdaLogLevel` | Log level for the Lambda function. Can be one of: `debug`, `info`, `warn`, `error`, `fatal`, `panic` | `info` | +| `httpEndpointDestinationIntervalInSeconds` | The length of time, in seconds, that Kinesis Data Firehose buffers incoming data before delivering it to the destination | `60` | +| `httpEndpointDestinationSizeInMBs` | The size of the buffer, in MBs, that Kinesis Data Firehose uses for incoming data before delivering it to the destination | `5` | > #### ⚠️ Important note ⚠️ @@ -98,6 +98,10 @@ Once new logs are added to your chosen log group, they will be sent to your Logz > If you've used the `services` field, you'll have to **wait 6 minutes** before creating new log groups for your chosen services. This is due to cold start and custom resource invocation, that can cause the Lambda to behave unexpectedly. ### Changelog: +- **0.3.0**: + - Support prefixes in `customLogGroups` via wildcard + - Upgrade go `1.19` >> `1.22` + - Parallelized subscription filter updates to improve performance - **0.2.1**: Add support for `aws-batch` service. - **0.2.0**: Option to provide `customLogGroups` exceeding 4KB. - **0.1.0**: diff --git a/cfn-lambda/handler/handler.go b/cfn-lambda/handler/handler.go index 5936007..350a76b 100644 --- a/cfn-lambda/handler/handler.go +++ b/cfn-lambda/handler/handler.go @@ -2,236 +2,142 @@ package handler import ( "context" + "encoding/json" "fmt" "github.com/aws/aws-lambda-go/cfn" "github.com/logzio/firehose-logs/common" - lp "github.com/logzio/firehose-logs/logger" + "github.com/logzio/firehose-logs/logger" "go.uber.org/zap" -) - -const ( - secretEnabledKey = "SecretEnabled" - customLogGroupsKey = "CustomLogGroups" - servicesKey = "Services" + "os" ) var sugLog *zap.SugaredLogger func HandleRequest(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { - logger := lp.GetLogger() - defer logger.Sync() - sugLog = logger.Sugar() + sugLog = logger.GetSugaredLogger() + sugLog.Info("Starting handling event...") sugLog.Debug("Handling event: ", event) - err = common.ValidateRequired() - if err != nil { - sugLog.Debug("Lambda finished with error") - return "", nil, err - } - // CloudFormation custom resource handling logic switch event.RequestType { case "Create": sugLog.Debug("Detected CloudFormation Create event") - return customResourceRun(ctx, event) + return createCustomResource(ctx, event) case "Update": sugLog.Debug("Detected CloudFormation Update event") - return customResourceRunUpdate(ctx, event) + return updateCustomResource(ctx, event) case "Delete": sugLog.Debug("Detected CloudFormation Delete event") - return customResourceRunDelete(ctx, event) + return deleteCustomResource(ctx, event) default: sugLog.Debug("Detected unsupported request type") - return customResourceRunDoNothing(ctx, event) + return } } -func generatePhysicalResourceId(event cfn.Event) string { - // Concatenate StackId and LogicalResourceId to form a unique PhysicalResourceId - physicalResourceId := fmt.Sprintf("%s-%s", event.StackID, event.LogicalResourceID) - sugLog.Debug("Generated physicalId with value: ", physicalResourceId) - return physicalResourceId -} - -func customResourceRunUpdate(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { - oldConfig := event.OldResourceProperties - newConfig := event.ResourceProperties - +func createCustomResource(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { physicalResourceID = generatePhysicalResourceId(event) - err = updateConfiguration(ctx, oldConfig, newConfig) - if err != nil { - sugLog.Error("Error during update: ", err) - return physicalResourceID, nil, err - } - - // Populate your data map as needed for the update - data = make(map[string]interface{}) - // Populate data as needed + payload := common.NewSubscriptionFilterEvent(common.RequestParameters{ + Action: common.AddSF, + NewServices: os.Getenv(common.EnvServices), + NewCustom: os.Getenv(common.EnvCustomGroups), + NewIsSecret: os.Getenv(common.EnvSecretEnabled), + }) + sugLog.Debug("Created SubscriptionFilter Event: ", payload) - return physicalResourceID, data, nil -} - -func updateConfiguration(ctx context.Context, oldConfig, newConfig map[string]interface{}) error { - sess, err := common.GetSession() + jsonPayload, err := json.Marshal(payload) if err != nil { - sugLog.Error("Error while creating session: ", err.Error()) - return err + sugLog.Error("Error marshalling payload: ", err.Error()) + return physicalResourceID, nil, err } - sugLog.Info("Extracting configuration strings...") - // Helper function to extract and validate configuration strings - extractConfigString := func(config map[string]interface{}, key string) (string, error) { - value, exists := config[key] - if !exists { - return "", nil - } - strValue, ok := value.(string) - if !ok { - sugLog.Errorf("Invalid type for %s; expected string", key) - return "", fmt.Errorf("invalid configuration type for %s", key) - } - return strValue, nil - } + stackName := event.ResourceProperties["StackName"].(string) - // Extract and validate services and custom log group strings from the configurations - oldServicesStr, err := extractConfigString(oldConfig, servicesKey) - if err != nil { - return err - } - newServicesStr, err := extractConfigString(newConfig, servicesKey) - if err != nil { - return err - } - oldCustomGroupsStr, err := extractConfigString(oldConfig, customLogGroupsKey) - if err != nil { - return err - } - oldSecretEnabledStr, err := extractConfigString(oldConfig, secretEnabledKey) - if err != nil { - return err - } - newCustomGroupsStr, err := extractConfigString(newConfig, customLogGroupsKey) + err = invokeLambdaAsynchronously(ctx, jsonPayload, stackName) if err != nil { - return err - } - newSecretEnabledStr, err := extractConfigString(newConfig, secretEnabledKey) - if err != nil { - return err + sugLog.Error("Error invoking lambda: ", err.Error()) + return physicalResourceID, nil, err } - oldCustomGroupsStr, err = common.GetCustomLogGroups(oldSecretEnabledStr, oldCustomGroupsStr) - if err != nil { - return err - } - newCustomGroupsStr, err = common.GetCustomLogGroups(newSecretEnabledStr, newCustomGroupsStr) - if err != nil { - return err - } + // no need to send back anything to the cfn stack, therefore we return empty map + return physicalResourceID, make(map[string]interface{}), nil +} - // Parse services and custom log groups - oldServices := common.ParseServices(oldServicesStr) - newServices := common.ParseServices(newServicesStr) - oldCustomGroups := common.ParseServices(oldCustomGroupsStr) - newCustomGroups := common.ParseServices(newCustomGroupsStr) +func updateCustomResource(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { + physicalResourceID = generatePhysicalResourceId(event) - // Find differences in services and custom log groups - servicesToAdd, servicesToRemove := common.FindDifferences(oldServices, newServices) - customGroupsToAdd, customGroupsToRemove := common.FindDifferences(oldCustomGroups, newCustomGroups) + oldConfig := event.OldResourceProperties + newConfig := event.ResourceProperties - // Update subscription filters - if err := common.UpdateSubscriptionFilters(sess, servicesToAdd, servicesToRemove, customGroupsToAdd, customGroupsToRemove); err != nil { - sugLog.Errorf("Error updating subscription filters: %v", err) - return err + getConfigItem := func(config map[string]interface{}, key string) string { + if value, ok := config[key].(string); ok { + return value + } + return "" } - return nil -} - -// Wrapper for first invocation from cloud formation custom resource -func customResourceRun(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { - physicalResourceID = generatePhysicalResourceId(event) + payload := common.NewSubscriptionFilterEvent(common.RequestParameters{ + Action: common.UpdateSF, + NewServices: getConfigItem(newConfig, common.EnvServices), + OldServices: getConfigItem(oldConfig, common.EnvServices), + NewCustom: getConfigItem(newConfig, common.EnvCustomGroups), + OldCustom: getConfigItem(oldConfig, common.EnvCustomGroups), + NewIsSecret: getConfigItem(newConfig, common.EnvSecretEnabled), + OldIsSecret: getConfigItem(oldConfig, common.EnvSecretEnabled), + }) + sugLog.Debug("Created SubscriptionFilter Event: ", payload) - err = handleFirstInvocation() + jsonPayload, err := json.Marshal(payload) if err != nil { - sugLog.Error("Error while handling first invocation: ", err.Error()) + sugLog.Error("Error marshalling payload: ", err.Error()) return physicalResourceID, nil, err } - // Populate your data map as needed for the update - data = make(map[string]interface{}) - // Populate data as needed + stackName := event.ResourceProperties["StackName"].(string) - return physicalResourceID, data, nil -} - -// Wrapper for invocation from cloudformation custom resource - for read, update -func customResourceRunDoNothing(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { - return -} - -// Wrapper for invocation from cloudformation custom resource - delete -func customResourceRunDelete(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { - sess, err := common.GetSession() + err = invokeLambdaAsynchronously(ctx, jsonPayload, stackName) if err != nil { - sugLog.Error("Error while creating session: ", err.Error()) - } - - deleted := make([]string, 0) - servicesToDelete := common.GetServices() - if servicesToDelete != nil { - newDeleted, err := common.DeleteServices(sess, servicesToDelete) - deleted = append(deleted, newDeleted...) - if err != nil { - sugLog.Error(err.Error()) - } - } - - pathsToDelete := common.GetCustomPaths() - if pathsToDelete != nil { - newDeleted, err := common.DeleteCustom(sess, pathsToDelete) - deleted = append(deleted, newDeleted...) - if err != nil { - sugLog.Error(err.Error()) - } + sugLog.Error("Error invoking lambda: ", err.Error()) + return physicalResourceID, nil, err } - sugLog.Info("Deleted subscription filters for the following log groups: ", deleted) + // no need to send back anything to the cfn stack, therefore we return empty map + return physicalResourceID, make(map[string]interface{}), nil +} +func deleteCustomResource(ctx context.Context, event cfn.Event) (physicalResourceID string, data map[string]interface{}, err error) { physicalResourceID = generatePhysicalResourceId(event) - // Populate your data map as needed for the update - data = make(map[string]interface{}) - // Populate data as needed - return physicalResourceID, data, nil -} + payload := common.NewSubscriptionFilterEvent(common.RequestParameters{ + Action: common.DeleteSF, + NewServices: os.Getenv(common.EnvServices), + NewCustom: os.Getenv(common.EnvCustomGroups), + NewIsSecret: os.Getenv(common.EnvSecretEnabled), + }) + sugLog.Debug("Created SubscriptionFilter Event: ", payload) -func handleFirstInvocation() error { - sess, err := common.GetSession() + jsonPayload, err := json.Marshal(payload) if err != nil { - return err + sugLog.Error("Error marshalling payload: ", err.Error()) + return physicalResourceID, nil, err } - added := make([]string, 0) - servicesToAdd := common.GetServices() - if servicesToAdd != nil { - newAdded, err := common.AddServices(sess, servicesToAdd) - added = append(added, newAdded...) - if err != nil { - sugLog.Error(err.Error()) - } - } + stackName := event.ResourceProperties["StackName"].(string) - pathsToAdd := common.GetCustomPaths() - if pathsToAdd != nil { - newAdded, err := common.AddCustom(sess, pathsToAdd, added) - added = append(added, newAdded...) - if err != nil { - sugLog.Error(err.Error()) - } + _, err = invokeLambdaSynchronously(ctx, jsonPayload, stackName) + if err != nil { + sugLog.Error("Error invoking lambda or executing function: ", err.Error()) + return physicalResourceID, nil, err } - sugLog.Info("Following these log groups: ", added) + // no need to send back anything to the cfn stack, therefore we return empty map + return physicalResourceID, make(map[string]interface{}), nil +} - return nil +func generatePhysicalResourceId(event cfn.Event) string { + // Concatenate StackId and LogicalResourceId to form a unique PhysicalResourceId + physicalResourceId := fmt.Sprintf("%s-%s", event.StackID, event.LogicalResourceID) + sugLog.Debug("Generated physicalId with value: ", physicalResourceId) + return physicalResourceId } diff --git a/cfn-lambda/handler/handler_test.go b/cfn-lambda/handler/handler_test.go index 007beec..83c5406 100644 --- a/cfn-lambda/handler/handler_test.go +++ b/cfn-lambda/handler/handler_test.go @@ -6,30 +6,10 @@ import ( lp "github.com/logzio/firehose-logs/logger" "github.com/stretchr/testify/assert" "go.uber.org/zap" - "os" "testing" ) -func stringPtr(s string) *string { - /* helper function */ - return &s -} - func setup(eventType string) (ctx context.Context, event cfn.Event, initLogger *zap.SugaredLogger) { - /* Setup needed env variables */ - err := os.Setenv("FIREHOSE_ARN", "test-arn") - if err != nil { - return - } - err = os.Setenv("ACCOUNT_ID", "aws-account-id") - if err != nil { - return - } - err = os.Setenv("AWS_PARTITION", "test-partition") - if err != nil { - return - } - /* Setup mock context and event */ mockEvent := cfn.Event{ RequestType: cfn.RequestType(eventType), @@ -67,55 +47,3 @@ func TestGeneratePhysicalResourceId(t *testing.T) { physicalId := generatePhysicalResourceId(mockEvent) assert.Equal(t, "arn:aws:cloudformation:us-west-2:EXAMPLE/stack-name/guid-MyTestResource", physicalId) } - -func TestCustomResourceRunUpdate(t *testing.T) { - ctx, mockEvent, sugLog := setup("Create") - sugLog.Info("init susLog") - - physicalId, data, err := customResourceRunUpdate(ctx, mockEvent) - assert.Equal(t, "arn:aws:cloudformation:us-west-2:EXAMPLE/stack-name/guid-MyTestResource", physicalId) - assert.Empty(t, data) - assert.Nil(t, err) -} - -func TestUpdateConfiguration(t *testing.T) { - tests := []struct { - name string - oldConfig map[string]interface{} - newConfig map[string]interface{} - expected *string - }{ - { - name: "invalid configuration value", - oldConfig: map[string]interface{}{"Services": 123}, - newConfig: map[string]interface{}{"Services": "elb", "CustomLogGroups": "rand"}, - expected: stringPtr("invalid configuration type for Services"), - }, - { - name: "no valid log groups configuration", - oldConfig: map[string]interface{}{}, - newConfig: map[string]interface{}{"Services": "elb", "CustomLogGroups": "rand"}, - expected: stringPtr("Could not retrieve any log groups"), - }, - { - name: "no changes", - oldConfig: map[string]interface{}{"Services": "rds", "CustomLogGroups": "rand"}, - newConfig: map[string]interface{}{"Services": "rds", "CustomLogGroups": "rand"}, - expected: nil, - }, - } - - ctx, _, _ := setup("Update") - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - result := updateConfiguration(ctx, test.oldConfig, test.newConfig) - - if test.expected == nil { - assert.Nil(t, result, "Expected nil, got %v", result) - } else { - assert.Equal(t, *test.expected, result.Error(), "Expected %v, got %v", *test.expected, result) - } - }) - } -} diff --git a/cfn-lambda/handler/lambda_ops.go b/cfn-lambda/handler/lambda_ops.go new file mode 100644 index 0000000..762a582 --- /dev/null +++ b/cfn-lambda/handler/lambda_ops.go @@ -0,0 +1,65 @@ +package handler + +import ( + "context" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/lambda" + "github.com/aws/aws-sdk-go/service/lambda/lambdaiface" + "github.com/logzio/firehose-logs/common" +) + +const lambdaToTrigger = "-log-group-events-lambda" + +type LambdaClient struct { + Function lambdaiface.LambdaAPI +} + +func invokeLambdaSynchronously(ctx context.Context, payload []byte, stackName string) (string, error) { + client, err := createLambdaClient() + if err != nil { + sugLog.Error("Error creating lambda client: ", err.Error()) + return "", err + } + + res, err := client.invokeLambda(ctx, stackName+lambdaToTrigger, "RequestResponse", payload) + if err != nil { + return "", err + } + return string(res.Payload), nil +} + +func invokeLambdaAsynchronously(ctx context.Context, payload []byte, stackName string) error { + client, err := createLambdaClient() + if err != nil { + sugLog.Error("Error creating lambda client: ", err.Error()) + return err + } + + _, err = client.invokeLambda(ctx, stackName+lambdaToTrigger, "Event", payload) + return err +} + +func (client *LambdaClient) invokeLambda(ctx context.Context, functionName, invocationType string, payload []byte) (*lambda.InvokeOutput, error) { + sugLog.Debugf("Invoking lambda %s %s", functionName, invocationType) + + res, err := client.Function.InvokeWithContext(ctx, &lambda.InvokeInput{ + FunctionName: aws.String(functionName), + InvocationType: aws.String(invocationType), + Payload: payload, + }) + + if err != nil { + sugLog.Error("Error while invoking lambda: ", err.Error()) + return nil, err + } + return res, nil +} + +func createLambdaClient() (*LambdaClient, error) { + sess, err := common.GetSession() + if err != nil { + sugLog.Error("Error while creating session: ", err.Error()) + return nil, err + } + return &LambdaClient{Function: lambda.New(sess)}, nil +} diff --git a/cfn-lambda/handler/lambda_ops_test.go b/cfn-lambda/handler/lambda_ops_test.go new file mode 100644 index 0000000..02938a5 --- /dev/null +++ b/cfn-lambda/handler/lambda_ops_test.go @@ -0,0 +1,97 @@ +package handler + +import ( + "context" + "encoding/json" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/lambda" + "github.com/aws/aws-sdk-go/service/lambda/lambdaiface" + "github.com/logzio/firehose-logs/common" + lp "github.com/logzio/firehose-logs/logger" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "testing" +) + +type mockLambdaClient struct { + mock.Mock + lambdaiface.LambdaAPI +} + +func (m *mockLambdaClient) InvokeWithContext(ctx aws.Context, input *lambda.InvokeInput, opts ...request.Option) (*lambda.InvokeOutput, error) { + args := m.Called(ctx, input) + return &lambda.InvokeOutput{ + StatusCode: aws.Int64(200), + FunctionError: nil, + }, args.Error(1) +} + +func setupLambdaTest(eventType common.ActionType) (ctx context.Context, payload []byte) { + /* Setup mock context and event */ + mockEvent := common.NewSubscriptionFilterEvent(common.RequestParameters{ + Action: eventType, + NewServices: "service1, service2", + OldServices: "service1, service3", + NewCustom: "log-group1, log-group2", + OldCustom: "log-group1", + NewIsSecret: "false", + OldIsSecret: "false", + }) + mockEventPayload, _ := json.Marshal(mockEvent) + + ctx = context.Background() + + /* Setup logger */ + sugLog = lp.GetSugaredLogger() + + return ctx, mockEventPayload +} + +func TestInvokeLambdaAsynchronously(t *testing.T) { + ctx, mockEvent := setupLambdaTest(common.AddSF) + + /* mock the lambda client */ + mockClient := new(mockLambdaClient) + mockClient.On("InvokeWithContext", ctx, &lambda.InvokeInput{ + FunctionName: aws.String(lambdaToTrigger), + InvocationType: aws.String("Event"), + Payload: mockEvent, + }).Return(&lambda.InvokeOutput{ + StatusCode: aws.Int64(200), + FunctionError: nil, + }, nil) + + /* test trigger */ + lambdaClient := &LambdaClient{Function: mockClient} + _, err := lambdaClient.invokeLambda(ctx, lambdaToTrigger, "Event", mockEvent) + + assert.NoError(t, err) + mockClient.AssertExpectations(t) +} + +func TestInvokeLambdaSynchronously(t *testing.T) { + ctx, mockEvent := setupLambdaTest(common.AddSF) + + /* mock the lambda client */ + mockClient := new(mockLambdaClient) + mockClient.On("InvokeWithContext", ctx, &lambda.InvokeInput{ + FunctionName: aws.String(lambdaToTrigger), + InvocationType: aws.String("RequestResponse"), + Payload: mockEvent, + }).Return(&lambda.InvokeOutput{ + StatusCode: aws.Int64(200), + FunctionError: nil, + }, nil) + + /* test trigger */ + lambdaClient := &LambdaClient{Function: mockClient} + res, err := lambdaClient.invokeLambda(ctx, lambdaToTrigger, "RequestResponse", mockEvent) + + assert.NoError(t, err) + assert.Equal(t, &lambda.InvokeOutput{ + StatusCode: aws.Int64(200), + FunctionError: nil, + }, res) + mockClient.AssertExpectations(t) +} diff --git a/cloudformation/sam-template.yaml b/cloudformation/sam-template.yaml index a6b3a74..2940023 100644 --- a/cloudformation/sam-template.yaml +++ b/cloudformation/sam-template.yaml @@ -78,7 +78,7 @@ Resources: # The lambda functions CfnLambdaFunction: Type: 'AWS::Lambda::Function' - DependsOn: logzioFirehose + DependsOn: LogGroupEventsLambdaFunction Properties: Code: S3Bucket: logzio-aws-integrations-<> @@ -86,7 +86,7 @@ Resources: FunctionName: !Join [ '-', [ !Ref AWS::StackName, 'cfn-lambda' ] ] Handler: bootstrap Runtime: provided.al2 - Role: !GetAtt lambdaExecutionRole.Arn + Role: !GetAtt cfnLambdaExecutionRole.Arn Timeout: !Ref triggerLambdaTimeout MemorySize: !Ref triggerLambdaMemory ReservedConcurrentExecutions: 1 @@ -101,17 +101,17 @@ Resources: LOG_LEVEL: !Ref triggerLambdaLogLevel PUT_SF_ROLE: !GetAtt firehosePutSubscriptionFilterRole.Arn - EventBridgeLambdaFunction: + LogGroupEventsLambdaFunction: Type: 'AWS::Lambda::Function' DependsOn: logzioFirehose Properties: Code: S3Bucket: logzio-aws-integrations-<> - S3Key: firehose-logs/<>/eventbridge-lambda.zip - FunctionName: !Join [ '-', [ !Ref AWS::StackName, 'eventbridge-lambda' ] ] + S3Key: firehose-logs/<>/log-group-events-lambda.zip + FunctionName: !Join [ '-', [ !Ref AWS::StackName, 'log-group-events-lambda' ] ] Handler: bootstrap Runtime: provided.al2 - Role: !GetAtt lambdaExecutionRole.Arn + Role: !GetAtt logGroupLambdaExecutionRole.Arn Timeout: !Ref triggerLambdaTimeout MemorySize: !Ref triggerLambdaMemory ReservedConcurrentExecutions: 1 @@ -127,10 +127,41 @@ Resources: PUT_SF_ROLE: !GetAtt firehosePutSubscriptionFilterRole.Arn # Lambda permissions for log groups and using firehose - lambdaExecutionRole: + cfnLambdaExecutionRole: + Type: 'AWS::IAM::Role' + Properties: + RoleName: !Join [ '-', [ 'logzioCfnLambdaRole', !Select [ 4, !Split [ '-', !Select [ 2, !Split [ '/', !Ref AWS::StackId ] ] ] ] ] ] + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - 'sts:AssumeRole' + Path: / + Policies: + - PolicyName: !Join [ '-', [ 'logzioRole', !Select [ 4, !Split [ '-', !Select [ 2, !Split [ '/', !Ref AWS::StackId ] ] ] ] ] ] + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - 'logs:CreateLogGroup' + - 'logs:CreateLogStream' + - 'logs:PutLogEvents' + Resource: '*' + - Effect: Allow + Action: + - 'lambda:InvokeFunction' + Resource: + - !GetAtt LogGroupEventsLambdaFunction.Arn + + logGroupLambdaExecutionRole: Type: 'AWS::IAM::Role' Properties: - RoleName: !Join [ '-', [ 'logzioRole', !Select [ 4, !Split [ '-', !Select [ 2, !Split [ '/', !Ref AWS::StackId ] ] ] ] ] ] + RoleName: !Join [ '-', [ 'logzioEventsLambdaRole', !Select [ 4, !Split [ '-', !Select [ 2, !Split [ '/', !Ref AWS::StackId ] ] ] ] ] ] AssumeRolePolicyDocument: Version: 2012-10-17 Statement: @@ -185,6 +216,7 @@ Resources: Services: !Ref services CustomLogGroups: !Ref customLogGroups SecretEnabled: !Ref useCustomLogGroupsFromSecret + StackName: !Ref AWS::StackName logGroupCreationEvent: Condition: createEventbridgeTrigger @@ -204,8 +236,8 @@ Resources: Name: !Join [ '-', [ 'logGroupCreated', !Select [ 4, !Split [ '-', !Select [ 2, !Split [ '/', !Ref AWS::StackId ] ] ] ] ] ] State: ENABLED Targets: - - Arn: !GetAtt EventBridgeLambdaFunction.Arn - Id: 'EventBridgeLambdaFunctionTarget' + - Arn: !GetAtt LogGroupEventsLambdaFunction.Arn + Id: 'LogGroupEventsLambdaFunctionTarget' secretChangeEvent: Condition: secretChangeEventsEnabled @@ -228,7 +260,7 @@ Resources: Name: !Join [ '-', [ 'customLogGroupsSecretChanged', !Select [ 4, !Split [ '-', !Select [ 2, !Split [ '/', !Ref AWS::StackId ] ] ] ] ] ] State: ENABLED Targets: - - Arn: !GetAtt EventBridgeLambdaFunction.Arn + - Arn: !GetAtt LogGroupEventsLambdaFunction.Arn Id: 'SecretChangeLambdaTarget' # Permissions to trigger events @@ -236,7 +268,7 @@ Resources: Condition: createEventbridgeTrigger Type: AWS::Lambda::Permission Properties: - FunctionName: !Ref EventBridgeLambdaFunction + FunctionName: !Ref LogGroupEventsLambdaFunction Action: 'lambda:InvokeFunction' Principal: 'events.amazonaws.com' SourceArn: !GetAtt logGroupCreationEvent.Arn @@ -246,7 +278,7 @@ Resources: Type: AWS::Lambda::Permission Properties: Action: 'lambda:InvokeFunction' - FunctionName: !Ref EventBridgeLambdaFunction + FunctionName: !Ref LogGroupEventsLambdaFunction Principal: 'events.amazonaws.com' SourceArn: !GetAtt secretChangeEvent.Arn diff --git a/common/log_management.go b/common/log_management.go deleted file mode 100644 index ddd273d..0000000 --- a/common/log_management.go +++ /dev/null @@ -1,55 +0,0 @@ -package common - -import ( - "fmt" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" - "go.uber.org/zap" - "os" -) - -var sugLog *zap.SugaredLogger - -// Ensure sugLog is safely initialized before use -func initLogger() { - // Basic logger initialization, replace with your actual logger configuration - logger, err := zap.NewProduction() - if err != nil { - fmt.Printf("Failed to initialize logger: %v\n", err) - os.Exit(1) // Or handle the error according to your application's requirements - } - sugLog = logger.Sugar() -} - -func GetSession() (*session.Session, error) { - sess, err := session.NewSessionWithOptions(session.Options{ - Config: aws.Config{ - Region: aws.String(os.Getenv(EnvAwsRegion)), - }, - }) - - if err != nil { - return nil, fmt.Errorf("error occurred while trying to create a connection to aws: %s. Aborting", err.Error()) - } - - return sess, nil -} - -func ValidateRequired() error { - destinationArn := os.Getenv(envFirehoseArn) - if destinationArn == emptyString { - return fmt.Errorf("destination ARN must be set") - } - - accountId := os.Getenv(envAccountId) - if accountId == emptyString { - return fmt.Errorf("account id must be set") - } - - awsPartition := os.Getenv(envAwsPartition) - if awsPartition == emptyString { - return fmt.Errorf("aws partition must be set") - } - - return nil -} diff --git a/common/log_management_test.go b/common/log_management_test.go deleted file mode 100644 index 4d1c3db..0000000 --- a/common/log_management_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package common - -import ( - "fmt" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudwatchlogs" - "github.com/stretchr/testify/assert" - "os" - "testing" -) - -func setup() (logsClient *cloudwatchlogs.CloudWatchLogs) { - err := os.Setenv(envFirehoseArn, "test-arn") - if err != nil { - return - } - - err = os.Setenv(envAccountId, "aws-account-id") - if err != nil { - return - } - - err = os.Setenv(envAwsPartition, "test-partition") - if err != nil { - return - } - - err = os.Setenv(EnvAwsRegion, "not-existing-region") - if err != nil { - return - } - - err = os.Setenv(envPutSubscriptionFilterRole, "not-existing-role-arn") - if err != nil { - return - } - - mockSession, _ := GetSession() - mockClient := cloudwatchlogs.New(mockSession) - - return mockClient -} - -func TestValidateRequired(t *testing.T) { - /* Missing all 3 required env variable */ - result := ValidateRequired() - assert.Equal(t, "destination ARN must be set", result.Error()) - - err := os.Setenv(envFirehoseArn, "test-arn") - if err != nil { - return - } - - /* Missing 2 required env variable */ - result = ValidateRequired() - assert.Equal(t, "account id must be set", result.Error()) - - err = os.Setenv(envAccountId, "aws-account-id") - if err != nil { - return - } - - /* Missing 1 required env variable */ - result = ValidateRequired() - assert.Equal(t, "aws partition must be set", result.Error()) - - /* Valid required env variables */ - err = os.Setenv(envAwsPartition, "test-partition") - if err != nil { - return - } - - result = ValidateRequired() - assert.Nil(t, result) -} - -func TestGetSession(t *testing.T) { - result, err := GetSession() - assert.IsType(t, (*session.Session)(nil), result) - assert.Nil(t, err) -} - -func TestPutSubscriptionFilter(t *testing.T) { - mockClient := setup() - logGroups := []string{"logGroup1", "logGroup2"} - - added := PutSubscriptionFilter(logGroups, mockClient) - fmt.Println(added) -} - -func TestDeleteSubscriptionFilter(t *testing.T) {} diff --git a/common/sf_event.go b/common/sf_event.go new file mode 100644 index 0000000..443c08c --- /dev/null +++ b/common/sf_event.go @@ -0,0 +1,55 @@ +package common + +import ( + "encoding/json" + "fmt" +) + +type ActionType string + +const ( + AddSF ActionType = "add" + UpdateSF ActionType = "update" + DeleteSF ActionType = "delete" +) + +type RequestParameters struct { + Action ActionType + NewServices string `json:"newServices,omitempty"` + OldServices string `json:"oldServices,omitempty"` + NewCustom string `json:"newCustom,omitempty"` + OldCustom string `json:"oldCustom,omitempty"` + NewIsSecret string `json:"newIsSecret,omitempty"` + OldIsSecret string `json:"oldIsSecret,omitempty"` +} + +type Detail struct { + EventName string `json:"eventName"` + RequestParameters RequestParameters `json:"requestParameters"` +} + +type SubscriptionFilterEvent struct { + Detail Detail `json:"detail"` +} + +func NewSubscriptionFilterEvent(params RequestParameters) SubscriptionFilterEvent { + return SubscriptionFilterEvent{ + Detail: Detail{ + EventName: "SubscriptionFilterEvent", + RequestParameters: params, + }, + } +} + +func ConvertToRequestParameters(obj interface{}) (RequestParameters, error) { + var rp RequestParameters + bytes, err := json.Marshal(obj) + if err != nil { + return rp, fmt.Errorf("error marshalling interface: %v", err) + } + err = json.Unmarshal(bytes, &rp) + if err != nil { + return rp, fmt.Errorf("error unmarshalling to RequestParameters: %v", err) + } + return rp, nil +} diff --git a/common/sf_event_test.go b/common/sf_event_test.go new file mode 100644 index 0000000..44e05bb --- /dev/null +++ b/common/sf_event_test.go @@ -0,0 +1,145 @@ +package common + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func stringPtr(s string) *string { + /* helper function */ + return &s +} + +func TestNewSubscriptionFilterEvent(t *testing.T) { + tests := []struct { + name string + params RequestParameters + expected SubscriptionFilterEvent + }{ + { + name: "Test New Add SF Event", + params: RequestParameters{ + Action: AddSF, + NewServices: "newServices", + NewCustom: "newCustom", + NewIsSecret: "false", + }, + expected: SubscriptionFilterEvent{ + Detail: Detail{ + EventName: "SubscriptionFilterEvent", + RequestParameters: RequestParameters{ + Action: AddSF, + NewServices: "newServices", + NewCustom: "newCustom", + NewIsSecret: "false", + }, + }}, + }, + { + name: "Test New Update SF Event", + params: RequestParameters{ + Action: UpdateSF, + NewServices: "newServices", + OldServices: "oldServices", + NewCustom: "newCustom", + OldCustom: "someSecret", + NewIsSecret: "false", + OldIsSecret: "true", + }, + expected: SubscriptionFilterEvent{ + Detail: Detail{ + EventName: "SubscriptionFilterEvent", + RequestParameters: RequestParameters{ + Action: UpdateSF, + NewServices: "newServices", + OldServices: "oldServices", + NewCustom: "newCustom", + OldCustom: "someSecret", + NewIsSecret: "false", + OldIsSecret: "true", + }, + }}, + }, + { + name: "Test New Delete SF Event", + params: RequestParameters{ + Action: DeleteSF, + NewServices: "services", + NewCustom: "custom", + NewIsSecret: "false", + }, + expected: SubscriptionFilterEvent{ + Detail: Detail{ + EventName: "SubscriptionFilterEvent", + RequestParameters: RequestParameters{ + Action: DeleteSF, + NewServices: "services", + NewCustom: "custom", + NewIsSecret: "false", + }, + }}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := NewSubscriptionFilterEvent(test.params) + assert.Equal(t, test.expected, result, "Expected %v, got %v", test.expected, result) + }) + } +} + +func TestConvertToRequestParameters(t *testing.T) { + tests := []struct { + name string + obj interface{} + expected RequestParameters + expectedErr *string + }{ + { + name: "Test Convert to Request Parameters", + obj: map[string]interface{}{ + "action": "add", + "newServices": "newServices", + "newCustom": "newCustom", + "newIsSecret": "false", + "oldServices": "oldServices", + "oldCustom": "oldCustom", + "oldIsSecret": "true", + "invalidField": "invalid", + }, + expected: RequestParameters{ + Action: AddSF, + NewServices: "newServices", + NewCustom: "newCustom", + NewIsSecret: "false", + OldServices: "oldServices", + OldCustom: "oldCustom", + OldIsSecret: "true", + }, + expectedErr: nil, + }, + { + name: "Test Convert to Request Parameters", + obj: map[string]interface{}{ + "action": 123, + "field": "val", + }, + expected: RequestParameters{}, + expectedErr: stringPtr("error unmarshalling to RequestParameters: json: cannot unmarshal number into Go struct field RequestParameters.Action of type common.ActionType"), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := ConvertToRequestParameters(test.obj) + + if test.expectedErr == nil { + assert.Equal(t, test.expected, result, "Expected %v, got %v", test.expected, result) + assert.Nil(t, err, "Expected nil, got %v", err) + } else { + assert.Equal(t, *test.expectedErr, err.Error(), "Expected %v, got %v", test.expectedErr, err.Error()) + } + }) + } +} diff --git a/common/subscription_filters_management.go b/common/subscription_filters_management.go deleted file mode 100644 index 53ae485..0000000 --- a/common/subscription_filters_management.go +++ /dev/null @@ -1,248 +0,0 @@ -package common - -import ( - "fmt" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/cloudwatchlogs" - "os" -) - -func getLogGroups(services []string, logsClient *cloudwatchlogs.CloudWatchLogs) []string { - if sugLog == nil { - initLogger() - } - logGroupsToAdd := make([]string, 0) - serviceToPrefix := GetServicesMap() - for _, service := range services { - if prefix, ok := serviceToPrefix[service]; ok { - sugLog.Debug("Working on prefix: ", prefix) - newLogGroups, err := logGroupsPagination(prefix, logsClient) - if err != nil { - sugLog.Errorf("Error while searching for log groups of %s: %s", service, err.Error()) - } - - logGroupsToAdd = append(logGroupsToAdd, newLogGroups...) - } else { - sugLog.Errorf("Service %s is not supported. Skipping.", service) - } - } - - return logGroupsToAdd -} - -func logGroupsPagination(prefix string, logsClient *cloudwatchlogs.CloudWatchLogs) ([]string, error) { - var nextToken *string - logGroups := make([]string, 0) - for { - describeOutput, err := logsClient.DescribeLogGroups(&cloudwatchlogs.DescribeLogGroupsInput{ - LogGroupNamePrefix: &prefix, - NextToken: nextToken, - }) - - if err != nil { - return nil, err - } - if describeOutput != nil { - nextToken = describeOutput.NextToken - for _, logGroup := range describeOutput.LogGroups { - // Prevent a situation where we put subscription filter on the trigger and shipper function - if *logGroup.LogGroupName != LambdaPrefix+os.Getenv(EnvFunctionName) { - logGroups = append(logGroups, *logGroup.LogGroupName) - } - } - } - - if nextToken == nil { - break - } - } - - return logGroups, nil -} - -func AddServices(sess *session.Session, servicesToAdd []string) ([]string, error) { - if sugLog == nil { - initLogger() - } - logsClient := cloudwatchlogs.New(sess) - logGroups := getLogGroups(servicesToAdd, logsClient) - if len(logGroups) > 0 { - sugLog.Debug("Detected the following services: ", logGroups) - newAdded := PutSubscriptionFilter(logGroups, logsClient) - return newAdded, nil - } else { - return nil, fmt.Errorf("Could not retrieve any log groups") - } -} - -func PutSubscriptionFilter(logGroups []string, logsClient *cloudwatchlogs.CloudWatchLogs) []string { - // Early return if logsClient is nil to avoid panic - if logsClient == nil { - fmt.Println("CloudWatch Logs client is nil") - return nil - } - - // Initialize logger if it's nil - if sugLog == nil { - initLogger() - } - - destinationArn := os.Getenv(envFirehoseArn) - roleArn := os.Getenv(envPutSubscriptionFilterRole) - filterPattern := "" - filterName := subscriptionFilterName - added := make([]string, 0) - for _, logGroup := range logGroups { - _, err := logsClient.PutSubscriptionFilter(&cloudwatchlogs.PutSubscriptionFilterInput{ - DestinationArn: &destinationArn, - FilterName: &filterName, - LogGroupName: &logGroup, - FilterPattern: &filterPattern, - RoleArn: &roleArn, - }) - - if err != nil { - sugLog.Error("Error while trying to add subscription filter for ", logGroup, ": ", err.Error()) - continue - } - - added = append(added, logGroup) - } - - return added -} - -func UpdateSubscriptionFilters(sess *session.Session, servicesToAdd, servicesToRemove, customGroupsToAdd, customGroupsToRemove []string) error { - if sugLog == nil { - initLogger() - } - // Add subscription filters for new services - if len(servicesToAdd) > 0 { - addedServices, err := AddServices(sess, servicesToAdd) - if err != nil { - sugLog.Errorf("Error adding subscriptions for services: %v", err) - return err - } - sugLog.Infof("Added subscriptions for services: %v", addedServices) - } - - // Add subscription filters for new custom log groups - if len(customGroupsToAdd) > 0 { - addedCustomGroups, err := AddCustom(sess, customGroupsToAdd, nil) // Assuming the third parameter is handled within the function - if err != nil { - sugLog.Errorf("Error adding subscriptions for custom log groups: %v", err) - return err - } - sugLog.Infof("Added subscriptions for custom log groups: %v", addedCustomGroups) - } - - // Remove subscription filters from services no longer needed - if len(servicesToRemove) > 0 { - removedServices, err := DeleteServices(sess, servicesToRemove) - if err != nil { - sugLog.Errorf("Error removing subscriptions for services: %v", err) - return err - } - sugLog.Infof("Removed subscriptions for services: %v", removedServices) - } - - // Remove subscription filters from custom log groups no longer needed - if len(customGroupsToRemove) > 0 { - removedCustomGroups, err := DeleteCustom(sess, customGroupsToRemove) - if err != nil { - sugLog.Errorf("Error removing subscriptions for custom log groups: %v", err) - return err - } - sugLog.Infof("Removed subscriptions for custom log groups: %v", removedCustomGroups) - } - - return nil -} - -func DeleteSubscriptionFilter(logGroups []string, logsClient *cloudwatchlogs.CloudWatchLogs) []string { - // Early return if logsClient is nil to avoid panic - if logsClient == nil { - fmt.Println("CloudWatch Logs client is nil") - return nil - } - - // Initialize logger if it's nil - if sugLog == nil { - initLogger() - } - - filterName := subscriptionFilterName - deleted := make([]string, 0) - for _, logGroup := range logGroups { - _, err := logsClient.DeleteSubscriptionFilter(&cloudwatchlogs.DeleteSubscriptionFilterInput{ - FilterName: &filterName, - LogGroupName: &logGroup, - }) - - if err != nil { - sugLog.Error("Error while trying to delete subscription filter for ", logGroup, ": ", err.Error()) - continue - } - - deleted = append(deleted, logGroup) - sugLog.Info("Detected the following services for deletion2: ", deleted) - - } - - return deleted -} - -func DeleteServices(sess *session.Session, servicesToDelete []string) ([]string, error) { - if sugLog == nil { - initLogger() - } - logsClient := cloudwatchlogs.New(sess) - logGroups := getLogGroups(servicesToDelete, logsClient) - - sugLog.Infow("Attempting to delete subscription filters", - "servicesToDelete", servicesToDelete, - "logGroups", logGroups) - - if len(logGroups) > 0 { - newDeleted := DeleteSubscriptionFilter(logGroups, logsClient) - sugLog.Infow("Deleted subscription filters", - "deletedLogGroups", newDeleted) - return newDeleted, nil - } else { - sugLog.Info("No log groups found for deletion") - return nil, fmt.Errorf("Could not delete any log groups") - } -} - -func AddCustom(sess *session.Session, customGroup, added []string) ([]string, error) { - logsClient := cloudwatchlogs.New(sess) - toAdd := make([]string, 0) - lambdaNameTrigger := LambdaPrefix + os.Getenv(EnvFunctionName) - for _, customLogGroup := range customGroup { - if !ListContains(customLogGroup, added) { - // Prevent a situation where we put subscription filter on the trigger function - if customLogGroup != lambdaNameTrigger { - toAdd = append(toAdd, customLogGroup) - } - } - } - - newAdded := PutSubscriptionFilter(toAdd, logsClient) - - return newAdded, nil -} - -func DeleteCustom(sess *session.Session, customGroup []string) ([]string, error) { - if sugLog == nil { - initLogger() - } - logsClient := cloudwatchlogs.New(sess) - - newDeleted := DeleteSubscriptionFilter(customGroup, logsClient) - - // Log the outcome of the deletion attempts - sugLog.Infow("Deleted custom subscription filters", - "deletedCustomLogGroups", newDeleted) - - return newDeleted, nil -} diff --git a/common/utils.go b/common/utils.go index 14a3ef3..8c11bb8 100644 --- a/common/utils.go +++ b/common/utils.go @@ -1,190 +1,29 @@ package common import ( - "encoding/json" "fmt" - "github.com/aws/aws-secretsmanager-caching-go/secretcache" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" "os" - "regexp" - "strings" ) const ( - envServices = "SERVICES" - EnvAwsRegion = "AWS_REGION" // reserved env - EnvFunctionName = "AWS_LAMBDA_FUNCTION_NAME" // reserved env - envFirehoseArn = "FIREHOSE_ARN" - envAccountId = "ACCOUNT_ID" - EnvCustomGroups = "CUSTOM_GROUPS" - envSecretEnabled = "SECRET_ENABLED" - envAwsPartition = "AWS_PARTITION" - envPutSubscriptionFilterRole = "PUT_SF_ROLE" - - valuesSeparator = "," - emptyString = "" - LambdaPrefix = "/aws/lambda/" - subscriptionFilterName = "logzio_firehose" + EnvServices = "SERVICES" + EnvAwsRegion = "AWS_REGION" // reserved env + EnvCustomGroups = "CUSTOM_GROUPS" + EnvSecretEnabled = "SECRET_ENABLED" ) -func GetServices() []string { - servicesStr := os.Getenv(envServices) - if servicesStr == emptyString { - return nil - } - - servicesStr = strings.ReplaceAll(servicesStr, " ", "") - return strings.Split(servicesStr, valuesSeparator) -} - -func GetServicesMap() map[string]string { - return map[string]string{ - "apigateway": "/aws/apigateway/", - "rds": "/aws/rds/", - "cloudhsm": "/aws/cloudhsm/", - "cloudtrail": "aws-cloudtrail-logs-", - "codebuild": "/aws/codebuild/", - "connect": "/aws/connect/", - "elasticbeanstalk": "/aws/elasticbeanstalk/", - "ecs": "/aws/ecs/", - "eks": "/aws/eks/", - "aws-glue": "/aws/aws-glue/", - "aws-iot": "AWSIotLogsV2", - "lambda": "/aws/lambda/", - "macie": "/aws/macie/", - "amazon-mq": "/aws/amazonmq/broker/", - "batch": "/aws/batch/", - } -} - -// FindDifferences finds elements in 'new' that are not in 'old', and vice versa. -func FindDifferences(old, new []string) (toAdd, toRemove []string) { - oldSet := make(map[string]struct{}) - newSet := make(map[string]struct{}) - - // Populate 'oldSet' with elements from the 'old' slice. - for _, item := range old { - oldSet[item] = struct{}{} - } - - for _, item := range new { - newSet[item] = struct{}{} - } - - // Find elements in 'new' that are not in 'old' and add them to 'toAdd'. - for item := range newSet { - _, exists := oldSet[item] // Check if 'item' exists in 'oldSet' - if !exists { - toAdd = append(toAdd, item) - } - } - - for item := range oldSet { - _, exists := newSet[item] - if !exists { - toRemove = append(toRemove, item) - } - } - - return toAdd, toRemove -} - -// GetSecretNameFromArn extracts a secret name from the given secret ARN -func GetSecretNameFromArn(secretArn string) string { - var secretName string - if sugLog == nil { - initLogger() - } - - sugLog.Debugf("Attempting to extract secret name from ARN: '%s'", secretArn) - - getSecretName := regexp.MustCompile(fmt.Sprintf(`^arn:aws:secretsmanager:%s:%s:secret:(?P\S+)-`, os.Getenv(EnvAwsRegion), os.Getenv(envAccountId))) - match := getSecretName.FindStringSubmatch(secretArn) - - for i, key := range getSecretName.SubexpNames() { - if key == "secretName" && len(match) > i { - secretName = match[i] - break - } - } - - return secretName -} +func GetSession() (*session.Session, error) { + sess, err := session.NewSessionWithOptions(session.Options{ + Config: aws.Config{ + Region: aws.String(os.Getenv(EnvAwsRegion)), + }, + }) -// ExtractCustomGroupsFromSecret extracts the custom log groups to monitor from the given secret value -func ExtractCustomGroupsFromSecret(secretId, result string) (string, error) { - var secretValues map[string]string - err := json.Unmarshal([]byte(result), &secretValues) if err != nil { - return "", err - } - - customLogGroups, ok := secretValues["logzioCustomLogGroups"] - if !ok { - return "", fmt.Errorf("did not find logzioCustomLogGroups key in the secret %s", secretId) - } - return customLogGroups, nil -} - -// GetCustomLogGroups returns the monitored custom log groups. If a secret was used, extracts the value from it and returns it -func GetCustomLogGroups(secretEnabled, customLogGroupsPrmVal string) (string, error) { - if sugLog == nil { - initLogger() - } - if secretEnabled == "true" { - sugLog.Debug("Attempting to get custom log groups from secret parameter: ", customLogGroupsPrmVal) - secretCache, err := secretcache.New() - if err != nil { - return "", err - } - - secretName := GetSecretNameFromArn(customLogGroupsPrmVal) - - result, err := secretCache.GetSecretString(secretName) - if err != nil { - return "", err - } - - return ExtractCustomGroupsFromSecret(customLogGroupsPrmVal, result) - } - - return customLogGroupsPrmVal, nil -} - -func GetCustomPaths() []string { - if sugLog == nil { - initLogger() - } - pathsStr := os.Getenv(EnvCustomGroups) - secretEnabled := os.Getenv(envSecretEnabled) - if pathsStr == emptyString { - return nil - } - sugLog.Debug("Getting custom log groups with information; secret enabled: ", secretEnabled) - customLogGroupsStr, err := GetCustomLogGroups(secretEnabled, pathsStr) - if err != nil { - sugLog.Errorf("Failed to get custom log groups from secret due to %s", err.Error()) - return nil - } - - customLogGroupsStr = strings.ReplaceAll(customLogGroupsStr, " ", "") - return strings.Split(customLogGroupsStr, valuesSeparator) -} - -func ParseServices(servicesStr string) []string { - if servicesStr == emptyString { - return nil - } - - servicesStr = strings.ReplaceAll(servicesStr, " ", "") - return strings.Split(servicesStr, valuesSeparator) -} - -func ListContains(s string, l []string) bool { - for _, item := range l { - if s == item { - return true - } + return nil, fmt.Errorf("error occurred while trying to create a connection to aws: %s. Aborting", err.Error()) } - return false + return sess, nil } diff --git a/common/utils_test.go b/common/utils_test.go index 9d660cf..2c734d1 100644 --- a/common/utils_test.go +++ b/common/utils_test.go @@ -1,178 +1,13 @@ package common import ( + "github.com/aws/aws-sdk-go/aws/session" "github.com/stretchr/testify/assert" - "os" - "sort" "testing" ) -func TestGetServicesNoServices(t *testing.T) { - result := GetServices() - assert.Nil(t, result) -} - -func TestGetServices(t *testing.T) { - err := os.Setenv(envServices, "rds, cloudwatch, custom") - if err != nil { - return - } - - result := GetServices() - assert.Equal(t, []string{"rds", "cloudwatch", "custom"}, result) -} - -func TestGetServicesMap(t *testing.T) { - result := GetServicesMap() - assert.NotNil(t, result) -} - -func TestGetCustomPathsNoPaths(t *testing.T) { - result := GetCustomPaths() - assert.Nil(t, result) -} - -func TestGetCustomPaths(t *testing.T) { - err := os.Setenv(EnvCustomGroups, "rand, custom") - if err != nil { - return - } - - result := GetCustomPaths() - assert.Equal(t, []string{"rand", "custom"}, result) -} - -func TestParseServices(t *testing.T) { - tests := []struct { - name string - servicesStr string - expected []string - }{ - { - name: "no service", - servicesStr: "", - expected: nil, - }, - { - name: "single service", - servicesStr: "service", - expected: []string{"service"}, - }, - { - name: "multiple services", - servicesStr: "service, another, oneMore", - expected: []string{"service", "another", "oneMore"}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - result := ParseServices(test.servicesStr) - assert.Equal(t, test.expected, result, "Expected %v, got %v", test.expected, result) - }) - } -} - -func TestListContains(t *testing.T) { - tests := []struct { - name string - str string - lst []string - expected bool - }{ - { - name: "empty list", - str: "some string", - lst: []string{}, - expected: false, - }, - { - name: "string not in the list", - str: "item3", - lst: []string{"item1", "item2"}, - expected: false, - }, - { - name: "string in the list", - str: "item1", - lst: []string{"item1", "item2"}, - expected: true, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - result := ListContains(test.str, test.lst) - assert.Equal(t, test.expected, result, "Expected %v, got %v", test.expected, result) - }) - } -} - -func TestFindDifferences(t *testing.T) { - tests := []struct { - name string - old []string - new []string - expectedToAdd []string - expectedToRemove []string - }{ - { - name: "no differences", - old: []string{"service1", "service2"}, - new: []string{"service1", "service2"}, - expectedToAdd: []string(nil), - expectedToRemove: []string(nil), - }, - { - name: "delete all", - old: []string{"service1", "service2"}, - new: []string{}, - expectedToAdd: []string(nil), - expectedToRemove: []string{"service1", "service2"}, - }, - { - name: "add to empty", - old: []string{}, - new: []string{"service1", "service2"}, - expectedToAdd: []string{"service1", "service2"}, - expectedToRemove: []string(nil), - }, - { - name: "delete some and add others", - old: []string{"service1", "service2"}, - new: []string{"service1", "service3"}, - expectedToAdd: []string{"service3"}, - expectedToRemove: []string{"service2"}, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - resultToAdd, resultToRemove := FindDifferences(test.old, test.new) - sort.Strings(resultToAdd) - sort.Strings(resultToRemove) - assert.Equal(t, test.expectedToAdd, resultToAdd, "Expected %v, got %v", test.expectedToAdd, resultToAdd) - assert.Equal(t, test.expectedToRemove, resultToRemove, "Expected %v, got %v", test.expectedToRemove, resultToRemove) - }) - } -} - -func TestGetSecretNameFromArn(t *testing.T) { - err := os.Setenv(EnvAwsRegion, "us-east-1") - if err != nil { - return - } - err = os.Setenv(envAccountId, "486140753397") - if err != nil { - return - } - - arn := "arn:aws:secretsmanager:us-east-1:486140753397:secret:testSecretName-56y7ud" - assert.Equal(t, "testSecretName", GetSecretNameFromArn(arn)) - - arn = "arn:aws:secretsmanager:us-east-1:486140753397:secret:random-name-56y7ud" - assert.Equal(t, "random-name", GetSecretNameFromArn(arn)) - - arn = "arn:aws:secretsmanager:us-east-1:486140753397:secret:now1with2numbers345-56y7ud" - assert.Equal(t, "now1with2numbers345", GetSecretNameFromArn(arn)) +func TestGetSession(t *testing.T) { + result, err := GetSession() + assert.IsType(t, (*session.Session)(nil), result) + assert.Nil(t, err) } diff --git a/eventbridge-lambda/handler/handler.go b/eventbridge-lambda/handler/handler.go deleted file mode 100644 index a472ae5..0000000 --- a/eventbridge-lambda/handler/handler.go +++ /dev/null @@ -1,197 +0,0 @@ -package handler - -import ( - "context" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/secretsmanager" - "github.com/aws/aws-sdk-go/service/cloudwatchlogs" - "github.com/logzio/firehose-logs/common" - lp "github.com/logzio/firehose-logs/logger" - "go.uber.org/zap" - "os" - "sort" - "strings" -) - -var sugLog *zap.SugaredLogger - -func HandleEventBridgeRequest(ctx context.Context, event map[string]interface{}) (string, error) { - logger := lp.GetLogger() - defer logger.Sync() - sugLog = logger.Sugar() - sugLog.Info("Starting handling EventBridge event...") - sugLog.Debug("Handling event: ", event) - err := common.ValidateRequired() - if err != nil { - return "Lambda finished with error", err - } - - // Extracted EventBridge event handling logic - if detail, ok := event["detail"].(map[string]interface{}); ok { - eventName := detail["eventName"] - - switch eventName { - case "CreateLogGroup": - if requestParameters, ok := detail["requestParameters"].(map[string]interface{}); ok { - if logGroupName, ok := requestParameters["logGroupName"].(string); ok { - newLogGroupCreated(logGroupName) - } else { - sugLog.Debug("log group name is not of type string or missing from EventBridge event") - } - } else { - sugLog.Debug("request parameters is not of type map[string]interface{} or missing from EventBridge event.") - } - case "PutSecretValue": - if requestParameters, ok := detail["requestParameters"].(map[string]interface{}); ok { - if secretId, ok := requestParameters["secretId"].(string); ok { - secretName := os.Getenv(common.EnvCustomGroups) - // make sure the secret that changed is the relevant secret - if secretId == secretName { - err := updateSecretCustomLogGroups(ctx, secretId) - if err != nil { - return "", err - } - } - if strings.Contains(secretId, secretName) { - err := updateSecretCustomLogGroups(ctx, secretId) - if err != nil { - return "", err - } - } else { - sugLog.Debug("The EventBridge event secretId is not the secret that has custom log groups in it. Skipping it.") - } - } else { - sugLog.Debug("secretId is not of string or missing from EventBridge event.") - } - } else { - sugLog.Debug("requestParameters is not of type map[string]interface{} or missing from EventBridge event.") - } - default: - sugLog.Debugf("Detected unsupported event type %s", eventName) - } - } else { - sugLog.Debug("detail is not of type map[string]interface{} or missing from EventBridge event.") - } - - return "EventBridge event processed", nil -} - -func newLogGroupCreated(logGroup string) { - // Prevent a situation where we put subscription filter on the trigger function - if logGroup == common.LambdaPrefix+os.Getenv(common.EnvFunctionName) { - return - } - - servicesToAdd := common.GetServices() - var added []string - if servicesToAdd != nil { - serviceToPrefix := common.GetServicesMap() - sess, err := common.GetSession() - if err != nil { - sugLog.Error("Could not create aws session: ", err.Error()) - return - } - logsClient := cloudwatchlogs.New(sess) - for _, service := range servicesToAdd { - if prefix, ok := serviceToPrefix[service]; ok { - if strings.Contains(logGroup, prefix) { - added = common.PutSubscriptionFilter([]string{logGroup}, logsClient) - if len(added) > 0 { - sugLog.Info("Added log group: ", logGroup) - return - } - } - } - } - } - - sugLog.Info("Log group ", logGroup, " does not match any of the selected services: ", servicesToAdd) -} - -func getPreviousSecretVersion(ctx context.Context, svc *secretsmanager.Client, secretId string) (*string, error) { - var latestOldVersionId *string - - listSecretVersionsInput := &secretsmanager.ListSecretVersionIdsInput{ - SecretId: &secretId, - } - - secretInfo, err := svc.ListSecretVersionIds(ctx, listSecretVersionsInput) - if err != nil { - sugLog.Error("Failed to list secret versions to update the monitored custom log groups") - return nil, err - } - - // Sort the versions based on created date - sort.Slice(secretInfo.Versions, func(i, j int) bool { - return secretInfo.Versions[i].CreatedDate.After(*secretInfo.Versions[j].CreatedDate) - }) - - if len(secretInfo.Versions) > 1 { - latestOldVersionId = secretInfo.Versions[1].VersionId - } else { - sugLog.Warn("Custom log groups secret doesn't have older version to apply changes in comparison to.") - } - - return latestOldVersionId, nil -} - -func getOldSecretValue(ctx context.Context, svc *secretsmanager.Client, secretId string, oldVersionId *string) (string, error) { - // get the old version value - getOldSecretValueInput := &secretsmanager.GetSecretValueInput{ - SecretId: &secretId, - VersionId: oldVersionId, - } - - oldSecret, err := svc.GetSecretValue(ctx, getOldSecretValueInput) - if err != nil { - sugLog.Error("Failed to get the old value of the custom log groups secret") - return "", err - } - oldSecretValueJson := oldSecret.SecretString - - return common.ExtractCustomGroupsFromSecret(secretId, *oldSecretValueJson) -} - -func updateSecretCustomLogGroups(ctx context.Context, secretId string) error { - // handle the event; get last version >> update according to it. - awsConf, err := config.LoadDefaultConfig(ctx, config.WithRegion(os.Getenv(common.EnvAwsRegion))) - if err != nil { - sugLog.Error("Failed to setup connection to get older custom log groups secret values.") - return err - } - svc := secretsmanager.NewFromConfig(awsConf) - - oldVersionId, err := getPreviousSecretVersion(ctx, svc, secretId) - if err != nil { - sugLog.Error("Failed to get the older custom log group secret version.") - return err - } - - oldSecretValueStr, err := getOldSecretValue(ctx, svc, secretId, oldVersionId) - if err != nil { - sugLog.Error("Failed to get the old custom log group secret version's value.") - return err - } - - newSecretValueStr, err := common.GetCustomLogGroups("true", secretId) - if err != nil { - sugLog.Error("Failed to get the current custom log group secret value.") - return err - } - - oldSecretValue := common.ParseServices(oldSecretValueStr) - newSecretValue := common.ParseServices(newSecretValueStr) - customGroupsToAdd, customGroupsToRemove := common.FindDifferences(oldSecretValue, newSecretValue) - - sess, err := common.GetSession() - if err != nil { - sugLog.Error("Error while creating session: ", err.Error()) - return err - } - - if err := common.UpdateSubscriptionFilters(sess, []string{}, []string{}, customGroupsToAdd, customGroupsToRemove); err != nil { - sugLog.Errorf("Error updating subscription filters: %v", err) - return err - } - return nil -} diff --git a/eventbridge-lambda/handler/handler_test.go b/eventbridge-lambda/handler/handler_test.go deleted file mode 100644 index 2647722..0000000 --- a/eventbridge-lambda/handler/handler_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package handler - -import ( - "context" - "github.com/stretchr/testify/assert" - "os" - "testing" -) - -func setup(includeDetail bool) (ctx context.Context, event map[string]interface{}) { - /* Setup needed env variables */ - err := os.Setenv("FIREHOSE_ARN", "test-arn") - if err != nil { - return - } - err = os.Setenv("ACCOUNT_ID", "aws-account-id") - if err != nil { - return - } - err = os.Setenv("AWS_PARTITION", "test-partition") - if err != nil { - return - } - - /* Setup mock context and event */ - mockEvent := map[string]interface{}{ - "version": "0", - "id": "12345678-1234-5678-1234-567812345678", - "detail-type": "MyCustomEvent", - "source": "my.custom.source", - "account": "123456789012", - "time": "2024-06-20T12:00:00Z", - "region": "us-west-2", - "resources": []string{ - "resource1", - "resource2", - }, - } - if includeDetail { - mockEvent["detail"] = map[string]interface{}{ - "key1": "value1", - "key2": "value2", - "nestedObject": map[string]interface{}{ - "nestedKey": "nestedValue", - }, - "requestParameters": map[string]interface{}{ - "logGroupName": "my-log-group", - }, - } - } - ctx = context.Background() - - return ctx, mockEvent -} - -func TestHandleNewLogGroupCreatedNoServices(t *testing.T) { - ctx, mockEvent := setup(true) - result, err := HandleEventBridgeRequest(ctx, mockEvent) - assert.Equal(t, "EventBridge event processed", result) - assert.Nil(t, err) -} - -func TestHandleNewLogGroupCreated(t *testing.T) { - err := os.Setenv("SERVICES", "rds, cloudtrail") - if err != nil { - return - } - ctx, mockEvent := setup(true) - result, err := HandleEventBridgeRequest(ctx, mockEvent) - assert.Equal(t, "EventBridge event processed", result) - assert.Nil(t, err) -} - -func TestHandleNoDetail(t *testing.T) { - ctx, mockEvent := setup(false) - result, err := HandleEventBridgeRequest(ctx, mockEvent) - assert.Equal(t, "EventBridge event processed", result) - assert.Nil(t, err) -} diff --git a/eventbridge-lambda/main.go b/eventbridge-lambda/main.go deleted file mode 100644 index 7955d05..0000000 --- a/eventbridge-lambda/main.go +++ /dev/null @@ -1,10 +0,0 @@ -package main - -import ( - "github.com/aws/aws-lambda-go/lambda" - handler "github.com/logzio/firehose-logs/eventbridge-lambda/handler" -) - -func main() { - lambda.Start(handler.HandleEventBridgeRequest) -} diff --git a/go.mod b/go.mod index 6b63850..ae13b13 100644 --- a/go.mod +++ b/go.mod @@ -1,33 +1,36 @@ module github.com/logzio/firehose-logs -go 1.19 +go 1.22 require ( github.com/aws/aws-lambda-go v1.47.0 - github.com/aws/aws-sdk-go v1.54.16 - github.com/aws/aws-sdk-go-v2/config v1.27.24 - github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.32.1 + github.com/aws/aws-sdk-go v1.55.5 + github.com/aws/aws-sdk-go-v2/config v1.27.43 + github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2 github.com/aws/aws-secretsmanager-caching-go v1.2.0 + github.com/hashicorp/go-multierror v1.1.1 github.com/stretchr/testify v1.9.0 go.uber.org/zap v1.27.0 ) require ( - github.com/aws/aws-sdk-go-v2 v1.30.1 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.17.24 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 // indirect - github.com/aws/smithy-go v1.20.3 // indirect + github.com/aws/aws-sdk-go-v2 v1.32.2 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 // indirect + github.com/aws/smithy-go v1.22.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 9f70994..37a5c27 100644 --- a/go.sum +++ b/go.sum @@ -1,41 +1,46 @@ github.com/aws/aws-lambda-go v1.47.0 h1:0H8s0vumYx/YKs4sE7YM0ktwL2eWse+kfopsRI1sXVI= github.com/aws/aws-lambda-go v1.47.0/go.mod h1:dpMpZgvWx5vuQJfBt0zqBha60q7Dd7RfgJv23DymV8A= github.com/aws/aws-sdk-go v1.47.10/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/aws/aws-sdk-go v1.54.16 h1:+B9zGaVwOUU6AO9Sy99VjTMDPthWx10HjB08hjaBHIc= -github.com/aws/aws-sdk-go v1.54.16/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/aws/aws-sdk-go-v2 v1.30.1 h1:4y/5Dvfrhd1MxRDD77SrfsDaj8kUkkljU7XE83NPV+o= -github.com/aws/aws-sdk-go-v2 v1.30.1/go.mod h1:nIQjQVp5sfpQcTc9mPSr1B0PaWK5ByX9MOoDadSN4lc= -github.com/aws/aws-sdk-go-v2/config v1.27.24 h1:NM9XicZ5o1CBU/MZaHwFtimRpWx9ohAUAqkG6AqSqPo= -github.com/aws/aws-sdk-go-v2/config v1.27.24/go.mod h1:aXzi6QJTuQRVVusAO8/NxpdTeTyr/wRcybdDtfUwJSs= -github.com/aws/aws-sdk-go-v2/credentials v1.17.24 h1:YclAsrnb1/GTQNt2nzv+756Iw4mF8AOzcDfweWwwm/M= -github.com/aws/aws-sdk-go-v2/credentials v1.17.24/go.mod h1:Hld7tmnAkoBQdTMNYZGzztzKRdA4fCdn9L83LOoigac= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9 h1:Aznqksmd6Rfv2HQN9cpqIV/lQRMaIpJkLLaJ1ZI76no= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.9/go.mod h1:WQr3MY7AxGNxaqAtsDWn+fBxmd4XvLkzeqQ8P1VM0/w= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13 h1:5SAoZ4jYpGH4721ZNoS1znQrhOfZinOhc4XuTXx/nVc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.13/go.mod h1:+rdA6ZLpaSeM7tSg/B0IEDinCIBJGmW8rKDFkYpP04g= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13 h1:WIijqeaAO7TYFLbhsZmi2rgLEAtWOC1LhxCAVTJlSKw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.13/go.mod h1:i+kbfa76PQbWw/ULoWnp51EYVWH4ENln76fLQE3lXT8= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3 h1:dT3MqvGhSoaIhRseqw2I0yH81l7wiR2vjs57O51EAm8= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.3/go.mod h1:GlAeCkHwugxdHaueRr4nhPuY+WW+gR8UjlcqzPr1SPI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15 h1:I9zMeF107l0rJrpnHpjEiiTSCKYAIw8mALiXcPsGBiA= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.15/go.mod h1:9xWJ3Q/S6Ojusz1UIkfycgD1mGirJfLLKqq3LPT7WN8= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.32.1 h1:ZoYRD8IJqPkzjBnpokiMNO6L/DQprtpVpD6k0YSaF5U= -github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.32.1/go.mod h1:GlRarZzIMl9VDi0mLQt+qQOuEkVFPnTkkjyugV1uVa8= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.1 h1:p1GahKIjyMDZtiKoIn0/jAj/TkMzfzndDv5+zi2Mhgc= -github.com/aws/aws-sdk-go-v2/service/sso v1.22.1/go.mod h1:/vWdhoIoYA5hYoPZ6fm7Sv4d8701PiG5VKe8/pPJL60= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2 h1:ORnrOK0C4WmYV/uYt3koHEWBLYsRDwk2Np+eEoyV4Z0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.26.2/go.mod h1:xyFHA4zGxgYkdD73VeezHt3vSKEG9EmFnGwoKlP00u4= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.1 h1:+woJ607dllHJQtsnJLi52ycuqHMwlW+Wqm2Ppsfp4nQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.30.1/go.mod h1:jiNR3JqT15Dm+QWq2SRgh0x0bCNSRP2L25+CqPNpJlQ= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go-v2 v1.32.2 h1:AkNLZEyYMLnx/Q/mSKkcMqwNFXMAvFto9bNsHqcTduI= +github.com/aws/aws-sdk-go-v2 v1.32.2/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2/config v1.27.43 h1:p33fDDihFC390dhhuv8nOmX419wjOSDQRb+USt20RrU= +github.com/aws/aws-sdk-go-v2/config v1.27.43/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 h1:UAsR3xA31QGf79WzpG/ixT9FZvQlh5HY1NRqSHBNOCk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21/go.mod h1:JNr43NFf5L9YaG3eKTm7HQzls9J+A9YYcGI5Quh1r2Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 h1:6jZVETqmYCadGFvrYEQfC5fAQmlo80CeL5psbno6r0s= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21/go.mod h1:1SR0GbLlnN3QUmYaflZNiH1ql+1qrSiB2vwcJ+4UM60= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2 h1:s7NA1SOw8q/5c0wr8477yOPp0z+uBaXBnLE0XYb0POA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.2/go.mod h1:fnjjWyAW/Pj5HYOxl9LJqWtEwS7W2qgcRLWP+uWbss0= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2 h1:Rrqru2wYkKQCS2IM5/JrgKUQIoNTqA6y/iuxkjzxC6M= +github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2/go.mod h1:QuCURO98Sqee2AXmqDNxKXYFm2OEDAVAPApMqO0Vqnc= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= github.com/aws/aws-secretsmanager-caching-go v1.2.0 h1:gUA+CVKvFLj4OUSknhIrnt4dF7Y37+JrChKqfaehJME= github.com/aws/aws-secretsmanager-caching-go v1.2.0/go.mod h1:6t2/zQIsigFMlnpOdGj503Dgaz24tMqIRhass9uoTBo= -github.com/aws/smithy-go v1.20.3 h1:ryHwveWzPV5BIof6fyDvor6V3iUL7nTfiTKXHiW05nE= -github.com/aws/smithy-go v1.20.3/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC3qS14E= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -43,10 +48,13 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= diff --git a/log-group-events-lambda/handler/config.go b/log-group-events-lambda/handler/config.go new file mode 100644 index 0000000..3a65e26 --- /dev/null +++ b/log-group-events-lambda/handler/config.go @@ -0,0 +1,56 @@ +package handler + +import ( + "fmt" + "github.com/logzio/firehose-logs/common" + "os" +) + +type Config struct { + awsPartition string + destinationArn string + roleArn string + accountId string + region string + thisFunctionLogGroup string + thisFunctionName string + customGroupsValue string + servicesValue string +} + +func NewConfig() *Config { + c := Config{ + awsPartition: os.Getenv(envAwsPartition), + destinationArn: os.Getenv(envFirehoseArn), + roleArn: os.Getenv(envPutSubscriptionFilterRole), + accountId: os.Getenv(envAccountId), + region: os.Getenv(common.EnvAwsRegion), + thisFunctionLogGroup: lambdaPrefix + os.Getenv(envFunctionName), + thisFunctionName: os.Getenv(envFunctionName), + customGroupsValue: os.Getenv(common.EnvCustomGroups), + servicesValue: os.Getenv(common.EnvServices), + } + + err := c.validateRequired() + if err != nil { + sugLog.Error("Error while validating required environment variables: ", err) + return nil + } + return &c +} + +func (c *Config) validateRequired() error { + if c.destinationArn == emptyString { + return fmt.Errorf("destination ARN must be set") + } + + if c.accountId == emptyString { + return fmt.Errorf("account id must be set") + } + + if c.awsPartition == emptyString { + return fmt.Errorf("aws partition must be set") + } + + return nil +} diff --git a/log-group-events-lambda/handler/config_test.go b/log-group-events-lambda/handler/config_test.go new file mode 100644 index 0000000..033303c --- /dev/null +++ b/log-group-events-lambda/handler/config_test.go @@ -0,0 +1,152 @@ +package handler + +import ( + "fmt" + "github.com/logzio/firehose-logs/logger" + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +func InitConfigTest() { + sugLog = logger.GetSugaredLogger() +} + +func TestNewConfigMissingEnv(t *testing.T) { + InitConfigTest() + conf := NewConfig() + assert.Nil(t, conf) +} + +func TestNewConfigValidRequired(t *testing.T) { + InitConfigTest() + + /* Setup required env variable */ + err := os.Setenv(envFirehoseArn, "test-arn") + if err != nil { + return + } + + err = os.Setenv(envAccountId, "aws-account-id") + if err != nil { + return + } + + err = os.Setenv(envAwsPartition, "test-partition") + if err != nil { + return + } + + conf := NewConfig() + assert.NotNil(t, conf) + assert.Equal(t, "test-arn", conf.destinationArn) + assert.Equal(t, "aws-account-id", conf.accountId) + assert.Equal(t, "", conf.region) + assert.Equal(t, "/aws/lambda/", conf.thisFunctionLogGroup) + assert.Equal(t, "", conf.thisFunctionName) + assert.Equal(t, "", conf.customGroupsValue) + assert.Equal(t, "", conf.servicesValue) +} + +func TestNewConfigValid(t *testing.T) { + InitConfigTest() + + /* Setup env variable */ + err := os.Setenv(envFirehoseArn, "test-arn") + if err != nil { + return + } + + err = os.Setenv(envAccountId, "aws-account-id") + if err != nil { + return + } + + err = os.Setenv(envAwsPartition, "test-partition") + if err != nil { + return + } + + err = os.Setenv(envFunctionName, "g2") + if err != nil { + return + } + + fmt.Println("test3") + + conf := NewConfig() + assert.NotNil(t, conf) + assert.Equal(t, "test-arn", conf.destinationArn) + assert.Equal(t, "aws-account-id", conf.accountId) + assert.Equal(t, "/aws/lambda/g2", conf.thisFunctionLogGroup) + assert.Equal(t, "g2", conf.thisFunctionName) + assert.Equal(t, "", conf.customGroupsValue) + assert.Equal(t, "", conf.servicesValue) + assert.Equal(t, "", conf.region) +} + +func TestValidateRequired(t *testing.T) { + /* Setup tests */ + InitConfigTest() + + tests := []struct { + name string + conf Config + expectedError bool + errorStr string + }{ + { + name: "missing all 3 required", + conf: Config{ + awsPartition: "", + destinationArn: "", + accountId: "", + }, + expectedError: true, + errorStr: "destination ARN must be set", + }, + { + name: "missing 2 required", + conf: Config{ + awsPartition: "partition", + destinationArn: "", + accountId: "", + }, + expectedError: true, + errorStr: "destination ARN must be set", + }, + { + name: "missing 1 required", + conf: Config{ + awsPartition: "partition", + destinationArn: "some-arn", + accountId: "", + }, + expectedError: true, + errorStr: "account id must be set", + }, + { + name: "valid", + conf: Config{ + awsPartition: "partition", + destinationArn: "some-arn", + accountId: "accountId", + }, + expectedError: false, + errorStr: "", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := test.conf.validateRequired() + if test.expectedError { + assert.NotNil(t, result) + assert.Equal(t, test.errorStr, result.Error()) + } else { + assert.Nil(t, result) + } + }) + + } +} diff --git a/log-group-events-lambda/handler/constants.go b/log-group-events-lambda/handler/constants.go new file mode 100644 index 0000000..9dbba37 --- /dev/null +++ b/log-group-events-lambda/handler/constants.go @@ -0,0 +1,16 @@ +package handler + +const ( + envFunctionName = "AWS_LAMBDA_FUNCTION_NAME" // reserved env + envAccountId = "ACCOUNT_ID" + envFirehoseArn = "FIREHOSE_ARN" + envAwsPartition = "AWS_PARTITION" + envPutSubscriptionFilterRole = "PUT_SF_ROLE" + + logzioSecretKeyName = "logzioCustomLogGroups" + valuesSeparator = "," + emptyString = "" + lambdaPrefix = "/aws/lambda/" + subscriptionFilterName = "logzio_firehose" + maxRetries = 10 +) diff --git a/log-group-events-lambda/handler/cw_ops.go b/log-group-events-lambda/handler/cw_ops.go new file mode 100644 index 0000000..3fc8b34 --- /dev/null +++ b/log-group-events-lambda/handler/cw_ops.go @@ -0,0 +1,198 @@ +package handler + +import ( + "errors" + "fmt" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" + "github.com/hashicorp/go-multierror" + "github.com/logzio/firehose-logs/common" + "sync" + "time" +) + +type CloudWatchLogsClient struct { + Client cloudwatchlogsiface.CloudWatchLogsAPI + Mutex sync.Mutex +} + +func getCloudWatchLogsClient() (*CloudWatchLogsClient, error) { + sess, err := common.GetSession() + if err != nil { + return nil, err + } + return &CloudWatchLogsClient{Client: cloudwatchlogs.New(sess)}, nil +} + +func (cwLogsClient *CloudWatchLogsClient) addSubscriptionFilter(logGroups []string) ([]string, error) { + if cwLogsClient == nil { + return nil, fmt.Errorf("CloudWatch Logs client is nil") + } + + destinationArn := envConfig.destinationArn + roleArn := envConfig.roleArn + filterPattern := "" + filterName := subscriptionFilterName + added := make([]string, 0, len(logGroups)) + var result *multierror.Error + + var wg sync.WaitGroup + + for _, logGroup := range logGroups { + // Prevent a situation where we put subscription filter on the trigger function + if logGroup == envConfig.thisFunctionLogGroup { + continue + } + + wg.Add(1) + go func(logGroup string) { + defer wg.Done() + + retries := 0 + for { + _, err := cwLogsClient.Client.PutSubscriptionFilter(&cloudwatchlogs.PutSubscriptionFilterInput{ + DestinationArn: &destinationArn, + FilterName: &filterName, + LogGroupName: &logGroup, + FilterPattern: &filterPattern, + RoleArn: &roleArn, + }) + + // retry mechanism + if err != nil { + var awsErr awserr.Error + ok := errors.As(err, &awsErr) + if ok && awsErr.Code() == "ThrottlingException" && retries < maxRetries { + time.Sleep(time.Second * time.Duration(retries*retries)) + retries++ + continue + } else { + sugLog.Errorf("Error while trying to add subscription filter for %s: %v", logGroup, err.Error()) + result = multierror.Append(result, err) + return + } + } + cwLogsClient.Mutex.Lock() + added = append(added, logGroup) + cwLogsClient.Mutex.Unlock() + return + } + }(logGroup) + } + wg.Wait() + + return added, result.ErrorOrNil() +} + +func (cwLogsClient *CloudWatchLogsClient) updateSubscriptionFilters(servicesToAdd, servicesToRemove, customGroupsToAdd, customGroupsToRemove []string) error { + var result *multierror.Error + + logGroupsToMonitor := getServicesLogGroups(servicesToAdd, cwLogsClient) + logGroupsToMonitor = append(logGroupsToMonitor, customGroupsToAdd...) + + if len(logGroupsToMonitor) > 0 { + added, err := cwLogsClient.addSubscriptionFilter(logGroupsToMonitor) + if err != nil { + result = multierror.Append(result, err) + } + sugLog.Info("Added subscription filters for the following log groups: ", added) + } else { + sugLog.Debug("No new log groups to monitor") + } + + logGroupsToUnMonitor := getServicesLogGroups(servicesToRemove, cwLogsClient) + logGroupsToUnMonitor = append(logGroupsToUnMonitor, customGroupsToRemove...) + + if len(logGroupsToUnMonitor) > 0 { + deleted, err := cwLogsClient.removeSubscriptionFilter(logGroupsToUnMonitor) + if err != nil { + result = multierror.Append(result, err) + } + sugLog.Info("Deleted subscription filters for the following log groups: ", deleted) + } else { + sugLog.Debug("No log groups to stop monitoring") + } + + return result.ErrorOrNil() +} + +func (cwLogsClient *CloudWatchLogsClient) removeSubscriptionFilter(logGroups []string) ([]string, error) { + if cwLogsClient == nil { + return nil, fmt.Errorf("CloudWatch Logs client is nil") + } + + filterName := subscriptionFilterName + deleted := make([]string, 0, len(logGroups)) + var result *multierror.Error + + var wg sync.WaitGroup + + for _, logGroup := range logGroups { + wg.Add(1) + go func(logGroup string) { + defer wg.Done() + + retries := 0 + for { + _, err := cwLogsClient.Client.DeleteSubscriptionFilter(&cloudwatchlogs.DeleteSubscriptionFilterInput{ + FilterName: &filterName, + LogGroupName: &logGroup, + }) + + // retry mechanism + if err != nil { + var awsErr awserr.Error + ok := errors.As(err, &awsErr) + if ok && awsErr.Code() == "ThrottlingException" && retries < maxRetries { + time.Sleep(time.Second * time.Duration(retries*retries)) + retries++ + continue + } else { + sugLog.Errorf("Error while trying to delete subscription filter for %s: %v", logGroup, err.Error()) + result = multierror.Append(result, err) + return + } + } + cwLogsClient.Mutex.Lock() + deleted = append(deleted, logGroup) + cwLogsClient.Mutex.Unlock() + return + } + }(logGroup) + } + wg.Wait() + + return deleted, result.ErrorOrNil() +} + +// getLogGroupsWithPrefix returns a list of log groups with the given prefix from cw client +func (cwLogsClient *CloudWatchLogsClient) getLogGroupsWithPrefix(prefix string) ([]string, error) { + var nextToken *string + logGroups := make([]string, 0) + for { + describeOutput, err := cwLogsClient.Client.DescribeLogGroups(&cloudwatchlogs.DescribeLogGroupsInput{ + LogGroupNamePrefix: &prefix, + NextToken: nextToken, + }) + + if err != nil { + return nil, err + } + if describeOutput != nil { + nextToken = describeOutput.NextToken + for _, logGroup := range describeOutput.LogGroups { + // Prevent a situation where we put subscription filter on the trigger and shipper function + if *logGroup.LogGroupName != envConfig.thisFunctionLogGroup { + logGroups = append(logGroups, *logGroup.LogGroupName) + } + } + } + + if nextToken == nil { + break + } + } + + return logGroups, nil +} diff --git a/log-group-events-lambda/handler/cw_ops_test.go b/log-group-events-lambda/handler/cw_ops_test.go new file mode 100644 index 0000000..5f3dd0b --- /dev/null +++ b/log-group-events-lambda/handler/cw_ops_test.go @@ -0,0 +1,331 @@ +package handler + +import ( + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs" + "github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface" + lp "github.com/logzio/firehose-logs/logger" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "os" + "sort" + "testing" +) + +type MockCloudWatchLogsClient struct { + mock.Mock + cloudwatchlogsiface.CloudWatchLogsAPI +} + +func (m *MockCloudWatchLogsClient) PutSubscriptionFilter(input *cloudwatchlogs.PutSubscriptionFilterInput) (*cloudwatchlogs.PutSubscriptionFilterOutput, error) { + if *input.LogGroupName == "errorGroup" { + return nil, fmt.Errorf("an error occurred") + } + + args := m.Called(input) + return args.Get(0).(*cloudwatchlogs.PutSubscriptionFilterOutput), args.Error(1) +} + +func (m *MockCloudWatchLogsClient) DeleteSubscriptionFilter(input *cloudwatchlogs.DeleteSubscriptionFilterInput) (*cloudwatchlogs.DeleteSubscriptionFilterOutput, error) { + if *input.LogGroupName == "errorGroup" { + return nil, fmt.Errorf("an error occurred") + } + + args := m.Called(input) + return args.Get(0).(*cloudwatchlogs.DeleteSubscriptionFilterOutput), args.Error(1) +} + +func (m *MockCloudWatchLogsClient) DescribeLogGroups(input *cloudwatchlogs.DescribeLogGroupsInput) (*cloudwatchlogs.DescribeLogGroupsOutput, error) { + switch *input.LogGroupNamePrefix { + case "/aws/apigateway/": + return &cloudwatchlogs.DescribeLogGroupsOutput{ + LogGroups: []*cloudwatchlogs.LogGroup{ + { + LogGroupName: aws.String("/aws/apigateway/g1"), + }}, + }, nil + case "/aws/lambda/": + return &cloudwatchlogs.DescribeLogGroupsOutput{ + LogGroups: []*cloudwatchlogs.LogGroup{{ + LogGroupName: aws.String("/aws/lambda/g1"), + }, + { + LogGroupName: aws.String("/aws/lambda/g2"), + }}, + }, nil + case "/aws/codebuild/": + return &cloudwatchlogs.DescribeLogGroupsOutput{ + LogGroups: []*cloudwatchlogs.LogGroup{}, + }, nil + case "/log/group1/": + return &cloudwatchlogs.DescribeLogGroupsOutput{ + LogGroups: []*cloudwatchlogs.LogGroup{{ + LogGroupName: aws.String("/log/group1/a"), + }, + { + LogGroupName: aws.String("/log/group1/b"), + }}, + }, nil + case "/log/group2/": + return &cloudwatchlogs.DescribeLogGroupsOutput{ + LogGroups: []*cloudwatchlogs.LogGroup{ + { + LogGroupName: aws.String("/log/group2/a"), + }}, + }, nil + case "/aws/error/test/": + return &cloudwatchlogs.DescribeLogGroupsOutput{ + LogGroups: []*cloudwatchlogs.LogGroup{}, + }, fmt.Errorf("an error occurred") + default: + return &cloudwatchlogs.DescribeLogGroupsOutput{ + LogGroups: []*cloudwatchlogs.LogGroup{{ + LogGroupName: aws.String("/random/log/group"), + }}, + }, nil + } +} + +func setupSFTest() { + err := os.Setenv(envFirehoseArn, "test-arn") + if err != nil { + return + } + + err = os.Setenv(envAccountId, "aws-account-id") + if err != nil { + return + } + + err = os.Setenv(envAwsPartition, "test-partition") + if err != nil { + return + } + + /* Setup config */ + envConfig = NewConfig() + + /* Setup logger */ + sugLog = lp.GetSugaredLogger() +} + +func TestAddSubscriptionFilter(t *testing.T) { + setupSFTest() + + tests := []struct { + name string + logGroups []string + expectedAdded []string + errorExpected bool + }{ + { + name: "All successful", + logGroups: []string{"group1", "group2"}, + expectedAdded: []string{"group1", "group2"}, + errorExpected: false, + }, + { + name: "Error on one group", + logGroups: []string{"group1", "errorGroup"}, + expectedAdded: []string{"group1"}, + errorExpected: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mockClient := new(MockCloudWatchLogsClient) + mockClient.On("PutSubscriptionFilter", mock.Anything).Return(&cloudwatchlogs.PutSubscriptionFilterOutput{}, nil).Times(len(test.logGroups)) + mockClient.On("PutSubscriptionFilter", mock.MatchedBy(func(input *cloudwatchlogs.PutSubscriptionFilterInput) bool { + return *input.LogGroupName == "errorGroup" + })).Return(nil, fmt.Errorf("an error occurred")) + + cwClient := &CloudWatchLogsClient{Client: mockClient} + added, err := cwClient.addSubscriptionFilter(test.logGroups) + sort.Strings(added) + + assert.Equal(t, test.expectedAdded, added, "Expected log groups to be added %v but got %v", test.expectedAdded, added) + + if test.errorExpected { + assert.NotNil(t, err, "Expected an error but got nil") + } else { + assert.Nil(t, err, "Expected error to be nil but got %v", err) + } + }) + } +} + +func TestUpdateSubscriptionFilters(t *testing.T) { + setupSFTest() + + tests := []struct { + name string + servicesToAdd []string + servicesToRemove []string + customGroupsToAdd []string + customGroupsToRemove []string + errorExpected bool + }{ + { + name: "new to add and delete", + servicesToAdd: []string{"apigateway", "lambda"}, + servicesToRemove: []string{"rds"}, + customGroupsToAdd: []string{"group1"}, + customGroupsToRemove: []string{"group2"}, + errorExpected: false, + }, + { + name: "no new to add", + servicesToAdd: []string{}, + servicesToRemove: []string{"lambda"}, + customGroupsToAdd: []string{}, + customGroupsToRemove: []string{"group2"}, + errorExpected: false, + }, + { + name: "no new to delete", + servicesToAdd: []string{"apigateway", "rds"}, + servicesToRemove: []string{}, + customGroupsToAdd: []string{"group1"}, + customGroupsToRemove: []string{}, + errorExpected: false, + }, + { + name: "error in add", + servicesToAdd: []string{}, + servicesToRemove: []string{}, + customGroupsToAdd: []string{"errorGroup"}, + customGroupsToRemove: []string{}, + errorExpected: true, + }, + { + name: "error in delete", + servicesToAdd: []string{}, + servicesToRemove: []string{}, + customGroupsToAdd: []string{}, + customGroupsToRemove: []string{"errorGroup"}, + errorExpected: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mockClient := new(MockCloudWatchLogsClient) + mockClient.On("PutSubscriptionFilter", mock.Anything).Return(&cloudwatchlogs.PutSubscriptionFilterOutput{}, nil) + mockClient.On("PutSubscriptionFilter", mock.MatchedBy(func(input *cloudwatchlogs.PutSubscriptionFilterInput) bool { + return *input.LogGroupName == "errorGroup" + })).Return(nil, fmt.Errorf("an error occurred")) + + mockClient.On("DeleteSubscriptionFilter", mock.Anything).Return(&cloudwatchlogs.DeleteSubscriptionFilterOutput{}, nil) + mockClient.On("DeleteSubscriptionFilter", mock.MatchedBy(func(input *cloudwatchlogs.DeleteSubscriptionFilterInput) bool { + return *input.LogGroupName == "errorGroup" + })).Return(nil, fmt.Errorf("an error occurred")) + + cwClient := &CloudWatchLogsClient{Client: mockClient} + err := cwClient.updateSubscriptionFilters(test.servicesToAdd, test.servicesToRemove, test.customGroupsToAdd, test.customGroupsToRemove) + + if test.errorExpected { + assert.NotNil(t, err, "Expected an error but got nil") + } else { + assert.Nil(t, err, "Expected error to be nil but got %v", err) + } + }) + } +} + +func TestDeleteSubscriptionFilters(t *testing.T) { + setupSFTest() + + tests := []struct { + name string + logGroups []string + expectedRemove []string + errorExpected bool + }{ + { + name: "All successful", + logGroups: []string{"group1", "group2"}, + expectedRemove: []string{"group1", "group2"}, + errorExpected: false, + }, + { + name: "error on one group", + logGroups: []string{"group1", "errorGroup"}, + expectedRemove: []string{"group1"}, + errorExpected: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mockClient := new(MockCloudWatchLogsClient) + mockClient.On("DeleteSubscriptionFilter", mock.Anything).Return(&cloudwatchlogs.DeleteSubscriptionFilterOutput{}, nil).Times(len(test.logGroups)) + mockClient.On("DeleteSubscriptionFilter", mock.MatchedBy(func(input *cloudwatchlogs.DeleteSubscriptionFilterInput) bool { + return *input.LogGroupName == "errorGroup" + })).Return(nil, fmt.Errorf("an error occurred")) + + cwClient := &CloudWatchLogsClient{Client: mockClient} + removed, err := cwClient.removeSubscriptionFilter(test.logGroups) + sort.Strings(removed) + + assert.Equal(t, test.expectedRemove, removed, "Expected log groups to be removed %v but got %v", test.expectedRemove, removed) + + if test.errorExpected { + assert.NotNil(t, err, "Expected an error but got nil") + } else { + assert.Nil(t, err, "Expected error to be nil but got %v", err) + } + }) + } +} + +func TestGetLogGroupsWithPrefix(t *testing.T) { + cwClient, _ := setupLGTest() + + tests := []struct { + name string + prefix string + expectedGroups []string + expectedError bool + }{ + { + name: "some prefix", + prefix: "/aws/apigateway/", + expectedGroups: []string{"/aws/apigateway/g1"}, + expectedError: false, + }, + { + name: "no log groups", + prefix: "/aws/codebuild/", + expectedGroups: []string{}, + expectedError: false, + }, + { + name: "don't return this function log group", + prefix: "/aws/lambda/", + expectedGroups: []string{"/aws/lambda/g1"}, + expectedError: false, + }, + { + name: "failed to get log groups", + prefix: "/aws/error/test/", + expectedGroups: nil, + expectedError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := cwClient.getLogGroupsWithPrefix(test.prefix) + sort.Strings(result) + assert.Equal(t, test.expectedGroups, result) + + if test.expectedError { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} diff --git a/log-group-events-lambda/handler/handler.go b/log-group-events-lambda/handler/handler.go new file mode 100644 index 0000000..731e05f --- /dev/null +++ b/log-group-events-lambda/handler/handler.go @@ -0,0 +1,238 @@ +package handler + +import ( + "context" + "fmt" + "github.com/logzio/firehose-logs/common" + "github.com/logzio/firehose-logs/logger" + "go.uber.org/zap" + "strings" +) + +var sugLog *zap.SugaredLogger +var envConfig *Config + +func HandleRequest(ctx context.Context, event map[string]interface{}) (string, error) { + sugLog = logger.GetSugaredLogger() + + envConfig = NewConfig() + if envConfig == nil { + return "Lambda finished with error", fmt.Errorf("error while validating required environment variables") + } + + sugLog.Info("Starting handling event...") + sugLog.Debug("Handling event: ", event) + + detail, ok := event["detail"].(map[string]interface{}) + if !ok { + sugLog.Error("`detail` is not of type map[string]interface{} or missing from the event.") + } + + eventName, ok := detail["eventName"].(string) + if !ok { + sugLog.Error("`eventName` is not of type string or missing from the event.") + } + + requestParameters, ok := detail["requestParameters"].(map[string]interface{}) + if !ok { + sugLog.Error("`requestParameters` is not of type map[string]interface{} or missing from the event.") + } + + switch eventName { + case "CreateLogGroup": + sugLog.Debug("Detected EventBridge CreateLogGroup event") + + logGroup, ok := requestParameters["logGroupName"].(string) + if !ok { + sugLog.Error("`logGroupName` is not of type string or missing from EventBridge event") + return "", fmt.Errorf("`logGroupName` is not of type string or missing from EventBridge event") + } + handleNewLogGroupEvent(ctx, logGroup) + + case "PutSecretValue": + sugLog.Debug("Detected EventBridge PutSecretValue event") + + secretId, ok := requestParameters["secretId"].(string) + if !ok { + sugLog.Error("`secretId` is not of type string or missing from EventBridge event") + return "", fmt.Errorf("`secretId` is not of type string or missing from EventBridge event") + } + err := handleSecretChangedEvent(ctx, secretId) + if err != nil { + return "", err + } + + case "SubscriptionFilterEvent": + sugLog.Debug("Detected SubscriptionFilterEvent event") + + var reqParams common.RequestParameters + reqParams, err := common.ConvertToRequestParameters(requestParameters) + if err != nil { + sugLog.Error("Error converting request parameters: ", err.Error()) + return "", err + } + + actionType := reqParams.Action + switch actionType { + case common.AddSF: + sugLog.Debug("Detected Add Subscription Filter event") + handleCreateEvent(ctx, reqParams) + case common.UpdateSF: + sugLog.Debug("Detected Update Subscription Filter event") + handleUpdateEvent(ctx, reqParams) + case common.DeleteSF: + sugLog.Debug("Detected Delete Subscription Filter event") + return handleDeleteEvent(ctx, reqParams) + default: + sugLog.Debug("Detected unsupported Subscription Filter event") + return "", fmt.Errorf("unsupported Subscription Filter event") + } + + default: + sugLog.Debug("Detected unsupported event") + return "", fmt.Errorf("unsupported event") + } + + return fmt.Sprintf("%s event handled successfully", eventName), nil +} + +func handleNewLogGroupEvent(ctx context.Context, newLogGroup string) { + // Prevent a situation where we put subscription filter on the trigger function + if newLogGroup == envConfig.thisFunctionLogGroup { + return + } + + // Check if the log group is of a monitored service + currMonitoredServices := getServices() + var added []string + if currMonitoredServices != nil { + serviceToPrefix := getServicesMap() + + cwClient, err := getCloudWatchLogsClient() + if err != nil { + sugLog.Error("Failed to get cloudwatch logs client") + } + + for _, service := range currMonitoredServices { + if prefix, ok := serviceToPrefix[service]; ok { + if strings.Contains(newLogGroup, prefix) { + added, _ = cwClient.addSubscriptionFilter([]string{newLogGroup}) + if len(added) > 0 { + sugLog.Info("Added subscription filter to log group: ", newLogGroup) + return + } + } + } + } + } + + // Check if the log group is of a monitored custom prefix + currCustomGroupsPrefixes := getCustomGroupsPrefixes() + if len(currCustomGroupsPrefixes) > 0 { + cwClient, err := getCloudWatchLogsClient() + if err != nil { + sugLog.Error("Failed to get cloudwatch logs client") + } + + for _, prefix := range currCustomGroupsPrefixes { + if strings.Contains(newLogGroup, prefix) { + added, _ = cwClient.addSubscriptionFilter([]string{newLogGroup}) + if len(added) > 0 { + sugLog.Info("Added subscription filter to log group: ", newLogGroup) + return + } + } + } + } +} + +func handleSecretChangedEvent(ctx context.Context, secretId string) error { + secretName := envConfig.customGroupsValue + + // make sure that the secret which changed is the relevant secret + if strings.Contains(secretId, secretName) { + err := updateSecretCustomLogGroups(ctx, secretId) + if err != nil { + sugLog.Error("Error while updating secret custom log groups: ", err.Error()) + return err + } + } else { + sugLog.Debug("The EventBridge event secretId is not the secret that has custom log groups in it. Skipping it.") + } + return nil +} + +func handleCreateEvent(ctx context.Context, event common.RequestParameters) { + cwClient, err := getCloudWatchLogsClient() + if err != nil { + sugLog.Error("Failed to get cloudwatch logs client") + return + } + + servicesToMonitor := convertStrToArr(event.NewServices) + logGroupsToMonitor := getServicesLogGroups(servicesToMonitor, cwClient) + + customLogGroupsToMonitor, err := getCustomLogGroups(event.NewIsSecret, event.NewCustom) + if err != nil { + sugLog.Error("Error while getting custom log groups: ", err.Error()) + } + logGroupsToMonitor = append(logGroupsToMonitor, customLogGroupsToMonitor...) + + added, _ := cwClient.addSubscriptionFilter(logGroupsToMonitor) + + sugLog.Info("Added subscription filters for the following log groups: ", added) +} + +func handleUpdateEvent(ctx context.Context, event common.RequestParameters) { + cwClient, err := getCloudWatchLogsClient() + if err != nil { + sugLog.Error("Failed to get cloudwatch logs client") + } + + oldServices := convertStrToArr(event.OldServices) + newServices := convertStrToArr(event.NewServices) + + oldCustomGroups, err := getCustomLogGroups(event.OldIsSecret, event.OldCustom) + if err != nil { + sugLog.Error("Error while getting old custom log groups: ", err.Error()) + } + newCustomGroups, err := getCustomLogGroups(event.NewIsSecret, event.NewCustom) + if err != nil { + sugLog.Error("Error while getting new custom log groups: ", err.Error()) + } + + servicesToAdd, servicesToRemove := findDifferences(oldServices, newServices) + customGroupsToAdd, customGroupsToRemove := findDifferences(oldCustomGroups, newCustomGroups) + + err = cwClient.updateSubscriptionFilters(servicesToAdd, servicesToRemove, customGroupsToAdd, customGroupsToRemove) + if err != nil { + sugLog.Error("Error while updating subscription filters: ", err.Error()) + } +} + +func handleDeleteEvent(ctx context.Context, event common.RequestParameters) (string, error) { + cwClient, err := getCloudWatchLogsClient() + if err != nil { + sugLog.Error("Failed to get cloudwatch logs client") + return "", err + } + + servicesToUnMonitor := convertStrToArr(event.NewServices) + logGroupsToUnMonitor := getServicesLogGroups(servicesToUnMonitor, cwClient) + + customLogGroupsToUnMonitor, err := getCustomLogGroups(event.NewIsSecret, event.NewCustom) + if err != nil { + sugLog.Error("Error while getting custom log groups: ", err.Error()) + return "", err + } + + logGroupsToUnMonitor = append(logGroupsToUnMonitor, customLogGroupsToUnMonitor...) + deleted, err := cwClient.removeSubscriptionFilter(logGroupsToUnMonitor) + if err != nil { + sugLog.Error("Error while removing subscription filters: ", err.Error()) + return "", err + } + + sugLog.Info("Deleted subscription filters for the following log groups: ", deleted) + return "Event handled successfully", nil +} diff --git a/log-group-events-lambda/handler/handler_test.go b/log-group-events-lambda/handler/handler_test.go new file mode 100644 index 0000000..cdfe2c8 --- /dev/null +++ b/log-group-events-lambda/handler/handler_test.go @@ -0,0 +1,112 @@ +package handler + +import ( + "context" + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +func setupHandlerTest() (ctx context.Context) { + /* Setup needed env variables */ + err := os.Setenv("FIREHOSE_ARN", "test-arn") + if err != nil { + return + } + err = os.Setenv("ACCOUNT_ID", "aws-account-id") + if err != nil { + return + } + err = os.Setenv("AWS_PARTITION", "test-partition") + if err != nil { + return + } + + ctx = context.Background() + + return ctx +} + +func TestUnsupportedEventHandling(t *testing.T) { + ctx := setupHandlerTest() + + tests := []struct { + name string + event map[string]interface{} + expectedOutputMsg string + expectedError bool + }{ + { + name: "Unsupported event with all required fields", + event: map[string]interface{}{ + "detail": map[string]interface{}{ + "eventName": "MyCustomEvent", + "requestParameters": map[string]interface{}{ + "logGroupName": "my-log-group", + }, + }, + }, + expectedOutputMsg: "", + expectedError: true, + }, + { + name: "Unsupported event with missing detail field", + event: map[string]interface{}{}, + expectedOutputMsg: "", + expectedError: true, + }, + { + name: "Unsupported event with missing eventName field", + event: map[string]interface{}{ + "detail": map[string]interface{}{ + "requestParameters": map[string]interface{}{ + "logGroupName": "my-log-group", + }, + }, + }, + expectedOutputMsg: "", + expectedError: true, + }, + { + name: "Unsupported event with missing requestParameters field", + event: map[string]interface{}{ + "detail": map[string]interface{}{ + "eventName": "MyCustomEvent", + }, + }, + expectedOutputMsg: "", + expectedError: true, + }, + { + name: "CreateLogGroup event with missing logGroup field", + event: map[string]interface{}{ + "detail": map[string]interface{}{ + "eventName": "CreateLogGroup", + "requestParameters": map[string]interface{}{}, + }, + }, + expectedOutputMsg: "", + expectedError: true, + }, + { + name: "PutSecretValue event with missing secretId field", + event: map[string]interface{}{ + "detail": map[string]interface{}{ + "eventName": "PutSecretValue", + "requestParameters": map[string]interface{}{}, + }, + }, + expectedOutputMsg: "", + expectedError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + res, err := HandleRequest(ctx, test.event) + + assert.NotNil(t, err) + assert.Equal(t, test.expectedOutputMsg, res) + }) + } +} diff --git a/log-group-events-lambda/handler/log_groups.go b/log-group-events-lambda/handler/log_groups.go new file mode 100644 index 0000000..a8b61ec --- /dev/null +++ b/log-group-events-lambda/handler/log_groups.go @@ -0,0 +1,141 @@ +package handler + +import ( + "github.com/hashicorp/go-multierror" + "strings" + "sync" +) + +// getServices returns a list of services to monitor +func getServices() []string { + servicesStr := envConfig.servicesValue + if servicesStr == emptyString { + return nil + } + return convertStrToArr(servicesStr) +} + +// getCustomGroupsPrefixes returns list of custom log groups which were defined with a wildcard, meaning as prefixes +func getCustomGroupsPrefixes() []string { + customGroupsStr := envConfig.customGroupsValue + if customGroupsStr == emptyString { + return nil + } + customGroups := convertStrToArr(customGroupsStr) + var wg sync.WaitGroup + var mu sync.Mutex + + prefixes := make([]string, 0) + for _, logGroup := range customGroups { + wg.Add(1) + go func(logGroup string) { + defer wg.Done() + + if strings.HasSuffix(logGroup, "*") { + mu.Lock() + prefixes = append(prefixes, strings.TrimSuffix(logGroup, "*")) + mu.Unlock() + } + }(logGroup) + } + wg.Wait() + + return prefixes +} + +// getServicesLogGroups returns a list of log groups to monitor based on the services +func getServicesLogGroups(services []string, cwLogsClient *CloudWatchLogsClient) []string { + servicesLogGroups := make([]string, 0) + serviceToPrefix := getServicesMap() + for _, service := range services { + if prefix, ok := serviceToPrefix[service]; ok { + currServiceLG, err := cwLogsClient.getLogGroupsWithPrefix(prefix) + if err != nil { + sugLog.Error("Failed to get log groups with prefix: ", prefix) + } + servicesLogGroups = append(servicesLogGroups, currServiceLG...) + } + } + return servicesLogGroups +} + +// getCustomLogGroups returns a list of custom log groups to monitor +func getCustomLogGroups(secretEnabled, customLogGroupsPrmVal string) ([]string, error) { + cwLogsClient, err := getCloudWatchLogsClient() + if err != nil { + sugLog.Error("Failed to get cloudwatch logs client") + } + + if secretEnabled == "true" { + secretCache, err := getSecretCacheClient() + if err != nil { + sugLog.Error("Failed to get secret cache client") + return nil, err + } + return getCustomLogGroupsFromSecret(customLogGroupsPrmVal, secretCache, cwLogsClient) + } + + return getCustomLogGroupsFromParam(convertStrToArr(customLogGroupsPrmVal), cwLogsClient) +} + +// getCustomLogGroupsFromSecret helper function of getCustomLogGroups, returns a list of custom log groups to monitor from secret value +func getCustomLogGroupsFromSecret(secretArn string, secretCache *SecretCacheClient, cwLogsClient *CloudWatchLogsClient) ([]string, error) { + secretName := getSecretNameFromArn(secretArn) + + secretStruct, err := secretCache.Client.GetSecretString(secretName) + if err != nil { + sugLog.Error("Error while getting secret value from cache.") + return nil, err + } + + customLogGroups, err := extractCustomGroupsFromSecret(secretArn, secretStruct) + if err != nil { + sugLog.Error("Error while extracting custom log groups from secret: ", err.Error()) + return nil, err + } + + if cwLogsClient != nil { + sugLog.Warn("Missing CloudWatch logs client, will not handle custom log group names with wildcards.") + return getCustomLogGroupsFromParam(convertStrToArr(customLogGroups), cwLogsClient) + } + return convertStrToArr(customLogGroups), nil +} + +// getCustomLogGroupsFromParam helper function of getCustomLogGroups, returns a list of custom log groups to monitor from parameter +func getCustomLogGroupsFromParam(logGroups []string, cwLogsClient *CloudWatchLogsClient) ([]string, error) { + customLogGroups := make([]string, 0, len(logGroups)) + var result *multierror.Error + var wg sync.WaitGroup + var mu sync.Mutex + + if cwLogsClient == nil { + // we shouldn't fail the entire process only if the cwLogsClient failed to get created + return logGroups, nil + } + + for i, logGroup := range logGroups { + wg.Add(1) + go func(i int, logGroup string) { + defer wg.Done() + + if strings.HasSuffix(logGroup, "*") { + newLogGroups, err := cwLogsClient.getLogGroupsWithPrefix(strings.TrimSuffix(logGroup, "*")) + if err != nil { + sugLog.Error("Failed to get log groups with prefix: ", logGroup) + result = multierror.Append(result, err) + return + } + mu.Lock() + customLogGroups = append(customLogGroups, newLogGroups...) + mu.Unlock() + + } else { + mu.Lock() + customLogGroups = append(customLogGroups, logGroup) + mu.Unlock() + } + }(i, logGroup) + } + wg.Wait() + return customLogGroups, result.ErrorOrNil() +} diff --git a/log-group-events-lambda/handler/log_groups_test.go b/log-group-events-lambda/handler/log_groups_test.go new file mode 100644 index 0000000..d1d312e --- /dev/null +++ b/log-group-events-lambda/handler/log_groups_test.go @@ -0,0 +1,277 @@ +package handler + +import ( + "fmt" + "github.com/logzio/firehose-logs/common" + lp "github.com/logzio/firehose-logs/logger" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "os" + "sort" + "testing" +) + +type MockSecretCacheClient struct { + mock.Mock + SecretCacheInterface +} + +func (m *MockSecretCacheClient) GetSecretString(secretName string) (string, error) { + if secretName == "errorSecret" { + return "", fmt.Errorf("an error occurred") + } + args := m.Called(secretName) + return args.String(0), args.Error(1) +} + +func setupLGTest() (cwClient *CloudWatchLogsClient, secretCacheClient *MockSecretCacheClient) { + err := os.Setenv(envFirehoseArn, "test-arn") + if err != nil { + return + } + + err = os.Setenv(envAccountId, "aws-account-id") + if err != nil { + return + } + + err = os.Setenv(envAwsPartition, "test-partition") + if err != nil { + return + } + + err = os.Setenv(envFunctionName, "g2") + if err != nil { + return + } + + /* Setup config */ + envConfig = NewConfig() + + /* Setup logger */ + sugLog = lp.GetSugaredLogger() + + /* Setup mock */ + mockCwClient := new(MockCloudWatchLogsClient) + return &CloudWatchLogsClient{Client: mockCwClient}, new(MockSecretCacheClient) +} + +func TestGetServices(t *testing.T) { + /* No services */ + err := os.Unsetenv(common.EnvServices) + if err != nil { + return + } + + setupLGTest() + + result := getServices() + assert.Nil(t, result) + + /* Has services */ + err = os.Setenv(common.EnvServices, "rds, cloudwatch, custom") + if err != nil { + return + } + setupLGTest() + + result = getServices() + assert.Equal(t, []string{"rds", "cloudwatch", "custom"}, result) +} + +func TestGetServicesLogGroups(t *testing.T) { + cwClient, _ := setupLGTest() + + tests := []struct { + name string + services []string + expectedLogGroups []string + }{ + { + name: "valid services", + services: []string{"cloudtrail", "apigateway"}, + expectedLogGroups: []string{"/aws/apigateway/g1", "/random/log/group"}, + }, + { + name: "invalid services", + services: []string{"svc1", "svc2"}, + expectedLogGroups: []string{}, + }, + { + name: "empty services", + services: []string{}, + expectedLogGroups: []string{}, + }, + { + name: "don't monitor this function's log group", + services: []string{"lambda"}, + expectedLogGroups: []string{"/aws/lambda/g1"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := getServicesLogGroups(test.services, cwClient) + sort.Strings(result) + assert.Equal(t, test.expectedLogGroups, result) + }) + } +} + +func TestGetCustomLogGroups(t *testing.T) { + setupLGTest() + tests := []struct { + name string + secretEnabled string + customLogGroupsPrmVal string + expectedGroups []string + }{ + { + name: "no log groups", + secretEnabled: "false", + customLogGroupsPrmVal: "", + expectedGroups: []string{}, + }, + { + name: "multiple log groups", + secretEnabled: "false", + customLogGroupsPrmVal: "g1, g2, g3", + expectedGroups: []string{"g1", "g2", "g3"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := getCustomLogGroups(test.secretEnabled, test.customLogGroupsPrmVal) + sort.Strings(result) + assert.Equal(t, test.expectedGroups, result) + assert.Nil(t, err) + }) + } +} + +func TestGetCustomLogGroupsFromSecret(t *testing.T) { + /* set needed env variables */ + err := os.Setenv(common.EnvAwsRegion, "us-west-2") + if err != nil { + return + } + + err = os.Setenv(envAccountId, "123456789012") + if err != nil { + return + } + + /* get mocks */ + _, mockSecretCacheClient := setupLGTest() + + /* tests */ + tests := []struct { + name string + secretArn string + secretData string + expectedData []string + expectedError bool + }{ + { + name: "no log groups", + secretArn: "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-secret", + secretData: `{"logzioCustomLogGroups": ""}`, + expectedData: nil, + expectedError: false, + }, + { + name: "multiple log groups", + secretArn: "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-secret", + secretData: `{"logzioCustomLogGroups": "g1, g2, g3"}`, + expectedData: []string{"g1", "g2", "g3"}, + expectedError: false, + }, + { + name: "error in getting secret", + secretArn: "arn:aws:secretsmanager:us-west-2:123456789012:errorSecret", + secretData: `{"someKey": "value"}`, + expectedData: nil, + expectedError: true, + }, + { + name: "invalid secret value", + secretArn: "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-secret", + secretData: `{"somKey": "g1, g2"}`, + expectedData: nil, + expectedError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + mockSecretCacheClient.On("GetSecretString", mock.Anything).Return(test.secretData, nil).Once() + secretCacheClient := &SecretCacheClient{Client: mockSecretCacheClient} + + result, err := getCustomLogGroupsFromSecret(test.secretArn, secretCacheClient, nil) + sort.Strings(result) + assert.Equal(t, test.expectedData, result) + + if test.expectedError { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} + +func TestGetCustomLogGroupsFromParam(t *testing.T) { + /* Note: + TestGetCustomLogGroups and TestGetCustomLogGroupsFromSecret tests this function for non prefix log groups. + This test will focus on testing the prefix via wildcard support capability. + */ + cwClient, _ := setupLGTest() + + tests := []struct { + name string + logGroups []string + expectedGroups []string + expectedError bool + }{ + { + name: "all wildcards", + logGroups: []string{"/log/group1/*", "/log/group2/*"}, + expectedGroups: []string{"/log/group1/a", "/log/group1/b", "/log/group2/a"}, + expectedError: false, + }, + { + name: "some wildcards, some not wildcards", + logGroups: []string{"/log/group1/*", "g1", "g2"}, + expectedGroups: []string{"/log/group1/a", "/log/group1/b", "g1", "g2"}, + expectedError: false, + }, + { + name: "no sub groups for prefix", + logGroups: []string{"/aws/codebuild/*"}, + expectedGroups: []string{}, + expectedError: false, + }, + { + name: "one error", + logGroups: []string{"/aws/error/test/*", "/log/group2/*", "g3"}, + expectedGroups: []string{"/log/group2/a", "g3"}, + expectedError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := getCustomLogGroupsFromParam(test.logGroups, cwClient) + sort.Strings(result) + assert.Equal(t, test.expectedGroups, result) + + if test.expectedError { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} diff --git a/log-group-events-lambda/handler/secret_ops.go b/log-group-events-lambda/handler/secret_ops.go new file mode 100644 index 0000000..a136c04 --- /dev/null +++ b/log-group-events-lambda/handler/secret_ops.go @@ -0,0 +1,175 @@ +package handler + +import ( + "context" + "encoding/json" + "fmt" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-secretsmanager-caching-go/secretcache" + "regexp" + "sort" +) + +// SecretsManagerAPIInterface AWS SDK v2 doesn't provide an interface for each service client like v1 +type SecretsManagerAPIInterface interface { + GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) + ListSecretVersionIds(ctx context.Context, params *secretsmanager.ListSecretVersionIdsInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.ListSecretVersionIdsOutput, error) +} + +// SecretManagerClient is a client for AWS Secrets Manager API +type SecretManagerClient struct { + Client SecretsManagerAPIInterface +} + +// getSecretManagerClient returns a client for AWS Secrets Manager API +func getSecretManagerClient(ctx context.Context) (*SecretManagerClient, error) { + awsConf, err := config.LoadDefaultConfig(ctx, config.WithRegion(envConfig.region)) + if err != nil { + sugLog.Error("Failed to setup connection to get older custom log groups secret values.") + return nil, err + } + return &SecretManagerClient{Client: secretsmanager.NewFromConfig(awsConf)}, nil +} + +// SecretCacheInterface is an interface for secret cache +type SecretCacheInterface interface { + GetSecretString(string) (string, error) +} + +// SecretCacheClient is a client for AWS Secret cache +type SecretCacheClient struct { + Client SecretCacheInterface +} + +// getSecretCacheClient returns a client for AWS Secret cache +func getSecretCacheClient() (*SecretCacheClient, error) { + secretCache, err := secretcache.New() + + return &SecretCacheClient{Client: secretCache}, err +} + +// updateSecretCustomLogGroups updates the custom log groups to monitor based on comparing the old secret value to the new one (helper of handleSecretChangedEvent) +func updateSecretCustomLogGroups(ctx context.Context, secretId string) error { + svc, err := getSecretManagerClient(ctx) + if err != nil { + return err + } + + oldSecretValue, err := svc.getOldSecretValue(ctx, secretId) + if err != nil { + sugLog.Error("Failed to get the old custom log group secret version's value.") + return err + } + + newSecretValue, err := getCustomLogGroups("true", secretId) + if err != nil { + sugLog.Error("Failed to get the new custom log group from secret") + return err + } + + customGroupsToAdd, customGroupsToRemove := findDifferences(oldSecretValue, newSecretValue) + + cwLogClient, err := getCloudWatchLogsClient() + if err != nil { + sugLog.Error("Failed to get the old custom log group secret version's value.") + return err + } + + if err := cwLogClient.updateSubscriptionFilters([]string{}, []string{}, customGroupsToAdd, customGroupsToRemove); err != nil { + return err + } + return nil +} + +// getSecretNameFromArn extracts a secret name from the given secret ARN +func getSecretNameFromArn(secretArn string) string { + var secretName string + + getSecretName := regexp.MustCompile(fmt.Sprintf(`^arn:aws:secretsmanager:%s:%s:secret:(?P\S+)-`, envConfig.region, envConfig.accountId)) + match := getSecretName.FindStringSubmatch(secretArn) + + for i, key := range getSecretName.SubexpNames() { + if key == "secretName" && len(match) > i { + secretName = match[i] + break + } + } + sugLog.Debugf("Found secret name %s, from secret ARN %s", secretName, secretArn) + return secretName +} + +// getOldSecretValue gets custom log groups value from the previous secret version +func (svc *SecretManagerClient) getOldSecretValue(ctx context.Context, secretId string) ([]string, error) { + // get the old version id + oldVersionId, err := svc.getPreviousSecretVersion(ctx, secretId) + if err != nil { + sugLog.Error("Failed to get the older custom log group secret version.") + return []string{}, err + } + + // get the old version value + getOldSecretValueInput := &secretsmanager.GetSecretValueInput{ + SecretId: &secretId, + VersionId: oldVersionId, + } + + oldSecret, err := svc.Client.GetSecretValue(ctx, getOldSecretValueInput) + if err != nil { + sugLog.Error("Failed to get the old value of the secret") + return []string{}, err + } + oldSecretValueJson := oldSecret.SecretString + + oldSecretValue, err := extractCustomGroupsFromSecret(secretId, *oldSecretValueJson) + if err != nil { + sugLog.Error("Failed to get the old value of the custom log groups from secret") + return []string{}, err + } + + return convertStrToArr(oldSecretValue), nil + +} + +// getPreviousSecretVersion returns the previous version id of the given secret +func (svc *SecretManagerClient) getPreviousSecretVersion(ctx context.Context, secretId string) (*string, error) { + var previousVersionId *string + + listSecretVersionsInput := &secretsmanager.ListSecretVersionIdsInput{ + SecretId: &secretId, + } + + secretInfo, err := svc.Client.ListSecretVersionIds(ctx, listSecretVersionsInput) + if err != nil { + sugLog.Error("Failed to list secret versions") + return nil, err + } + + // Sort the versions based on created date + sort.Slice(secretInfo.Versions, func(i, j int) bool { + return secretInfo.Versions[i].CreatedDate.After(*secretInfo.Versions[j].CreatedDate) + }) + + if len(secretInfo.Versions) > 1 { + previousVersionId = secretInfo.Versions[1].VersionId + } else { + return nil, fmt.Errorf("secret %s doesn't have an older version", secretId) + } + + return previousVersionId, nil +} + +// extractCustomGroupsFromSecret extracts the custom log groups to monitor from the given secret value +func extractCustomGroupsFromSecret(secretId, result string) (string, error) { + var secretValues map[string]string + err := json.Unmarshal([]byte(result), &secretValues) + if err != nil { + return "", err + } + + customLogGroups, ok := secretValues[logzioSecretKeyName] + if !ok { + return "", fmt.Errorf("did not find logzioCustomLogGroups key in the secret %s", secretId) + } + return customLogGroups, nil +} diff --git a/log-group-events-lambda/handler/secret_ops_test.go b/log-group-events-lambda/handler/secret_ops_test.go new file mode 100644 index 0000000..a34861e --- /dev/null +++ b/log-group-events-lambda/handler/secret_ops_test.go @@ -0,0 +1,296 @@ +package handler + +import ( + "context" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager" + "github.com/aws/aws-sdk-go-v2/service/secretsmanager/types" + "github.com/logzio/firehose-logs/common" + lp "github.com/logzio/firehose-logs/logger" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "os" + "testing" + "time" +) + +func stringPtr(s string) *string { + /* helper function */ + return &s +} + +func timePtr(t time.Time) *time.Time { + /* helper function */ + return &t +} + +type MockSecretManagerClient struct { + mock.Mock + SecretsManagerAPIInterface +} + +func (m *MockSecretManagerClient) GetSecretValue(ctx context.Context, params *secretsmanager.GetSecretValueInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.GetSecretValueOutput, error) { + args := m.Called(params) + return args.Get(0).(*secretsmanager.GetSecretValueOutput), args.Error(1) + +} + +func (m *MockSecretManagerClient) ListSecretVersionIds(ctx context.Context, params *secretsmanager.ListSecretVersionIdsInput, optFns ...func(*secretsmanager.Options)) (*secretsmanager.ListSecretVersionIdsOutput, error) { + args := m.Called(params) + return args.Get(0).(*secretsmanager.ListSecretVersionIdsOutput), args.Error(1) +} + +func setupSecretTest() (ctx context.Context, mockClient *MockSecretManagerClient) { + err := os.Setenv(common.EnvAwsRegion, "us-east-1") + if err != nil { + return + } + err = os.Setenv(envAccountId, "486140753397") + if err != nil { + return + } + + /* Setup config */ + envConfig = NewConfig() + + /* Setup logger */ + sugLog = lp.GetSugaredLogger() + + return context.Background(), new(MockSecretManagerClient) +} + +func TestGetSecretNameFromArn(t *testing.T) { + setupSecretTest() + tests := []struct { + name string + arn string + expectedSecretName string + }{ + { + name: "camel case secret name", + arn: "arn:aws:secretsmanager:us-east-1:486140753397:secret:testSecretName-56y7ud", + expectedSecretName: "testSecretName", + }, + { + name: "kebab case secret name", + arn: "arn:aws:secretsmanager:us-east-1:486140753397:secret:random-name-56y7ud", + expectedSecretName: "random-name", + }, + { + name: "snake case secret name", + arn: "arn:aws:secretsmanager:us-east-1:486140753397:secret:random_name-56y7ud", + expectedSecretName: "random_name", + }, + { + name: "name with numbers", + arn: "arn:aws:secretsmanager:us-east-1:486140753397:secret:now1with2numbers345-56y7ud", + expectedSecretName: "now1with2numbers345", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.expectedSecretName, getSecretNameFromArn(test.arn)) + }) + } +} + +func TestGetOldSecretValue(t *testing.T) { + ctx, mockClient := setupSecretTest() + + tests := []struct { + name string + SecretString string + versions []types.SecretVersionsListEntry + expectedOutput []string + expectedError bool + }{ + { + name: "valid secret with history", + SecretString: `{"logzioCustomLogGroups": "g1, g2, g3"}`, + versions: []types.SecretVersionsListEntry{ + { + VersionId: stringPtr("v2"), + CreatedDate: timePtr(time.Date(2023, 10, 8, 12, 0, 0, 0, time.UTC)), + }, + { + VersionId: stringPtr("v1"), + CreatedDate: timePtr(time.Date(2023, 5, 8, 12, 0, 0, 0, time.UTC)), + }, + }, + expectedOutput: []string{"g1", "g2", "g3"}, + expectedError: false, + }, + { + name: "valid secret with no history", + SecretString: "", + versions: []types.SecretVersionsListEntry{ + { + VersionId: stringPtr("v1"), + CreatedDate: timePtr(time.Date(2023, 5, 8, 12, 0, 0, 0, time.UTC)), + }, + }, + expectedOutput: []string{}, + expectedError: true, + }, + { + name: "invalid last secret", + SecretString: `{"someKey": "g1, g2, g3"}`, + versions: []types.SecretVersionsListEntry{ + { + VersionId: stringPtr("v2"), + CreatedDate: timePtr(time.Date(2023, 10, 8, 12, 0, 0, 0, time.UTC)), + }, + { + VersionId: stringPtr("v1"), + CreatedDate: timePtr(time.Date(2023, 5, 8, 12, 0, 0, 0, time.UTC)), + }, + }, + expectedOutput: []string{}, + expectedError: true, + }, + } + + arn := "arn:aws:secretsmanager:us-east-1:486140753397:secret:testSecretName-56y7ud" + secretName := "testSecretName" + oldVer := "v1" + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mockClient.On("GetSecretValue", mock.Anything).Return( + &secretsmanager.GetSecretValueOutput{ + ARN: stringPtr(arn), + Name: stringPtr(secretName), + VersionId: stringPtr(oldVer), + SecretString: stringPtr(test.SecretString), + }, nil).Once() + mockClient.On("ListSecretVersionIds", mock.Anything).Return( + &secretsmanager.ListSecretVersionIdsOutput{ + ARN: stringPtr(arn), + Name: stringPtr(secretName), + Versions: test.versions, + }, nil).Once() + + secretClient := &SecretManagerClient{Client: mockClient} + result, err := secretClient.getOldSecretValue(ctx, secretName) + + assert.Equal(t, test.expectedOutput, result) + if test.expectedError { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} + +func TestGetPreviousSecretVersion(t *testing.T) { + ctx, mockClient := setupSecretTest() + + tests := []struct { + name string + versions []types.SecretVersionsListEntry + expectedOutput *string + expectedError bool + }{ + { + name: "secret with history", + versions: []types.SecretVersionsListEntry{ + { + VersionId: stringPtr("v2"), + CreatedDate: timePtr(time.Date(2023, 10, 8, 12, 0, 0, 0, time.UTC)), + }, + { + VersionId: stringPtr("v1"), + CreatedDate: timePtr(time.Date(2023, 5, 8, 12, 0, 0, 0, time.UTC)), + }, + }, + expectedOutput: stringPtr("v1"), + expectedError: false, + }, + { + name: "secret with no history", + versions: []types.SecretVersionsListEntry{ + { + VersionId: stringPtr("v1"), + CreatedDate: timePtr(time.Date(2023, 5, 8, 12, 0, 0, 0, time.UTC)), + }, + }, + expectedOutput: nil, + expectedError: true, + }, + } + + arn := "arn:aws:secretsmanager:us-east-1:486140753397:secret:testSecretName-56y7ud" + secretName := "testSecretName" + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + mockClient.On("ListSecretVersionIds", mock.Anything).Return( + &secretsmanager.ListSecretVersionIdsOutput{ + ARN: stringPtr(arn), + Name: stringPtr(secretName), + Versions: test.versions, + }, nil).Once() + secretClient := &SecretManagerClient{Client: mockClient} + result, err := secretClient.getPreviousSecretVersion(ctx, secretName) + + assert.Equal(t, test.expectedOutput, result) + if test.expectedError { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} + +func TestExtractCustomGroupsFromSecret(t *testing.T) { + tests := []struct { + name string + secretId string + result string + expectedOutput string + expectedError bool + }{ + { + name: "valid secret", + secretId: "testSecret", + result: `{"logzioCustomLogGroups": "g1, g2, g3"}`, + expectedOutput: "g1, g2, g3", + expectedError: false, + }, + { + name: "empty secret", + secretId: "testSecret", + result: ``, + expectedOutput: "", + expectedError: true, + }, + { + name: "missing `logzioSecretKeyName` as key", + secretId: "testSecret", + result: `{"someKey": "g1, g2"}`, + expectedOutput: "", + expectedError: true, + }, + { + name: "edge case, invalid json", + secretId: "testSecret", + result: `{""someKey": "g1, g2"}`, + expectedOutput: "", + expectedError: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := extractCustomGroupsFromSecret(test.secretId, test.result) + assert.Equal(t, test.expectedOutput, result) + if test.expectedError { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} diff --git a/log-group-events-lambda/handler/utils.go b/log-group-events-lambda/handler/utils.go new file mode 100644 index 0000000..32034c3 --- /dev/null +++ b/log-group-events-lambda/handler/utils.go @@ -0,0 +1,66 @@ +package handler + +import ( + "strings" +) + +func getServicesMap() map[string]string { + return map[string]string{ + "apigateway": "/aws/apigateway/", + "rds": "/aws/rds/", + "cloudhsm": "/aws/cloudhsm/", + "cloudtrail": "aws-cloudtrail-logs-", + "codebuild": "/aws/codebuild/", + "connect": "/aws/connect/", + "elasticbeanstalk": "/aws/elasticbeanstalk/", + "ecs": "/aws/ecs/", + "eks": "/aws/eks/", + "aws-glue": "/aws/aws-glue/", + "aws-iot": "AWSIotLogsV2", + "lambda": "/aws/lambda/", + "macie": "/aws/macie/", + "amazon-mq": "/aws/amazonmq/broker/", + "batch": "/aws/batch/", + } +} + +func convertStrToArr(s string) []string { + if s == emptyString { + return nil + } + + s = strings.ReplaceAll(s, " ", "") + return strings.Split(s, valuesSeparator) +} + +// findDifferences finds elements in 'new' that are not in 'old', and vice versa. +func findDifferences(old, new []string) (toAdd, toRemove []string) { + oldSet := make(map[string]struct{}) + newSet := make(map[string]struct{}) + + // Populate 'oldSet' with elements from the 'old' slice. + for _, item := range old { + oldSet[item] = struct{}{} + } + + for _, item := range new { + newSet[item] = struct{}{} + } + + // Find elements in 'new' that are not in 'old' and add them to 'toAdd'. + for item := range newSet { + _, exists := oldSet[item] // Check if 'item' exists in 'oldSet' + if !exists { + toAdd = append(toAdd, item) + } + } + + for item := range oldSet { + _, exists := newSet[item] + if !exists { + toRemove = append(toRemove, item) + } + } + + return toAdd, toRemove +} diff --git a/log-group-events-lambda/handler/utils_test.go b/log-group-events-lambda/handler/utils_test.go new file mode 100644 index 0000000..d24bea6 --- /dev/null +++ b/log-group-events-lambda/handler/utils_test.go @@ -0,0 +1,92 @@ +package handler + +import ( + "github.com/stretchr/testify/assert" + "sort" + "testing" +) + +func TestGetServicesMap(t *testing.T) { + result := getServicesMap() + assert.NotNil(t, result) +} + +func TestConvertStrToArr(t *testing.T) { + tests := []struct { + name string + input string + expected []string + }{ + { + name: "empty string", + input: "", + expected: []string(nil), + }, + { + name: "single element", + input: "service1", + expected: []string{"service1"}, + }, + { + name: "multiple elements", + input: "service1, service2", + expected: []string{"service1", "service2"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := convertStrToArr(test.input) + assert.Equal(t, test.expected, result, "Expected %v, got %v", test.expected, result) + }) + } +} + +func TestFindDifferences(t *testing.T) { + tests := []struct { + name string + old []string + new []string + expectedToAdd []string + expectedToRemove []string + }{ + { + name: "no differences", + old: []string{"service1", "service2"}, + new: []string{"service1", "service2"}, + expectedToAdd: []string(nil), + expectedToRemove: []string(nil), + }, + { + name: "delete all", + old: []string{"service1", "service2"}, + new: []string{}, + expectedToAdd: []string(nil), + expectedToRemove: []string{"service1", "service2"}, + }, + { + name: "add to empty", + old: []string{}, + new: []string{"service1", "service2"}, + expectedToAdd: []string{"service1", "service2"}, + expectedToRemove: []string(nil), + }, + { + name: "delete some and add others", + old: []string{"service1", "service2"}, + new: []string{"service1", "service3"}, + expectedToAdd: []string{"service3"}, + expectedToRemove: []string{"service2"}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + resultToAdd, resultToRemove := findDifferences(test.old, test.new) + sort.Strings(resultToAdd) + sort.Strings(resultToRemove) + assert.Equal(t, test.expectedToAdd, resultToAdd, "Expected %v, got %v", test.expectedToAdd, resultToAdd) + assert.Equal(t, test.expectedToRemove, resultToRemove, "Expected %v, got %v", test.expectedToRemove, resultToRemove) + }) + } +} diff --git a/log-group-events-lambda/main.go b/log-group-events-lambda/main.go new file mode 100644 index 0000000..b2bc9d9 --- /dev/null +++ b/log-group-events-lambda/main.go @@ -0,0 +1,10 @@ +package main + +import ( + "github.com/aws/aws-lambda-go/lambda" + handler "github.com/logzio/firehose-logs/log-group-events-lambda/handler" +) + +func main() { + lambda.Start(handler.HandleRequest) +} diff --git a/logger/logger.go b/logger/logger.go index 9421e26..bd53cb0 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -19,6 +19,13 @@ const ( defaultLogLevel = LogLevelInfo ) +func GetSugaredLogger() *zap.SugaredLogger { + // Basic logger initialization, replace with your actual logger configuration + logger := GetLogger() + logger.Sync() + return logger.Sugar() +} + func GetLogger() *zap.Logger { logLevel := getLogLevel() cfg := zap.Config{