Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

POC for Partial Batch Error Handling #1518

Draft
wants to merge 7 commits into
base: main
Choose a base branch
from
Draft

POC for Partial Batch Error Handling #1518

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2cbc92b
POC
samchungy Apr 3, 2024
a870e0d
fix createHandler
samchungy Apr 3, 2024
008088a
Merge branch 'main' of github.com:seek-oss/skuba into batch-workers
samchungy Jun 8, 2024
9ba2415
update distribution
samchungy Jun 8, 2024
7a89b7b
update tests
samchungy Jun 8, 2024
537e904
update serverless file
samchungy Jun 8, 2024
de256be
upgrade template
samchungy Jun 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions template/lambda-sqs-worker-cdk/.env
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
ENVIRONMENT=local
76 changes: 72 additions & 4 deletions template/lambda-sqs-worker-cdk/infra/__snapshots__/appStack.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,32 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
},
"Type": "AWS::IAM::Policy",
},
"destinationtopicDCE2E0B8": {
"Properties": {
"KmsMasterKeyId": {
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition",
},
":kms:",
{
"Ref": "AWS::Region",
},
":",
{
"Ref": "AWS::AccountId",
},
":alias/aws/sns",
],
],
},
"TopicName": "<%- serviceName %>",
},
"Type": "AWS::SNS::Topic",
},
"kmskey49FBC3B3": {
"DeletionPolicy": "Retain",
"Properties": {
Expand Down Expand Up @@ -448,6 +474,9 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
"Description": "Updated at 1212-12-12T12:12:12.121Z",
"Environment": {
"Variables": {
"DESTINATION_SNS_TOPIC_ARN": {
"Ref": "destinationtopicDCE2E0B8",
},
"ENVIRONMENT": "dev",
"NODE_ENV": "production",
"NODE_OPTIONS": "--enable-source-maps",
Expand All @@ -463,6 +492,7 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
"Arn",
],
},
"MemorySize": 512,
"ReservedConcurrentExecutions": 2,
"Role": {
"Fn::GetAtt": [
Expand Down Expand Up @@ -509,6 +539,7 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
},
"workerAliasliveSqsEventSourceappStackworkerqueue8281B9F42DEDB47B": {
"Properties": {
"BatchSize": 5,
"EventSourceArn": {
"Fn::GetAtt": [
"workerqueueA05CE5C6",
Expand Down Expand Up @@ -536,6 +567,9 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
],
],
},
"FunctionResponseTypes": [
"ReportBatchItemFailures",
],
"ScalingConfig": {
"MaximumConcurrency": 2,
},
Expand Down Expand Up @@ -651,7 +685,7 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
},
},
"FunctionName": "serviceName-post-hook",
"Handler": "index.handler",
"Handler": "index.post",
"KmsKeyArn": {
"Fn::GetAtt": [
"kmskey49FBC3B3",
Expand Down Expand Up @@ -805,7 +839,7 @@ exports[`returns expected CloudFormation stack for dev 1`] = `
},
},
"FunctionName": "serviceName-pre-hook",
"Handler": "index.handler",
"Handler": "index.pre",
"KmsKeyArn": {
"Fn::GetAtt": [
"kmskey49FBC3B3",
Expand Down Expand Up @@ -1350,6 +1384,32 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
},
"Type": "AWS::IAM::Policy",
},
"destinationtopicDCE2E0B8": {
"Properties": {
"KmsMasterKeyId": {
"Fn::Join": [
"",
[
"arn:",
{
"Ref": "AWS::Partition",
},
":kms:",
{
"Ref": "AWS::Region",
},
":",
{
"Ref": "AWS::AccountId",
},
":alias/aws/sns",
],
],
},
"TopicName": "<%- serviceName %>",
},
"Type": "AWS::SNS::Topic",
},
"kmskey49FBC3B3": {
"DeletionPolicy": "Retain",
"Properties": {
Expand Down Expand Up @@ -1496,6 +1556,9 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
"Description": "Updated at 1212-12-12T12:12:12.121Z",
"Environment": {
"Variables": {
"DESTINATION_SNS_TOPIC_ARN": {
"Ref": "destinationtopicDCE2E0B8",
},
"ENVIRONMENT": "prod",
"NODE_ENV": "production",
"NODE_OPTIONS": "--enable-source-maps",
Expand All @@ -1511,6 +1574,7 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
"Arn",
],
},
"MemorySize": 512,
"ReservedConcurrentExecutions": 20,
"Role": {
"Fn::GetAtt": [
Expand Down Expand Up @@ -1557,6 +1621,7 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
},
"workerAliasliveSqsEventSourceappStackworkerqueue8281B9F42DEDB47B": {
"Properties": {
"BatchSize": 5,
"EventSourceArn": {
"Fn::GetAtt": [
"workerqueueA05CE5C6",
Expand Down Expand Up @@ -1584,6 +1649,9 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
],
],
},
"FunctionResponseTypes": [
"ReportBatchItemFailures",
],
"ScalingConfig": {
"MaximumConcurrency": 20,
},
Expand Down Expand Up @@ -1699,7 +1767,7 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
},
},
"FunctionName": "serviceName-post-hook",
"Handler": "index.handler",
"Handler": "index.post",
"KmsKeyArn": {
"Fn::GetAtt": [
"kmskey49FBC3B3",
Expand Down Expand Up @@ -1853,7 +1921,7 @@ exports[`returns expected CloudFormation stack for prod 1`] = `
},
},
"FunctionName": "serviceName-pre-hook",
"Handler": "index.handler",
"Handler": "index.pre",
"KmsKeyArn": {
"Fn::GetAtt": [
"kmskey49FBC3B3",
Expand Down
21 changes: 19 additions & 2 deletions template/lambda-sqs-worker-cdk/infra/appStack.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,17 @@ export class AppStack extends Stack {

topic.addSubscription(new aws_sns_subscriptions.SqsSubscription(queue));

const snsKey = aws_kms.Alias.fromAliasName(
this,
'alias-aws-sns',
'alias/aws/sns',
);

const destinationTopic = new aws_sns.Topic(this, 'destination-topic', {
masterKey: snsKey,
topicName: '<%- serviceName %>',
});

const architecture = '<%- lambdaCdkArchitecture %>';

const defaultWorkerConfig: aws_lambda_nodejs.NodejsFunctionProps = {
Expand Down Expand Up @@ -88,9 +99,11 @@ export class AppStack extends Stack {
timeout: Duration.seconds(30),
bundling: defaultWorkerBundlingConfig,
functionName: '<%- serviceName %>',
memorySize: 512,
environment: {
...defaultWorkerEnvironment,
...config.workerLambda.environment,
DESTINATION_SNS_TOPIC_ARN: destinationTopic.topicArn,
},
// https://github.com/aws/aws-cdk/issues/28237
// This forces the lambda to be updated on every deployment
Expand All @@ -106,6 +119,8 @@ export class AppStack extends Stack {
alias.addEventSource(
new aws_lambda_event_sources.SqsEventSource(queue, {
maxConcurrency: config.workerLambda.reservedConcurrency,
batchSize: 5,
reportBatchItemFailures: true,
}),
);

Expand All @@ -114,7 +129,8 @@ export class AppStack extends Stack {
'worker-pre-hook',
{
...defaultWorkerConfig,
entry: './src/preHook.ts',
entry: './src/hooks.ts',
handler: 'pre',
timeout: Duration.seconds(120),
bundling: defaultWorkerBundlingConfig,
functionName: '<%- serviceName %>-pre-hook',
Expand All @@ -133,7 +149,8 @@ export class AppStack extends Stack {
'worker-post-hook',
{
...defaultWorkerConfig,
entry: './src/postHook.ts',
entry: './src/hooks.ts',
handler: 'post',
timeout: Duration.seconds(30),
bundling: defaultWorkerBundlingConfig,
functionName: '<%- serviceName %>-post-hook',
Expand Down
6 changes: 6 additions & 0 deletions template/lambda-sqs-worker-cdk/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,20 @@
"@aws-sdk/client-lambda": "^3.363.0",
"@aws-sdk/client-sns": "^3.363.0",
"@seek/logger": "^6.0.0",
"datadog-lambda-js": "^8.0.0",
"dd-trace": "^5.0.0",
"skuba-dive": "^2.0.0",
"zod": "^3.19.1"
},
"devDependencies": {
"@types/aws-lambda": "^8.10.82",
"@types/chance": "^1.1.3",
"@types/node": "^20.9.0",
"aws-cdk": "^2.109.0",
"aws-cdk-lib": "^2.109.0",
"aws-sdk-client-mock": "^4.0.0",
"aws-sdk-client-mock-jest": "^4.0.0",
"chance": "^1.1.8",
"constructs": "^10.0.17",
"pino-pretty": "^11.0.0",
"skuba": "*"
Expand Down
Loading
Loading