diff --git a/packages/@aws-cdk/aws-glue-alpha/README.md b/packages/@aws-cdk/aws-glue-alpha/README.md index 051044a74c8ff..00c5cee28765f 100644 --- a/packages/@aws-cdk/aws-glue-alpha/README.md +++ b/packages/@aws-cdk/aws-glue-alpha/README.md @@ -594,3 +594,236 @@ new glue.DataQualityRuleset(this, 'MyDataQualityRuleset', { ``` For more information, see [AWS Glue Data Quality](https://docs.aws.amazon.com/glue/latest/dg/glue-data-quality.html). + +## Workflow + +A `Workflow` is a collection of multiple Glue jobs and crawlers that are executed in a DAG. For example, to create a workflow: + +```ts +new glue.Workflow(this, 'MyWorkflow', { + workflowName: 'my_workflow', + description: 'description', + maxConcurrentRuns: 5, + defaultRunProperties: { + key1: 'value1', + key2: 'value2', + }, +}); +``` + +### Add Triggers + +A glue `Workflow` requires triggers to be appeneded to the DAG. These triggers are executed at different stages of the DAG, depending on the type of trigger enabled. + +#### On Demand Triggers + +On Demand triggers are executed manually by the user, or by a third-party service. For example, to add an On Demand trigger to a workflow: + +```ts +declare const myWorkflow: glue.Workflow; +declare const myJob: glue.IJob; +declare const myCrawler: glueCfn.CfnCrawler; +declare const securityConfiguration: glue.ISecurityConfiguration; + +myWorkflow.addOnDemandTrigger('OnDemandTrigger', { + triggerName: 'on_demand_trigger', + description: 'description', + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); +``` + +#### Schedule Triggers + +Schedule triggers are executed at a specified time or interval. For example, to add a Schedule trigger to a workflow: + +```ts +declare const myWorkflow: glue.Workflow; +declare const myJob: glue.IJob; +declare const myCrawler: glueCfn.CfnCrawler; +declare const securityConfiguration: glue.ISecurityConfiguration; +declare const schedule: events.Schedule; + +myWorkflow.addCustomScheduleTrigger('ScheduleTrigger', { + triggerName: 'schedule_trigger', + description: 'description', + enabled: true, + schedule, + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); +``` + +Convinience methods are available to add triggers to a workflow, for daily, weekly, and monthly schedules: + +```ts +declare const myWorkflow: glue.Workflow; +declare const myJob: glue.IJob; +declare const myCrawler: glueCfn.CfnCrawler; +declare const securityConfiguration: glue.ISecurityConfiguration; + +myWorkflow.addDailyScheduleTrigger('DailyScheduleTrigger', { + triggerName: 'daily_schedule_trigger', + description: 'description', + enabled: true, + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); + +myWorkflow.addWeeklyScheduleTrigger('WeeklyScheduleTrigger', { + triggerName: 'weekly_schedule_trigger', + description: 'description', + enabled: true, + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); + +myWorkflow.addMonthlyScheduleTrigger('MonthlyScheduleTrigger', { + triggerName: 'monthly_schedule_trigger', + description: 'description', + enabled: true, + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); +``` + +#### Event Triggers + +Event triggers are executed after a number of events have reached. For example, to add an Event trigger to a workflow: + +```ts +declare const myWorkflow: glue.Workflow; +declare const myJob: glue.IJob; +declare const myCrawler: glueCfn.CfnCrawler; +declare const securityConfiguration: glue.ISecurityConfiguration; + +myWorkflow.addNotifyEventTrigger('EventTrigger', { + triggerName: 'event_trigger', + description: 'description', + batchSize: 10, + batchWindow: cdk.Duration.minutes(5), + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); +``` + +#### Conditional Triggers + +Conditional triggers are executed based on a condition. For example, to add a Conditional trigger to a workflow: + +```ts +declare const myWorkflow: glue.Workflow; +declare const myJob: glue.IJob; +declare const myCrawler: glueCfn.CfnCrawler; +declare const predicateJob: glue.IJob; +declare const predicateCrawler: glueCfn.CfnCrawler; +declare const securityConfiguration: glue.ISecurityConfiguration; + +myWorkflow.addConditionalTrigger('ConditionalTrigger', { + triggerName: 'conditional_trigger', + description: 'description', + enabled: true, + predicateCondition: glue.TriggerPredicateCondition.AND, + jobPredicates: [{ + job: predicateJob, + state: glue.PredicateState.SUCCEEDED, + }], + crawlerPredicates: [{ + crawler: predicateCrawler, + state: glue.PredicateState.SUCCEEDED, + }], + actions: [ + { + job: myJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + key1: 'value1', + key2: 'value2', + }, + securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: myCrawler, + } + ], +}); +``` diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/index.ts b/packages/@aws-cdk/aws-glue-alpha/lib/index.ts index 1b9514c14625e..cd7dee8e635dd 100644 --- a/packages/@aws-cdk/aws-glue-alpha/lib/index.ts +++ b/packages/@aws-cdk/aws-glue-alpha/lib/index.ts @@ -14,3 +14,4 @@ export * from './security-configuration'; export * from './storage-parameter'; export * from './table-base'; export * from './table-deprecated'; +export * from './workflow'; diff --git a/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts new file mode 100644 index 0000000000000..4b9be8024c7e5 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/lib/workflow.ts @@ -0,0 +1,578 @@ +import * as cdk from 'aws-cdk-lib'; +import { IResource, Resource } from 'aws-cdk-lib'; +import { Schedule } from 'aws-cdk-lib/aws-events'; +import { CfnCrawler, CfnTrigger, CfnWorkflow } from 'aws-cdk-lib/aws-glue'; +import { Construct } from 'constructs'; +import { IJob } from './job'; +import { ISecurityConfiguration } from './security-configuration'; + +/** + * The condition to be evaluated. + */ +export enum TriggerPredicateCondition { + /** + * Defines when all predicates are met. + */ + AND = 'AND', + + /** + * Defines when any predicate is met. + */ + OR = 'ANY', +} + +/** + * The state of the predicate. + */ +export enum PredicateState { + /** + * The predicate is in a state of SUCCEEDED. + */ + SUCCEEDED = 'SUCCEEDED', + + /** + * The predicate is in a state of STOPPED. + */ + STOPPED = 'STOPPED', + + /** + * The predicate is in a state of TIMEOUT. + */ + TIMEOUT = 'TIMEOUT', + + /** + * The predicate is in a state of FAILED. + */ + FAILED = 'FAILED', +} + +/** + * The job predicates to be evaluated. + */ +export interface JobPredicate { + /** + * The job to be evaluated. + */ + readonly job: IJob; + + /** + * The state of the predicate. + */ + readonly state: PredicateState; +} + +/** + * The crawler predicates to be evaluated. + */ +export interface CrawlerPredicate { + /** + * The crawler to be evaluated. + */ + readonly crawler: CfnCrawler; + + /** + * The state of the predicate. + */ + readonly state: PredicateState; +} + +/** + * The action to be executed when the trigger fires. + */ +export interface Action { + /** + * The job to be executed, either this or `crawler` must be specified. + * + * @default - A job will not be executed. + */ + readonly job?: IJob; + + /** + * The crawler to be executed, either this or `job` must be specified. + * + * @default - A crawler will not be executed. + */ + readonly crawler?: CfnCrawler; + + /** + * After a job run starts, the `Duration` to wait before glue will trigger a CloudWatch event, in minutes. + * + * @default - A notification will not be sent. + */ + readonly delayCloudwatchEvent?: cdk.Duration; + + /** + * The job arguments specifically for this run. + * + * @default - No arguments. + */ + readonly arguments?: {[key: string]: string}; + + /** + * A Security Configuration to be used for the action. + * + * @default - No security configuration. + */ + readonly securityConfiguration?: ISecurityConfiguration; + + /** + * The timeout for a job run, in minutes. + * + * @default - No timeout. + */ + readonly timeout?: cdk.Duration; +} + +/** + * Properties for defining a new trigger to be assgined to a workflow. + */ +export interface TriggerProps { + /** + * The name of the trigger. + * + * @default - A CloudFormation generated name. + */ + readonly triggerName?: string; + + /** + * The description of the trigger. + * + * @default - No description. + */ + readonly description?: string; + + /** + * The actions to be executed when this trigger fires. + */ + readonly actions: Action[]; +} + +/** + * Properties for a trigger that can be enabled or disabled. + */ +export interface ActivatableTriggerProps extends TriggerProps { + /** + * Whether the trigger is enabled. + * + * @default true + */ + readonly enabled?: boolean; +} + +/** + * Properties for defining a new schedule trigger, to be assigned to a workflow. + */ +export interface ScheduleTriggerProps extends ActivatableTriggerProps { + /** + * The schedule for this trigger. + */ + readonly schedule: Schedule; +} + +/** + * Properties for defining a new notification trigger, to be assigned to a workflow. + */ +export interface NotificationTriggerProps extends TriggerProps { + /** + * The number of events that should be recieved from Amazon EventBridge before the EventBridge trigger is fired. + * + * @default - 1 + */ + readonly batchSize?: number; + + /** + * The Window duration, after which the EventBridge trigger fires. + * + * @default - 900 seconds + */ + readonly batchWindow?: cdk.Duration; +} + +/** + * Properties for defining a new conditional trigger, to be assigned to a workflow. + */ +export interface ConditionalTriggerProps extends ActivatableTriggerProps { + /** + * The condition to be evaluated. + * + * @default - TriggerPredicateCondition.AND + */ + readonly predicateCondition?: TriggerPredicateCondition; + + /** + * The job predicates to be evaluated. + * + * @default - No job predicates. + */ + readonly jobPredicates?: JobPredicate[]; + + /** + * The crawler predicates to be evaluated. + * + * @default - No crawler predicates. + */ + readonly crawlerPredicates?: CrawlerPredicate[]; +} + +/** + * Properties representing either a new or imported workflow. + */ +export interface IWorkflow extends IResource { + /** + * The ARN of the workflow. + * + * @attribute + */ + readonly workflowArn: string; + + /** + * The name of the workflow. + * + * @attribute + */ + readonly workflowName: string; + + /** + * Adds a trigger to the workflow to run on demand. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addOnDemandTrigger(id: string, props: TriggerProps): CfnTrigger; + + /** + * Adds a trigger to the workflow to run on a daily schedule. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addDailyScheduleTrigger(id: string, props: ActivatableTriggerProps): CfnTrigger; + + /** + * Adds a trigger to the workflow to run on a weekly schedule. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addWeeklyScheduleTrigger(id: string, props: ActivatableTriggerProps): CfnTrigger; + + /** + * Adds a trigger to the workflow to run on a monthly schedule. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addMonthlyScheduleTrigger(id: string, props: ActivatableTriggerProps): CfnTrigger; + + /** + * Adds a trigger to the workflow to run on a custom schedule. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addCustomScheduleTrigger(id: string, props: ScheduleTriggerProps): CfnTrigger; + + /** + * Adds a trigger to the workflow to run on a notification event. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addNotifyEventTrigger(id: string, props: NotificationTriggerProps): CfnTrigger; + + /** + * Adds a trigger to the workflow to run on a conditional event. + * + * @param id The identifier for the trigger. + * @param props The properties for the trigger. + */ + addConditionalTrigger(id: string, props: ConditionalTriggerProps): CfnTrigger; +} + +/** + * Properties for defining a new workflow. + */ +export interface WorkflowProps { + /** + * The name of the workflow. + * + * @default - A CloudFormation generated name. + */ + readonly workflowName?: string; + + /** + * A description of the workflow. + * + * @default - No description. + */ + readonly description?: string; + + /** + * The maximum concurrent runs allowed for the workflow. + * + * @default - No limit. + */ + readonly maxConcurrentRuns?: number; + + /** + * The default run properties for the workflow. + * + * @default - No default run properties. + */ + readonly defaultRunProperties?: {[key: string]: string}; +} + +abstract class WorkflowBase extends Resource implements IWorkflow { + /** + * The ARN of the workflow. + */ + public abstract readonly workflowArn: string; + + /** + * The name of the workflow. + */ + public abstract readonly workflowName: string; + + /** + * Adds a trigger to the workflow to run on demand. + */ + public addOnDemandTrigger(id: string, props: TriggerProps): CfnTrigger { + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'ON_DEMAND', + workflowName: this.workflowName, + actions: props.actions?.map(action => this.renderAction(action)), + }); + } + + /** + * Adds a trigger to the workflow to run on a daily schedule. + */ + public addDailyScheduleTrigger(id: string, props: ActivatableTriggerProps): CfnTrigger { + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'SCHEDULED', + schedule: 'cron(0 1 * * ? *)', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)), + }); + } + + /** + * Adds a trigger to the workflow to run on a weekly schedule. + */ + public addWeeklyScheduleTrigger(id: string, props: ActivatableTriggerProps): CfnTrigger { + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'SCHEDULED', + schedule: 'cron(0 1 ? * MON *)', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)), + }); + } + + /** + * Adds a trigger to the workflow to run on a monthly schedule. + */ + public addMonthlyScheduleTrigger(id: string, props: ActivatableTriggerProps): CfnTrigger { + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'SCHEDULED', + schedule: 'cron(0 1 1 * ? *)', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)) ?? [], + }); + } + + /** + * Adds a trigger to the workflow to run on a custom schedule. + */ + public addCustomScheduleTrigger(id: string, props: ScheduleTriggerProps): CfnTrigger { + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'SCHEDULED', + schedule: props.schedule.expressionString, + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)) ?? [], + }); + } + + /** + * Adds a trigger to the workflow to run on a notification event. + */ + public addNotifyEventTrigger(id: string, props: NotificationTriggerProps): CfnTrigger { + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'EVENT', + workflowName: this.workflowName, + actions: props.actions?.map(action => this.renderAction(action)) ?? [], + eventBatchingCondition: { + batchSize: props.batchSize ?? 1, + batchWindow: props.batchWindow?.toSeconds() ?? 900, + }, + }); + } + + /** + * Adds a trigger to the workflow to run on a conditional event. + */ + public addConditionalTrigger(id: string, props: ConditionalTriggerProps): CfnTrigger { + this.validatePredicates(props.jobPredicates, props.crawlerPredicates); + const conditions: CfnTrigger.ConditionProperty[] = []; + + if (props.jobPredicates) { + conditions.push( + ...props.jobPredicates.map(predicate => ({ + logical: props.predicateCondition, + jobName: predicate.job.jobName, + state: predicate.state, + logicalOperator: 'EQUALS', + })), + ); + } + if (props.crawlerPredicates) { + conditions.push( + ...props.crawlerPredicates.map(predicate => ({ + logical: props.predicateCondition, + crawlerName: predicate.crawler.name ?? predicate.crawler.ref, + crawlState: predicate.state, + logicalOperator: 'EQUALS', + })), + ); + } + + return new CfnTrigger(this, id, { + name: props.triggerName, + type: 'CONDITIONAL', + workflowName: this.workflowName, + startOnCreation: props.enabled ?? true, + actions: props.actions?.map(action => this.renderAction(action)), + predicate: { + logical: props.predicateCondition, + conditions: conditions.length > 0 ? conditions : undefined, + }, + }); + } + + protected renderAction(action: Action): CfnTrigger.ActionProperty { + if (!action.job && !action.crawler) { + throw new Error('Either job or crawler must be specified in an action'); + } + if (action.job && action.crawler) { + throw new Error( + 'Only one of job or crawler can be specified in an action', + ); + } + + return { + jobName: action.job?.jobName, + crawlerName: action.crawler?.name ?? action.crawler?.ref, + arguments: action.arguments, + securityConfiguration: + action.securityConfiguration?.securityConfigurationName, + timeout: action.timeout?.toMinutes(), + notificationProperty: { + notifyDelayAfter: action.delayCloudwatchEvent?.toMinutes(), + }, + }; + } + + protected validatePredicates( + jobPredicates?: JobPredicate[], + crawlerPredicates?: CrawlerPredicate[], + ) { + if (!jobPredicates && !crawlerPredicates) { + throw new Error( + 'At least one job predicate or crawler predicate must be specified.', + ); + } + if (!jobPredicates?.length && !crawlerPredicates?.length) { + throw new Error( + 'At least one job predicate or crawler predicate must be specified.', + ); + } + } +} + +/** + * A Glue workflow. + */ +export class Workflow extends WorkflowBase { + /** + * Import an existing workflow, using its ARN. + */ + public static fromWorkflowArn( + scope: Construct, + id: string, + workflowArn: string, + ): IWorkflow { + class Import extends WorkflowBase { + public readonly workflowArn = workflowArn; + public readonly workflowName = cdk.Arn.extractResourceName( + workflowArn, + 'workflow', + ); + } + + return new Import(scope, id); + } + + /** + * Import an existing workflow, using its name. + */ + public static fromWorkflowName( + scope: Construct, + id: string, + workflowName: string, + ): IWorkflow { + class Import extends WorkflowBase { + public readonly workflowArn = Workflow.buildWorkflowArn( + scope, + workflowName, + ); + public readonly workflowName = workflowName; + } + + return new Import(scope, id); + } + + private static buildWorkflowArn( + scope: Construct, + workflowName: string, + ): string { + return cdk.Stack.of(scope).formatArn({ + service: 'glue', + resource: 'workflow', + resourceName: workflowName, + }); + } + + /** + * Name of this workflow. + */ + public readonly workflowName: string; + + /** + * ARN of this workflow. + */ + public readonly workflowArn: string; + + constructor(scope: Construct, id: string, props?: WorkflowProps) { + super(scope, id); + + const resource = new CfnWorkflow(this, 'Resource', { + name: props?.workflowName, + description: props?.description, + maxConcurrentRuns: props?.maxConcurrentRuns, + defaultRunProperties: props?.defaultRunProperties, + }); + + const resourceName = this.getResourceNameAttribute(resource.ref); + this.workflowName = resourceName; + this.workflowArn = Workflow.buildWorkflowArn(scope, this.workflowName); + } +} diff --git a/packages/@aws-cdk/aws-glue-alpha/rosetta/default.ts-fixture b/packages/@aws-cdk/aws-glue-alpha/rosetta/default.ts-fixture index 047a3eee7b647..e8e34dcaf9a84 100644 --- a/packages/@aws-cdk/aws-glue-alpha/rosetta/default.ts-fixture +++ b/packages/@aws-cdk/aws-glue-alpha/rosetta/default.ts-fixture @@ -7,6 +7,9 @@ import * as s3 from 'aws-cdk-lib/aws-s3'; import * as ec2 from 'aws-cdk-lib/aws-ec2'; import * as kms from 'aws-cdk-lib/aws-kms'; import * as rds from 'aws-cdk-lib/aws-rds'; +import * as cdk from 'aws-cdk-lib'; +import * as glueCfn from 'aws-cdk-lib/aws-glue'; +import * as events from 'aws-cdk-lib/aws-events'; class Fixture extends Stack { constructor(scope: Construct, id: string) { diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py new file mode 100644 index 0000000000000..e75154b7c390f --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py @@ -0,0 +1 @@ +print("hello world") \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.assets.json new file mode 100644 index 0000000000000..0bd473a04b737 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.assets.json @@ -0,0 +1,32 @@ +{ + "version": "36.0.0", + "files": { + "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855": { + "source": { + "path": "asset.432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "085a2e2b10fd8c98d1a4d022657be795839a4fe0286e2c41082219f45aecbbec": { + "source": { + "path": "aws-cdk-glue-workflow.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "085a2e2b10fd8c98d1a4d022657be795839a4fe0286e2c41082219f45aecbbec.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.template.json new file mode 100644 index 0000000000000..498850985e37d --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/aws-cdk-glue-workflow.template.json @@ -0,0 +1,642 @@ +{ + "Resources": { + "MyWorkflowTask5BAF3B32": { + "Type": "AWS::Glue::Workflow" + }, + "Role1ABCC5F0": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "MyJobServiceRoleE6853162": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "MyJobServiceRoleDefaultPolicy3BDBBD28": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "MyJobServiceRoleDefaultPolicy3BDBBD28", + "Roles": [ + { + "Ref": "MyJobServiceRoleE6853162" + } + ] + } + }, + "MyJob8719E923": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "pythonshell", + "PythonVersion": "3", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "DefaultArguments": { + "--job-language": "python" + }, + "GlueVersion": "1.0", + "Role": { + "Fn::GetAtt": [ + "MyJobServiceRoleE6853162", + "Arn" + ] + } + } + }, + "MyCrawler": { + "Type": "AWS::Glue::Crawler", + "Properties": { + "DatabaseName": "my_database", + "Role": { + "Fn::GetAtt": [ + "Role1ABCC5F0", + "Arn" + ] + }, + "Targets": { + "S3Targets": [ + { + "Path": "s3://my_bucket" + } + ] + } + } + }, + "MyPredicateJobServiceRoleE1F838CC": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "MyPredicateJobServiceRoleDefaultPolicy87CB55E2": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "MyPredicateJobServiceRoleDefaultPolicy87CB55E2", + "Roles": [ + { + "Ref": "MyPredicateJobServiceRoleE1F838CC" + } + ] + } + }, + "MyPredicateJobF30A7B45": { + "Type": "AWS::Glue::Job", + "Properties": { + "Command": { + "Name": "pythonshell", + "PythonVersion": "3", + "ScriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + } + }, + "DefaultArguments": { + "--job-language": "python" + }, + "GlueVersion": "1.0", + "Role": { + "Fn::GetAtt": [ + "MyPredicateJobServiceRoleE1F838CC", + "Arn" + ] + } + } + }, + "MyPredicateCrawler": { + "Type": "AWS::Glue::Crawler", + "Properties": { + "DatabaseName": "my_database", + "Role": { + "Fn::GetAtt": [ + "Role1ABCC5F0", + "Arn" + ] + }, + "Targets": { + "S3Targets": [ + { + "Path": "s3://my_bucket" + } + ] + } + } + }, + "MySecurityConfiguration4AB9DD39": { + "Type": "AWS::Glue::SecurityConfiguration", + "Properties": { + "EncryptionConfiguration": { + "S3Encryptions": [ + { + "S3EncryptionMode": "SSE-S3" + } + ] + }, + "Name": "awscdkglueworkflowMySecurityConfigurationBBF7B904" + } + }, + "MyWorkflowBA72662B": { + "Type": "AWS::Glue::Workflow", + "Properties": { + "Description": "my_on_demand_workflow_description", + "Name": "my_on_demand_workflow" + } + }, + "MyWorkflowMyOnDemandTrigger2A682A63": { + "Type": "AWS::Glue::Trigger", + "Properties": { + "Actions": [ + { + "Arguments": { + "--arg1": "value1" + }, + "JobName": { + "Ref": "MyJob8719E923" + }, + "NotificationProperty": {}, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + }, + { + "CrawlerName": { + "Ref": "MyCrawler" + }, + "NotificationProperty": {}, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + } + ], + "Name": "my_on_demand_trigger", + "Type": "ON_DEMAND", + "WorkflowName": { + "Ref": "MyWorkflowBA72662B" + } + } + }, + "MyWorkflowMyConditionalTrigger389C8949": { + "Type": "AWS::Glue::Trigger", + "Properties": { + "Actions": [ + { + "Arguments": { + "--arg1": "value1" + }, + "JobName": { + "Ref": "MyJob8719E923" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + }, + { + "CrawlerName": { + "Ref": "MyCrawler" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + } + ], + "Name": "my_conditional_trigger", + "Predicate": { + "Conditions": [ + { + "JobName": { + "Ref": "MyPredicateJobF30A7B45" + }, + "LogicalOperator": "EQUALS", + "State": "SUCCEEDED" + }, + { + "CrawlState": "SUCCEEDED", + "CrawlerName": { + "Ref": "MyPredicateCrawler" + }, + "LogicalOperator": "EQUALS" + } + ], + "Logical": "AND" + }, + "StartOnCreation": true, + "Type": "CONDITIONAL", + "WorkflowName": { + "Ref": "MyWorkflowBA72662B" + } + } + }, + "MyDailyWorkflow3684C8E1": { + "Type": "AWS::Glue::Workflow", + "Properties": { + "Description": "my_daily_workflow_description", + "Name": "my_daily_workflow" + } + }, + "MyDailyWorkflowMyDailySchedule2DF51EB8": { + "Type": "AWS::Glue::Trigger", + "Properties": { + "Actions": [ + { + "Arguments": { + "--arg1": "value1" + }, + "JobName": { + "Ref": "MyJob8719E923" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + }, + { + "CrawlerName": { + "Ref": "MyCrawler" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + } + ], + "Name": "my_daily_schedule", + "Schedule": "cron(0 1 * * ? *)", + "StartOnCreation": true, + "Type": "SCHEDULED", + "WorkflowName": { + "Ref": "MyDailyWorkflow3684C8E1" + } + } + }, + "MyWeeklyWorkflowFC4A11C3": { + "Type": "AWS::Glue::Workflow", + "Properties": { + "Description": "my_weekly_workflow_description", + "Name": "my_weekly_workflow" + } + }, + "MyWeeklyWorkflowMyWeeklySchedule66671664": { + "Type": "AWS::Glue::Trigger", + "Properties": { + "Actions": [ + { + "Arguments": { + "--arg1": "value1" + }, + "JobName": { + "Ref": "MyJob8719E923" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + }, + { + "CrawlerName": { + "Ref": "MyCrawler" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + } + ], + "Name": "my_weekly_schedule", + "Schedule": "cron(0 1 ? * MON *)", + "StartOnCreation": true, + "Type": "SCHEDULED", + "WorkflowName": { + "Ref": "MyWeeklyWorkflowFC4A11C3" + } + } + }, + "MyMonthlyWorkflow5B4B0907": { + "Type": "AWS::Glue::Workflow", + "Properties": { + "Description": "my_monthly_workflow_description", + "Name": "my_monthly_workflow" + } + }, + "MyMonthlyWorkflowMyMonthlySchedule14F546AC": { + "Type": "AWS::Glue::Trigger", + "Properties": { + "Actions": [ + { + "Arguments": { + "--arg1": "value1" + }, + "JobName": { + "Ref": "MyJob8719E923" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + }, + { + "CrawlerName": { + "Ref": "MyCrawler" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + } + ], + "Name": "my_monthly_schedule", + "Schedule": "cron(0 1 1 * ? *)", + "StartOnCreation": true, + "Type": "SCHEDULED", + "WorkflowName": { + "Ref": "MyMonthlyWorkflow5B4B0907" + } + } + }, + "MyEventWorkflow0AA02AF3": { + "Type": "AWS::Glue::Workflow", + "Properties": { + "Description": "my_event_workflow_description", + "Name": "my_event_workflow" + } + }, + "MyEventWorkflowMyNotifyEvent4590A017": { + "Type": "AWS::Glue::Trigger", + "Properties": { + "Actions": [ + { + "Arguments": { + "--arg1": "value1" + }, + "JobName": { + "Ref": "MyJob8719E923" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + }, + { + "CrawlerName": { + "Ref": "MyCrawler" + }, + "NotificationProperty": { + "NotifyDelayAfter": 5 + }, + "SecurityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "Timeout": 10 + } + ], + "EventBatchingCondition": { + "BatchSize": 10, + "BatchWindow": 600 + }, + "Name": "my_notify_event", + "Type": "EVENT", + "WorkflowName": { + "Ref": "MyEventWorkflow0AA02AF3" + } + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json new file mode 100644 index 0000000000000..cc6198ec6ee39 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json @@ -0,0 +1,19 @@ +{ + "version": "36.0.0", + "files": { + "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { + "source": { + "path": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json new file mode 100644 index 0000000000000..ad9d0fb73d1dd --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json @@ -0,0 +1,36 @@ +{ + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/cdk.out b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/cdk.out new file mode 100644 index 0000000000000..1f0068d32659a --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/cdk.out @@ -0,0 +1 @@ +{"version":"36.0.0"} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/integ.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/integ.json new file mode 100644 index 0000000000000..94388279220a8 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/integ.json @@ -0,0 +1,12 @@ +{ + "version": "36.0.0", + "testCases": { + "aws-cdk-glue-table-integ/DefaultTest": { + "stacks": [ + "aws-cdk-glue-workflow" + ], + "assertionStack": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert", + "assertionStackName": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/manifest.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/manifest.json new file mode 100644 index 0000000000000..2101fabf1d9ae --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/manifest.json @@ -0,0 +1,239 @@ +{ + "version": "36.0.0", + "artifacts": { + "aws-cdk-glue-workflow.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "aws-cdk-glue-workflow.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "aws-cdk-glue-workflow": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "aws-cdk-glue-workflow.template.json", + "terminationProtection": false, + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/085a2e2b10fd8c98d1a4d022657be795839a4fe0286e2c41082219f45aecbbec.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "aws-cdk-glue-workflow.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "aws-cdk-glue-workflow.assets" + ], + "metadata": { + "/aws-cdk-glue-workflow/MyWorkflowTask/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyWorkflowTask5BAF3B32" + } + ], + "/aws-cdk-glue-workflow/Role/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "Role1ABCC5F0" + } + ], + "/aws-cdk-glue-workflow/MyJob/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyJobServiceRoleE6853162" + } + ], + "/aws-cdk-glue-workflow/MyJob/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyJobServiceRoleDefaultPolicy3BDBBD28" + } + ], + "/aws-cdk-glue-workflow/MyJob/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyJob8719E923" + } + ], + "/aws-cdk-glue-workflow/MyCrawler": [ + { + "type": "aws:cdk:logicalId", + "data": "MyCrawler" + } + ], + "/aws-cdk-glue-workflow/MyPredicateJob/ServiceRole/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyPredicateJobServiceRoleE1F838CC" + } + ], + "/aws-cdk-glue-workflow/MyPredicateJob/ServiceRole/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyPredicateJobServiceRoleDefaultPolicy87CB55E2" + } + ], + "/aws-cdk-glue-workflow/MyPredicateJob/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyPredicateJobF30A7B45" + } + ], + "/aws-cdk-glue-workflow/MyPredicateCrawler": [ + { + "type": "aws:cdk:logicalId", + "data": "MyPredicateCrawler" + } + ], + "/aws-cdk-glue-workflow/MySecurityConfiguration/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MySecurityConfiguration4AB9DD39" + } + ], + "/aws-cdk-glue-workflow/MyWorkflow/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyWorkflowBA72662B" + } + ], + "/aws-cdk-glue-workflow/MyWorkflow/MyOnDemandTrigger": [ + { + "type": "aws:cdk:logicalId", + "data": "MyWorkflowMyOnDemandTrigger2A682A63" + } + ], + "/aws-cdk-glue-workflow/MyWorkflow/MyConditionalTrigger": [ + { + "type": "aws:cdk:logicalId", + "data": "MyWorkflowMyConditionalTrigger389C8949" + } + ], + "/aws-cdk-glue-workflow/MyDailyWorkflow/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyDailyWorkflow3684C8E1" + } + ], + "/aws-cdk-glue-workflow/MyDailyWorkflow/MyDailySchedule": [ + { + "type": "aws:cdk:logicalId", + "data": "MyDailyWorkflowMyDailySchedule2DF51EB8" + } + ], + "/aws-cdk-glue-workflow/MyWeeklyWorkflow/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyWeeklyWorkflowFC4A11C3" + } + ], + "/aws-cdk-glue-workflow/MyWeeklyWorkflow/MyWeeklySchedule": [ + { + "type": "aws:cdk:logicalId", + "data": "MyWeeklyWorkflowMyWeeklySchedule66671664" + } + ], + "/aws-cdk-glue-workflow/MyMonthlyWorkflow/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyMonthlyWorkflow5B4B0907" + } + ], + "/aws-cdk-glue-workflow/MyMonthlyWorkflow/MyMonthlySchedule": [ + { + "type": "aws:cdk:logicalId", + "data": "MyMonthlyWorkflowMyMonthlySchedule14F546AC" + } + ], + "/aws-cdk-glue-workflow/MyEventWorkflow/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "MyEventWorkflow0AA02AF3" + } + ], + "/aws-cdk-glue-workflow/MyEventWorkflow/MyNotifyEvent": [ + { + "type": "aws:cdk:logicalId", + "data": "MyEventWorkflowMyNotifyEvent4590A017" + } + ], + "/aws-cdk-glue-workflow/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/aws-cdk-glue-workflow/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "aws-cdk-glue-workflow" + }, + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets": { + "type": "cdk:asset-manifest", + "properties": { + "file": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70": { + "type": "aws:cloudformation:stack", + "environment": "aws://unknown-account/unknown-region", + "properties": { + "templateFile": "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.template.json", + "terminationProtection": false, + "validateOnSynth": false, + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}", + "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}", + "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json", + "requiresBootstrapStackVersion": 6, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version", + "additionalDependencies": [ + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets" + ], + "lookupRole": { + "arn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-lookup-role-${AWS::AccountId}-${AWS::Region}", + "requiresBootstrapStackVersion": 8, + "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version" + } + }, + "dependencies": [ + "awscdkgluetableintegDefaultTestDeployAssert8BFB5B70.assets" + ], + "metadata": { + "/aws-cdk-glue-table-integ/DefaultTest/DeployAssert/BootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "BootstrapVersion" + } + ], + "/aws-cdk-glue-table-integ/DefaultTest/DeployAssert/CheckBootstrapVersion": [ + { + "type": "aws:cdk:logicalId", + "data": "CheckBootstrapVersion" + } + ] + }, + "displayName": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert" + }, + "Tree": { + "type": "cdk:tree", + "properties": { + "file": "tree.json" + } + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/tree.json b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/tree.json new file mode 100644 index 0000000000000..41f0f64c70296 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.js.snapshot/tree.json @@ -0,0 +1,1090 @@ +{ + "version": "tree-0.1", + "tree": { + "id": "App", + "path": "", + "children": { + "aws-cdk-glue-workflow": { + "id": "aws-cdk-glue-workflow", + "path": "aws-cdk-glue-workflow", + "children": { + "MyWorkflowTask": { + "id": "MyWorkflowTask", + "path": "aws-cdk-glue-workflow/MyWorkflowTask", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyWorkflowTask/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Workflow", + "aws:cdk:cloudformation:props": {} + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnWorkflow", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Workflow", + "version": "0.0.0" + } + }, + "Role": { + "id": "Role", + "path": "aws-cdk-glue-workflow/Role", + "children": { + "ImportRole": { + "id": "ImportRole", + "path": "aws-cdk-glue-workflow/Role/ImportRole", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/Role/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "MyJob": { + "id": "MyJob", + "path": "aws-cdk-glue-workflow/MyJob", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-cdk-glue-workflow/MyJob/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-cdk-glue-workflow/MyJob/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyJob/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-cdk-glue-workflow/MyJob/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyJob/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "MyJobServiceRoleDefaultPolicy3BDBBD28", + "roles": [ + { + "Ref": "MyJobServiceRoleE6853162" + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "Code90269e4a92c3a84619f8820dfa764340": { + "id": "Code90269e4a92c3a84619f8820dfa764340", + "path": "aws-cdk-glue-workflow/MyJob/Code90269e4a92c3a84619f8820dfa764340", + "children": { + "Stage": { + "id": "Stage", + "path": "aws-cdk-glue-workflow/MyJob/Code90269e4a92c3a84619f8820dfa764340/Stage", + "constructInfo": { + "fqn": "aws-cdk-lib.AssetStaging", + "version": "0.0.0" + } + }, + "AssetBucket": { + "id": "AssetBucket", + "path": "aws-cdk-glue-workflow/MyJob/Code90269e4a92c3a84619f8820dfa764340/AssetBucket", + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3.BucketBase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_s3_assets.Asset", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyJob/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "pythonshell", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3" + }, + "defaultArguments": { + "--job-language": "python" + }, + "glueVersion": "1.0", + "role": { + "Fn::GetAtt": [ + "MyJobServiceRoleE6853162", + "Arn" + ] + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" + } + }, + "MyCrawler": { + "id": "MyCrawler", + "path": "aws-cdk-glue-workflow/MyCrawler", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Crawler", + "aws:cdk:cloudformation:props": { + "databaseName": "my_database", + "role": { + "Fn::GetAtt": [ + "Role1ABCC5F0", + "Arn" + ] + }, + "targets": { + "s3Targets": [ + { + "path": "s3://my_bucket" + } + ] + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnCrawler", + "version": "0.0.0" + } + }, + "MyPredicateJob": { + "id": "MyPredicateJob", + "path": "aws-cdk-glue-workflow/MyPredicateJob", + "children": { + "ServiceRole": { + "id": "ServiceRole", + "path": "aws-cdk-glue-workflow/MyPredicateJob/ServiceRole", + "children": { + "ImportServiceRole": { + "id": "ImportServiceRole", + "path": "aws-cdk-glue-workflow/MyPredicateJob/ServiceRole/ImportServiceRole", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyPredicateJob/ServiceRole/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Role", + "aws:cdk:cloudformation:props": { + "assumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "glue.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "managedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSGlueServiceRole" + ] + ] + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnRole", + "version": "0.0.0" + } + }, + "DefaultPolicy": { + "id": "DefaultPolicy", + "path": "aws-cdk-glue-workflow/MyPredicateJob/ServiceRole/DefaultPolicy", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyPredicateJob/ServiceRole/DefaultPolicy/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::IAM::Policy", + "aws:cdk:cloudformation:props": { + "policyDocument": { + "Statement": [ + { + "Action": [ + "s3:GetBucket*", + "s3:GetObject*", + "s3:List*" + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/*" + ] + ] + }, + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":s3:::", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "policyName": "MyPredicateJobServiceRoleDefaultPolicy87CB55E2", + "roles": [ + { + "Ref": "MyPredicateJobServiceRoleE1F838CC" + } + ] + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.CfnPolicy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Policy", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_iam.Role", + "version": "0.0.0" + } + }, + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyPredicateJob/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Job", + "aws:cdk:cloudformation:props": { + "command": { + "name": "pythonshell", + "scriptLocation": { + "Fn::Join": [ + "", + [ + "s3://", + { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "/432033e3218068a915d2532fa9be7858a12b228a2ae6e5c10faccd9097b1e855.py" + ] + ] + }, + "pythonVersion": "3" + }, + "defaultArguments": { + "--job-language": "python" + }, + "glueVersion": "1.0", + "role": { + "Fn::GetAtt": [ + "MyPredicateJobServiceRoleE1F838CC", + "Arn" + ] + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnJob", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Job", + "version": "0.0.0" + } + }, + "MyPredicateCrawler": { + "id": "MyPredicateCrawler", + "path": "aws-cdk-glue-workflow/MyPredicateCrawler", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Crawler", + "aws:cdk:cloudformation:props": { + "databaseName": "my_database", + "role": { + "Fn::GetAtt": [ + "Role1ABCC5F0", + "Arn" + ] + }, + "targets": { + "s3Targets": [ + { + "path": "s3://my_bucket" + } + ] + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnCrawler", + "version": "0.0.0" + } + }, + "MySecurityConfiguration": { + "id": "MySecurityConfiguration", + "path": "aws-cdk-glue-workflow/MySecurityConfiguration", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MySecurityConfiguration/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::SecurityConfiguration", + "aws:cdk:cloudformation:props": { + "encryptionConfiguration": { + "s3Encryptions": [ + { + "s3EncryptionMode": "SSE-S3" + } + ] + }, + "name": "awscdkglueworkflowMySecurityConfigurationBBF7B904" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnSecurityConfiguration", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.SecurityConfiguration", + "version": "0.0.0" + } + }, + "MyWorkflow": { + "id": "MyWorkflow", + "path": "aws-cdk-glue-workflow/MyWorkflow", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyWorkflow/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Workflow", + "aws:cdk:cloudformation:props": { + "description": "my_on_demand_workflow_description", + "name": "my_on_demand_workflow" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnWorkflow", + "version": "0.0.0" + } + }, + "MyOnDemandTrigger": { + "id": "MyOnDemandTrigger", + "path": "aws-cdk-glue-workflow/MyWorkflow/MyOnDemandTrigger", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Trigger", + "aws:cdk:cloudformation:props": { + "actions": [ + { + "jobName": { + "Ref": "MyJob8719E923" + }, + "arguments": { + "--arg1": "value1" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": {} + }, + { + "crawlerName": { + "Ref": "MyCrawler" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": {} + } + ], + "name": "my_on_demand_trigger", + "type": "ON_DEMAND", + "workflowName": { + "Ref": "MyWorkflowBA72662B" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTrigger", + "version": "0.0.0" + } + }, + "MyConditionalTrigger": { + "id": "MyConditionalTrigger", + "path": "aws-cdk-glue-workflow/MyWorkflow/MyConditionalTrigger", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Trigger", + "aws:cdk:cloudformation:props": { + "actions": [ + { + "jobName": { + "Ref": "MyJob8719E923" + }, + "arguments": { + "--arg1": "value1" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + }, + { + "crawlerName": { + "Ref": "MyCrawler" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + } + ], + "name": "my_conditional_trigger", + "predicate": { + "logical": "AND", + "conditions": [ + { + "logical": "AND", + "jobName": { + "Ref": "MyPredicateJobF30A7B45" + }, + "state": "SUCCEEDED", + "logicalOperator": "EQUALS" + }, + { + "logical": "AND", + "crawlerName": { + "Ref": "MyPredicateCrawler" + }, + "crawlState": "SUCCEEDED", + "logicalOperator": "EQUALS" + } + ] + }, + "startOnCreation": true, + "type": "CONDITIONAL", + "workflowName": { + "Ref": "MyWorkflowBA72662B" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTrigger", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Workflow", + "version": "0.0.0" + } + }, + "MyDailyWorkflow": { + "id": "MyDailyWorkflow", + "path": "aws-cdk-glue-workflow/MyDailyWorkflow", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyDailyWorkflow/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Workflow", + "aws:cdk:cloudformation:props": { + "description": "my_daily_workflow_description", + "name": "my_daily_workflow" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnWorkflow", + "version": "0.0.0" + } + }, + "MyDailySchedule": { + "id": "MyDailySchedule", + "path": "aws-cdk-glue-workflow/MyDailyWorkflow/MyDailySchedule", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Trigger", + "aws:cdk:cloudformation:props": { + "actions": [ + { + "jobName": { + "Ref": "MyJob8719E923" + }, + "arguments": { + "--arg1": "value1" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + }, + { + "crawlerName": { + "Ref": "MyCrawler" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + } + ], + "name": "my_daily_schedule", + "schedule": "cron(0 1 * * ? *)", + "startOnCreation": true, + "type": "SCHEDULED", + "workflowName": { + "Ref": "MyDailyWorkflow3684C8E1" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTrigger", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Workflow", + "version": "0.0.0" + } + }, + "MyWeeklyWorkflow": { + "id": "MyWeeklyWorkflow", + "path": "aws-cdk-glue-workflow/MyWeeklyWorkflow", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyWeeklyWorkflow/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Workflow", + "aws:cdk:cloudformation:props": { + "description": "my_weekly_workflow_description", + "name": "my_weekly_workflow" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnWorkflow", + "version": "0.0.0" + } + }, + "MyWeeklySchedule": { + "id": "MyWeeklySchedule", + "path": "aws-cdk-glue-workflow/MyWeeklyWorkflow/MyWeeklySchedule", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Trigger", + "aws:cdk:cloudformation:props": { + "actions": [ + { + "jobName": { + "Ref": "MyJob8719E923" + }, + "arguments": { + "--arg1": "value1" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + }, + { + "crawlerName": { + "Ref": "MyCrawler" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + } + ], + "name": "my_weekly_schedule", + "schedule": "cron(0 1 ? * MON *)", + "startOnCreation": true, + "type": "SCHEDULED", + "workflowName": { + "Ref": "MyWeeklyWorkflowFC4A11C3" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTrigger", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Workflow", + "version": "0.0.0" + } + }, + "MyMonthlyWorkflow": { + "id": "MyMonthlyWorkflow", + "path": "aws-cdk-glue-workflow/MyMonthlyWorkflow", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyMonthlyWorkflow/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Workflow", + "aws:cdk:cloudformation:props": { + "description": "my_monthly_workflow_description", + "name": "my_monthly_workflow" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnWorkflow", + "version": "0.0.0" + } + }, + "MyMonthlySchedule": { + "id": "MyMonthlySchedule", + "path": "aws-cdk-glue-workflow/MyMonthlyWorkflow/MyMonthlySchedule", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Trigger", + "aws:cdk:cloudformation:props": { + "actions": [ + { + "jobName": { + "Ref": "MyJob8719E923" + }, + "arguments": { + "--arg1": "value1" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + }, + { + "crawlerName": { + "Ref": "MyCrawler" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + } + ], + "name": "my_monthly_schedule", + "schedule": "cron(0 1 1 * ? *)", + "startOnCreation": true, + "type": "SCHEDULED", + "workflowName": { + "Ref": "MyMonthlyWorkflow5B4B0907" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTrigger", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Workflow", + "version": "0.0.0" + } + }, + "MyEventWorkflow": { + "id": "MyEventWorkflow", + "path": "aws-cdk-glue-workflow/MyEventWorkflow", + "children": { + "Resource": { + "id": "Resource", + "path": "aws-cdk-glue-workflow/MyEventWorkflow/Resource", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Workflow", + "aws:cdk:cloudformation:props": { + "description": "my_event_workflow_description", + "name": "my_event_workflow" + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnWorkflow", + "version": "0.0.0" + } + }, + "MyNotifyEvent": { + "id": "MyNotifyEvent", + "path": "aws-cdk-glue-workflow/MyEventWorkflow/MyNotifyEvent", + "attributes": { + "aws:cdk:cloudformation:type": "AWS::Glue::Trigger", + "aws:cdk:cloudformation:props": { + "actions": [ + { + "jobName": { + "Ref": "MyJob8719E923" + }, + "arguments": { + "--arg1": "value1" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + }, + { + "crawlerName": { + "Ref": "MyCrawler" + }, + "securityConfiguration": { + "Ref": "MySecurityConfiguration4AB9DD39" + }, + "timeout": 10, + "notificationProperty": { + "notifyDelayAfter": 5 + } + } + ], + "eventBatchingCondition": { + "batchSize": 10, + "batchWindow": 600 + }, + "name": "my_notify_event", + "type": "EVENT", + "workflowName": { + "Ref": "MyEventWorkflow0AA02AF3" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.aws_glue.CfnTrigger", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/aws-glue-alpha.Workflow", + "version": "0.0.0" + } + }, + "imported-workflow-name": { + "id": "imported-workflow-name", + "path": "aws-cdk-glue-workflow/imported-workflow-name", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "imported-workflow-arn": { + "id": "imported-workflow-arn", + "path": "aws-cdk-glue-workflow/imported-workflow-arn", + "constructInfo": { + "fqn": "aws-cdk-lib.Resource", + "version": "0.0.0" + } + }, + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "aws-cdk-glue-workflow/BootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "aws-cdk-glue-workflow/CheckBootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" + } + }, + "aws-cdk-glue-table-integ": { + "id": "aws-cdk-glue-table-integ", + "path": "aws-cdk-glue-table-integ", + "children": { + "DefaultTest": { + "id": "DefaultTest", + "path": "aws-cdk-glue-table-integ/DefaultTest", + "children": { + "Default": { + "id": "Default", + "path": "aws-cdk-glue-table-integ/DefaultTest/Default", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + }, + "DeployAssert": { + "id": "DeployAssert", + "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert", + "children": { + "BootstrapVersion": { + "id": "BootstrapVersion", + "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert/BootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnParameter", + "version": "0.0.0" + } + }, + "CheckBootstrapVersion": { + "id": "CheckBootstrapVersion", + "path": "aws-cdk-glue-table-integ/DefaultTest/DeployAssert/CheckBootstrapVersion", + "constructInfo": { + "fqn": "aws-cdk-lib.CfnRule", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.Stack", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests-alpha.IntegTestCase", + "version": "0.0.0" + } + } + }, + "constructInfo": { + "fqn": "@aws-cdk/integ-tests-alpha.IntegTest", + "version": "0.0.0" + } + }, + "Tree": { + "id": "Tree", + "path": "Tree", + "constructInfo": { + "fqn": "constructs.Construct", + "version": "10.3.0" + } + } + }, + "constructInfo": { + "fqn": "aws-cdk-lib.App", + "version": "0.0.0" + } + } +} \ No newline at end of file diff --git a/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts new file mode 100644 index 0000000000000..f93fdf4754c1a --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/integ.workflow.ts @@ -0,0 +1,229 @@ +import * as cdk from 'aws-cdk-lib'; +import * as glueCfn from 'aws-cdk-lib/aws-glue'; +import * as path from 'path'; +import * as glue from '../lib'; +import * as integ from '@aws-cdk/integ-tests-alpha'; +import * as iam from 'aws-cdk-lib/aws-iam'; + +const app = new cdk.App(); + +const stack = new cdk.Stack(app, 'aws-cdk-glue-workflow'); + +new glue.Workflow(stack, 'MyWorkflowTask'); +const glueRole = new iam.Role(stack, 'Role', { + assumedBy: new iam.ServicePrincipal('glue.amazonaws.com'), +}); +const script = glue.Code.fromAsset(path.join(__dirname, 'job-script', 'hello_world.py')); +const glueJob = new glue.Job(stack, 'MyJob', { + executable: glue.JobExecutable.pythonShell({ + glueVersion: glue.GlueVersion.V1_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), +}); +const crawler = new glueCfn.CfnCrawler(stack, 'MyCrawler', { + databaseName: 'my_database', + targets: { + s3Targets: [{ path: 's3://my_bucket' }], + }, + role: glueRole.roleArn, +}); +const predicateGlueJob = new glue.Job(stack, 'MyPredicateJob', { + executable: glue.JobExecutable.pythonShell({ + glueVersion: glue.GlueVersion.V1_0, + pythonVersion: glue.PythonVersion.THREE, + script, + }), +}); +const predicateCrawler = new glueCfn.CfnCrawler(stack, 'MyPredicateCrawler', { + databaseName: 'my_database', + targets: { + s3Targets: [{ path: 's3://my_bucket' }], + }, + role: glueRole.roleArn, +}); +const securityConfiguration = new glue.SecurityConfiguration(stack, 'MySecurityConfiguration', { + s3Encryption: { + mode: glue.S3EncryptionMode.S3_MANAGED, + }, +}); + +const onDemandWorkflow = new glue.Workflow(stack, 'MyWorkflow', { + workflowName: 'my_on_demand_workflow', + description: 'my_on_demand_workflow_description', +}); + +onDemandWorkflow.addOnDemandTrigger('MyOnDemandTrigger', { + triggerName: 'my_on_demand_trigger', + description: 'my_on_demand_trigger_description', + actions: [ + { + job: glueJob, + arguments: { + '--arg1': 'value1', + }, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: crawler, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + ], +}); + +onDemandWorkflow.addConditionalTrigger('MyConditionalTrigger', { + triggerName: 'my_conditional_trigger', + description: 'my_conditional_trigger_description', + enabled: true, + predicateCondition: glue.TriggerPredicateCondition.AND, + jobPredicates: [{ + job: predicateGlueJob, + state: glue.PredicateState.SUCCEEDED, + }], + crawlerPredicates: [{ + crawler: predicateCrawler, + state: glue.PredicateState.SUCCEEDED, + }], + actions: [ + { + job: glueJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + '--arg1': 'value1', + }, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: crawler, + delayCloudwatchEvent: cdk.Duration.minutes(5), + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + ], +}); + +const dailyWorkflow = new glue.Workflow(stack, 'MyDailyWorkflow', { + workflowName: 'my_daily_workflow', + description: 'my_daily_workflow_description', +}); + +dailyWorkflow.addDailyScheduleTrigger('MyDailySchedule', { + triggerName: 'my_daily_schedule', + description: 'my_daily_schedule_description', + enabled: true, + actions: [ + { + job: glueJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + '--arg1': 'value1', + }, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: crawler, + delayCloudwatchEvent: cdk.Duration.minutes(5), + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + ], +}); + +const weeklyWorkflow = new glue.Workflow(stack, 'MyWeeklyWorkflow', { + workflowName: 'my_weekly_workflow', + description: 'my_weekly_workflow_description', +}); + +weeklyWorkflow.addWeeklyScheduleTrigger('MyWeeklySchedule', { + triggerName: 'my_weekly_schedule', + description: 'my_weekly_schedule_description', + enabled: true, + actions: [ + { + job: glueJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + '--arg1': 'value1', + }, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: crawler, + delayCloudwatchEvent: cdk.Duration.minutes(5), + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + ], +}); + +const monthlyWorkflow = new glue.Workflow(stack, 'MyMonthlyWorkflow', { + workflowName: 'my_monthly_workflow', + description: 'my_monthly_workflow_description', +}); + +monthlyWorkflow.addMonthlyScheduleTrigger('MyMonthlySchedule', { + triggerName: 'my_monthly_schedule', + description: 'my_monthly_schedule_description', + enabled: true, + actions: [ + { + job: glueJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + '--arg1': 'value1', + }, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: crawler, + delayCloudwatchEvent: cdk.Duration.minutes(5), + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + ], +}); + +const eventWorkflow = new glue.Workflow(stack, 'MyEventWorkflow', { + workflowName: 'my_event_workflow', + description: 'my_event_workflow_description', +}); + +eventWorkflow.addNotifyEventTrigger('MyNotifyEvent', { + triggerName: 'my_notify_event', + description: 'my_notify_event_description', + batchSize: 10, + batchWindow: cdk.Duration.minutes(10), + actions: [ + { + job: glueJob, + delayCloudwatchEvent: cdk.Duration.minutes(5), + arguments: { + '--arg1': 'value1', + }, + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + { + crawler: crawler, + delayCloudwatchEvent: cdk.Duration.minutes(5), + securityConfiguration: securityConfiguration, + timeout: cdk.Duration.minutes(10), + }, + ], +}); + +const importedWorkflowByName = glue.Workflow.fromWorkflowName(stack, 'imported-workflow-name', monthlyWorkflow.workflowName); + +glue.Workflow.fromWorkflowArn(stack, 'imported-workflow-arn', importedWorkflowByName.workflowArn); + +new integ.IntegTest(app, 'aws-cdk-glue-table-integ', { + testCases: [stack], +}); + +app.synth(); diff --git a/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts new file mode 100644 index 0000000000000..e7a0e130a18b4 --- /dev/null +++ b/packages/@aws-cdk/aws-glue-alpha/test/workflow.test.ts @@ -0,0 +1,531 @@ +import * as cdk from 'aws-cdk-lib'; +import { Template } from 'aws-cdk-lib/assertions'; +import { Schedule } from 'aws-cdk-lib/aws-events'; +import * as glueCfn from 'aws-cdk-lib/aws-glue'; +import { Bucket } from 'aws-cdk-lib/aws-s3'; +import * as glue from '../lib'; + +test('workflow', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + new glue.Workflow(stack, 'Workflow'); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Workflow', {}); +}); + +test('workflow with props', () => { + // GIVEN + const stack = new cdk.Stack(); + + // WHEN + new glue.Workflow(stack, 'myWorkflow', { + workflowName: 'myWorkflow', + description: 'myDescription', + maxConcurrentRuns: 1, + defaultRunProperties: { + foo: 'bar', + key: 'value', + }, + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Workflow', { + Name: 'myWorkflow', + Description: 'myDescription', + MaxConcurrentRuns: 1, + DefaultRunProperties: { + foo: 'bar', + key: 'value', + }, + }); +}); + +describe('imported workflow', () => { + let stack: cdk.Stack; + let job: glue.Job; + let securityConfiguration: glue.SecurityConfiguration; + let triggerProps: glue.TriggerProps; + + beforeEach(() => { + stack = new cdk.Stack(); + job = new glue.Job(stack, 'myJob', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V2_0, + pythonVersion: glue.PythonVersion.THREE, + script: glue.Code.fromBucket(Bucket.fromBucketName(stack, 'myBucket', 'my-bucket'), 'myKey'), + }), + }); + securityConfiguration = new glue.SecurityConfiguration(stack, 'mySecurityConfiguration', { + s3Encryption: { + mode: glue.S3EncryptionMode.S3_MANAGED, + }, + }); + triggerProps = { + actions: [{ + job: job, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(60), + }], + }; + }); + + test('fromWorkflowArn', () => { + // GIVEN + const workflowArn = 'arn:aws:glue:us-east-1:123456789012:workflow/myWorkflow'; + const predicateJob = new glue.Job(stack, 'myJob2', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V2_0, + pythonVersion: glue.PythonVersion.THREE, + script: glue.Code.fromBucket(Bucket.fromBucketName(stack, 'myPredicateBucket', 'my-bucket'), 'myKey'), + }), + }); + + // WHEN + const imported = glue.Workflow.fromWorkflowArn(stack, 'myWorkflow', workflowArn); + imported.addConditionalTrigger('ConditionalTrigger', { + ...triggerProps, + predicateCondition: glue.TriggerPredicateCondition.AND, + jobPredicates: [{ + job: predicateJob, + state: glue.PredicateState.SUCCEEDED, + }], + }); + + // THEN + expect(imported.workflowArn).toEqual(workflowArn); + expect(imported.workflowName).toEqual('myWorkflow'); + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { + Type: 'CONDITIONAL', + Actions: [{ + JobName: { Ref: 'myJob9A6589B3' }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 5, + NotificationProperty: { + NotifyDelayAfter: 1, + }, + }], + Predicate: { + Conditions: [ + { + JobName: { Ref: 'myJob2902BF595' }, + LogicalOperator: 'EQUALS', + State: 'SUCCEEDED', + }, + ], + Logical: 'AND', + }, + WorkflowName: 'myWorkflow', + }); + }); + + test('fromWorkflowName', () => { + // WHEN + const imported = glue.Workflow.fromWorkflowName(stack, 'myWorkflow', 'myWorkflow'); + imported.addOnDemandTrigger('OnDemandTrigger', triggerProps); + + // THEN + expect(imported.workflowName).toEqual('myWorkflow'); + expect(imported.workflowArn).toMatch(/^arn:.+:glue:.+:.+:workflow\/myWorkflow$/); + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { + Type: 'ON_DEMAND', + Actions: [{ + JobName: { Ref: 'myJob9A6589B3' }, + SecurityConfiguration: { Ref: 'mySecurityConfiguration58B0C573' }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 5, + NotificationProperty: { + NotifyDelayAfter: 1, + }, + }], + WorkflowName: 'myWorkflow', + }); + }); +}); + +describe('workflow with triggers', () => { + let stack: cdk.Stack; + let workflow: glue.Workflow; + let job: glue.Job; + let crawler: glueCfn.CfnCrawler; + let securityConfiguration: glue.SecurityConfiguration; + + beforeEach(() => { + stack = new cdk.Stack(); + workflow = new glue.Workflow(stack, 'myWorkflow'); + job = new glue.Job(stack, 'myJob', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V2_0, + pythonVersion: glue.PythonVersion.THREE, + script: glue.Code.fromBucket(Bucket.fromBucketName(stack, 'myBucket', 'my-bucket'), 'myKey'), + }), + }); + crawler = new glueCfn.CfnCrawler(stack, 'myCrawler', { + role: 'myRole', + databaseName: 'myDatabase', + targets: { + s3Targets: [{ path: 'myPath' }], + }, + }); + securityConfiguration = new glue.SecurityConfiguration(stack, 'mySecurityConfiguration', { + s3Encryption: { + mode: glue.S3EncryptionMode.S3_MANAGED, + }, + }); + }); + + test('onDemand', () => { + // WHEN + workflow.addOnDemandTrigger('OnDemandTrigger', { + triggerName: 'OnDemandTrigger', + actions: [{ + job: job, + arguments: { + foo: 'bar', + key: 'value', + }, + delayCloudwatchEvent: cdk.Duration.seconds(60), + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + }], + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { + Name: 'OnDemandTrigger', + Type: 'ON_DEMAND', + Actions: [{ + JobName: { Ref: 'myJob9A6589B3' }, + SecurityConfiguration: { Ref: 'mySecurityConfiguration58B0C573' }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 5, + NotificationProperty: { + NotifyDelayAfter: 1, + }, + }], + WorkflowName: { Ref: 'myWorkflow931F3265' }, + }); + }); + + test('dailySchedule', () => { + // WHEN + workflow.addDailyScheduleTrigger('DailyScheduleTrigger', { + actions: [{ + crawler: crawler, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(60), + }], + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { + Type: 'SCHEDULED', + Schedule: 'cron(0 1 * * ? *)', + Actions: [{ + CrawlerName: { Ref: 'myCrawler' }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 5, + SecurityConfiguration: { Ref: 'mySecurityConfiguration58B0C573' }, + NotificationProperty: { + NotifyDelayAfter: 1, + }, + }], + WorkflowName: { Ref: 'myWorkflow931F3265' }, + }); + }); + + test('weeklySchedule', () => { + // WHEN + workflow.addWeeklyScheduleTrigger('WeeklyScheduleTrigger', { + actions: [{ + job: job, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(60), + }], + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { + Type: 'SCHEDULED', + Schedule: 'cron(0 1 ? * MON *)', + Actions: [{ + JobName: { Ref: 'myJob9A6589B3' }, + SecurityConfiguration: { Ref: 'mySecurityConfiguration58B0C573' }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 5, + NotificationProperty: { + NotifyDelayAfter: 1, + }, + }], + WorkflowName: { Ref: 'myWorkflow931F3265' }, + }); + }); + + test('monthlySchedule', () => { + // WHEN + workflow.addMonthlyScheduleTrigger('MonthlyScheduleTrigger', { + actions: [{ + crawler: crawler, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(60), + }], + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { + Type: 'SCHEDULED', + Schedule: 'cron(0 1 1 * ? *)', + Actions: [{ + CrawlerName: { Ref: 'myCrawler' }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 5, + NotificationProperty: { + NotifyDelayAfter: 1, + }, + }], + WorkflowName: { Ref: 'myWorkflow931F3265' }, + }); + }); + + test('customSchedule', () => { + // WHEN + workflow.addCustomScheduleTrigger('CustomScheduleTrigger', { + schedule: Schedule.cron({ minute: '0', hour: '1', day: '1', month: 'JAN', year: '?' }), + actions: [{ + job: job, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(60), + }], + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { + Type: 'SCHEDULED', + Schedule: 'cron(0 1 1 JAN ? ?)', + Actions: [{ + JobName: { Ref: 'myJob9A6589B3' }, + SecurityConfiguration: { Ref: 'mySecurityConfiguration58B0C573' }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 5, + NotificationProperty: { + NotifyDelayAfter: 1, + }, + }], + WorkflowName: { Ref: 'myWorkflow931F3265' }, + }); + }); + + test('notifyEvent', () => { + // WHEN + workflow.addNotifyEventTrigger('OnDemandTrigger', { + batchSize: 50, + batchWindow: cdk.Duration.minutes(5), + actions: [{ + job: job, + arguments: { + foo: 'bar', + key: 'value', + }, + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(60), + timeout: cdk.Duration.seconds(5 * 60), + }], + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { + Type: 'EVENT', + Actions: [{ + JobName: { Ref: 'myJob9A6589B3' }, + Arguments: { + foo: 'bar', + key: 'value', + }, + NotificationProperty: { + NotifyDelayAfter: 1, + }, + Timeout: 5, + }], + EventBatchingCondition: { + BatchSize: 50, + BatchWindow: 300, + }, + WorkflowName: { Ref: 'myWorkflow931F3265' }, + }); + }); + + test('conditional', () => { + // WHEN + workflow.addConditionalTrigger('ConditionalTrigger', { + actions: [{ + crawler: crawler, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(60), + }], + predicateCondition: glue.TriggerPredicateCondition.AND, + jobPredicates: [{ + job: job, + state: glue.PredicateState.SUCCEEDED, + }], + crawlerPredicates: [{ + crawler: crawler, + state: glue.PredicateState.SUCCEEDED, + }], + }); + + // THEN + Template.fromStack(stack).hasResourceProperties('AWS::Glue::Trigger', { + Type: 'CONDITIONAL', + Actions: [{ + CrawlerName: { Ref: 'myCrawler' }, + Arguments: { + foo: 'bar', + key: 'value', + }, + Timeout: 5, + NotificationProperty: { + NotifyDelayAfter: 1, + }, + }], + Predicate: { + Conditions: [ + { + JobName: { Ref: 'myJob9A6589B3' }, + LogicalOperator: 'EQUALS', + State: 'SUCCEEDED', + }, + { + CrawlerName: { Ref: 'myCrawler' }, + LogicalOperator: 'EQUALS', + CrawlState: 'SUCCEEDED', + }, + ], + Logical: 'AND', + }, + WorkflowName: { Ref: 'myWorkflow931F3265' }, + }); + }); + + test('conditional with no predicates', () => { + // WHEN + expect(() => + workflow.addConditionalTrigger('ConditionalTrigger', { + actions: [{ + crawler: crawler, + arguments: { + foo: 'bar', + key: 'value', + }, + timeout: cdk.Duration.seconds(5 * 60), + securityConfiguration: securityConfiguration, + delayCloudwatchEvent: cdk.Duration.seconds(5), + }], + }), + + // THEN + ).toThrow(/At least one job predicate or crawler predicate must be specified./); + }); +}); + +test('workflow with trigger that has both job and crawler as one action', () => { + // GIVEN + const stack = new cdk.Stack(); + const workflow = new glue.Workflow(stack, 'myWorkflow'); + const job = new glue.Job(stack, 'myJob', { + executable: glue.JobExecutable.pythonEtl({ + glueVersion: glue.GlueVersion.V2_0, + pythonVersion: glue.PythonVersion.THREE, + script: glue.Code.fromBucket(Bucket.fromBucketName(stack, 'myBucket', 'my-bucket'), 'myKey'), + }), + }); + const crawler = new glueCfn.CfnCrawler(stack, 'myCrawler', { + role: 'myRole', + databaseName: 'myDatabase', + targets: { + s3Targets: [{ path: 'myPath' }], + }, + }); + + // WHEN + expect(() => workflow.addOnDemandTrigger('OnDemandTrigger', { + actions: [{ + job: job, + crawler: crawler, + }], + }), + + // THEN + ).toThrow(/Only one of job or crawler can be specified in an action/); +}); + +test('workflow with trigger that has neither job nor crawler as one action', () => { + // GIVEN + const stack = new cdk.Stack(); + const workflow = new glue.Workflow(stack, 'myWorkflow'); + + // WHEN + expect(() => workflow.addOnDemandTrigger('OnDemandTrigger', { + actions: [{ + arguments: { + foo: 'bar', + key: 'value', + }, + }], + }), + + // THEN + ).toThrow(/Either job or crawler must be specified in an action/); +}); \ No newline at end of file