From 8942757d2605b8cef38ac42a0e4ac71514b943fa Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Fri, 5 Jul 2024 12:27:32 +0100 Subject: [PATCH 01/11] Initial commit for AWS permissions modules Signed-off-by: Jim Enright --- modules/terraform-aws-cred-permissions/data.tf | 0 .../terraform-aws-cred-permissions/defaults.tf | 0 .../doc_fragments/header.md | 15 +++++++++++++++ modules/terraform-aws-cred-permissions/main.tf | 0 modules/terraform-aws-cred-permissions/outputs.tf | 0 .../terraform-aws-cred-permissions/provider.tf | 0 .../terraform-aws-cred-permissions/variables.tf | 0 modules/terraform-aws-permissions/data.tf | 0 modules/terraform-aws-permissions/defaults.tf | 0 .../doc_fragments/header.md | 15 +++++++++++++++ modules/terraform-aws-permissions/main.tf | 0 modules/terraform-aws-permissions/outputs.tf | 0 modules/terraform-aws-permissions/provider.tf | 0 modules/terraform-aws-permissions/variables.tf | 0 14 files changed, 30 insertions(+) create mode 100644 modules/terraform-aws-cred-permissions/data.tf create mode 100644 modules/terraform-aws-cred-permissions/defaults.tf create mode 100644 modules/terraform-aws-cred-permissions/doc_fragments/header.md create mode 100644 modules/terraform-aws-cred-permissions/main.tf create mode 100644 modules/terraform-aws-cred-permissions/outputs.tf create mode 100644 modules/terraform-aws-cred-permissions/provider.tf create mode 100644 modules/terraform-aws-cred-permissions/variables.tf create mode 100644 modules/terraform-aws-permissions/data.tf create mode 100644 modules/terraform-aws-permissions/defaults.tf create mode 100644 modules/terraform-aws-permissions/doc_fragments/header.md create mode 100644 modules/terraform-aws-permissions/main.tf create mode 100644 modules/terraform-aws-permissions/outputs.tf create mode 100644 modules/terraform-aws-permissions/provider.tf create mode 100644 modules/terraform-aws-permissions/variables.tf diff --git a/modules/terraform-aws-cred-permissions/data.tf b/modules/terraform-aws-cred-permissions/data.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/terraform-aws-cred-permissions/defaults.tf b/modules/terraform-aws-cred-permissions/defaults.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/terraform-aws-cred-permissions/doc_fragments/header.md b/modules/terraform-aws-cred-permissions/doc_fragments/header.md new file mode 100644 index 0000000..a797ec4 --- /dev/null +++ b/modules/terraform-aws-cred-permissions/doc_fragments/header.md @@ -0,0 +1,15 @@ +# Terraform Module for CDP Prerequisites on AWS + +This module contains resource files and example variable definition files for creation of the pre-requisite AWS cloud resources required for Cloudera Data Platform (CDP) Public Cloud. + +## Usage + +The [examples](./examples) directory has example AWS Cloud Service Provider deployments for different scenarios: + +* `ex01-minimal-inputs` uses the minimum set of inputs for the module. + +* `ex02-existing-vpc` creates a VPC and subnets outside of the module and passes this as an additional input. CDP deployment then uses these network assets. + +* `ex03-create-keypair` creates the AWS EC2 Keypair in the module caller and passes this as an additional input. + +In each directory an example `terraform.tfvars.sample` values file is included to show input variable values. diff --git a/modules/terraform-aws-cred-permissions/main.tf b/modules/terraform-aws-cred-permissions/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/terraform-aws-cred-permissions/outputs.tf b/modules/terraform-aws-cred-permissions/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/terraform-aws-cred-permissions/provider.tf b/modules/terraform-aws-cred-permissions/provider.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/terraform-aws-cred-permissions/variables.tf b/modules/terraform-aws-cred-permissions/variables.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/terraform-aws-permissions/data.tf b/modules/terraform-aws-permissions/data.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/terraform-aws-permissions/defaults.tf b/modules/terraform-aws-permissions/defaults.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/terraform-aws-permissions/doc_fragments/header.md b/modules/terraform-aws-permissions/doc_fragments/header.md new file mode 100644 index 0000000..a797ec4 --- /dev/null +++ b/modules/terraform-aws-permissions/doc_fragments/header.md @@ -0,0 +1,15 @@ +# Terraform Module for CDP Prerequisites on AWS + +This module contains resource files and example variable definition files for creation of the pre-requisite AWS cloud resources required for Cloudera Data Platform (CDP) Public Cloud. + +## Usage + +The [examples](./examples) directory has example AWS Cloud Service Provider deployments for different scenarios: + +* `ex01-minimal-inputs` uses the minimum set of inputs for the module. + +* `ex02-existing-vpc` creates a VPC and subnets outside of the module and passes this as an additional input. CDP deployment then uses these network assets. + +* `ex03-create-keypair` creates the AWS EC2 Keypair in the module caller and passes this as an additional input. + +In each directory an example `terraform.tfvars.sample` values file is included to show input variable values. diff --git a/modules/terraform-aws-permissions/main.tf b/modules/terraform-aws-permissions/main.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/terraform-aws-permissions/outputs.tf b/modules/terraform-aws-permissions/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/terraform-aws-permissions/provider.tf b/modules/terraform-aws-permissions/provider.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/terraform-aws-permissions/variables.tf b/modules/terraform-aws-permissions/variables.tf new file mode 100644 index 0000000..e69de29 From 52feb3b95e1cb7ab66f1c239aca1fbce79348a01 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 15 Aug 2024 12:39:42 +0100 Subject: [PATCH 02/11] Update AWS storage and environment permissions Signed-off-by: Jim Enright --- modules/terraform-aws-permissions/data.tf | 16 + modules/terraform-aws-permissions/defaults.tf | 96 ++++ modules/terraform-aws-permissions/main.tf | 382 +++++++++++++++ modules/terraform-aws-permissions/outputs.tf | 61 +++ .../terraform-aws-permissions/variables.tf | 249 ++++++++++ modules/terraform-cdp-aws-pre-reqs/data.tf | 33 -- .../terraform-cdp-aws-pre-reqs/defaults.tf | 75 --- modules/terraform-cdp-aws-pre-reqs/main.tf | 436 ++---------------- modules/terraform-cdp-aws-pre-reqs/outputs.tf | 16 +- .../terraform-cdp-aws-pre-reqs/variables.tf | 25 +- .../examples/ex01-aws-basic/main.tf | 84 ++-- 11 files changed, 928 insertions(+), 545 deletions(-) diff --git a/modules/terraform-aws-permissions/data.tf b/modules/terraform-aws-permissions/data.tf index e69de29..749c865 100644 --- a/modules/terraform-aws-permissions/data.tf +++ b/modules/terraform-aws-permissions/data.tf @@ -0,0 +1,16 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Find the AWS account details +data "aws_caller_identity" "current" {} diff --git a/modules/terraform-aws-permissions/defaults.tf b/modules/terraform-aws-permissions/defaults.tf index e69de29..6f940ca 100644 --- a/modules/terraform-aws-permissions/defaults.tf +++ b/modules/terraform-aws-permissions/defaults.tf @@ -0,0 +1,96 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + + # ------- Global settings ------- + caller_account_id = data.aws_caller_identity.current.account_id + + # ------- Policies ------- + # Process placeholders in policy documents + + # Bucket Access Policy + # ...process placeholders in the policy doc + data_bucket_access_policy_doc_processed = replace( + replace( + var.data_bucket_access_policy_doc, "$${ARN_PARTITION}", var.arn_partition), + "$${DATALAKE_BUCKET}", "${var.data_storage_bucket}") + + log_bucket_access_policy_doc_processed = replace( + replace( + var.log_bucket_access_policy_doc, "$${ARN_PARTITION}", var.arn_partition), + "$${DATALAKE_BUCKET}", "${var.log_storage_bucket}") + + backup_bucket_access_policy_doc_processed = replace( + replace( + var.backup_bucket_access_policy_doc, "$${ARN_PARTITION}", var.arn_partition), + "$${DATALAKE_BUCKET}", "${var.backup_storage_bucket}") + + # ...then assign either input or processed policy doc to var used in resource + data_bucket_access_policy_doc = var.process_policy_placeholders ? local.data_bucket_access_policy_doc_processed : var.data_bucket_access_policy_doc + log_bucket_access_policy_doc = var.process_policy_placeholders ? local.log_bucket_access_policy_doc_processed : var.log_bucket_access_policy_doc + backup_bucket_access_policy_doc = var.process_policy_placeholders ? local.backup_bucket_access_policy_doc_processed : var.backup_bucket_access_policy_doc + + # Datalake Admin + # ...process placeholders in the policy doc + datalake_admin_s3_policy_doc_processed = replace( + replace( + var.datalake_admin_s3_policy_doc, "$${ARN_PARTITION}", var.arn_partition), + "$${STORAGE_LOCATION_BASE}", var.storage_location_base) + # ...then assign either input or processed policy doc to var used in resource + datalake_admin_s3_policy_doc = var.process_policy_placeholders ? local.datalake_admin_s3_policy_doc_processed : var.datalake_admin_s3_policy_doc + + # Backup policy + # ...process placeholders in the policy doc + datalake_backup_policy_doc_processed = replace( + replace( + var.datalake_backup_policy_doc, "$${ARN_PARTITION}", var.arn_partition), + "$${BACKUP_LOCATION_BASE}", var.backup_location_base) + # ...then assign either input or processed policy doc to var used in resource + datalake_backup_policy_doc = var.process_policy_placeholders ? local.datalake_backup_policy_doc_processed : var.datalake_backup_policy_doc + + # Restore policy + # ...process placeholders in the policy doc + datalake_restore_policy_doc_processed = replace( + replace( + replace( + var.datalake_restore_policy_doc, "$${ARN_PARTITION}", var.arn_partition), + "$${BACKUP_LOCATION_BASE}", var.backup_location_base), + "$${BACKUP_BUCKET}", var.backup_storage_bucket) + # ...then assign either input or processed policy doc to var used in resource + datalake_restore_policy_doc = var.process_policy_placeholders ? local.datalake_restore_policy_doc_processed : var.datalake_restore_policy_doc + + # Log policy + # ...process placeholders in the policy doc + log_data_access_policy_doc_processed = replace( + replace( + replace( + var.log_data_access_policy_doc, "$${ARN_PARTITION}", var.arn_partition), + "$${LOGS_BUCKET}", var.log_storage_bucket), + "$${LOGS_LOCATION_BASE}", var.log_location_base) + # ...then assign either input or processed policy doc to var used in resource + log_data_access_policy_doc = var.process_policy_placeholders ? local.log_data_access_policy_doc_processed : var.log_data_access_policy_doc + + # Ranger Audit + # ...process placeholders in the policy doc + ranger_audit_s3_policy_doc_processed = replace( + replace( + replace( + var.ranger_audit_s3_policy_doc, "$${ARN_PARTITION}", var.arn_partition), + "$${STORAGE_LOCATION_BASE}", var.storage_location_base), + "$${DATALAKE_BUCKET}", var.data_storage_bucket) + # ...then assign either input or processed policy doc to var used in resource + ranger_audit_s3_policy_doc = var.process_policy_placeholders ? local.ranger_audit_s3_policy_doc_processed : var.ranger_audit_s3_policy_doc + +} \ No newline at end of file diff --git a/modules/terraform-aws-permissions/main.tf b/modules/terraform-aws-permissions/main.tf index e69de29..9ba89d0 100644 --- a/modules/terraform-aws-permissions/main.tf +++ b/modules/terraform-aws-permissions/main.tf @@ -0,0 +1,382 @@ +# ------- CDP IDBroker Assume Role policy ------- +resource "aws_iam_policy" "cdp_idbroker_policy" { + name = var.idbroker_policy_name + # description = "CDP IDBroker Assume Role policy for ${var.env_prefix}" + + tags = merge(var.tags, { Name = var.idbroker_policy_name }) + + policy = var.idbroker_policy_doc +} + +# ------- CDP Data Access Policies - Log ------- +resource "aws_iam_policy" "cdp_log_data_access_policy" { + name = var.log_data_access_policy_name + # description = "CDP Log Location Access policy for ${var.env_prefix}" + + tags = merge(var.tags, { Name = var.log_data_access_policy_name }) + + policy = local.log_data_access_policy_doc + +} +# ------- CDP Data Access Policies - ranger_audit_s3 ------- +resource "aws_iam_policy" "cdp_ranger_audit_s3_data_access_policy" { + name = var.ranger_audit_s3_policy_name + # description = "CDP Ranger Audit S3 Access policy for ${var.env_prefix}" + + tags = merge(var.tags, { Name = var.ranger_audit_s3_policy_name }) + + policy = local.ranger_audit_s3_policy_doc +} + +# ------- CDP Data Access Policies - datalake_admin_s3 ------- +resource "aws_iam_policy" "cdp_datalake_admin_s3_data_access_policy" { + name = var.datalake_admin_s3_policy_name + # description = "CDP Datalake Admin S3 Access policy for ${var.env_prefix}" + + tags = merge(var.tags, { Name = var.datalake_admin_s3_policy_name }) + + policy = local.datalake_admin_s3_policy_doc + +} + +# ------- CDP Data Access Policies - bucket_access ------- +# Policy for Data bucket +resource "aws_iam_policy" "cdp_data_bucket_data_access_policy" { + name = var.data_bucket_access_policy_name + # description = "CDP Data Bucket S3 Access policy for ${var.env_prefix}" + + tags = merge(var.tags, { Name = var.data_bucket_access_policy_name }) + + policy = local.data_bucket_access_policy_doc +} + +# Policy for Log bucket...Only required if log bucket different from data bucket +resource "aws_iam_policy" "cdp_log_bucket_data_access_policy" { + + # count = (( + # var.log_storage_bucket != var.data_storage_bucket) ? + # 1 : 0 + # ) + + name = var.log_bucket_access_policy_name + # description = "CDP Log Bucket S3 Access policy for ${var.env_prefix}" + + tags = merge(var.tags, { Name = var.log_bucket_access_policy_name }) + + policy = local.log_bucket_access_policy_doc +} +# Policy for backup bucket...requied only if different from backup and log bucket +resource "aws_iam_policy" "cdp_backup_bucket_data_access_policy" { + + # count = (( + # var.backup_storage_bucket != var.data_storage_bucket + # || + # var.backup_storage_bucket != var.log_storage_bucket) ? + # 1 : 0 + # ) + + name = var.backup_bucket_access_policy_name + # description = "CDP Backup Bucket S3 Access policy for ${var.env_prefix}" + + tags = merge(var.tags, { Name = var.backup_bucket_access_policy_name }) + + policy = local.backup_bucket_access_policy_doc +} + +# ------- CDP Data Access Policies - datalake_backup_policy ------- +resource "aws_iam_policy" "cdp_datalake_backup_policy" { + name = var.datalake_backup_policy_name + # description = "CDP Datalake Backup policy for ${var.env_prefix}" + + tags = merge(var.tags, { Name = var.datalake_backup_policy_name }) + + policy = local.datalake_backup_policy_doc +} + +# ------- CDP Data Access Policies - datalake_restore_policy ------- +resource "aws_iam_policy" "cdp_datalake_restore_policy" { + name = var.datalake_restore_policy_name + # description = "CDP Datalake Restore policy for ${var.env_prefix}" + + tags = merge(var.tags, { Name = var.datalake_restore_policy_name }) + + policy = local.datalake_restore_policy_doc +} + +# ------- AWS Service Roles - CDP IDBroker ------- +# First create the Assume role policy document +data "aws_iam_policy_document" "cdp_idbroker_role_policy_doc" { + version = "2012-10-17" + + statement { + actions = ["sts:AssumeRole"] + effect = "Allow" + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +# Create the IAM role that uses the above assume_role_policy document +resource "aws_iam_role" "cdp_idbroker_role" { + name = var.idbroker_role_name + # description = "CDP IDBroker role for ${var.env_prefix}" + + assume_role_policy = data.aws_iam_policy_document.cdp_idbroker_role_policy_doc.json + + tags = merge(var.tags, { Name = var.idbroker_role_name }) +} + +# Create an instance profile for the iam_role +resource "aws_iam_instance_profile" "cdp_idbroker_role_instance_profile" { + name = var.idbroker_role_name + role = aws_iam_role.cdp_idbroker_role.name +} + +# Attach CDP IDBroker Assume Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_idbroker_role_attach1" { + role = aws_iam_role.cdp_idbroker_role.name + policy_arn = aws_iam_policy.cdp_idbroker_policy.arn +} + +# Attach AWS Log Location Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_idbroker_role_attach2" { + + role = aws_iam_role.cdp_idbroker_role.name + policy_arn = aws_iam_policy.cdp_log_data_access_policy.arn +} + +# ------- AWS Service Roles - CDP Log ------- +# First create the Assume role policy document +data "aws_iam_policy_document" "cdp_log_role_policy_doc" { + version = "2012-10-17" + + statement { + actions = ["sts:AssumeRole"] + effect = "Allow" + + principals { + type = "Service" + identifiers = ["ec2.amazonaws.com"] + } + } +} + +# Create the IAM role that uses the above assume_role_policy document +resource "aws_iam_role" "cdp_log_role" { + name = var.log_role_name + # description = "CDP Log role for ${var.env_prefix}" + + assume_role_policy = data.aws_iam_policy_document.cdp_log_role_policy_doc.json + + tags = merge(var.tags, { Name = var.log_role_name }) +} + +# Create an instance profile for the iam_role +resource "aws_iam_instance_profile" "cdp_log_role_instance_profile" { + name = var.log_role_name + role = aws_iam_role.cdp_log_role.name +} + +# Attach AWS Log Location Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_log_role_attach1" { + + role = aws_iam_role.cdp_log_role.name + policy_arn = aws_iam_policy.cdp_log_data_access_policy.arn +} + +# Attach AWS Datalake Restore Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_log_role_attach2" { + + role = aws_iam_role.cdp_log_role.name + policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn +} + +# Attach AWS Datalake Backup Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_log_role_attach3" { + + role = aws_iam_role.cdp_log_role.name + policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn +} + +# ------- AWS Data Access Roles - CDP Datalake Admin ------- +# First create the Assume role policy document +data "aws_iam_policy_document" "cdp_datalake_admin_role_policy_doc" { + version = "2012-10-17" + + statement { + actions = ["sts:AssumeRole"] + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${local.caller_account_id}:role/${aws_iam_role.cdp_idbroker_role.name}"] + } + } +} + +# Create the IAM role that uses the above assume_role_policy document +resource "aws_iam_role" "cdp_datalake_admin_role" { + name = var.datalake_admin_role_name + # description = "CDP Datalake Admin role for ${var.env_prefix}" + + assume_role_policy = data.aws_iam_policy_document.cdp_datalake_admin_role_policy_doc.json + + tags = merge(var.tags, { Name = var.datalake_admin_role_name }) +} + +# Create an instance profile for the iam_role +resource "aws_iam_instance_profile" "cdp_datalake_admin_role_instance_profile" { + name = var.datalake_admin_role_name + role = aws_iam_role.cdp_datalake_admin_role.name +} + +# Attach AWS Datalake Admin S3 Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach1" { + + role = aws_iam_role.cdp_datalake_admin_role.name + policy_arn = aws_iam_policy.cdp_datalake_admin_s3_data_access_policy.arn +} + +# Attach AWS Bucket Access Policy to the Role +# ..data bucket policy +resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach2" { + + role = aws_iam_role.cdp_datalake_admin_role.name + policy_arn = aws_iam_policy.cdp_data_bucket_data_access_policy.arn +} + +# ..log bucket policy, if required +resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach3" { + + # count = (( + # var.log_storage_bucket != var.data_storage_bucket) ? + # 1 : 0 + # ) + + role = aws_iam_role.cdp_datalake_admin_role.name + # policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy[0].arn + policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy.arn +} + +# ..backup bucket policy, if required +resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach4" { + + # count = (( + # var.backup_storage_bucket != var.data_storage_bucket + # || + # var.backup_storage_bucket != var.log_storage_bucket) ? + # 1 : 0 + # ) + + role = aws_iam_role.cdp_datalake_admin_role.name + # policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy[0].arn + policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy.arn +} + +# Attach AWS Datalake Backup Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach5" { + + role = aws_iam_role.cdp_datalake_admin_role.name + policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn +} + +# Attach AWS Datalake Restore Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach6" { + + role = aws_iam_role.cdp_datalake_admin_role.name + policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn +} + + +# ------- AWS Data Access Roles - CDP Ranger Audit ------- +# First create the Assume role policy document +data "aws_iam_policy_document" "cdp_ranger_audit_role_policy_doc" { + version = "2012-10-17" + + statement { + actions = ["sts:AssumeRole"] + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${local.caller_account_id}:role/${aws_iam_role.cdp_idbroker_role.name}"] + } + } +} + +# Create the IAM role that uses the above assume_role_policy document +resource "aws_iam_role" "cdp_ranger_audit_role" { + name = var.ranger_audit_role_name + # description = "CDP Ranger Audit role for ${var.env_prefix}" + + assume_role_policy = data.aws_iam_policy_document.cdp_ranger_audit_role_policy_doc.json + + tags = merge(var.tags, { Name = var.ranger_audit_role_name }) +} + +# Create an instance profile for the iam_role +resource "aws_iam_instance_profile" "cdp_ranger_audit_role_instance_profile" { + name = var.ranger_audit_role_name + role = aws_iam_role.cdp_ranger_audit_role.name +} + +# Attach AWS Ranger Audit S3 Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach1" { + + role = aws_iam_role.cdp_ranger_audit_role.name + policy_arn = aws_iam_policy.cdp_ranger_audit_s3_data_access_policy.arn +} + +# Attach AWS Bucket Access Policies to the Role +# ..data bucket policy +resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach2" { + + role = aws_iam_role.cdp_ranger_audit_role.name + policy_arn = aws_iam_policy.cdp_data_bucket_data_access_policy.arn +} + +# ..log bucket policy, if required +resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach3" { + + # count = (( + # var.log_storage_bucket != var.data_storage_bucket) ? + # 1 : 0 + # ) + + role = aws_iam_role.cdp_ranger_audit_role.name + # policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy[0].arn + policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy.arn +} + +# ..backup bucket policy, if required +resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach4" { + + # count = (( + # var.backup_storage_bucket != var.data_storage_bucket + # || + # var.backup_storage_bucket != var.log_storage_bucket) ? + # 1 : 0 + # ) + + role = aws_iam_role.cdp_ranger_audit_role.name + # policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy[0].arn + policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy.arn +} + +# Attach AWS Datalake Backup Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach5" { + + role = aws_iam_role.cdp_ranger_audit_role.name + policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn +} + +# Attach AWS Datalake Restore Policy to the Role +resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach6" { + + role = aws_iam_role.cdp_ranger_audit_role.name + policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn +} + diff --git a/modules/terraform-aws-permissions/outputs.tf b/modules/terraform-aws-permissions/outputs.tf index e69de29..2858202 100644 --- a/modules/terraform-aws-permissions/outputs.tf +++ b/modules/terraform-aws-permissions/outputs.tf @@ -0,0 +1,61 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "aws_datalake_admin_role_arn" { + value = aws_iam_role.cdp_datalake_admin_role.arn + + description = "Datalake Admin role ARN" +} + +output "aws_log_role_name" { + value = aws_iam_role.cdp_log_role.name + + description = "Log role Name" +} + +output "aws_ranger_audit_role_name" { + value = aws_iam_role.cdp_ranger_audit_role.name + + description = "Ranger Audit role Name" +} + +output "aws_ranger_audit_role_arn" { + value = aws_iam_role.cdp_ranger_audit_role.arn + + description = "Ranger Audit role ARN" +} + +output "aws_log_instance_profile_arn" { + value = aws_iam_instance_profile.cdp_log_role_instance_profile.arn + + description = "Log instance profile ARN" +} + +output "aws_idbroker_role_name" { + value = aws_iam_role.cdp_idbroker_role.name + + description = "IDBroker role Name" +} + +output "aws_idbroker_instance_profile_arn" { + value = aws_iam_instance_profile.cdp_idbroker_role_instance_profile.arn + + description = "IDBroker instance profile ARN" +} + +output "aws_datalake_admin_role_name" { + value = aws_iam_role.cdp_datalake_admin_role.name + + description = "Datalake Admin role Name" +} diff --git a/modules/terraform-aws-permissions/variables.tf b/modules/terraform-aws-permissions/variables.tf index e69de29..8a65283 100644 --- a/modules/terraform-aws-permissions/variables.tf +++ b/modules/terraform-aws-permissions/variables.tf @@ -0,0 +1,249 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "tags" { + type = map(any) + description = "Tags applied to provised resources" + + default = null +} + +variable "process_policy_placeholders" { + type = bool + description = "Flag to enable replacement of the standard placeholders in the AWS CDP Policy documents" + + default = true +} + +variable "arn_partition" { + type = string + description = "The string used to subsitute ARN_PARTITION placeholder in policy documents." + + default = "aws" + +} + +# ------- CDP Environment Deployment ------- +variable "enable_raz" { + type = bool + + description = "Flag to enable Ranger Authorization Service (RAZ)" + + default = true +} + +# ------- Policies ------- +# CDP IDBroker Assume Role policy +variable "idbroker_policy_name" { + type = string + description = "IDBroker Policy name" + +} + +variable "idbroker_policy_doc" { + type = string + description = "Contents of IDBroker Assumer Policy Document." +} + +# CDP Data Access Policies - Log +variable "log_data_access_policy_name" { + type = string + description = "Log Data Access Policy Name" + +} + +variable "log_data_access_policy_doc" { + type = string + description = "Contents of Log Data Access Policy" + +} + +# CDP Data Access Policies - ranger_audit_s3 +variable "ranger_audit_s3_policy_name" { + type = string + description = "Ranger S3 Audit Data Access Policy Name" +} + +variable "ranger_audit_s3_policy_doc" { + type = string + description = "Contents of Ranger S3 Audit Data Access Policy" +} + +# CDP Data Access Policies - datalake_admin_s3 +variable "datalake_admin_s3_policy_name" { + type = string + description = "Datalake Admin S3 Data Access Policy Name" + +} + +variable "datalake_admin_s3_policy_doc" { + type = string + description = "Contents of Datalake Admin S3 Data Access Policy" + +} + +variable "datalake_backup_policy_doc" { + type = string + description = "Contents of Datalake Backup Data Access Policy" + +} + +variable "datalake_restore_policy_doc" { + type = string + description = "Contents of Datalake Restore Data Access Policy" + + default = null +} + +# CDP Data Access Policies - bucket_access +variable "data_bucket_access_policy_name" { + type = string + description = "Data Bucket Access Data Access Policy Name" + +} +variable "log_bucket_access_policy_name" { + type = string + description = "Log Bucket Access Data Access Policy Name" + +} + +variable "backup_bucket_access_policy_name" { + type = string + description = "Backup Bucket Access Data Access Policy Name" + +} + +# CDP Datalake restore Policies - datalake +variable "datalake_restore_policy_name" { + type = string + description = "Datalake restore Data Access Policy Name" + + default = null +} + +# CDP Datalake backup Policies - datalake +variable "datalake_backup_policy_name" { + type = string + description = "Datalake backup Data Access Policy Name" + +} + +variable "data_bucket_access_policy_doc" { + type = string + description = "Data Bucket Access Data Access Policy" + +} + +variable "log_bucket_access_policy_doc" { + type = string + description = "Contents of Log Bucket Access Data Access Policy" + +} +variable "backup_bucket_access_policy_doc" { + type = string + description = "Contents of Backup Bucket Access Data Access Policy" + +} + +# Ranger Raz +variable "ranger_raz_policy_name" { + type = string + description = "Ranger Raz Policy Name" +} + +variable "ranger_raz_policy_doc" { + type = string + description = "Contents of Ranger Raz Policy" +} + +# ------- Roles ------- +# IDBroker service role +variable "idbroker_role_name" { + type = string + description = "IDBroker service role Name" + +} + +# Log service role +variable "log_role_name" { + type = string + description = "Log service role Name" + +} + +# CDP Datalake Admin role +variable "datalake_admin_role_name" { + type = string + description = "Datalake Admin role Name" + +} + +# CDP Ranger Audit role +variable "ranger_audit_role_name" { + type = string + description = "Ranger Audit role Name" + +} + +# CDP Ranger Raz role +variable "ranger_raz_role_name" { + type = string + description = "Ranger Raz role Name. Required is Raz is to be enabled." + + default = null +} + +# ------- Buckets and Storage Locations ------- +variable "data_storage_bucket" { + type = string + + description = "Name of the Data storage bucket" +} + +variable "log_storage_bucket" { + type = string + + description = "Name of the Log storage bucket" +} + +variable "backup_storage_bucket" { + type = string + + description = "Name of the Backup storage bucket" +} + +variable "storage_location_base" { + type = string + + description = "The bucket and path to the Data Lake storage directory. Should be specified as /" + + default = null +} + +variable "log_location_base" { + type = string + + description = "The bucket and path to the location for log storage. Should be specified as /" + + default = null +} + +variable "backup_location_base" { + type = string + + description = "The bucket and path to the location used for FreeIPA and Datalake backups. Should be specified as /" + + default = null +} \ No newline at end of file diff --git a/modules/terraform-cdp-aws-pre-reqs/data.tf b/modules/terraform-cdp-aws-pre-reqs/data.tf index 0754fda..a253c78 100644 --- a/modules/terraform-cdp-aws-pre-reqs/data.tf +++ b/modules/terraform-cdp-aws-pre-reqs/data.tf @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Find the AWS account details -data "aws_caller_identity" "current" {} - # Find details about S3 Gateway endpoint services data "aws_vpc_endpoint_service" "gateway_endpoints" { for_each = var.create_vpc && var.create_vpc_endpoints ? toset(var.vpc_endpoint_gateway_services) : [] @@ -36,33 +33,3 @@ data "aws_vpc_endpoint_service" "interface_endpoints" { data "http" "xaccount_account_policy_doc" { url = "https://raw.githubusercontent.com/hortonworks/cloudbreak/CB-2.73.0/cloud-aws-common/src/main/resources/definitions/aws-cb-policy.json" } - -# ..CDP Log Data Access Policies -data "http" "log_data_access_policy_doc" { - url = "https://raw.githubusercontent.com/hortonworks/cloudbreak/CB-2.73.0/cloud-aws-common/src/main/resources/definitions/cdp/aws-cdp-log-policy.json" -} - -# ..CDP ranger_audit_s3 Data Access Policies -data "http" "ranger_audit_s3_policy_doc" { - url = "https://raw.githubusercontent.com/hortonworks/cloudbreak/CB-2.73.0/cloud-aws-common/src/main/resources/definitions/cdp/aws-cdp-ranger-audit-s3-policy.json" -} - -# ..CDP datalake_admin_s3 Data Access Policies -data "http" "datalake_admin_s3_policy_doc" { - url = "https://raw.githubusercontent.com/hortonworks/cloudbreak/CB-2.73.0/cloud-aws-common/src/main/resources/definitions/cdp/aws-cdp-datalake-admin-s3-policy.json" -} - -# ..CDP bucket_access Data Access Policies -data "http" "bucket_access_policy_doc" { - url = "https://raw.githubusercontent.com/hortonworks/cloudbreak/CB-2.73.0/cloud-aws-common/src/main/resources/definitions/cdp/aws-cdp-bucket-access-policy.json" -} - -# ..CDP Data Lake Backup Policies -data "http" "datalake_backup_policy_doc" { - url = "https://raw.githubusercontent.com/hortonworks/cloudbreak/CB-2.73.0/cloud-aws-cloudformation/src/main/resources/definitions/aws-datalake-backup-policy.json" -} - -# ..CDP Data Lake Restore Policies -data "http" "datalake_restore_policy_doc" { - url = "https://raw.githubusercontent.com/hortonworks/cloudbreak/CB-2.73.0/cloud-aws-cloudformation/src/main/resources/definitions/aws-datalake-restore-policy.json" -} diff --git a/modules/terraform-cdp-aws-pre-reqs/defaults.tf b/modules/terraform-cdp-aws-pre-reqs/defaults.tf index 646dca0..0598e6b 100644 --- a/modules/terraform-cdp-aws-pre-reqs/defaults.tf +++ b/modules/terraform-cdp-aws-pre-reqs/defaults.tf @@ -20,8 +20,6 @@ locals { )) ) - caller_account_id = data.aws_caller_identity.current.account_id - # ------- Network Resources ------- vpc_name = coalesce(var.vpc_name, "${var.env_prefix}-net") @@ -80,97 +78,24 @@ locals { # CDP Data Access Policies - Log log_data_access_policy_name = coalesce(var.log_data_access_policy_name, "${var.env_prefix}-logs-policy") - # log_data_access_policy_doc - # ...first process placeholders in the downloaded policy doc - log_data_access_policy_doc_processed = replace( - replace( - replace( - data.http.log_data_access_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), - "$${LOGS_BUCKET}", "${local.log_storage.log_storage_bucket}${local.storage_suffix}"), - "$${LOGS_LOCATION_BASE}", "${local.log_storage.log_storage_bucket}${local.storage_suffix}") - - # ...then assign either input or downloaded policy doc to var used in resource - log_data_access_policy_doc = coalesce(var.log_data_access_policy_doc, local.log_data_access_policy_doc_processed) - # CDP Data Access Policies - ranger_audit_s3 ranger_audit_s3_policy_name = coalesce(var.ranger_audit_s3_policy_name, "${var.env_prefix}-audit-policy") - # ranger_audit_s3_policy_doc - # ...first process placeholders in the downloaded policy doc - ranger_audit_s3_policy_doc_processed = replace( - replace( - replace( - data.http.ranger_audit_s3_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), - "$${STORAGE_LOCATION_BASE}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}/${replace(local.data_storage.data_storage_object, "/", "")}"), - "$${DATALAKE_BUCKET}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}") - - # ...then assign either input or downloaded policy doc to var used in resource - ranger_audit_s3_policy_doc = coalesce(var.ranger_audit_s3_policy_doc, local.ranger_audit_s3_policy_doc_processed) - # CDP Data Access Policies - datalake_admin_s3 datalake_admin_s3_policy_name = coalesce(var.datalake_admin_s3_policy_name, "${var.env_prefix}-dladmin-policy") - # datalake_admin_s3_policy_doc - # ...first process placeholders in the downloaded policy doc - datalake_admin_s3_policy_doc_processed = replace( - replace( - data.http.datalake_admin_s3_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), - "$${STORAGE_LOCATION_BASE}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}/${replace(local.data_storage.data_storage_object, "/", "")}") - - # ...then assign either input or downloaded policy doc to var used in resource - datalake_admin_s3_policy_doc = coalesce(var.datalake_admin_s3_policy_doc, local.datalake_admin_s3_policy_doc_processed) - # CDP Data Access Policies - bucket_access # Note - separate policies for data, log and backup buckets data_bucket_access_policy_name = coalesce(var.data_bucket_access_policy_name, "${var.env_prefix}-data-bucket-access-policy") log_bucket_access_policy_name = coalesce(var.log_bucket_access_policy_name, "${var.env_prefix}-log-bucket-access-policy") backup_bucket_access_policy_name = coalesce(var.backup_bucket_access_policy_name, "${var.env_prefix}-backup-bucket-access-policy") - # ...first process placeholders in the downloaded policy doc - data_bucket_access_policy_doc_processed = replace( - replace( - data.http.bucket_access_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), - "$${DATALAKE_BUCKET}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}") - log_bucket_access_policy_doc_processed = replace( - replace( - data.http.bucket_access_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), - "$${DATALAKE_BUCKET}", "${local.log_storage.log_storage_bucket}${local.storage_suffix}") - backup_bucket_access_policy_doc_processed = replace( - replace( - data.http.bucket_access_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), - "$${DATALAKE_BUCKET}", "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}") - - # ...then assign either input or downloaded policy doc to var used in resource - data_bucket_access_policy_doc = coalesce(var.data_bucket_access_policy_doc, local.data_bucket_access_policy_doc_processed) - log_bucket_access_policy_doc = coalesce(var.log_bucket_access_policy_doc, local.log_bucket_access_policy_doc_processed) - backup_bucket_access_policy_doc = coalesce(var.backup_bucket_access_policy_doc, local.backup_bucket_access_policy_doc_processed) - # CDP Datalake backup Policy datalake_backup_policy_name = coalesce(var.datalake_backup_policy_name, "${var.env_prefix}-datalake-backup-policy") - # datalake_backup_policy_doc - # ...first process placeholders in the downloaded policy doc - datalake_backup_policy_doc_processed = replace( - replace( - data.http.datalake_backup_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), - "$${BACKUP_LOCATION_BASE}", "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${replace(local.backup_storage.backup_storage_object, "/", "")}") - - # ...then assign either input or downloaded policy doc to var used in resource - datalake_backup_policy_doc = coalesce(var.datalake_backup_policy_doc, local.datalake_backup_policy_doc_processed) - # CDP Datalake restore Policy datalake_restore_policy_name = coalesce(var.datalake_restore_policy_name, "${var.env_prefix}-datalake-restore-policy") - # datalake_restore_policy_doc - # ...first process placeholders in the downloaded policy doc - datalake_restore_policy_doc_processed = replace( - replace( - data.http.datalake_restore_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), - "$${BACKUP_LOCATION_BASE}", "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${replace(local.backup_storage.backup_storage_object, "/", "")}") - - # ...then assign either input or downloaded policy doc to var used in resource - datalake_restore_policy_doc = coalesce(var.datalake_restore_policy_doc, local.datalake_restore_policy_doc_processed) - # ------- Roles ------- xaccount_role_name = coalesce(var.xaccount_role_name, "${var.env_prefix}-xaccount-role") diff --git a/modules/terraform-cdp-aws-pre-reqs/main.tf b/modules/terraform-cdp-aws-pre-reqs/main.tf index 3922fe2..2d61e2b 100644 --- a/modules/terraform-cdp-aws-pre-reqs/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/main.tf @@ -334,125 +334,6 @@ resource "aws_s3_object" "cdp_backup_storage_object" { ] } -# ------- CDP IDBroker Assume Role policy ------- -# First create the assume role policy document -data "aws_iam_policy_document" "cdp_idbroker_policy_doc" { - version = "2012-10-17" - - statement { - sid = "VisualEditor0" - actions = ["sts:AssumeRole"] - effect = "Allow" - resources = ["*"] #tfsec:ignore:aws-iam-no-policy-wildcards - } -} - -# Then create the policy using the document -resource "aws_iam_policy" "cdp_idbroker_policy" { - name = local.idbroker_policy_name - description = "CDP IDBroker Assume Role policy for ${var.env_prefix}" - - tags = merge(local.env_tags, { Name = local.idbroker_policy_name }) - - policy = data.aws_iam_policy_document.cdp_idbroker_policy_doc.json -} - -# ------- CDP Data Access Policies - Log ------- -resource "aws_iam_policy" "cdp_log_data_access_policy" { - name = local.log_data_access_policy_name - description = "CDP Log Location Access policy for ${var.env_prefix}" - - tags = merge(local.env_tags, { Name = local.log_data_access_policy_name }) - - policy = local.log_data_access_policy_doc - -} - -# ------- CDP Data Access Policies - ranger_audit_s3 ------- -resource "aws_iam_policy" "cdp_ranger_audit_s3_data_access_policy" { - name = local.ranger_audit_s3_policy_name - description = "CDP Ranger Audit S3 Access policy for ${var.env_prefix}" - - tags = merge(local.env_tags, { Name = local.ranger_audit_s3_policy_name }) - - policy = local.ranger_audit_s3_policy_doc -} - -# ------- CDP Data Access Policies - datalake_admin_s3 ------- -resource "aws_iam_policy" "cdp_datalake_admin_s3_data_access_policy" { - name = local.datalake_admin_s3_policy_name - description = "CDP Datalake Admin S3 Access policy for ${var.env_prefix}" - - tags = merge(local.env_tags, { Name = local.datalake_admin_s3_policy_name }) - - policy = local.datalake_admin_s3_policy_doc - -} - -# ------- CDP Data Access Policies - bucket_access ------- -# Policy for Data bucket -resource "aws_iam_policy" "cdp_data_bucket_data_access_policy" { - name = local.data_bucket_access_policy_name - description = "CDP Data Bucket S3 Access policy for ${var.env_prefix}" - - tags = merge(local.env_tags, { Name = local.data_bucket_access_policy_name }) - - policy = local.data_bucket_access_policy_doc -} - -# Policy for Log bucket...Only required if log bucket different from data bucket -resource "aws_iam_policy" "cdp_log_bucket_data_access_policy" { - - count = (( - local.log_storage.log_storage_bucket != local.data_storage.data_storage_bucket) ? - 1 : 0 - ) - - name = local.log_bucket_access_policy_name - description = "CDP Log Bucket S3 Access policy for ${var.env_prefix}" - - tags = merge(local.env_tags, { Name = local.log_bucket_access_policy_name }) - - policy = local.log_bucket_access_policy_doc -} -# Policy for backup bucket...requied only if different from backup and log bucket -resource "aws_iam_policy" "cdp_backup_bucket_data_access_policy" { - - count = (( - local.backup_storage.backup_storage_bucket != local.data_storage.data_storage_bucket - || - local.backup_storage.backup_storage_bucket != local.log_storage.log_storage_bucket) ? - 1 : 0 - ) - - name = local.backup_bucket_access_policy_name - description = "CDP Backup Bucket S3 Access policy for ${var.env_prefix}" - - tags = merge(local.env_tags, { Name = local.backup_bucket_access_policy_name }) - - policy = local.backup_bucket_access_policy_doc -} - -# ------- CDP Data Access Policies - datalake_backup_policy ------- -resource "aws_iam_policy" "cdp_datalake_backup_policy" { - name = local.datalake_backup_policy_name - description = "CDP Datalake Backup policy for ${var.env_prefix}" - - tags = merge(local.env_tags, { Name = local.datalake_backup_policy_name }) - - policy = local.datalake_backup_policy_doc -} - -# ------- CDP Data Access Policies - datalake_restore_policy ------- -resource "aws_iam_policy" "cdp_datalake_restore_policy" { - name = local.datalake_restore_policy_name - description = "CDP Datalake Restore policy for ${var.env_prefix}" - - tags = merge(local.env_tags, { Name = local.datalake_restore_policy_name }) - - policy = local.datalake_restore_policy_doc -} - # ------- Cross Account Role ------- # First create the assume role policy document data "aws_iam_policy_document" "cdp_xaccount_role_policy_doc" { @@ -501,277 +382,6 @@ resource "time_sleep" "iam_propagation" { create_duration = "45s" } -# ------- AWS Service Roles - CDP IDBroker ------- -# First create the Assume role policy document -data "aws_iam_policy_document" "cdp_idbroker_role_policy_doc" { - version = "2012-10-17" - - statement { - actions = ["sts:AssumeRole"] - effect = "Allow" - - principals { - type = "Service" - identifiers = ["ec2.amazonaws.com"] - } - } -} - -# Create the IAM role that uses the above assume_role_policy document -resource "aws_iam_role" "cdp_idbroker_role" { - name = local.idbroker_role_name - description = "CDP IDBroker role for ${var.env_prefix}" - - assume_role_policy = data.aws_iam_policy_document.cdp_idbroker_role_policy_doc.json - - tags = merge(local.env_tags, { Name = local.idbroker_role_name }) -} - -# Create an instance profile for the iam_role -resource "aws_iam_instance_profile" "cdp_idbroker_role_instance_profile" { - name = local.idbroker_role_name - role = aws_iam_role.cdp_idbroker_role.name -} - -# Attach CDP IDBroker Assume Policy to the Role -resource "aws_iam_role_policy_attachment" "cdp_idbroker_role_attach1" { - role = aws_iam_role.cdp_idbroker_role.name - policy_arn = aws_iam_policy.cdp_idbroker_policy.arn -} - -# Attach AWS Log Location Policy to the Role -resource "aws_iam_role_policy_attachment" "cdp_idbroker_role_attach2" { - - role = aws_iam_role.cdp_idbroker_role.name - policy_arn = aws_iam_policy.cdp_log_data_access_policy.arn -} - -# ------- AWS Service Roles - CDP Log ------- -# First create the Assume role policy document -data "aws_iam_policy_document" "cdp_log_role_policy_doc" { - version = "2012-10-17" - - statement { - actions = ["sts:AssumeRole"] - effect = "Allow" - - principals { - type = "Service" - identifiers = ["ec2.amazonaws.com"] - } - } -} - -# Create the IAM role that uses the above assume_role_policy document -resource "aws_iam_role" "cdp_log_role" { - name = local.log_role_name - description = "CDP Log role for ${var.env_prefix}" - - assume_role_policy = data.aws_iam_policy_document.cdp_log_role_policy_doc.json - - tags = merge(local.env_tags, { Name = local.log_role_name }) -} - -# Create an instance profile for the iam_role -resource "aws_iam_instance_profile" "cdp_log_role_instance_profile" { - name = local.log_role_name - role = aws_iam_role.cdp_log_role.name -} - -# Attach AWS Log Location Policy to the Role -resource "aws_iam_role_policy_attachment" "cdp_log_role_attach1" { - - role = aws_iam_role.cdp_log_role.name - policy_arn = aws_iam_policy.cdp_log_data_access_policy.arn -} - -# Attach AWS Datalake Restore Policy to the Role -resource "aws_iam_role_policy_attachment" "cdp_log_role_attach2" { - - role = aws_iam_role.cdp_log_role.name - policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn -} - -# Attach AWS Datalake Backup Policy to the Role -resource "aws_iam_role_policy_attachment" "cdp_log_role_attach3" { - - role = aws_iam_role.cdp_log_role.name - policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn -} - -# ------- AWS Data Access Roles - CDP Datalake Admin ------- -# First create the Assume role policy document -data "aws_iam_policy_document" "cdp_datalake_admin_role_policy_doc" { - version = "2012-10-17" - - statement { - actions = ["sts:AssumeRole"] - effect = "Allow" - - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${local.caller_account_id}:role/${aws_iam_role.cdp_idbroker_role.name}"] - } - } -} - -# Create the IAM role that uses the above assume_role_policy document -resource "aws_iam_role" "cdp_datalake_admin_role" { - name = local.datalake_admin_role_name - description = "CDP Datalake Admin role for ${var.env_prefix}" - - assume_role_policy = data.aws_iam_policy_document.cdp_datalake_admin_role_policy_doc.json - - tags = merge(local.env_tags, { Name = local.datalake_admin_role_name }) -} - -# Create an instance profile for the iam_role -resource "aws_iam_instance_profile" "cdp_datalake_admin_role_instance_profile" { - name = local.datalake_admin_role_name - role = aws_iam_role.cdp_datalake_admin_role.name -} - -# Attach AWS Datalake Admin S3 Policy to the Role -resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach1" { - - role = aws_iam_role.cdp_datalake_admin_role.name - policy_arn = aws_iam_policy.cdp_datalake_admin_s3_data_access_policy.arn -} - -# Attach AWS Bucket Access Policy to the Role -# ..data bucket policy -resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach2" { - - role = aws_iam_role.cdp_datalake_admin_role.name - policy_arn = aws_iam_policy.cdp_data_bucket_data_access_policy.arn -} - -# ..log bucket policy, if required -resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach3" { - - count = (( - local.log_storage.log_storage_bucket != local.data_storage.data_storage_bucket) ? - 1 : 0 - ) - - role = aws_iam_role.cdp_datalake_admin_role.name - policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy[0].arn -} - -# ..backup bucket policy, if required -resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach4" { - - count = (( - local.backup_storage.backup_storage_bucket != local.data_storage.data_storage_bucket - || - local.backup_storage.backup_storage_bucket != local.log_storage.log_storage_bucket) ? - 1 : 0 - ) - - role = aws_iam_role.cdp_datalake_admin_role.name - policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy[0].arn -} - -# Attach AWS Datalake Backup Policy to the Role -resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach5" { - - role = aws_iam_role.cdp_datalake_admin_role.name - policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn -} - -# Attach AWS Datalake Restore Policy to the Role -resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach6" { - - role = aws_iam_role.cdp_datalake_admin_role.name - policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn -} - -# ------- AWS Data Access Roles - CDP Ranger Audit ------- -# First create the Assume role policy document -data "aws_iam_policy_document" "cdp_ranger_audit_role_policy_doc" { - version = "2012-10-17" - - statement { - actions = ["sts:AssumeRole"] - effect = "Allow" - - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${local.caller_account_id}:role/${aws_iam_role.cdp_idbroker_role.name}"] - } - } -} - -# Create the IAM role that uses the above assume_role_policy document -resource "aws_iam_role" "cdp_ranger_audit_role" { - name = local.ranger_audit_role_name - description = "CDP Ranger Audit role for ${var.env_prefix}" - - assume_role_policy = data.aws_iam_policy_document.cdp_ranger_audit_role_policy_doc.json - - tags = merge(local.env_tags, { Name = local.ranger_audit_role_name }) -} - -# Create an instance profile for the iam_role -resource "aws_iam_instance_profile" "cdp_ranger_audit_role_instance_profile" { - name = local.ranger_audit_role_name - role = aws_iam_role.cdp_ranger_audit_role.name -} - -# Attach AWS Ranger Audit S3 Policy to the Role -resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach1" { - - role = aws_iam_role.cdp_ranger_audit_role.name - policy_arn = aws_iam_policy.cdp_ranger_audit_s3_data_access_policy.arn -} - -# Attach AWS Bucket Access Policies to the Role -# ..data bucket policy -resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach2" { - - role = aws_iam_role.cdp_ranger_audit_role.name - policy_arn = aws_iam_policy.cdp_data_bucket_data_access_policy.arn -} - -# ..log bucket policy, if required -resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach3" { - - count = (( - local.log_storage.log_storage_bucket != local.data_storage.data_storage_bucket) ? - 1 : 0 - ) - - role = aws_iam_role.cdp_ranger_audit_role.name - policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy[0].arn -} - -# ..backup bucket policy, if required -resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach4" { - - count = (( - local.backup_storage.backup_storage_bucket != local.data_storage.data_storage_bucket - || - local.backup_storage.backup_storage_bucket != local.log_storage.log_storage_bucket) ? - 1 : 0 - ) - - role = aws_iam_role.cdp_ranger_audit_role.name - policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy[0].arn -} - -# Attach AWS Datalake Backup Policy to the Role -resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach5" { - - role = aws_iam_role.cdp_ranger_audit_role.name - policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn -} - -# Attach AWS Datalake Restore Policy to the Role -resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach6" { - - role = aws_iam_role.cdp_ranger_audit_role.name - policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn -} # ------- Add missing iam:Tag* permissions to Cross-Account Policy ------- # First create the extra policy document @@ -801,3 +411,49 @@ resource "aws_iam_role_policy" "cdp_extra_xaccount_policy" { policy = data.aws_iam_policy_document.cdp_extra_xaccount_policy_doc[0].json } + +# ------- Permissions ------- +module "aws_cdp_permissions" { + + source = "../terraform-aws-permissions" + + tags = local.env_tags + + idbroker_policy_name = local.idbroker_policy_name + idbroker_policy_doc = var.idbroker_policy_doc + + log_data_access_policy_name = local.log_data_access_policy_name + log_data_access_policy_doc = var.log_data_access_policy_doc + + ranger_audit_s3_policy_name = local.ranger_audit_s3_policy_name + ranger_audit_s3_policy_doc = var.ranger_audit_s3_policy_doc + + datalake_admin_s3_policy_name = local.datalake_admin_s3_policy_name + datalake_admin_s3_policy_doc = var.datalake_admin_s3_policy_doc + + data_bucket_access_policy_name = local.data_bucket_access_policy_name + data_bucket_access_policy_doc = var.data_bucket_access_policy_doc + log_bucket_access_policy_name = local.log_bucket_access_policy_name + log_bucket_access_policy_doc = var.log_bucket_access_policy_doc + backup_bucket_access_policy_name = local.backup_bucket_access_policy_name + backup_bucket_access_policy_doc = var.backup_bucket_access_policy_doc + + data_storage_bucket = "${local.data_storage.data_storage_bucket}${local.storage_suffix}" + log_storage_bucket = "${local.log_storage.log_storage_bucket}${local.storage_suffix}" + backup_storage_bucket = "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}" + + storage_location_base = "${local.data_storage.data_storage_bucket}${local.storage_suffix}/${replace(local.data_storage.data_storage_object, "/", "")}" + log_location_base = "${local.log_storage.log_storage_bucket}${local.storage_suffix}" + backup_location_base = "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${replace(local.backup_storage.backup_storage_object, "/", "")}" + + datalake_backup_policy_name = local.datalake_backup_policy_name + datalake_backup_policy_doc = var.datalake_backup_policy_doc + datalake_restore_policy_name = local.datalake_restore_policy_name + datalake_restore_policy_doc = var.datalake_restore_policy_doc + + idbroker_role_name = local.idbroker_role_name + log_role_name = local.log_role_name + datalake_admin_role_name = local.datalake_admin_role_name + + depends_on = [ aws_s3_bucket.cdp_storage_locations ] +} diff --git a/modules/terraform-cdp-aws-pre-reqs/outputs.tf b/modules/terraform-cdp-aws-pre-reqs/outputs.tf index 289ae9d..a8174c6 100644 --- a/modules/terraform-cdp-aws-pre-reqs/outputs.tf +++ b/modules/terraform-cdp-aws-pre-reqs/outputs.tf @@ -134,13 +134,13 @@ output "aws_security_group_knox_id" { } output "aws_datalake_admin_role_arn" { - value = aws_iam_role.cdp_datalake_admin_role.arn + value = module.aws_cdp_permissions.aws_datalake_admin_role_arn description = "Datalake Admin role ARN" } output "aws_ranger_audit_role_arn" { - value = aws_iam_role.cdp_ranger_audit_role.arn + value = module.aws_cdp_permissions.aws_ranger_audit_role_arn description = "Ranger Audit role ARN" } @@ -158,37 +158,37 @@ output "aws_xaccount_role_name" { } output "aws_log_role_name" { - value = aws_iam_role.cdp_log_role.name + value = module.aws_cdp_permissions.aws_log_role_name description = "Log role Name" } output "aws_idbroker_role_name" { - value = aws_iam_role.cdp_idbroker_role.name + value = module.aws_cdp_permissions.aws_idbroker_role_name description = "IDBroker role Name" } output "aws_datalake_admin_role_name" { - value = aws_iam_role.cdp_datalake_admin_role.name + value = module.aws_cdp_permissions.aws_datalake_admin_role_name description = "Datalake Admin role Name" } output "aws_ranger_audit_role_name" { - value = aws_iam_role.cdp_ranger_audit_role.name + value = module.aws_cdp_permissions.aws_ranger_audit_role_name description = "Ranger Audit role Name" } output "aws_log_instance_profile_arn" { - value = aws_iam_instance_profile.cdp_log_role_instance_profile.arn + value = module.aws_cdp_permissions.aws_log_instance_profile_arn description = "Log instance profile ARN" } output "aws_idbroker_instance_profile_arn" { - value = aws_iam_instance_profile.cdp_idbroker_role_instance_profile.arn + value = module.aws_cdp_permissions.aws_idbroker_instance_profile_arn description = "IDBroker instance profile ARN" } diff --git a/modules/terraform-cdp-aws-pre-reqs/variables.tf b/modules/terraform-cdp-aws-pre-reqs/variables.tf index b2fdb03..502045b 100644 --- a/modules/terraform-cdp-aws-pre-reqs/variables.tf +++ b/modules/terraform-cdp-aws-pre-reqs/variables.tf @@ -59,13 +59,13 @@ variable "deployment_template" { } } -# variable "enable_raz" { -# type = bool +variable "enable_raz" { + type = bool -# description = "Flag to enable Ranger Authorization Service (RAZ)" + description = "Flag to enable Ranger Authorization Service (RAZ)" -# default = true -# } + default = true +} # ------- Network Resources ------- variable "create_vpc" { @@ -391,6 +391,11 @@ variable "create_extra_xaccount_policy" { } # CDP IDBroker Assume Role policy +variable "idbroker_policy_doc" { + type = string + description = "Contents of IDBroker Assumer Policy Document." +} + variable "idbroker_policy_name" { type = string description = "IDBroker Policy name" @@ -410,7 +415,6 @@ variable "log_data_access_policy_doc" { type = string description = "Contents of Log Data Access Policy" - default = null } # CDP Data Access Policies - ranger_audit_s3 @@ -425,7 +429,6 @@ variable "ranger_audit_s3_policy_doc" { type = string description = "Contents of Ranger S3 Audit Data Access Policy" - default = null } # CDP Data Access Policies - datalake_admin_s3 @@ -440,21 +443,18 @@ variable "datalake_admin_s3_policy_doc" { type = string description = "Contents of Datalake Admin S3 Data Access Policy" - default = null } variable "datalake_backup_policy_doc" { type = string description = "Contents of Datalake Backup Data Access Policy" - default = null } variable "datalake_restore_policy_doc" { type = string description = "Contents of Datalake Restore Data Access Policy" - default = null } # CDP Data Access Policies - bucket_access @@ -498,19 +498,17 @@ variable "data_bucket_access_policy_doc" { type = string description = "Data Bucket Access Data Access Policy" - default = null } + variable "log_bucket_access_policy_doc" { type = string description = "Contents of Log Bucket Access Data Access Policy" - default = null } variable "backup_bucket_access_policy_doc" { type = string description = "Contents of Backup Bucket Access Data Access Policy" - default = null } # ------- Roles ------- @@ -565,3 +563,4 @@ variable "ranger_audit_role_name" { default = null } + diff --git a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf index 27f732c..cb5d2e5 100644 --- a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf +++ b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf @@ -55,6 +55,21 @@ module "cdp_aws_prereqs" { xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + # Policy documents from CDP TF Provider cred pre-reqs + idbroker_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Idbroker_Assumer"]) + + data_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + log_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + backup_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + + datalake_admin_s3_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Admin"]) + datalake_backup_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Backup"]) + datalake_restore_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Restore"]) + + log_data_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Log_Policy"]) + ranger_audit_s3_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Ranger_Audit"]) + ranger_raz_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Ranger_Raz"]) + # Inputs for BYO-VPC create_vpc = var.create_vpc cdp_vpc_id = var.cdp_vpc_id @@ -69,45 +84,62 @@ module "cdp_aws_prereqs" { } -module "cdp_deploy" { - source = "../.." +# module "cdp_deploy" { +# source = "../.." - env_prefix = var.env_prefix - infra_type = "aws" - region = var.aws_region - keypair_name = var.aws_key_pair - deployment_template = var.deployment_template +# env_prefix = var.env_prefix +# infra_type = "aws" +# region = var.aws_region +# keypair_name = var.aws_key_pair +# deployment_template = var.deployment_template - environment_async_creation = var.environment_async_creation - datalake_async_creation = var.datalake_async_creation +# environment_async_creation = var.environment_async_creation +# datalake_async_creation = var.datalake_async_creation - # From pre-reqs module output - aws_vpc_id = module.cdp_aws_prereqs.aws_vpc_id - aws_public_subnet_ids = module.cdp_aws_prereqs.aws_public_subnet_ids - aws_private_subnet_ids = module.cdp_aws_prereqs.aws_private_subnet_ids +# # From pre-reqs module output +# aws_vpc_id = module.cdp_aws_prereqs.aws_vpc_id +# aws_public_subnet_ids = module.cdp_aws_prereqs.aws_public_subnet_ids +# aws_private_subnet_ids = module.cdp_aws_prereqs.aws_private_subnet_ids - aws_security_group_default_id = module.cdp_aws_prereqs.aws_security_group_default_id - aws_security_group_knox_id = module.cdp_aws_prereqs.aws_security_group_knox_id +# aws_security_group_default_id = module.cdp_aws_prereqs.aws_security_group_default_id +# aws_security_group_knox_id = module.cdp_aws_prereqs.aws_security_group_knox_id - data_storage_location = module.cdp_aws_prereqs.aws_data_storage_location - log_storage_location = module.cdp_aws_prereqs.aws_log_storage_location - backup_storage_location = module.cdp_aws_prereqs.aws_backup_storage_location +# data_storage_location = module.cdp_aws_prereqs.aws_data_storage_location +# log_storage_location = module.cdp_aws_prereqs.aws_log_storage_location +# backup_storage_location = module.cdp_aws_prereqs.aws_backup_storage_location aws_xaccount_role_arn = module.cdp_aws_prereqs.aws_xaccount_role_arn aws_datalake_admin_role_arn = module.cdp_aws_prereqs.aws_datalake_admin_role_arn aws_ranger_audit_role_arn = module.cdp_aws_prereqs.aws_ranger_audit_role_arn aws_raz_role_arn = module.cdp_aws_prereqs.aws_datalake_admin_role_arn - aws_log_instance_profile_arn = module.cdp_aws_prereqs.aws_log_instance_profile_arn - aws_idbroker_instance_profile_arn = module.cdp_aws_prereqs.aws_idbroker_instance_profile_arn +# aws_log_instance_profile_arn = module.cdp_aws_prereqs.aws_log_instance_profile_arn +# aws_idbroker_instance_profile_arn = module.cdp_aws_prereqs.aws_idbroker_instance_profile_arn - # Tags to apply resources (omitted by default) - env_tags = var.env_tags +# # Tags to apply resources (omitted by default) +# env_tags = var.env_tags - depends_on = [ - module.cdp_aws_prereqs - ] -} +# depends_on = [ +# module.cdp_aws_prereqs +# ] +# } # Use the CDP Terraform Provider to find the xaccount account and external ids data "cdp_environments_aws_credential_prerequisites" "cdp_prereqs" {} + + +# Policies +output "Ranger_Raz" { + value = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Ranger_Raz"]) +} + +# output "Idbroker_Assumer" { +# value = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Idbroker_Assumer"]) +# } +# output "Environment" { +# value = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Environment"]) +# } + +# output "policies" { +# value = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies) +# } \ No newline at end of file From 7feee27eecdfca853f0f35326a1bcf7a30e1956b Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 15 Aug 2024 13:08:59 +0100 Subject: [PATCH 03/11] Fix formatting Signed-off-by: Jim Enright --- modules/terraform-aws-permissions/defaults.tf | 19 +++++---- modules/terraform-aws-permissions/main.tf | 34 +++++++-------- .../terraform-aws-permissions/variables.tf | 42 ++++--------------- modules/terraform-cdp-aws-pre-reqs/main.tf | 29 ++++++------- .../terraform-cdp-aws-pre-reqs/variables.tf | 9 ---- .../examples/ex01-aws-basic/main.tf | 12 +++--- 6 files changed, 54 insertions(+), 91 deletions(-) diff --git a/modules/terraform-aws-permissions/defaults.tf b/modules/terraform-aws-permissions/defaults.tf index 6f940ca..c609f84 100644 --- a/modules/terraform-aws-permissions/defaults.tf +++ b/modules/terraform-aws-permissions/defaults.tf @@ -19,27 +19,27 @@ locals { # ------- Policies ------- # Process placeholders in policy documents - + # Bucket Access Policy # ...process placeholders in the policy doc data_bucket_access_policy_doc_processed = replace( replace( var.data_bucket_access_policy_doc, "$${ARN_PARTITION}", var.arn_partition), "$${DATALAKE_BUCKET}", "${var.data_storage_bucket}") - + log_bucket_access_policy_doc_processed = replace( replace( var.log_bucket_access_policy_doc, "$${ARN_PARTITION}", var.arn_partition), "$${DATALAKE_BUCKET}", "${var.log_storage_bucket}") - + backup_bucket_access_policy_doc_processed = replace( replace( var.backup_bucket_access_policy_doc, "$${ARN_PARTITION}", var.arn_partition), "$${DATALAKE_BUCKET}", "${var.backup_storage_bucket}") - + # ...then assign either input or processed policy doc to var used in resource - data_bucket_access_policy_doc = var.process_policy_placeholders ? local.data_bucket_access_policy_doc_processed : var.data_bucket_access_policy_doc - log_bucket_access_policy_doc = var.process_policy_placeholders ? local.log_bucket_access_policy_doc_processed : var.log_bucket_access_policy_doc + data_bucket_access_policy_doc = var.process_policy_placeholders ? local.data_bucket_access_policy_doc_processed : var.data_bucket_access_policy_doc + log_bucket_access_policy_doc = var.process_policy_placeholders ? local.log_bucket_access_policy_doc_processed : var.log_bucket_access_policy_doc backup_bucket_access_policy_doc = var.process_policy_placeholders ? local.backup_bucket_access_policy_doc_processed : var.backup_bucket_access_policy_doc # Datalake Admin @@ -64,9 +64,9 @@ locals { # ...process placeholders in the policy doc datalake_restore_policy_doc_processed = replace( replace( - replace( - var.datalake_restore_policy_doc, "$${ARN_PARTITION}", var.arn_partition), - "$${BACKUP_LOCATION_BASE}", var.backup_location_base), + replace( + var.datalake_restore_policy_doc, "$${ARN_PARTITION}", var.arn_partition), + "$${BACKUP_LOCATION_BASE}", var.backup_location_base), "$${BACKUP_BUCKET}", var.backup_storage_bucket) # ...then assign either input or processed policy doc to var used in resource datalake_restore_policy_doc = var.process_policy_placeholders ? local.datalake_restore_policy_doc_processed : var.datalake_restore_policy_doc @@ -93,4 +93,5 @@ locals { # ...then assign either input or processed policy doc to var used in resource ranger_audit_s3_policy_doc = var.process_policy_placeholders ? local.ranger_audit_s3_policy_doc_processed : var.ranger_audit_s3_policy_doc + } \ No newline at end of file diff --git a/modules/terraform-aws-permissions/main.tf b/modules/terraform-aws-permissions/main.tf index 9ba89d0..65a0a88 100644 --- a/modules/terraform-aws-permissions/main.tf +++ b/modules/terraform-aws-permissions/main.tf @@ -1,6 +1,6 @@ # ------- CDP IDBroker Assume Role policy ------- resource "aws_iam_policy" "cdp_idbroker_policy" { - name = var.idbroker_policy_name + name = var.idbroker_policy_name # description = "CDP IDBroker Assume Role policy for ${var.env_prefix}" tags = merge(var.tags, { Name = var.idbroker_policy_name }) @@ -10,7 +10,7 @@ resource "aws_iam_policy" "cdp_idbroker_policy" { # ------- CDP Data Access Policies - Log ------- resource "aws_iam_policy" "cdp_log_data_access_policy" { - name = var.log_data_access_policy_name + name = var.log_data_access_policy_name # description = "CDP Log Location Access policy for ${var.env_prefix}" tags = merge(var.tags, { Name = var.log_data_access_policy_name }) @@ -20,7 +20,7 @@ resource "aws_iam_policy" "cdp_log_data_access_policy" { } # ------- CDP Data Access Policies - ranger_audit_s3 ------- resource "aws_iam_policy" "cdp_ranger_audit_s3_data_access_policy" { - name = var.ranger_audit_s3_policy_name + name = var.ranger_audit_s3_policy_name # description = "CDP Ranger Audit S3 Access policy for ${var.env_prefix}" tags = merge(var.tags, { Name = var.ranger_audit_s3_policy_name }) @@ -30,7 +30,7 @@ resource "aws_iam_policy" "cdp_ranger_audit_s3_data_access_policy" { # ------- CDP Data Access Policies - datalake_admin_s3 ------- resource "aws_iam_policy" "cdp_datalake_admin_s3_data_access_policy" { - name = var.datalake_admin_s3_policy_name + name = var.datalake_admin_s3_policy_name # description = "CDP Datalake Admin S3 Access policy for ${var.env_prefix}" tags = merge(var.tags, { Name = var.datalake_admin_s3_policy_name }) @@ -42,7 +42,7 @@ resource "aws_iam_policy" "cdp_datalake_admin_s3_data_access_policy" { # ------- CDP Data Access Policies - bucket_access ------- # Policy for Data bucket resource "aws_iam_policy" "cdp_data_bucket_data_access_policy" { - name = var.data_bucket_access_policy_name + name = var.data_bucket_access_policy_name # description = "CDP Data Bucket S3 Access policy for ${var.env_prefix}" tags = merge(var.tags, { Name = var.data_bucket_access_policy_name }) @@ -58,7 +58,7 @@ resource "aws_iam_policy" "cdp_log_bucket_data_access_policy" { # 1 : 0 # ) - name = var.log_bucket_access_policy_name + name = var.log_bucket_access_policy_name # description = "CDP Log Bucket S3 Access policy for ${var.env_prefix}" tags = merge(var.tags, { Name = var.log_bucket_access_policy_name }) @@ -75,7 +75,7 @@ resource "aws_iam_policy" "cdp_backup_bucket_data_access_policy" { # 1 : 0 # ) - name = var.backup_bucket_access_policy_name + name = var.backup_bucket_access_policy_name # description = "CDP Backup Bucket S3 Access policy for ${var.env_prefix}" tags = merge(var.tags, { Name = var.backup_bucket_access_policy_name }) @@ -85,7 +85,7 @@ resource "aws_iam_policy" "cdp_backup_bucket_data_access_policy" { # ------- CDP Data Access Policies - datalake_backup_policy ------- resource "aws_iam_policy" "cdp_datalake_backup_policy" { - name = var.datalake_backup_policy_name + name = var.datalake_backup_policy_name # description = "CDP Datalake Backup policy for ${var.env_prefix}" tags = merge(var.tags, { Name = var.datalake_backup_policy_name }) @@ -95,7 +95,7 @@ resource "aws_iam_policy" "cdp_datalake_backup_policy" { # ------- CDP Data Access Policies - datalake_restore_policy ------- resource "aws_iam_policy" "cdp_datalake_restore_policy" { - name = var.datalake_restore_policy_name + name = var.datalake_restore_policy_name # description = "CDP Datalake Restore policy for ${var.env_prefix}" tags = merge(var.tags, { Name = var.datalake_restore_policy_name }) @@ -121,7 +121,7 @@ data "aws_iam_policy_document" "cdp_idbroker_role_policy_doc" { # Create the IAM role that uses the above assume_role_policy document resource "aws_iam_role" "cdp_idbroker_role" { - name = var.idbroker_role_name + name = var.idbroker_role_name # description = "CDP IDBroker role for ${var.env_prefix}" assume_role_policy = data.aws_iam_policy_document.cdp_idbroker_role_policy_doc.json @@ -166,7 +166,7 @@ data "aws_iam_policy_document" "cdp_log_role_policy_doc" { # Create the IAM role that uses the above assume_role_policy document resource "aws_iam_role" "cdp_log_role" { - name = var.log_role_name + name = var.log_role_name # description = "CDP Log role for ${var.env_prefix}" assume_role_policy = data.aws_iam_policy_document.cdp_log_role_policy_doc.json @@ -219,7 +219,7 @@ data "aws_iam_policy_document" "cdp_datalake_admin_role_policy_doc" { # Create the IAM role that uses the above assume_role_policy document resource "aws_iam_role" "cdp_datalake_admin_role" { - name = var.datalake_admin_role_name + name = var.datalake_admin_role_name # description = "CDP Datalake Admin role for ${var.env_prefix}" assume_role_policy = data.aws_iam_policy_document.cdp_datalake_admin_role_policy_doc.json @@ -256,7 +256,7 @@ resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach3" { # 1 : 0 # ) - role = aws_iam_role.cdp_datalake_admin_role.name + role = aws_iam_role.cdp_datalake_admin_role.name # policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy[0].arn policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy.arn } @@ -271,7 +271,7 @@ resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach4" { # 1 : 0 # ) - role = aws_iam_role.cdp_datalake_admin_role.name + role = aws_iam_role.cdp_datalake_admin_role.name # policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy[0].arn policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy.arn } @@ -309,7 +309,7 @@ data "aws_iam_policy_document" "cdp_ranger_audit_role_policy_doc" { # Create the IAM role that uses the above assume_role_policy document resource "aws_iam_role" "cdp_ranger_audit_role" { - name = var.ranger_audit_role_name + name = var.ranger_audit_role_name # description = "CDP Ranger Audit role for ${var.env_prefix}" assume_role_policy = data.aws_iam_policy_document.cdp_ranger_audit_role_policy_doc.json @@ -346,7 +346,7 @@ resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach3" { # 1 : 0 # ) - role = aws_iam_role.cdp_ranger_audit_role.name + role = aws_iam_role.cdp_ranger_audit_role.name # policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy[0].arn policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy.arn } @@ -361,7 +361,7 @@ resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach4" { # 1 : 0 # ) - role = aws_iam_role.cdp_ranger_audit_role.name + role = aws_iam_role.cdp_ranger_audit_role.name # policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy[0].arn policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy.arn } diff --git a/modules/terraform-aws-permissions/variables.tf b/modules/terraform-aws-permissions/variables.tf index 8a65283..7eb90f7 100644 --- a/modules/terraform-aws-permissions/variables.tf +++ b/modules/terraform-aws-permissions/variables.tf @@ -21,27 +21,18 @@ variable "tags" { } variable "process_policy_placeholders" { - type = bool + type = bool description = "Flag to enable replacement of the standard placeholders in the AWS CDP Policy documents" - + default = true } variable "arn_partition" { - type = string + type = string description = "The string used to subsitute ARN_PARTITION placeholder in policy documents." default = "aws" - -} - -# ------- CDP Environment Deployment ------- -variable "enable_raz" { - type = bool - description = "Flag to enable Ranger Authorization Service (RAZ)" - - default = true } # ------- Policies ------- @@ -54,7 +45,7 @@ variable "idbroker_policy_name" { variable "idbroker_policy_doc" { type = string - description = "Contents of IDBroker Assumer Policy Document." + description = "Contents of IDBroker Assumer Policy Document." } # CDP Data Access Policies - Log @@ -157,17 +148,6 @@ variable "backup_bucket_access_policy_doc" { } -# Ranger Raz -variable "ranger_raz_policy_name" { - type = string - description = "Ranger Raz Policy Name" -} - -variable "ranger_raz_policy_doc" { - type = string - description = "Contents of Ranger Raz Policy" -} - # ------- Roles ------- # IDBroker service role variable "idbroker_role_name" { @@ -197,31 +177,23 @@ variable "ranger_audit_role_name" { } -# CDP Ranger Raz role -variable "ranger_raz_role_name" { - type = string - description = "Ranger Raz role Name. Required is Raz is to be enabled." - - default = null -} - # ------- Buckets and Storage Locations ------- variable "data_storage_bucket" { type = string - description = "Name of the Data storage bucket" + description = "Name of the Data storage bucket" } variable "log_storage_bucket" { type = string - description = "Name of the Log storage bucket" + description = "Name of the Log storage bucket" } variable "backup_storage_bucket" { type = string - description = "Name of the Backup storage bucket" + description = "Name of the Backup storage bucket" } variable "storage_location_base" { diff --git a/modules/terraform-cdp-aws-pre-reqs/main.tf b/modules/terraform-cdp-aws-pre-reqs/main.tf index 2d61e2b..aa46880 100644 --- a/modules/terraform-cdp-aws-pre-reqs/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/main.tf @@ -418,25 +418,25 @@ module "aws_cdp_permissions" { source = "../terraform-aws-permissions" tags = local.env_tags - + idbroker_policy_name = local.idbroker_policy_name idbroker_policy_doc = var.idbroker_policy_doc log_data_access_policy_name = local.log_data_access_policy_name - log_data_access_policy_doc = var.log_data_access_policy_doc + log_data_access_policy_doc = var.log_data_access_policy_doc ranger_audit_s3_policy_name = local.ranger_audit_s3_policy_name - ranger_audit_s3_policy_doc = var.ranger_audit_s3_policy_doc + ranger_audit_s3_policy_doc = var.ranger_audit_s3_policy_doc datalake_admin_s3_policy_name = local.datalake_admin_s3_policy_name datalake_admin_s3_policy_doc = var.datalake_admin_s3_policy_doc - data_bucket_access_policy_name = local.data_bucket_access_policy_name - data_bucket_access_policy_doc = var.data_bucket_access_policy_doc - log_bucket_access_policy_name = local.log_bucket_access_policy_name - log_bucket_access_policy_doc = var.log_bucket_access_policy_doc + data_bucket_access_policy_name = local.data_bucket_access_policy_name + data_bucket_access_policy_doc = var.data_bucket_access_policy_doc + log_bucket_access_policy_name = local.log_bucket_access_policy_name + log_bucket_access_policy_doc = var.log_bucket_access_policy_doc backup_bucket_access_policy_name = local.backup_bucket_access_policy_name - backup_bucket_access_policy_doc = var.backup_bucket_access_policy_doc + backup_bucket_access_policy_doc = var.backup_bucket_access_policy_doc data_storage_bucket = "${local.data_storage.data_storage_bucket}${local.storage_suffix}" log_storage_bucket = "${local.log_storage.log_storage_bucket}${local.storage_suffix}" @@ -444,16 +444,17 @@ module "aws_cdp_permissions" { storage_location_base = "${local.data_storage.data_storage_bucket}${local.storage_suffix}/${replace(local.data_storage.data_storage_object, "/", "")}" log_location_base = "${local.log_storage.log_storage_bucket}${local.storage_suffix}" - backup_location_base = "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${replace(local.backup_storage.backup_storage_object, "/", "")}" + backup_location_base = "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${replace(local.backup_storage.backup_storage_object, "/", "")}" - datalake_backup_policy_name = local.datalake_backup_policy_name - datalake_backup_policy_doc = var.datalake_backup_policy_doc + datalake_backup_policy_name = local.datalake_backup_policy_name + datalake_backup_policy_doc = var.datalake_backup_policy_doc datalake_restore_policy_name = local.datalake_restore_policy_name datalake_restore_policy_doc = var.datalake_restore_policy_doc - idbroker_role_name = local.idbroker_role_name - log_role_name = local.log_role_name + idbroker_role_name = local.idbroker_role_name + log_role_name = local.log_role_name datalake_admin_role_name = local.datalake_admin_role_name + ranger_audit_role_name = local.ranger_audit_role_name - depends_on = [ aws_s3_bucket.cdp_storage_locations ] + depends_on = [aws_s3_bucket.cdp_storage_locations] } diff --git a/modules/terraform-cdp-aws-pre-reqs/variables.tf b/modules/terraform-cdp-aws-pre-reqs/variables.tf index 502045b..410085e 100644 --- a/modules/terraform-cdp-aws-pre-reqs/variables.tf +++ b/modules/terraform-cdp-aws-pre-reqs/variables.tf @@ -59,14 +59,6 @@ variable "deployment_template" { } } -variable "enable_raz" { - type = bool - - description = "Flag to enable Ranger Authorization Service (RAZ)" - - default = true -} - # ------- Network Resources ------- variable "create_vpc" { type = bool @@ -563,4 +555,3 @@ variable "ranger_audit_role_name" { default = null } - diff --git a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf index cb5d2e5..e32b777 100644 --- a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf +++ b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf @@ -58,8 +58,8 @@ module "cdp_aws_prereqs" { # Policy documents from CDP TF Provider cred pre-reqs idbroker_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Idbroker_Assumer"]) - data_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) - log_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + data_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + log_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) backup_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) datalake_admin_s3_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Admin"]) @@ -68,7 +68,6 @@ module "cdp_aws_prereqs" { log_data_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Log_Policy"]) ranger_audit_s3_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Ranger_Audit"]) - ranger_raz_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Ranger_Raz"]) # Inputs for BYO-VPC create_vpc = var.create_vpc @@ -127,11 +126,10 @@ module "cdp_aws_prereqs" { # Use the CDP Terraform Provider to find the xaccount account and external ids data "cdp_environments_aws_credential_prerequisites" "cdp_prereqs" {} - # Policies -output "Ranger_Raz" { - value = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Ranger_Raz"]) -} +# output "Ranger_Raz" { +# value = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Ranger_Raz"]) +# } # output "Idbroker_Assumer" { # value = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Idbroker_Assumer"]) From e214c1cdc9a9c48585b061cae072036e9ba5b1aa Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 15 Aug 2024 13:12:14 +0100 Subject: [PATCH 04/11] Fix tflint issues in aws-premissions module Signed-off-by: Jim Enright --- modules/terraform-aws-permissions/defaults.tf | 6 ++--- modules/terraform-aws-permissions/provider.tf | 24 +++++++++++++++++++ 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/modules/terraform-aws-permissions/defaults.tf b/modules/terraform-aws-permissions/defaults.tf index c609f84..e721543 100644 --- a/modules/terraform-aws-permissions/defaults.tf +++ b/modules/terraform-aws-permissions/defaults.tf @@ -25,17 +25,17 @@ locals { data_bucket_access_policy_doc_processed = replace( replace( var.data_bucket_access_policy_doc, "$${ARN_PARTITION}", var.arn_partition), - "$${DATALAKE_BUCKET}", "${var.data_storage_bucket}") + "$${DATALAKE_BUCKET}", var.data_storage_bucket) log_bucket_access_policy_doc_processed = replace( replace( var.log_bucket_access_policy_doc, "$${ARN_PARTITION}", var.arn_partition), - "$${DATALAKE_BUCKET}", "${var.log_storage_bucket}") + "$${DATALAKE_BUCKET}", var.log_storage_bucket) backup_bucket_access_policy_doc_processed = replace( replace( var.backup_bucket_access_policy_doc, "$${ARN_PARTITION}", var.arn_partition), - "$${DATALAKE_BUCKET}", "${var.backup_storage_bucket}") + "$${DATALAKE_BUCKET}", var.backup_storage_bucket) # ...then assign either input or processed policy doc to var used in resource data_bucket_access_policy_doc = var.process_policy_placeholders ? local.data_bucket_access_policy_doc_processed : var.data_bucket_access_policy_doc diff --git a/modules/terraform-aws-permissions/provider.tf b/modules/terraform-aws-permissions/provider.tf index e69de29..9539876 100644 --- a/modules/terraform-aws-permissions/provider.tf +++ b/modules/terraform-aws-permissions/provider.tf @@ -0,0 +1,24 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.30" + } + } + + required_version = ">= 1.3.0" +} \ No newline at end of file From 6a2e14c629d19f5f5aea9ad9ed49e923f89961b8 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 15 Aug 2024 19:06:09 +0100 Subject: [PATCH 05/11] Update docs and add example to aws permissions module Signed-off-by: Jim Enright --- .../doc_fragments/header.md | 10 +-- .../examples/ex01-minimal-inputs/main.tf | 75 +++++++++++++++++++ .../terraform.tfvars.sample | 32 ++++++++ .../examples/ex01-minimal-inputs/variables.tf | 50 +++++++++++++ modules/terraform-cdp-aws-pre-reqs/main.tf | 2 +- 5 files changed, 161 insertions(+), 8 deletions(-) create mode 100644 modules/terraform-aws-permissions/examples/ex01-minimal-inputs/main.tf create mode 100644 modules/terraform-aws-permissions/examples/ex01-minimal-inputs/terraform.tfvars.sample create mode 100644 modules/terraform-aws-permissions/examples/ex01-minimal-inputs/variables.tf diff --git a/modules/terraform-aws-permissions/doc_fragments/header.md b/modules/terraform-aws-permissions/doc_fragments/header.md index a797ec4..032d741 100644 --- a/modules/terraform-aws-permissions/doc_fragments/header.md +++ b/modules/terraform-aws-permissions/doc_fragments/header.md @@ -1,15 +1,11 @@ -# Terraform Module for CDP Prerequisites on AWS +# Terraform Module for CDP Environment Permissions on AWS -This module contains resource files and example variable definition files for creation of the pre-requisite AWS cloud resources required for Cloudera Data Platform (CDP) Public Cloud. +This module contains resource files and example variable definition files for creation the AWS IAM permissions required for Cloudera Data Platform (CDP) Public Cloud environment and datalake deployment. ## Usage -The [examples](./examples) directory has example AWS Cloud Service Provider deployments for different scenarios: +The [examples](./examples) directory has the following examples for AWS Cloud permission deployments: * `ex01-minimal-inputs` uses the minimum set of inputs for the module. -* `ex02-existing-vpc` creates a VPC and subnets outside of the module and passes this as an additional input. CDP deployment then uses these network assets. - -* `ex03-create-keypair` creates the AWS EC2 Keypair in the module caller and passes this as an additional input. - In each directory an example `terraform.tfvars.sample` values file is included to show input variable values. diff --git a/modules/terraform-aws-permissions/examples/ex01-minimal-inputs/main.tf b/modules/terraform-aws-permissions/examples/ex01-minimal-inputs/main.tf new file mode 100644 index 0000000..c1de61c --- /dev/null +++ b/modules/terraform-aws-permissions/examples/ex01-minimal-inputs/main.tf @@ -0,0 +1,75 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5.7" + required_providers { + cdp = { + source = "cloudera/cdp" + version = "~> 0.6.1" + } + aws = { + source = "hashicorp/aws" + version = "~>5.30" + } + } +} + +provider "aws" { + region = var.aws_region +} + +module "ex01_minimal_inputs" { + source = "../.." + + tags = var.tags + + # Policy documents from CDP TF Provider cred pre-reqs + idbroker_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Idbroker_Assumer"]) + data_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + log_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + backup_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + datalake_admin_s3_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Admin"]) + datalake_backup_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Backup"]) + datalake_restore_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Restore"]) + log_data_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Log_Policy"]) + ranger_audit_s3_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Ranger_Audit"]) + + data_storage_bucket = var.data_storage_bucket + log_storage_bucket = var.log_storage_bucket + backup_storage_bucket = var.backup_storage_bucket + + storage_location_base = "${var.data_storage_bucket}/data" + log_location_base = "${var.log_storage_bucket}/logs" + backup_location_base = "${var.backup_storage_bucket}/backups" + + idbroker_policy_name = "${var.env_prefix}-idbroker-policy" + log_data_access_policy_name = "${var.env_prefix}-logs-policy" + ranger_audit_s3_policy_name = "${var.env_prefix}-audit-policy" + datalake_admin_s3_policy_name = "${var.env_prefix}-dladmin-policy" + data_bucket_access_policy_name = "${var.env_prefix}-data-bucket-access-policy" + log_bucket_access_policy_name = "${var.env_prefix}-log-bucket-access-policy" + backup_bucket_access_policy_name = "${var.env_prefix}-backup-bucket-access-policy" + datalake_backup_policy_name = "${var.env_prefix}-datalake-backup-policy" + datalake_restore_policy_name = "${var.env_prefix}-datalake-restore-policy" + + idbroker_role_name = "${var.env_prefix}-idbroker-role" + log_role_name = "${var.env_prefix}-logs-role" + datalake_admin_role_name = "${var.env_prefix}-dladmin-role" + ranger_audit_role_name = "${var.env_prefix}-audit-role" + +} + +# Use the CDP Terraform Provider to find the xaccount account and external ids +data "cdp_environments_aws_credential_prerequisites" "cdp_prereqs" {} diff --git a/modules/terraform-aws-permissions/examples/ex01-minimal-inputs/terraform.tfvars.sample b/modules/terraform-aws-permissions/examples/ex01-minimal-inputs/terraform.tfvars.sample new file mode 100644 index 0000000..0e6dd1c --- /dev/null +++ b/modules/terraform-aws-permissions/examples/ex01-minimal-inputs/terraform.tfvars.sample @@ -0,0 +1,32 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +env_prefix = "" # Required name prefix for cloud and CDP resources, e.g. cldr1 + +# ------- Cloud Settings ------- +aws_region = "" # Change this to specify Cloud Provider region, e.g. eu-west-1 + +# ------- Resource Tagging ------- +# **NOTE: An example of how to specify tags is below; uncomment & edit if required +tags = { + owner = "" + project = "" + enddate = "" +} + +# ------- AWS Storage settings for CDP ------- +data_storage_bucket = "" # Name of the Data storage bucket +log_storage_bucket = "" # Name of the Log storage bucket +backup_storage_bucket = "" # Name of the Backup storage bucket \ No newline at end of file diff --git a/modules/terraform-aws-permissions/examples/ex01-minimal-inputs/variables.tf b/modules/terraform-aws-permissions/examples/ex01-minimal-inputs/variables.tf new file mode 100644 index 0000000..4a3679f --- /dev/null +++ b/modules/terraform-aws-permissions/examples/ex01-minimal-inputs/variables.tf @@ -0,0 +1,50 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "aws_region" { + type = string + description = "Region which Cloud resources will be created" +} + +variable "env_prefix" { + type = string + description = "Shorthand name for the environment. Used in resource descriptions" +} + +variable "tags" { + type = map(any) + description = "Tags applied to provised resources" + + default = null +} + +# ------- AWS Storage settings for CDP ------- +variable "data_storage_bucket" { + type = string + + description = "Name of the Data storage bucket" +} + +variable "log_storage_bucket" { + type = string + + description = "Name of the Log storage bucket" +} + +variable "backup_storage_bucket" { + type = string + + description = "Name of the Backup storage bucket" +} diff --git a/modules/terraform-cdp-aws-pre-reqs/main.tf b/modules/terraform-cdp-aws-pre-reqs/main.tf index aa46880..46abda0 100644 --- a/modules/terraform-cdp-aws-pre-reqs/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/main.tf @@ -443,7 +443,7 @@ module "aws_cdp_permissions" { backup_storage_bucket = "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}" storage_location_base = "${local.data_storage.data_storage_bucket}${local.storage_suffix}/${replace(local.data_storage.data_storage_object, "/", "")}" - log_location_base = "${local.log_storage.log_storage_bucket}${local.storage_suffix}" + log_location_base = "${local.log_storage.log_storage_bucket}${local.storage_suffix}/${replace(local.log_storage.log_storage_object, "/", "")}" backup_location_base = "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${replace(local.backup_storage.backup_storage_object, "/", "")}" datalake_backup_policy_name = local.datalake_backup_policy_name From 3b53ace6f2e31dacc2a101443d547d11d8d7a8d5 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 15 Aug 2024 22:28:24 +0100 Subject: [PATCH 06/11] Add AWS cross account credential permissions module Signed-off-by: Jim Enright --- README.md | 2 + .../.terraform-docs.yaml | 21 ++++ .../terraform-aws-cred-permissions/README.md | 64 +++++++++++ .../terraform-aws-cred-permissions/data.tf | 20 ++++ .../defaults.tf | 19 ++++ .../doc_fragments/header.md | 14 +-- .../examples/ex01-minimal-inputs/main.tf | 62 +++++++++++ .../terraform.tfvars.sample | 27 +++++ .../examples/ex01-minimal-inputs/variables.tf | 31 ++++++ .../examples/ex02-existing-role/main.tf | 50 +++++++++ .../terraform.tfvars.sample | 19 ++++ .../examples/ex02-existing-role/variables.tf | 24 +++++ .../terraform-aws-cred-permissions/main.tf | 100 ++++++++++++++++++ .../terraform-aws-cred-permissions/outputs.tf | 25 +++++ .../provider.tf | 28 +++++ .../variables.tf | 68 ++++++++++++ 16 files changed, 567 insertions(+), 7 deletions(-) create mode 100644 modules/terraform-aws-cred-permissions/.terraform-docs.yaml create mode 100644 modules/terraform-aws-cred-permissions/README.md create mode 100644 modules/terraform-aws-cred-permissions/examples/ex01-minimal-inputs/main.tf create mode 100644 modules/terraform-aws-cred-permissions/examples/ex01-minimal-inputs/terraform.tfvars.sample create mode 100644 modules/terraform-aws-cred-permissions/examples/ex01-minimal-inputs/variables.tf create mode 100644 modules/terraform-aws-cred-permissions/examples/ex02-existing-role/main.tf create mode 100644 modules/terraform-aws-cred-permissions/examples/ex02-existing-role/terraform.tfvars.sample create mode 100644 modules/terraform-aws-cred-permissions/examples/ex02-existing-role/variables.tf diff --git a/README.md b/README.md index 1d5fd51..e3877ad 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ This repository contains a number of Terraform modules for creation of the pre-r | [terraform-cdp-azure-pre-reqs](modules/terraform-cdp-azure-pre-reqs/README.md) | For all Azure pre-requisite Cloud resources | | [terraform-cdp-gcp-pre-reqs](modules/terraform-cdp-gcp-pre-reqs/README.md) | For all GCP pre-requisite Cloud resources | | [terraform-cdp-deploy](modules/terraform-cdp-deploy/README.md) | For deployment of CDP on AWS, Azure or GCP. | +| [terraform-aws-cred-permissions](modules/terraform-aws-cred-permissions/README.md) | Module for creation of the Cross Account Credential pre-requisite on AWS. Note that this module is called from the terraform-cdp-aws-prereqs module. | +| [terraform-aws-permissions](modules/terraform-aws-permissions/README.md) | Module for creation of the AWS IAM permissions required by the (CDP) Public Cloud environment and datalake deployment. Note that this module is called from the terraform-cdp-aws-prereqs module. | | [terraform-aws-vpc](modules/terraform-aws-vpc/README.md) | Module for creation of the VPC networking resources on AWS suitable. Can be used to create the CDP VPC and Subnets. Note that this module is called from the terraform-cdp-aws-prereqs module. | | [terraform-aws-tgw](modules/terraform-aws-tgw/README.md) | Module for creation of AWS Transity Gateway (TGW) and attaching a specified list of VPCs via the TGW. This module can be used to assist in deploying Cloudera Data Platform (CDP) Public Cloud in a fully private networking configuration where a CDP VPC and Networking VPC are connected using the Transit Gateway. | | [terraform-aws-proxy](modules/terraform-aws-proxy/README.md) | Module to create and configure and EC2 Auto-Scaling Group for a highly available Squid Proxy service with Network Load Balancer (NLB) to forward traffic to the proxy instances. This module can be used to assist in deploying Cloudera Data Platform (CDP) Public Cloud in a fully private networking configuration where a the CDP Environments uses a proxy config via the NLB. | diff --git a/modules/terraform-aws-cred-permissions/.terraform-docs.yaml b/modules/terraform-aws-cred-permissions/.terraform-docs.yaml new file mode 100644 index 0000000..0936036 --- /dev/null +++ b/modules/terraform-aws-cred-permissions/.terraform-docs.yaml @@ -0,0 +1,21 @@ +formatter: markdown +header-from: doc_fragments/header.md +settings: + anchor: true + color: true + default: true + escape: true + html: true + indent: 2 + required: true + sensitive: true + type: true + + +sort: + enabled: true + by: required + +output: + file: README.md + mode: replace \ No newline at end of file diff --git a/modules/terraform-aws-cred-permissions/README.md b/modules/terraform-aws-cred-permissions/README.md new file mode 100644 index 0000000..059a9c6 --- /dev/null +++ b/modules/terraform-aws-cred-permissions/README.md @@ -0,0 +1,64 @@ + +# Terraform Module for CDP Credential Prerequisites on AWS + +This module contains resource files and example variable definition files for creation of the Cloudera Data Platform (CDP) Public Cloud Cross Account Credential pre-requisite on AWS. + +Support for using a pre-existing Cross Account Role is provided via the `existing_xaccount_role_name` input variable. When this is set no policy or role resources are created. Instead a lookup of the details of the existing role takes place and the role ARN is returned. + +## Usage + +The [examples](./examples) directory has the following examples for Cross Account Credentials on AWS: + +* `ex01-minimal-inputs` uses the minimum set of inputs for the module where the Cross Account policy and roles are to be created. + +* `ex02-existing-role` passes a pre-existing Cross Account role to the module. In this case no resources are created. + +In each directory an example `terraform.tfvars.sample` values file is included to show input variable values. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [aws](#requirement\_aws) | ~>5.30 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | ~>5.30 | +| [time](#provider\_time) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_role.cdp_xaccount_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy.cdp_xaccount_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | +| [time_sleep.iam_propagation](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | +| [aws_iam_policy_document.cdp_xaccount_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_role.existing_xaccount_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [existing\_xaccount\_role\_name](#input\_existing\_xaccount\_role\_name) | Name of existing CDP Cross Account Role. If set then no policy or role resources are created. | `string` | `null` | no | +| [tags](#input\_tags) | Tags applied to provised resources | `map(any)` | `null` | no | +| [xaccount\_account\_id](#input\_xaccount\_account\_id) | Account ID of the cross account. Required if xaccount resources are to be created. | `string` | `null` | no | +| [xaccount\_account\_policy\_doc](#input\_xaccount\_account\_policy\_doc) | Contents of cross acount policy document. Required if xaccount resources are to be created. | `string` | `null` | no | +| [xaccount\_external\_id](#input\_xaccount\_external\_id) | External ID of the cross account. Required if xaccount resources are to be created. | `string` | `null` | no | +| [xaccount\_policy\_name](#input\_xaccount\_policy\_name) | Cross Account Policy name. Required if xaccount resources are to be created. | `string` | `null` | no | +| [xaccount\_role\_name](#input\_xaccount\_role\_name) | Cross account Assume role Name. Required if xaccount resources are to be created. | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [aws\_xaccount\_role\_arn](#output\_aws\_xaccount\_role\_arn) | Cross Account role ARN | +| [aws\_xaccount\_role\_name](#output\_aws\_xaccount\_role\_name) | Cross Account role name | + \ No newline at end of file diff --git a/modules/terraform-aws-cred-permissions/data.tf b/modules/terraform-aws-cred-permissions/data.tf index e69de29..b8ab748 100644 --- a/modules/terraform-aws-cred-permissions/data.tf +++ b/modules/terraform-aws-cred-permissions/data.tf @@ -0,0 +1,20 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +data "aws_iam_role" "existing_xaccount_role" { + + count = local.create_xaccount_resources ? 0 : 1 + + name = var.existing_xaccount_role_name +} \ No newline at end of file diff --git a/modules/terraform-aws-cred-permissions/defaults.tf b/modules/terraform-aws-cred-permissions/defaults.tf index e69de29..ba4ac73 100644 --- a/modules/terraform-aws-cred-permissions/defaults.tf +++ b/modules/terraform-aws-cred-permissions/defaults.tf @@ -0,0 +1,19 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # ------- Determine if resources should be created ------- + create_xaccount_resources = (var.existing_xaccount_role_name == null) + +} \ No newline at end of file diff --git a/modules/terraform-aws-cred-permissions/doc_fragments/header.md b/modules/terraform-aws-cred-permissions/doc_fragments/header.md index a797ec4..dbe4373 100644 --- a/modules/terraform-aws-cred-permissions/doc_fragments/header.md +++ b/modules/terraform-aws-cred-permissions/doc_fragments/header.md @@ -1,15 +1,15 @@ -# Terraform Module for CDP Prerequisites on AWS +# Terraform Module for CDP Credential Prerequisites on AWS -This module contains resource files and example variable definition files for creation of the pre-requisite AWS cloud resources required for Cloudera Data Platform (CDP) Public Cloud. +This module contains resource files and example variable definition files for creation of the Cloudera Data Platform (CDP) Public Cloud Cross Account Credential pre-requisite on AWS. -## Usage +Support for using a pre-existing Cross Account Role is provided via the `existing_xaccount_role_name` input variable. When this is set no policy or role resources are created. Instead a lookup of the details of the existing role takes place and the role ARN is returned. -The [examples](./examples) directory has example AWS Cloud Service Provider deployments for different scenarios: +## Usage -* `ex01-minimal-inputs` uses the minimum set of inputs for the module. +The [examples](./examples) directory has the following examples for Cross Account Credentials on AWS: -* `ex02-existing-vpc` creates a VPC and subnets outside of the module and passes this as an additional input. CDP deployment then uses these network assets. +* `ex01-minimal-inputs` uses the minimum set of inputs for the module where the Cross Account policy and roles are to be created. -* `ex03-create-keypair` creates the AWS EC2 Keypair in the module caller and passes this as an additional input. +* `ex02-existing-role` passes a pre-existing Cross Account role to the module. In this case no resources are created. In each directory an example `terraform.tfvars.sample` values file is included to show input variable values. diff --git a/modules/terraform-aws-cred-permissions/examples/ex01-minimal-inputs/main.tf b/modules/terraform-aws-cred-permissions/examples/ex01-minimal-inputs/main.tf new file mode 100644 index 0000000..02164cc --- /dev/null +++ b/modules/terraform-aws-cred-permissions/examples/ex01-minimal-inputs/main.tf @@ -0,0 +1,62 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5.7" + required_providers { + cdp = { + source = "cloudera/cdp" + version = "~> 0.6.1" + } + aws = { + source = "hashicorp/aws" + version = "~>5.30" + } + } +} + +provider "aws" { + region = var.aws_region +} + +module "ex01_minimal_inputs" { + source = "../.." + + tags = var.tags + + # Using CDP TF Provider cred pre-reqs data source for values of xaccount account_id and external_id + xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id + xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + + xaccount_policy_name = "${var.env_prefix}-xaccount-policy" + xaccount_account_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policy) + + xaccount_role_name = "${var.env_prefix}-xaccount-role" + +} + +# Use the CDP Terraform Provider to find the xaccount account and external ids +data "cdp_environments_aws_credential_prerequisites" "cdp_prereqs" {} + +# ------- Outputs ------- +output "xaccount_role_arn" { + value = module.ex01_minimal_inputs.aws_xaccount_role_arn + + description = "The ARN of the created Cross Account Role" +} +output "xaccount_role_name" { + value = module.ex01_minimal_inputs.aws_xaccount_role_name + + description = "The name of the created Cross Account Role" +} \ No newline at end of file diff --git a/modules/terraform-aws-cred-permissions/examples/ex01-minimal-inputs/terraform.tfvars.sample b/modules/terraform-aws-cred-permissions/examples/ex01-minimal-inputs/terraform.tfvars.sample new file mode 100644 index 0000000..e2b9fcb --- /dev/null +++ b/modules/terraform-aws-cred-permissions/examples/ex01-minimal-inputs/terraform.tfvars.sample @@ -0,0 +1,27 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +env_prefix = "" # Required name prefix for cloud and CDP resources, e.g. cldr1 + +# ------- Cloud Settings ------- +aws_region = "" # Change this to specify Cloud Provider region, e.g. eu-west-1 + +# ------- Resource Tagging ------- +# **NOTE: An example of how to specify tags is below; uncomment & edit if required +tags = { + owner = "" + project = "" + enddate = "" +} diff --git a/modules/terraform-aws-cred-permissions/examples/ex01-minimal-inputs/variables.tf b/modules/terraform-aws-cred-permissions/examples/ex01-minimal-inputs/variables.tf new file mode 100644 index 0000000..9594990 --- /dev/null +++ b/modules/terraform-aws-cred-permissions/examples/ex01-minimal-inputs/variables.tf @@ -0,0 +1,31 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "aws_region" { + type = string + description = "Region which Cloud resources will be created" +} + +variable "env_prefix" { + type = string + description = "Shorthand name for the environment. Used in resource descriptions" +} + +variable "tags" { + type = map(any) + description = "Tags applied to provised resources" + + default = null +} diff --git a/modules/terraform-aws-cred-permissions/examples/ex02-existing-role/main.tf b/modules/terraform-aws-cred-permissions/examples/ex02-existing-role/main.tf new file mode 100644 index 0000000..237349b --- /dev/null +++ b/modules/terraform-aws-cred-permissions/examples/ex02-existing-role/main.tf @@ -0,0 +1,50 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.5.7" + required_providers { + cdp = { + source = "cloudera/cdp" + version = "~> 0.6.1" + } + aws = { + source = "hashicorp/aws" + version = "~>5.30" + } + } +} + +provider "aws" { + region = var.aws_region +} + +module "ex01_existing_role" { + source = "../.." + + existing_xaccount_role_name = var.existing_xaccount_role_name + +} + +# ------- Outputs ------- +output "xaccount_role_arn" { + value = module.ex01_existing_role.aws_xaccount_role_arn + + description = "The ARN of the created Cross Account Role" +} +output "xaccount_role_name" { + value = module.ex01_existing_role.aws_xaccount_role_name + + description = "The name of the created Cross Account Role" +} \ No newline at end of file diff --git a/modules/terraform-aws-cred-permissions/examples/ex02-existing-role/terraform.tfvars.sample b/modules/terraform-aws-cred-permissions/examples/ex02-existing-role/terraform.tfvars.sample new file mode 100644 index 0000000..c6dc6e1 --- /dev/null +++ b/modules/terraform-aws-cred-permissions/examples/ex02-existing-role/terraform.tfvars.sample @@ -0,0 +1,19 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +existing_xaccount_role_name = "" # Name of pre-existing cross account role, e.g. cldr1-xaccount-role + +# ------- Cloud Settings ------- +aws_region = "" # Change this to specify Cloud Provider region, e.g. eu-west-1 diff --git a/modules/terraform-aws-cred-permissions/examples/ex02-existing-role/variables.tf b/modules/terraform-aws-cred-permissions/examples/ex02-existing-role/variables.tf new file mode 100644 index 0000000..30b32c1 --- /dev/null +++ b/modules/terraform-aws-cred-permissions/examples/ex02-existing-role/variables.tf @@ -0,0 +1,24 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "aws_region" { + type = string + description = "Region which Cloud resources will be created" +} + +variable "existing_xaccount_role_name" { + type = string + description = "Name of pre-existing Cross Account Role" +} diff --git a/modules/terraform-aws-cred-permissions/main.tf b/modules/terraform-aws-cred-permissions/main.tf index e69de29..9e32f7e 100644 --- a/modules/terraform-aws-cred-permissions/main.tf +++ b/modules/terraform-aws-cred-permissions/main.tf @@ -0,0 +1,100 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Cross Account Policy ------- +# Create assume role policy document for the Cross Account +data "aws_iam_policy_document" "cdp_xaccount_role_policy_doc" { + + count = local.create_xaccount_resources ? 1 : 0 + + version = "2012-10-17" + + statement { + actions = ["sts:AssumeRole"] + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${var.xaccount_account_id}:root"] + } + + condition { + test = "StringEquals" + variable = "sts:ExternalId" + + values = [var.xaccount_external_id] + } + } +} + +# Create the IAM role that uses the above assume_role_policy document +resource "aws_iam_role" "cdp_xaccount_role" { + + count = local.create_xaccount_resources ? 1 : 0 + + name = var.xaccount_role_name + # description = "CDP Cross Account role for ${var.env_prefix}" + + assume_role_policy = data.aws_iam_policy_document.cdp_xaccount_role_policy_doc[0].json + + tags = merge(var.tags, { Name = var.xaccount_role_name }) +} + +# Create AWS Cross Account Inline Policy +resource "aws_iam_role_policy" "cdp_xaccount_policy" { + + count = local.create_xaccount_resources ? 1 : 0 + + name = var.xaccount_policy_name + role = aws_iam_role.cdp_xaccount_role[0].id + + policy = var.xaccount_account_policy_doc +} + +# Wait for propagation of IAM xaccount role. +# Required for CDP credential +resource "time_sleep" "iam_propagation" { + depends_on = [aws_iam_role.cdp_xaccount_role] + create_duration = "45s" +} + + +# # ------- Add missing iam:Tag* permissions to Cross-Account Policy ------- +# # First create the extra policy document +# data "aws_iam_policy_document" "cdp_extra_xaccount_policy_doc" { +# count = var.create_extra_xaccount_policy ? 1 : 0 + +# version = "2012-10-17" + +# statement { +# sid = "AllowIAMTagRole" + +# actions = ["iam:TagRole"] +# effect = "Allow" + +# resources = [ +# "*", +# ] +# } +# } + +# # Then create the inline policy using the document +# resource "aws_iam_role_policy" "cdp_extra_xaccount_policy" { +# count = var.create_extra_xaccount_policy ? 1 : 0 + +# name = "${var.env_prefix}-cross-account-extra" +# role = aws_iam_role.cdp_xaccount_role.id + +# policy = data.aws_iam_policy_document.cdp_extra_xaccount_policy_doc[0].json +# } diff --git a/modules/terraform-aws-cred-permissions/outputs.tf b/modules/terraform-aws-cred-permissions/outputs.tf index e69de29..3d28b86 100644 --- a/modules/terraform-aws-cred-permissions/outputs.tf +++ b/modules/terraform-aws-cred-permissions/outputs.tf @@ -0,0 +1,25 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "aws_xaccount_role_arn" { + value = local.create_xaccount_resources ? aws_iam_role.cdp_xaccount_role[0].arn : data.aws_iam_role.existing_xaccount_role[0].arn + + description = "Cross Account role ARN" +} + +output "aws_xaccount_role_name" { + value = local.create_xaccount_resources ? aws_iam_role.cdp_xaccount_role[0].name : data.aws_iam_role.existing_xaccount_role[0].name + + description = "Cross Account role name" +} diff --git a/modules/terraform-aws-cred-permissions/provider.tf b/modules/terraform-aws-cred-permissions/provider.tf index e69de29..8b78993 100644 --- a/modules/terraform-aws-cred-permissions/provider.tf +++ b/modules/terraform-aws-cred-permissions/provider.tf @@ -0,0 +1,28 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~>5.30" + } + time = { + source = "hashicorp/time" + version = "0.9.1" + } + } + + required_version = ">= 1.3.0" +} \ No newline at end of file diff --git a/modules/terraform-aws-cred-permissions/variables.tf b/modules/terraform-aws-cred-permissions/variables.tf index e69de29..f2231e3 100644 --- a/modules/terraform-aws-cred-permissions/variables.tf +++ b/modules/terraform-aws-cred-permissions/variables.tf @@ -0,0 +1,68 @@ +# Copyright 2024 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "tags" { + type = map(any) + description = "Tags applied to provised resources" + + default = null +} + +# ------- CDP Settings ------- +variable "xaccount_account_id" { + type = string + description = "Account ID of the cross account. Required if xaccount resources are to be created." + + default = null +} + +variable "xaccount_external_id" { + type = string + description = "External ID of the cross account. Required if xaccount resources are to be created." + + default = null + +} + +# ------- Policies ------- +variable "xaccount_policy_name" { + type = string + description = "Cross Account Policy name. Required if xaccount resources are to be created." + + default = null +} + +variable "xaccount_account_policy_doc" { + type = string + description = "Contents of cross acount policy document. Required if xaccount resources are to be created." + + default = null +} + +# ------- Roles ------- +variable "xaccount_role_name" { + type = string + description = "Cross account Assume role Name. Required if xaccount resources are to be created." + + default = null +} + +# ------- Support for existing Cross Account Role ------- +variable "existing_xaccount_role_name" { + type = string + description = "Name of existing CDP Cross Account Role. If set then no policy or role resources are created." + + default = null +} From 70cb6c0c0b8e35d4892d69c4eca051c85e1a70be Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 15 Aug 2024 22:29:24 +0100 Subject: [PATCH 07/11] Add docs to AWS IAM permissions module Signed-off-by: Jim Enright --- .../.terraform-docs.yaml | 21 +++ modules/terraform-aws-permissions/README.md | 123 ++++++++++++++++++ .../examples/ex01-minimal-inputs/main.tf | 4 +- modules/terraform-aws-permissions/provider.tf | 2 +- 4 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 modules/terraform-aws-permissions/.terraform-docs.yaml create mode 100644 modules/terraform-aws-permissions/README.md diff --git a/modules/terraform-aws-permissions/.terraform-docs.yaml b/modules/terraform-aws-permissions/.terraform-docs.yaml new file mode 100644 index 0000000..0936036 --- /dev/null +++ b/modules/terraform-aws-permissions/.terraform-docs.yaml @@ -0,0 +1,21 @@ +formatter: markdown +header-from: doc_fragments/header.md +settings: + anchor: true + color: true + default: true + escape: true + html: true + indent: 2 + required: true + sensitive: true + type: true + + +sort: + enabled: true + by: required + +output: + file: README.md + mode: replace \ No newline at end of file diff --git a/modules/terraform-aws-permissions/README.md b/modules/terraform-aws-permissions/README.md new file mode 100644 index 0000000..a796992 --- /dev/null +++ b/modules/terraform-aws-permissions/README.md @@ -0,0 +1,123 @@ + +# Terraform Module for CDP Environment Permissions on AWS + +This module contains resource files and example variable definition files for creation the AWS IAM permissions required for Cloudera Data Platform (CDP) Public Cloud environment and datalake deployment. + +## Usage + +The [examples](./examples) directory has the following examples for AWS Cloud permission deployments: + +* `ex01-minimal-inputs` uses the minimum set of inputs for the module. + +In each directory an example `terraform.tfvars.sample` values file is included to show input variable values. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [aws](#requirement\_aws) | ~>5.30 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | ~>5.30 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_instance_profile.cdp_datalake_admin_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_instance_profile.cdp_idbroker_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_instance_profile.cdp_log_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_instance_profile.cdp_ranger_audit_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | +| [aws_iam_policy.cdp_backup_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_data_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_datalake_admin_s3_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_datalake_backup_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_datalake_restore_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_idbroker_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_log_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_log_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_ranger_audit_s3_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.cdp_datalake_admin_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.cdp_idbroker_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.cdp_log_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role.cdp_ranger_audit_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach4](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach5](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach6](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_idbroker_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_idbroker_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_log_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_log_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_log_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach4](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach5](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach6](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.cdp_datalake_admin_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cdp_idbroker_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cdp_log_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cdp_ranger_audit_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [backup\_bucket\_access\_policy\_doc](#input\_backup\_bucket\_access\_policy\_doc) | Contents of Backup Bucket Access Data Access Policy | `string` | n/a | yes | +| [backup\_bucket\_access\_policy\_name](#input\_backup\_bucket\_access\_policy\_name) | Backup Bucket Access Data Access Policy Name | `string` | n/a | yes | +| [backup\_storage\_bucket](#input\_backup\_storage\_bucket) | Name of the Backup storage bucket | `string` | n/a | yes | +| [data\_bucket\_access\_policy\_doc](#input\_data\_bucket\_access\_policy\_doc) | Data Bucket Access Data Access Policy | `string` | n/a | yes | +| [data\_bucket\_access\_policy\_name](#input\_data\_bucket\_access\_policy\_name) | Data Bucket Access Data Access Policy Name | `string` | n/a | yes | +| [data\_storage\_bucket](#input\_data\_storage\_bucket) | Name of the Data storage bucket | `string` | n/a | yes | +| [datalake\_admin\_role\_name](#input\_datalake\_admin\_role\_name) | Datalake Admin role Name | `string` | n/a | yes | +| [datalake\_admin\_s3\_policy\_doc](#input\_datalake\_admin\_s3\_policy\_doc) | Contents of Datalake Admin S3 Data Access Policy | `string` | n/a | yes | +| [datalake\_admin\_s3\_policy\_name](#input\_datalake\_admin\_s3\_policy\_name) | Datalake Admin S3 Data Access Policy Name | `string` | n/a | yes | +| [datalake\_backup\_policy\_doc](#input\_datalake\_backup\_policy\_doc) | Contents of Datalake Backup Data Access Policy | `string` | n/a | yes | +| [datalake\_backup\_policy\_name](#input\_datalake\_backup\_policy\_name) | Datalake backup Data Access Policy Name | `string` | n/a | yes | +| [idbroker\_policy\_doc](#input\_idbroker\_policy\_doc) | Contents of IDBroker Assumer Policy Document. | `string` | n/a | yes | +| [idbroker\_policy\_name](#input\_idbroker\_policy\_name) | IDBroker Policy name | `string` | n/a | yes | +| [idbroker\_role\_name](#input\_idbroker\_role\_name) | IDBroker service role Name | `string` | n/a | yes | +| [log\_bucket\_access\_policy\_doc](#input\_log\_bucket\_access\_policy\_doc) | Contents of Log Bucket Access Data Access Policy | `string` | n/a | yes | +| [log\_bucket\_access\_policy\_name](#input\_log\_bucket\_access\_policy\_name) | Log Bucket Access Data Access Policy Name | `string` | n/a | yes | +| [log\_data\_access\_policy\_doc](#input\_log\_data\_access\_policy\_doc) | Contents of Log Data Access Policy | `string` | n/a | yes | +| [log\_data\_access\_policy\_name](#input\_log\_data\_access\_policy\_name) | Log Data Access Policy Name | `string` | n/a | yes | +| [log\_role\_name](#input\_log\_role\_name) | Log service role Name | `string` | n/a | yes | +| [log\_storage\_bucket](#input\_log\_storage\_bucket) | Name of the Log storage bucket | `string` | n/a | yes | +| [ranger\_audit\_role\_name](#input\_ranger\_audit\_role\_name) | Ranger Audit role Name | `string` | n/a | yes | +| [ranger\_audit\_s3\_policy\_doc](#input\_ranger\_audit\_s3\_policy\_doc) | Contents of Ranger S3 Audit Data Access Policy | `string` | n/a | yes | +| [ranger\_audit\_s3\_policy\_name](#input\_ranger\_audit\_s3\_policy\_name) | Ranger S3 Audit Data Access Policy Name | `string` | n/a | yes | +| [arn\_partition](#input\_arn\_partition) | The string used to subsitute ARN\_PARTITION placeholder in policy documents. | `string` | `"aws"` | no | +| [backup\_location\_base](#input\_backup\_location\_base) | The bucket and path to the location used for FreeIPA and Datalake backups. Should be specified as / | `string` | `null` | no | +| [datalake\_restore\_policy\_doc](#input\_datalake\_restore\_policy\_doc) | Contents of Datalake Restore Data Access Policy | `string` | `null` | no | +| [datalake\_restore\_policy\_name](#input\_datalake\_restore\_policy\_name) | Datalake restore Data Access Policy Name | `string` | `null` | no | +| [log\_location\_base](#input\_log\_location\_base) | The bucket and path to the location for log storage. Should be specified as / | `string` | `null` | no | +| [process\_policy\_placeholders](#input\_process\_policy\_placeholders) | Flag to enable replacement of the standard placeholders in the AWS CDP Policy documents | `bool` | `true` | no | +| [storage\_location\_base](#input\_storage\_location\_base) | The bucket and path to the Data Lake storage directory. Should be specified as / | `string` | `null` | no | +| [tags](#input\_tags) | Tags applied to provised resources | `map(any)` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [aws\_datalake\_admin\_role\_arn](#output\_aws\_datalake\_admin\_role\_arn) | Datalake Admin role ARN | +| [aws\_datalake\_admin\_role\_name](#output\_aws\_datalake\_admin\_role\_name) | Datalake Admin role Name | +| [aws\_idbroker\_instance\_profile\_arn](#output\_aws\_idbroker\_instance\_profile\_arn) | IDBroker instance profile ARN | +| [aws\_idbroker\_role\_name](#output\_aws\_idbroker\_role\_name) | IDBroker role Name | +| [aws\_log\_instance\_profile\_arn](#output\_aws\_log\_instance\_profile\_arn) | Log instance profile ARN | +| [aws\_log\_role\_name](#output\_aws\_log\_role\_name) | Log role Name | +| [aws\_ranger\_audit\_role\_arn](#output\_aws\_ranger\_audit\_role\_arn) | Ranger Audit role ARN | +| [aws\_ranger\_audit\_role\_name](#output\_aws\_ranger\_audit\_role\_name) | Ranger Audit role Name | + \ No newline at end of file diff --git a/modules/terraform-aws-permissions/examples/ex01-minimal-inputs/main.tf b/modules/terraform-aws-permissions/examples/ex01-minimal-inputs/main.tf index c1de61c..2b0338f 100644 --- a/modules/terraform-aws-permissions/examples/ex01-minimal-inputs/main.tf +++ b/modules/terraform-aws-permissions/examples/ex01-minimal-inputs/main.tf @@ -61,8 +61,8 @@ module "ex01_minimal_inputs" { data_bucket_access_policy_name = "${var.env_prefix}-data-bucket-access-policy" log_bucket_access_policy_name = "${var.env_prefix}-log-bucket-access-policy" backup_bucket_access_policy_name = "${var.env_prefix}-backup-bucket-access-policy" - datalake_backup_policy_name = "${var.env_prefix}-datalake-backup-policy" - datalake_restore_policy_name = "${var.env_prefix}-datalake-restore-policy" + datalake_backup_policy_name = "${var.env_prefix}-datalake-backup-policy" + datalake_restore_policy_name = "${var.env_prefix}-datalake-restore-policy" idbroker_role_name = "${var.env_prefix}-idbroker-role" log_role_name = "${var.env_prefix}-logs-role" diff --git a/modules/terraform-aws-permissions/provider.tf b/modules/terraform-aws-permissions/provider.tf index 9539876..b6ed8e9 100644 --- a/modules/terraform-aws-permissions/provider.tf +++ b/modules/terraform-aws-permissions/provider.tf @@ -16,7 +16,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.30" + version = "~>5.30" } } From 3dbd275b6a850462b9877a7a8b20fcf9c84f33f1 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Thu, 15 Aug 2024 22:31:54 +0100 Subject: [PATCH 08/11] Update aws pre-reqs module examples for new modules Signed-off-by: Jim Enright --- modules/terraform-cdp-aws-pre-reqs/README.md | 78 +---- modules/terraform-cdp-aws-pre-reqs/data.tf | 6 - .../terraform-cdp-aws-pre-reqs/defaults.tf | 2 - .../examples/ex01-minimal_inputs/main.tf | 22 +- .../examples/ex01-minimal_inputs/variables.tf | 8 - .../examples/ex02-existing-vpc/main.tf | 22 +- .../examples/ex02-existing-vpc/variables.tf | 8 - .../examples/ex02-existing-vpc/vpc.tf | 4 +- .../examples/ex03-create-keypair/main.tf | 22 +- .../examples/ex03-create-keypair/variables.tf | 8 - .../ex04-all_inputs_specified/main.tf | 98 ------- .../aws-cdp-bucket-access-policy.json | 52 ---- .../aws-cdp-datalake-admin-s3-policy.json | 51 ---- .../policy_docs/aws-cdp-log-policy.json | 21 -- .../aws-cdp-ranger-audit-s3-policy.json | 24 -- .../ex04-all_inputs_specified/variables.tf | 273 ------------------ modules/terraform-cdp-aws-pre-reqs/main.tf | 80 +---- modules/terraform-cdp-aws-pre-reqs/outputs.tf | 4 +- .../terraform-cdp-aws-pre-reqs/provider.tf | 4 - .../terraform-cdp-aws-pre-reqs/variables.tf | 24 +- 20 files changed, 89 insertions(+), 722 deletions(-) delete mode 100644 modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/main.tf delete mode 100644 modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-bucket-access-policy.json delete mode 100644 modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-datalake-admin-s3-policy.json delete mode 100644 modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-log-policy.json delete mode 100644 modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-ranger-audit-s3-policy.json delete mode 100644 modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/variables.tf diff --git a/modules/terraform-cdp-aws-pre-reqs/README.md b/modules/terraform-cdp-aws-pre-reqs/README.md index 44c3bc4..09bb789 100644 --- a/modules/terraform-cdp-aws-pre-reqs/README.md +++ b/modules/terraform-cdp-aws-pre-reqs/README.md @@ -30,57 +30,20 @@ In each directory an example `terraform.tfvars.sample` values file is included t | Name | Version | |------|---------| | [aws](#provider\_aws) | >= 5.30 | -| [http](#provider\_http) | 3.2.1 | | [random](#provider\_random) | 3.4.3 | -| [time](#provider\_time) | 0.9.1 | ## Modules | Name | Source | Version | |------|--------|---------| +| [aws\_cdp\_cred\_permissions](#module\_aws\_cdp\_cred\_permissions) | ../terraform-aws-cred-permissions | n/a | +| [aws\_cdp\_permissions](#module\_aws\_cdp\_permissions) | ../terraform-aws-permissions | n/a | | [aws\_cdp\_vpc](#module\_aws\_cdp\_vpc) | ../terraform-aws-vpc | n/a | ## Resources | Name | Type | |------|------| -| [aws_iam_instance_profile.cdp_datalake_admin_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | -| [aws_iam_instance_profile.cdp_idbroker_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | -| [aws_iam_instance_profile.cdp_log_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | -| [aws_iam_instance_profile.cdp_ranger_audit_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | -| [aws_iam_policy.cdp_backup_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_data_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_datalake_admin_s3_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_datalake_backup_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_datalake_restore_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_idbroker_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_log_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_log_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_ranger_audit_s3_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_role.cdp_datalake_admin_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role.cdp_idbroker_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role.cdp_log_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role.cdp_ranger_audit_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role.cdp_xaccount_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role_policy.cdp_extra_xaccount_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | -| [aws_iam_role_policy.cdp_xaccount_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | -| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach4](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach5](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach6](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_idbroker_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_idbroker_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_log_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_log_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_log_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach4](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach5](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach6](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_kms_alias.cdp_kms_alias](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_alias) | resource | | [aws_kms_key.cdp_kms_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | | [aws_s3_bucket.cdp_storage_locations](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | @@ -104,36 +67,29 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [aws_vpc_endpoint.gateway_endpoints](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint) | resource | | [aws_vpc_endpoint.interface_endpoints](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint) | resource | | [random_id.bucket_suffix](https://registry.terraform.io/providers/hashicorp/random/3.4.3/docs/resources/id) | resource | -| [time_sleep.iam_propagation](https://registry.terraform.io/providers/hashicorp/time/0.9.1/docs/resources/sleep) | resource | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_iam_policy_document.cdp_datalake_admin_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cdp_extra_xaccount_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cdp_idbroker_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cdp_idbroker_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cdp_log_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cdp_ranger_audit_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cdp_xaccount_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_vpc_endpoint_service.gateway_endpoints](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc_endpoint_service) | data source | | [aws_vpc_endpoint_service.interface_endpoints](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc_endpoint_service) | data source | -| [http_http.bucket_access_policy_doc](https://registry.terraform.io/providers/hashicorp/http/3.2.1/docs/data-sources/http) | data source | -| [http_http.datalake_admin_s3_policy_doc](https://registry.terraform.io/providers/hashicorp/http/3.2.1/docs/data-sources/http) | data source | -| [http_http.datalake_backup_policy_doc](https://registry.terraform.io/providers/hashicorp/http/3.2.1/docs/data-sources/http) | data source | -| [http_http.datalake_restore_policy_doc](https://registry.terraform.io/providers/hashicorp/http/3.2.1/docs/data-sources/http) | data source | -| [http_http.log_data_access_policy_doc](https://registry.terraform.io/providers/hashicorp/http/3.2.1/docs/data-sources/http) | data source | -| [http_http.ranger_audit_s3_policy_doc](https://registry.terraform.io/providers/hashicorp/http/3.2.1/docs/data-sources/http) | data source | -| [http_http.xaccount_account_policy_doc](https://registry.terraform.io/providers/hashicorp/http/3.2.1/docs/data-sources/http) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [backup\_bucket\_access\_policy\_doc](#input\_backup\_bucket\_access\_policy\_doc) | Contents of Backup Bucket Access Data Access Policy | `string` | n/a | yes | +| [data\_bucket\_access\_policy\_doc](#input\_data\_bucket\_access\_policy\_doc) | Data Bucket Access Data Access Policy | `string` | n/a | yes | +| [datalake\_admin\_s3\_policy\_doc](#input\_datalake\_admin\_s3\_policy\_doc) | Contents of Datalake Admin S3 Data Access Policy | `string` | n/a | yes | +| [datalake\_backup\_policy\_doc](#input\_datalake\_backup\_policy\_doc) | Contents of Datalake Backup Data Access Policy | `string` | n/a | yes | +| [datalake\_restore\_policy\_doc](#input\_datalake\_restore\_policy\_doc) | Contents of Datalake Restore Data Access Policy | `string` | n/a | yes | | [deployment\_template](#input\_deployment\_template) | Deployment Pattern to use for Cloud resources and CDP | `string` | n/a | yes | | [env\_prefix](#input\_env\_prefix) | Shorthand name for the environment. Used in resource descriptions | `string` | n/a | yes | +| [idbroker\_policy\_doc](#input\_idbroker\_policy\_doc) | Contents of IDBroker Assumer Policy Document. | `string` | n/a | yes | +| [log\_bucket\_access\_policy\_doc](#input\_log\_bucket\_access\_policy\_doc) | Contents of Log Bucket Access Data Access Policy | `string` | n/a | yes | +| [log\_data\_access\_policy\_doc](#input\_log\_data\_access\_policy\_doc) | Contents of Log Data Access Policy | `string` | n/a | yes | +| [ranger\_audit\_s3\_policy\_doc](#input\_ranger\_audit\_s3\_policy\_doc) | Contents of Ranger S3 Audit Data Access Policy | `string` | n/a | yes | | [xaccount\_account\_id](#input\_xaccount\_account\_id) | Account ID of the cross account | `string` | n/a | yes | +| [xaccount\_account\_policy\_doc](#input\_xaccount\_account\_policy\_doc) | Contents of cross acount policy document | `string` | n/a | yes | | [xaccount\_external\_id](#input\_xaccount\_external\_id) | External ID of the cross account | `string` | n/a | yes | | [agent\_source\_tag](#input\_agent\_source\_tag) | Tag to identify deployment source | `map(any)` | 
{
  "agent_source": "tf-cdp-module"
}
| no | | [aws\_region](#input\_aws\_region) | Region which Cloud resources will be created | `string` | `null` | no | -| [backup\_bucket\_access\_policy\_doc](#input\_backup\_bucket\_access\_policy\_doc) | Contents of Backup Bucket Access Data Access Policy | `string` | `null` | no | | [backup\_bucket\_access\_policy\_name](#input\_backup\_bucket\_access\_policy\_name) | Backup Bucket Access Data Access Policy Name | `string` | `null` | no | | [backup\_storage](#input\_backup\_storage) | Optional Backup location for CDP environment. If not provided follow the data\_storage variable | 
object({
    backup_storage_bucket = string
    backup_storage_object = string
  })
| `null` | no | | [cdp\_default\_sg\_egress\_cidrs](#input\_cdp\_default\_sg\_egress\_cidrs) | List of egress CIDR blocks for CDP Default Security Group Egress rule | `list(string)` | 
[
  "0.0.0.0/0"
]
| no | @@ -142,28 +98,22 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [cdp\_private\_subnet\_ids](#input\_cdp\_private\_subnet\_ids) | List of private subnet ids. Required if create\_vpc is false. | `list(any)` | `null` | no | | [cdp\_public\_subnet\_ids](#input\_cdp\_public\_subnet\_ids) | List of public subnet ids. Required if create\_vpc is false. | `list(any)` | `null` | no | | [cdp\_vpc\_id](#input\_cdp\_vpc\_id) | VPC ID for CDP environment. Required if create\_vpc is false. | `string` | `null` | no | -| [create\_extra\_xaccount\_policy](#input\_create\_extra\_xaccount\_policy) | Create extra Cross-Account Policy for missing iam:Tag* permissions required for Data Services. | `bool` | `true` | no | | [create\_vpc](#input\_create\_vpc) | Flag to specify if the VPC should be created | `bool` | `true` | no | | [create\_vpc\_endpoints](#input\_create\_vpc\_endpoints) | Flag to specify if VPC Endpoints should be created | `bool` | `true` | no | -| [data\_bucket\_access\_policy\_doc](#input\_data\_bucket\_access\_policy\_doc) | Data Bucket Access Data Access Policy | `string` | `null` | no | | [data\_bucket\_access\_policy\_name](#input\_data\_bucket\_access\_policy\_name) | Data Bucket Access Data Access Policy Name | `string` | `null` | no | | [data\_storage](#input\_data\_storage) | Data storage locations for CDP environment | 
object({
    data_storage_bucket = string
    data_storage_object = string
  })
| `null` | no | | [datalake\_admin\_role\_name](#input\_datalake\_admin\_role\_name) | Datalake Admin role Name | `string` | `null` | no | -| [datalake\_admin\_s3\_policy\_doc](#input\_datalake\_admin\_s3\_policy\_doc) | Contents of Datalake Admin S3 Data Access Policy | `string` | `null` | no | | [datalake\_admin\_s3\_policy\_name](#input\_datalake\_admin\_s3\_policy\_name) | Datalake Admin S3 Data Access Policy Name | `string` | `null` | no | -| [datalake\_backup\_policy\_doc](#input\_datalake\_backup\_policy\_doc) | Contents of Datalake Backup Data Access Policy | `string` | `null` | no | | [datalake\_backup\_policy\_name](#input\_datalake\_backup\_policy\_name) | Datalake backup Data Access Policy Name | `string` | `null` | no | -| [datalake\_restore\_policy\_doc](#input\_datalake\_restore\_policy\_doc) | Contents of Datalake Restore Data Access Policy | `string` | `null` | no | | [datalake\_restore\_policy\_name](#input\_datalake\_restore\_policy\_name) | Datalake restore Data Access Policy Name | `string` | `null` | no | | [enable\_bucket\_versioning](#input\_enable\_bucket\_versioning) | Flag to enable versioning of S3 buckets. | `bool` | `true` | no | | [enable\_kms\_bucket\_encryption](#input\_enable\_kms\_bucket\_encryption) | Flag to create AWS KMS for encryption of S3 buckets. Currently disabled as further settings needed for successful CDP deployment. | `bool` | `false` | no | | [env\_tags](#input\_env\_tags) | Tags applied to provised resources | `map(any)` | `null` | no | +| [existing\_xaccount\_role\_name](#input\_existing\_xaccount\_role\_name) | Name of existing CDP Cross Account Role. If set then no Cross Account policy or role resources are created. | `string` | `null` | no | | [idbroker\_policy\_name](#input\_idbroker\_policy\_name) | IDBroker Policy name | `string` | `null` | no | | [idbroker\_role\_name](#input\_idbroker\_role\_name) | IDBroker service role Name | `string` | `null` | no | | [ingress\_extra\_cidrs\_and\_ports](#input\_ingress\_extra\_cidrs\_and\_ports) | List of extra CIDR blocks and ports to include in Security Group Ingress rules | 
object({
    cidrs = list(string)
    ports = list(number)
  })
| 
{
  "cidrs": [],
  "ports": []
}
| no | -| [log\_bucket\_access\_policy\_doc](#input\_log\_bucket\_access\_policy\_doc) | Contents of Log Bucket Access Data Access Policy | `string` | `null` | no | | [log\_bucket\_access\_policy\_name](#input\_log\_bucket\_access\_policy\_name) | Log Bucket Access Data Access Policy Name | `string` | `null` | no | -| [log\_data\_access\_policy\_doc](#input\_log\_data\_access\_policy\_doc) | Contents of Log Data Access Policy | `string` | `null` | no | | [log\_data\_access\_policy\_name](#input\_log\_data\_access\_policy\_name) | Log Data Access Policy Name | `string` | `null` | no | | [log\_role\_name](#input\_log\_role\_name) | Log service role Name | `string` | `null` | no | | [log\_storage](#input\_log\_storage) | Optional log locations for CDP environment. If not provided follow the data\_storage variable | 
object({
    log_storage_bucket = string
    log_storage_object = string
  })
| `null` | no | @@ -172,7 +122,6 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [public\_cidr\_range](#input\_public\_cidr\_range) | Size of each public subnet. Required if create\_vpc is true. Number of subnets will be automatically selected to match on the number of Availability Zones in the selected AWS region. (Depending on the selected deployment pattern, one subnet will be created per region.) | `number` | `24` | no | | [random\_id\_for\_bucket](#input\_random\_id\_for\_bucket) | Create a random suffix for the bucket names | `bool` | `true` | no | | [ranger\_audit\_role\_name](#input\_ranger\_audit\_role\_name) | Ranger Audit role Name | `string` | `null` | no | -| [ranger\_audit\_s3\_policy\_doc](#input\_ranger\_audit\_s3\_policy\_doc) | Contents of Ranger S3 Audit Data Access Policy | `string` | `null` | no | | [ranger\_audit\_s3\_policy\_name](#input\_ranger\_audit\_s3\_policy\_name) | Ranger S3 Audit Data Access Policy Name | `string` | `null` | no | | [security\_group\_default\_name](#input\_security\_group\_default\_name) | Default Security Group for CDP environment | `string` | `null` | no | | [security\_group\_endpoint\_name](#input\_security\_group\_endpoint\_name) | Security Group for VPC Endpoints | `string` | `null` | no | @@ -186,7 +135,6 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [vpc\_public\_inbound\_acl\_rules](#input\_vpc\_public\_inbound\_acl\_rules) | Inbound network ACLs for Public subnets. Exposes default value of VPC module variable to allow for overriding. Only used when create\_vpc is true. | `list(map(string))` | 
[
  {
    "cidr_block": "0.0.0.0/0",
    "from_port": 0,
    "protocol": "-1",
    "rule_action": "allow",
    "rule_number": 100,
    "to_port": 0
  }
]
| no | | [vpc\_public\_outbound\_acl\_rules](#input\_vpc\_public\_outbound\_acl\_rules) | Public subnets outbound network ACLs. Exposes default value of VPC module variable to allow for overriding. Only used when create\_vpc is true. | `list(map(string))` | 
[
  {
    "cidr_block": "0.0.0.0/0",
    "from_port": 0,
    "protocol": "-1",
    "rule_action": "allow",
    "rule_number": 100,
    "to_port": 0
  }
]
| no | | [vpc\_public\_subnets\_map\_public\_ip\_on\_launch](#input\_vpc\_public\_subnets\_map\_public\_ip\_on\_launch) | Auto-assign public IP on launch for instances created in Public Subnets. Exposes default value of VPC module variable to allow for overriding. Only used when create\_vpc is true. | `bool` | `true` | no | -| [xaccount\_account\_policy\_doc](#input\_xaccount\_account\_policy\_doc) | Contents of cross acount policy document | `string` | `null` | no | | [xaccount\_policy\_name](#input\_xaccount\_policy\_name) | Cross Account Policy name | `string` | `null` | no | | [xaccount\_role\_name](#input\_xaccount\_role\_name) | Cross account Assume role Name | `string` | `null` | no | diff --git a/modules/terraform-cdp-aws-pre-reqs/data.tf b/modules/terraform-cdp-aws-pre-reqs/data.tf index a253c78..ff60c50 100644 --- a/modules/terraform-cdp-aws-pre-reqs/data.tf +++ b/modules/terraform-cdp-aws-pre-reqs/data.tf @@ -27,9 +27,3 @@ data "aws_vpc_endpoint_service" "interface_endpoints" { service = each.key service_type = "Interface" } - -# HTTP get request to download policy documents -# ..Cross Account Policy -data "http" "xaccount_account_policy_doc" { - url = "https://raw.githubusercontent.com/hortonworks/cloudbreak/CB-2.73.0/cloud-aws-common/src/main/resources/definitions/aws-cb-policy.json" -} diff --git a/modules/terraform-cdp-aws-pre-reqs/defaults.tf b/modules/terraform-cdp-aws-pre-reqs/defaults.tf index 0598e6b..17f5440 100644 --- a/modules/terraform-cdp-aws-pre-reqs/defaults.tf +++ b/modules/terraform-cdp-aws-pre-reqs/defaults.tf @@ -70,8 +70,6 @@ locals { # Cross Account Policy (name and document) xaccount_policy_name = coalesce(var.xaccount_policy_name, "${var.env_prefix}-xaccount-policy") - xaccount_account_policy_doc = coalesce(var.xaccount_account_policy_doc, data.http.xaccount_account_policy_doc.response_body) - # CDP IDBroker Assume Role policy idbroker_policy_name = coalesce(var.idbroker_policy_name, "${var.env_prefix}-idbroker-policy") diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf index 39bed77..23afbde 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf @@ -13,8 +13,7 @@ # limitations under the License. provider "aws" { - profile = var.aws_profile - region = var.aws_region + region = var.aws_region } module "ex01_minimal_inputs" { @@ -28,8 +27,23 @@ module "ex01_minimal_inputs" { ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports # Using CDP TF Provider cred pre-reqs data source for values of xaccount account_id and external_id - xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id - xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id + xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + xaccount_account_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policy) + + # Policy documents from CDP TF Provider cred pre-reqs + idbroker_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Idbroker_Assumer"]) + + data_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + log_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + backup_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + + datalake_admin_s3_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Admin"]) + datalake_backup_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Backup"]) + datalake_restore_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Restore"]) + + log_data_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Log_Policy"]) + ranger_audit_s3_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Ranger_Audit"]) } diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/variables.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/variables.tf index aa021db..04f9d67 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/variables.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/variables.tf @@ -13,14 +13,6 @@ # limitations under the License. # ------- Global settings ------- -variable "aws_profile" { - type = string - description = "Profile for AWS cloud credentials" - - # Profile is default unless explicitly specified - default = "default" -} - variable "aws_region" { type = string description = "Region which Cloud resources will be created" diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf index 25be5d1..7e7614c 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf @@ -13,8 +13,7 @@ # limitations under the License. provider "aws" { - profile = var.aws_profile - region = var.aws_region + region = var.aws_region } module "ex02_existing_vpc" { @@ -28,8 +27,23 @@ module "ex02_existing_vpc" { ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports # Using CDP TF Provider cred pre-reqs data source for values of xaccount account_id and external_id - xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id - xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id + xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + xaccount_account_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policy) + + # Policy documents from CDP TF Provider cred pre-reqs + idbroker_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Idbroker_Assumer"]) + + data_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + log_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + backup_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + + datalake_admin_s3_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Admin"]) + datalake_backup_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Backup"]) + datalake_restore_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Restore"]) + + log_data_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Log_Policy"]) + ranger_audit_s3_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Ranger_Audit"]) create_vpc = var.create_vpc cdp_vpc_id = aws_vpc.cdp_vpc.id diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/variables.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/variables.tf index ed27ab5..28bc4f7 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/variables.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/variables.tf @@ -13,14 +13,6 @@ # limitations under the License. # ------- Global settings ------- -variable "aws_profile" { - type = string - description = "Profile for AWS cloud credentials" - - # Profile is default unless explicitly specified - default = "default" -} - variable "aws_region" { type = string description = "Region which Cloud resources will be created" diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/vpc.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/vpc.tf index e278234..027926e 100755 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/vpc.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/vpc.tf @@ -131,8 +131,8 @@ resource "aws_eip" "cdp_nat_gateway_eip" { for_each = { for idx, subnet in local.public_subnets : idx => subnet } - vpc = true - tags = { Name = format("%s-%s-%02d", "${var.env_prefix}-ngw", "eip", index(local.public_subnets, each.value) + 1) } + domain = "vpc" + tags = { Name = format("%s-%s-%02d", "${var.env_prefix}-ngw", "eip", index(local.public_subnets, each.value) + 1) } } # Network Gateways (NAT) diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf index 1a84bb6..e6872cd 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf @@ -13,8 +13,7 @@ # limitations under the License. provider "aws" { - profile = var.aws_profile - region = var.aws_region + region = var.aws_region } # Create and save a RSA key @@ -48,8 +47,23 @@ module "ex01_create_keypair" { ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports # Using CDP TF Provider cred pre-reqs data source for values of xaccount account_id and external_id - xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id - xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id + xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + xaccount_account_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policy) + + # Policy documents from CDP TF Provider cred pre-reqs + idbroker_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Idbroker_Assumer"]) + + data_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + log_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + backup_bucket_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Bucket_Access"]) + + datalake_admin_s3_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Admin"]) + datalake_backup_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Backup"]) + datalake_restore_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Datalake_Restore"]) + + log_data_access_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Log_Policy"]) + ranger_audit_s3_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Ranger_Audit"]) } diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/variables.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/variables.tf index aa021db..04f9d67 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/variables.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/variables.tf @@ -13,14 +13,6 @@ # limitations under the License. # ------- Global settings ------- -variable "aws_profile" { - type = string - description = "Profile for AWS cloud credentials" - - # Profile is default unless explicitly specified - default = "default" -} - variable "aws_region" { type = string description = "Region which Cloud resources will be created" diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/main.tf deleted file mode 100644 index 7a8fe85..0000000 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/main.tf +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -provider "aws" { - profile = var.profile - region = var.region -} - -# ------- Read Policy Documents into local_file data resource ------- -data "local_file" "log_data_access_policy_doc" { - filename = var.log_data_access_policy_doc_location -} - -data "local_file" "ranger_audit_s3_policy_doc" { - filename = var.ranger_audit_s3_policy_doc_location -} - -data "local_file" "datalake_admin_s3_policy_doc" { - filename = var.datalake_admin_s3_policy_doc_location -} - -data "local_file" "bucket_access_policy_doc" { - filename = var.bucket_access_policy_doc_location -} - -# ------- Call CDP module ------- -module "ex04_cdp_pre_reqs_example" { - source = "../.." - - profile = var.profile - region = var.region - - env_prefix = var.env_prefix - - # TODO: Figure out how to best specify keypair - aws_key_pair = var.aws_key_pair - - env_tags = var.env_tags - - deploy_cdp = var.deploy_cdp - deployment_template = var.deployment_template - - vpc_name = var.vpc_name - vpc_cidr = var.vpc_cidr - - igw_name = var.igw_name - public_subnets = var.public_subnets - public_route_table_name = var.public_route_table_name - - private_subnets = var.private_subnets - private_route_table_name = var.private_route_table_name - nat_gateway_name = var.nat_gateway_name - - security_group_default_name = var.security_group_default_name - security_group_knox_name = var.security_group_knox_name - ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports - - random_id_for_bucket = var.random_id_for_bucket - data_storage = var.data_storage - log_storage = var.log_storage - - xaccount_policy_name = var.xaccount_policy_name - xaccount_account_policy_doc = var.xaccount_account_policy_doc - - idbroker_policy_name = var.idbroker_policy_name - - log_data_access_policy_name = var.log_data_access_policy_name - log_data_access_policy_doc = data.local_file.log_data_access_policy_doc.content - - ranger_audit_s3_policy_name = var.ranger_audit_s3_policy_name - ranger_audit_s3_policy_doc = data.local_file.ranger_audit_s3_policy_doc.content - - datalake_admin_s3_policy_name = var.datalake_admin_s3_policy_name - datalake_admin_s3_policy_doc = data.local_file.datalake_admin_s3_policy_doc.content - - bucket_access_policy_name = var.bucket_access_policy_name - bucket_access_policy_doc = data.local_file.bucket_access_policy_doc.content - - xaccount_role_name = var.xaccount_role_name - xaccount_account_id = var.xaccount_account_id - xaccount_external_id = var.xaccount_external_id - - idbroker_role_name = var.idbroker_role_name - log_role_name = var.log_role_name - datalake_admin_role_name = var.datalake_admin_role_name - ranger_audit_role_name = var.ranger_audit_role_name -} diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-bucket-access-policy.json b/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-bucket-access-policy.json deleted file mode 100644 index b45b218..0000000 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-bucket-access-policy.json +++ /dev/null @@ -1,52 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:CreateJob", - "s3:GetAccountPublicAccessBlock", - "s3:ListJobs" - ], - "Resource": "*" - }, - { - "Sid": "AllowListingOfDataLakeFolder", - "Effect": "Allow", - "Action": [ - "s3:GetAccelerateConfiguration", - "s3:GetAnalyticsConfiguration", - "s3:GetBucketAcl", - "s3:GetBucketCORS", - "s3:GetBucketLocation", - "s3:GetBucketLogging", - "s3:GetBucketNotification", - "s3:GetBucketPolicy", - "s3:GetBucketPolicyStatus", - "s3:GetBucketPublicAccessBlock", - "s3:GetBucketRequestPayment", - "s3:GetBucketTagging", - "s3:GetBucketVersioning", - "s3:GetBucketWebsite", - "s3:GetEncryptionConfiguration", - "s3:GetInventoryConfiguration", - "s3:GetLifecycleConfiguration", - "s3:GetMetricsConfiguration", - "s3:GetObject", - "s3:GetObjectAcl", - "s3:GetObjectTagging", - "s3:GetObjectVersion", - "s3:GetObjectVersionAcl", - "s3:GetObjectVersionTagging", - "s3:GetReplicationConfiguration", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:ListMultipartUploadParts" - ], - "Resource": [ - "arn:aws:s3:::ex4-eet2", - "arn:aws:s3:::ex4-eet2/*" - ] - } - ] -} diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-datalake-admin-s3-policy.json b/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-datalake-admin-s3-policy.json deleted file mode 100644 index 912ac86..0000000 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-datalake-admin-s3-policy.json +++ /dev/null @@ -1,51 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor3", - "Effect": "Allow", - "Action": [ - "s3:AbortMultipartUpload", - "s3:DeleteObject", - "s3:DeleteObjectVersion", - "s3:GetAccelerateConfiguration", - "s3:GetAnalyticsConfiguration", - "s3:GetBucketAcl", - "s3:GetBucketCORS", - "s3:GetBucketLocation", - "s3:GetBucketLogging", - "s3:GetBucketNotification", - "s3:GetBucketObjectLockConfiguration", - "s3:GetBucketPolicy", - "s3:GetBucketPolicyStatus", - "s3:GetBucketPublicAccessBlock", - "s3:GetBucketRequestPayment", - "s3:GetBucketTagging", - "s3:GetBucketVersioning", - "s3:GetBucketWebsite", - "s3:GetEncryptionConfiguration", - "s3:GetInventoryConfiguration", - "s3:GetLifecycleConfiguration", - "s3:GetMetricsConfiguration", - "s3:GetObject", - "s3:GetObjectAcl", - "s3:GetObjectLegalHold", - "s3:GetObjectRetention", - "s3:GetObjectTagging", - "s3:GetObjectVersion", - "s3:GetObjectVersionAcl", - "s3:GetObjectVersionTagging", - "s3:GetReplicationConfiguration", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:ListBucketVersions", - "s3:ListMultipartUploadParts", - "s3:PutObject" - ], - "Resource": [ - "arn:aws:s3:::ex4-eet2", - "arn:aws:s3:::ex4-eet2/*" - ] - } - ] -} diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-log-policy.json b/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-log-policy.json deleted file mode 100644 index 305a7d7..0000000 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-log-policy.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": [ - "s3:ListBucket" - ], - "Resource": "arn:aws:s3:::ex4-eet2" - }, - { - "Effect": "Allow", - "Action": [ - "s3:AbortMultipartUpload", - "s3:ListMultipartUploadParts", - "s3:PutObject" - ], - "Resource": "arn:aws:s3:::ex4-eet2/*" - } - ] -} \ No newline at end of file diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-ranger-audit-s3-policy.json b/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-ranger-audit-s3-policy.json deleted file mode 100644 index 3ce9db9..0000000 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/policy_docs/aws-cdp-ranger-audit-s3-policy.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "FullObjectAccessUnderAuditDir", - "Effect": "Allow", - "Action": [ - "s3:GetObject", - "s3:PutObject" - ], - "Resource": "arn:aws:s3:::ex4-eet2/ranger/audit/*" - }, - { - "Sid": "LimitedAccessToDataLakeBucket", - "Effect": "Allow", - "Action": [ - "s3:AbortMultipartUpload", - "s3:ListBucket", - "s3:ListBucketMultipartUploads" - ], - "Resource": "arn:aws:s3:::ex4-eet2" - } - ] -} diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/variables.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/variables.tf deleted file mode 100644 index db988be..0000000 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex04-all_inputs_specified/variables.tf +++ /dev/null @@ -1,273 +0,0 @@ -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# ------- Global settings ------- -variable "profile" { - type = string - description = "Profile for AWS cloud credentials" - - # Profile is default unless explicitly specified - default = "default" -} - -variable "region" { - type = string - description = "Region which Cloud resources will be created" -} - -variable "env_tags" { - type = map(any) - description = "Tags applied to provised resources" -} - -variable "env_prefix" { - type = string - description = "Shorthand name for the environment. Used in resource descriptions" -} - -variable "public_keypair" { - type = string - - description = "Name of the Public SSH key for the CDP environment" - -} - -# ------- CDP Environment Deployment ------- -variable "deployment_type" { - type = string - -} - -variable "deploy_cdp" { - type = bool - - description = "Deploy the CDP environment as part of Terraform" - -} - -# ------- Network Resources ------- -variable "vpc_name" { - type = string - description = "VPC name" -} - -variable "vpc_cidr" { - type = string - description = "VPC CIDR Block" -} - -variable "igw_name" { - type = string - description = "Internet Gateway" -} - -# Public Network infrastructure -variable "public_subnets" { - type = list(object({ - name = string - cidr = string - az = string - tags = map(string) - })) - - description = "List of Public Subnets" -} - -variable "public_route_table_name" { - type = string - description = "Public Route Table Name" -} - -# Private Network infrastructure -variable "private_subnets" { - type = list(object({ - name = string - cidr = string - az = string - tags = map(string) - })) - - description = "List of Private Subnets" -} - -variable "nat_gateway_name" { - type = string - - description = "Nat Gateway" -} - -variable "private_route_table_name" { - type = string - - description = "Private Route Table" -} - -variable "security_group_default_name" { - type = string - - description = "Default Security Group for CDP environment" -} - -variable "security_group_knox_name" { - type = string - - description = "Knox Security Group for CDP environment" -} - -variable "ingress_extra_cidrs_and_ports" { - type = object({ - cidrs = list(string) - ports = list(number) - }) - description = "List of extra CIDR blocks and ports to include in Security Group Ingress rules" -} - -# ------- Storage Resources ------- -variable "random_id_for_bucket" { - type = bool - -} - -variable "data_storage" { - type = object({ - data_storage_bucket = string - data_storage_object = string - }) - - description = "Storage locations for CDP environment" - -} - -variable "log_storage" { - type = object({ - log_storage_bucket = string - log_storage_objects = list(string) - }) - - description = "Optional log locations for CDP environment. If not provided follow the data_storage variable" - -} - -# ------- Policies ------- -# Cross Account Policy (name and document) -variable "xaccount_policy_name" { - type = string - description = "Cross Account Policy name" - -} - -variable "xaccount_account_policy_doc" { - type = string - description = "Location of cross acount policy document" - -} -# CDP IDBroker Assume Role policy -variable "idbroker_policy_name" { - type = string - description = "IDBroker Policy name" - -} - -# CDP Data Access Policies - Log -variable "log_data_access_policy_name" { - type = string - description = "Log Data Access Policy Name" - -} - -variable "log_data_access_policy_doc_location" { - type = string - description = "Location of Log Data Access Policy file." - -} - -# CDP Data Access Policies - ranger_audit_s3 -variable "ranger_audit_s3_policy_name" { - type = string - description = "Ranger S3 Audit Data Access Policy Name" - -} - -variable "ranger_audit_s3_policy_doc_location" { - type = string - description = "Location of Ranger S3 Audit Data Access Policy file." - -} - -# CDP Data Access Policies - datalake_admin_s3 -variable "datalake_admin_s3_policy_name" { - type = string - description = "Datalake Admin S3 Data Access Policy Name" - -} - -variable "datalake_admin_s3_policy_doc_location" { - type = string - description = "Location of Datalake Admin S3 Data Access Policy file." - -} - -# CDP Data Access Policies - bucket_access -variable "bucket_access_policy_name" { - type = string - description = "Bucket Access Data Access Policy Name" - -} - -variable "bucket_access_policy_doc_location" { - type = string - description = "Location of Bucket Access Data Access Policy file." - -} - -# ------- Roles ------- -# Cross Account Role (name and id) -variable "xaccount_role_name" { - type = string - description = "Cross account Assume role Name" -} - -variable "xaccount_account_id" { - type = string - description = "Account ID of the cross account" -} - -variable "xaccount_external_id" { - type = string - description = "External ID of the cross account" -} - -# IDBroker service role -variable "idbroker_role_name" { - type = string - description = "IDBroker service role Name" -} - -# Log service role -variable "log_role_name" { - type = string - description = "Log service role Name" -} - -# CDP Datalake Admin role -variable "datalake_admin_role_name" { - type = string - description = "Datalake Admin role Name" -} - -# CDP Ranger Audit role -variable "ranger_audit_role_name" { - type = string - description = "Ranger Audit role Name" -} diff --git a/modules/terraform-cdp-aws-pre-reqs/main.tf b/modules/terraform-cdp-aws-pre-reqs/main.tf index 46abda0..232afbb 100644 --- a/modules/terraform-cdp-aws-pre-reqs/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/main.tf @@ -334,82 +334,22 @@ resource "aws_s3_object" "cdp_backup_storage_object" { ] } -# ------- Cross Account Role ------- -# First create the assume role policy document -data "aws_iam_policy_document" "cdp_xaccount_role_policy_doc" { - version = "2012-10-17" - - statement { - actions = ["sts:AssumeRole"] - effect = "Allow" - - principals { - type = "AWS" - identifiers = ["arn:aws:iam::${var.xaccount_account_id}:root"] - } - - condition { - test = "StringEquals" - variable = "sts:ExternalId" +# ------- Credential Permissions - Cross Account Role ------- +module "aws_cdp_cred_permissions" { - values = [var.xaccount_external_id] - } - } -} + source = "../terraform-aws-cred-permissions" -# Create the IAM role that uses the above assume_role_policy document -resource "aws_iam_role" "cdp_xaccount_role" { - name = local.xaccount_role_name - description = "CDP Cross Account role for ${var.env_prefix}" + xaccount_role_name = local.xaccount_role_name + xaccount_account_id = var.xaccount_account_id + xaccount_external_id = var.xaccount_external_id - assume_role_policy = data.aws_iam_policy_document.cdp_xaccount_role_policy_doc.json + xaccount_policy_name = local.xaccount_policy_name + xaccount_account_policy_doc = var.xaccount_account_policy_doc - tags = merge(local.env_tags, { Name = local.xaccount_role_name }) -} - -# Create AWS Cross Account Inline Policy -resource "aws_iam_role_policy" "cdp_xaccount_policy" { - name = local.xaccount_policy_name - role = aws_iam_role.cdp_xaccount_role.id - - policy = local.xaccount_account_policy_doc -} - -# Wait for propagation of IAM xaccount role. -# Required for CDP credential -resource "time_sleep" "iam_propagation" { - depends_on = [aws_iam_role.cdp_xaccount_role] - create_duration = "45s" -} + existing_xaccount_role_name = var.existing_xaccount_role_name + tags = local.env_tags -# ------- Add missing iam:Tag* permissions to Cross-Account Policy ------- -# First create the extra policy document -data "aws_iam_policy_document" "cdp_extra_xaccount_policy_doc" { - count = var.create_extra_xaccount_policy ? 1 : 0 - - version = "2012-10-17" - - statement { - sid = "AllowIAMTagRole" - - actions = ["iam:TagRole"] - effect = "Allow" - - resources = [ - "*", - ] - } -} - -# Then create the inline policy using the document -resource "aws_iam_role_policy" "cdp_extra_xaccount_policy" { - count = var.create_extra_xaccount_policy ? 1 : 0 - - name = "${var.env_prefix}-cross-account-extra" - role = aws_iam_role.cdp_xaccount_role.id - - policy = data.aws_iam_policy_document.cdp_extra_xaccount_policy_doc[0].json } # ------- Permissions ------- diff --git a/modules/terraform-cdp-aws-pre-reqs/outputs.tf b/modules/terraform-cdp-aws-pre-reqs/outputs.tf index a8174c6..e30c7cf 100644 --- a/modules/terraform-cdp-aws-pre-reqs/outputs.tf +++ b/modules/terraform-cdp-aws-pre-reqs/outputs.tf @@ -146,13 +146,13 @@ output "aws_ranger_audit_role_arn" { } output "aws_xaccount_role_arn" { - value = aws_iam_role.cdp_xaccount_role.arn + value = module.aws_cdp_cred_permissions.aws_xaccount_role_arn description = "Cross Account role ARN" } output "aws_xaccount_role_name" { - value = aws_iam_role.cdp_xaccount_role.name + value = module.aws_cdp_cred_permissions.aws_xaccount_role_name description = "Cross Account role name" } diff --git a/modules/terraform-cdp-aws-pre-reqs/provider.tf b/modules/terraform-cdp-aws-pre-reqs/provider.tf index bbcb7e9..4c31747 100644 --- a/modules/terraform-cdp-aws-pre-reqs/provider.tf +++ b/modules/terraform-cdp-aws-pre-reqs/provider.tf @@ -26,10 +26,6 @@ terraform { source = "hashicorp/random" version = "~> 3.4.3" } - time = { - source = "hashicorp/time" - version = "~> 0.9.1" - } } required_version = ">= 1.3.0" diff --git a/modules/terraform-cdp-aws-pre-reqs/variables.tf b/modules/terraform-cdp-aws-pre-reqs/variables.tf index 410085e..d2bd3eb 100644 --- a/modules/terraform-cdp-aws-pre-reqs/variables.tf +++ b/modules/terraform-cdp-aws-pre-reqs/variables.tf @@ -40,14 +40,6 @@ variable "env_prefix" { } # ------- CDP Environment Deployment ------- -# variable "cdp_control_plane_region" { -# type = string -# description = "CDP Control Plane Region" - -# # Region is us-west-1 unless explicitly specified -# default = "us-west-1" -# } - variable "deployment_template" { type = string @@ -372,14 +364,6 @@ variable "xaccount_account_policy_doc" { type = string description = "Contents of cross acount policy document" - default = null -} - -variable "create_extra_xaccount_policy" { - type = bool - description = "Create extra Cross-Account Policy for missing iam:Tag* permissions required for Data Services." - - default = true } # CDP IDBroker Assume Role policy @@ -555,3 +539,11 @@ variable "ranger_audit_role_name" { default = null } + +# ------- Support for pre-existing roles ------- +variable "existing_xaccount_role_name" { + type = string + description = "Name of existing CDP Cross Account Role. If set then no Cross Account policy or role resources are created." + + default = null +} From 52dad333d85df6d2cdd68497b8a0fcd81e4f17a5 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Fri, 16 Aug 2024 15:02:28 +0100 Subject: [PATCH 09/11] Update AWS cdp-deploy example to use permissions module Signed-off-by: Jim Enright --- .../examples/ex01-aws-basic/main.tf | 73 ++++++++----------- 1 file changed, 29 insertions(+), 44 deletions(-) diff --git a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf index e32b777..8e7d0a3 100644 --- a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf +++ b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf @@ -52,8 +52,9 @@ module "cdp_aws_prereqs" { ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports # Using CDP TF Provider cred pre-reqs data source for values of xaccount account_id and external_id - xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id - xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id + xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + xaccount_account_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policy) # Policy documents from CDP TF Provider cred pre-reqs idbroker_policy_doc = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Idbroker_Assumer"]) @@ -83,61 +84,45 @@ module "cdp_aws_prereqs" { } -# module "cdp_deploy" { -# source = "../.." +module "cdp_deploy" { + source = "../.." -# env_prefix = var.env_prefix -# infra_type = "aws" -# region = var.aws_region -# keypair_name = var.aws_key_pair -# deployment_template = var.deployment_template + env_prefix = var.env_prefix + infra_type = "aws" + region = var.aws_region + keypair_name = var.aws_key_pair + deployment_template = var.deployment_template -# environment_async_creation = var.environment_async_creation -# datalake_async_creation = var.datalake_async_creation + environment_async_creation = var.environment_async_creation + datalake_async_creation = var.datalake_async_creation -# # From pre-reqs module output -# aws_vpc_id = module.cdp_aws_prereqs.aws_vpc_id -# aws_public_subnet_ids = module.cdp_aws_prereqs.aws_public_subnet_ids -# aws_private_subnet_ids = module.cdp_aws_prereqs.aws_private_subnet_ids + # From pre-reqs module output + aws_vpc_id = module.cdp_aws_prereqs.aws_vpc_id + aws_public_subnet_ids = module.cdp_aws_prereqs.aws_public_subnet_ids + aws_private_subnet_ids = module.cdp_aws_prereqs.aws_private_subnet_ids -# aws_security_group_default_id = module.cdp_aws_prereqs.aws_security_group_default_id -# aws_security_group_knox_id = module.cdp_aws_prereqs.aws_security_group_knox_id + aws_security_group_default_id = module.cdp_aws_prereqs.aws_security_group_default_id + aws_security_group_knox_id = module.cdp_aws_prereqs.aws_security_group_knox_id -# data_storage_location = module.cdp_aws_prereqs.aws_data_storage_location -# log_storage_location = module.cdp_aws_prereqs.aws_log_storage_location -# backup_storage_location = module.cdp_aws_prereqs.aws_backup_storage_location + data_storage_location = module.cdp_aws_prereqs.aws_data_storage_location + log_storage_location = module.cdp_aws_prereqs.aws_log_storage_location + backup_storage_location = module.cdp_aws_prereqs.aws_backup_storage_location aws_xaccount_role_arn = module.cdp_aws_prereqs.aws_xaccount_role_arn aws_datalake_admin_role_arn = module.cdp_aws_prereqs.aws_datalake_admin_role_arn aws_ranger_audit_role_arn = module.cdp_aws_prereqs.aws_ranger_audit_role_arn aws_raz_role_arn = module.cdp_aws_prereqs.aws_datalake_admin_role_arn -# aws_log_instance_profile_arn = module.cdp_aws_prereqs.aws_log_instance_profile_arn -# aws_idbroker_instance_profile_arn = module.cdp_aws_prereqs.aws_idbroker_instance_profile_arn + aws_log_instance_profile_arn = module.cdp_aws_prereqs.aws_log_instance_profile_arn + aws_idbroker_instance_profile_arn = module.cdp_aws_prereqs.aws_idbroker_instance_profile_arn -# # Tags to apply resources (omitted by default) -# env_tags = var.env_tags + # Tags to apply resources (omitted by default) + env_tags = var.env_tags -# depends_on = [ -# module.cdp_aws_prereqs -# ] -# } + depends_on = [ + module.cdp_aws_prereqs + ] +} # Use the CDP Terraform Provider to find the xaccount account and external ids data "cdp_environments_aws_credential_prerequisites" "cdp_prereqs" {} - -# Policies -# output "Ranger_Raz" { -# value = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Ranger_Raz"]) -# } - -# output "Idbroker_Assumer" { -# value = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Idbroker_Assumer"]) -# } -# output "Environment" { -# value = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies["Environment"]) -# } - -# output "policies" { -# value = base64decode(data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.policies) -# } \ No newline at end of file From 2b5c4624c34dbd57b0d79be205459d6f6fc8f383 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Fri, 16 Aug 2024 15:47:14 +0100 Subject: [PATCH 10/11] Fix tflint issues are rebase Signed-off-by: Jim Enright --- modules/terraform-aws-cred-permissions/README.md | 5 +++-- modules/terraform-aws-cred-permissions/provider.tf | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/terraform-aws-cred-permissions/README.md b/modules/terraform-aws-cred-permissions/README.md index 059a9c6..7ab2859 100644 --- a/modules/terraform-aws-cred-permissions/README.md +++ b/modules/terraform-aws-cred-permissions/README.md @@ -21,13 +21,14 @@ In each directory an example `terraform.tfvars.sample` values file is included t |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.0 | | [aws](#requirement\_aws) | ~>5.30 | +| [time](#requirement\_time) | 0.9.1 | ## Providers | Name | Version | |------|---------| | [aws](#provider\_aws) | ~>5.30 | -| [time](#provider\_time) | n/a | +| [time](#provider\_time) | 0.9.1 | ## Modules @@ -39,7 +40,7 @@ No modules. |------|------| | [aws_iam_role.cdp_xaccount_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy.cdp_xaccount_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | -| [time_sleep.iam_propagation](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | +| [time_sleep.iam_propagation](https://registry.terraform.io/providers/hashicorp/time/0.9.1/docs/resources/sleep) | resource | | [aws_iam_policy_document.cdp_xaccount_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_role.existing_xaccount_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | diff --git a/modules/terraform-aws-cred-permissions/provider.tf b/modules/terraform-aws-cred-permissions/provider.tf index 8b78993..35aaddf 100644 --- a/modules/terraform-aws-cred-permissions/provider.tf +++ b/modules/terraform-aws-cred-permissions/provider.tf @@ -21,7 +21,7 @@ terraform { time = { source = "hashicorp/time" version = "0.9.1" - } + } } required_version = ">= 1.3.0" From 0ac647c2499e094ca7eed08f40b30cdedaa339ba Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Fri, 16 Aug 2024 17:35:00 +0100 Subject: [PATCH 11/11] Fix formatting and remove unused code Signed-off-by: Jim Enright --- .../terraform-aws-cred-permissions/main.tf | 30 ----- modules/terraform-aws-permissions/main.tf | 118 +++++------------- 2 files changed, 30 insertions(+), 118 deletions(-) diff --git a/modules/terraform-aws-cred-permissions/main.tf b/modules/terraform-aws-cred-permissions/main.tf index 9e32f7e..9135751 100644 --- a/modules/terraform-aws-cred-permissions/main.tf +++ b/modules/terraform-aws-cred-permissions/main.tf @@ -68,33 +68,3 @@ resource "time_sleep" "iam_propagation" { depends_on = [aws_iam_role.cdp_xaccount_role] create_duration = "45s" } - - -# # ------- Add missing iam:Tag* permissions to Cross-Account Policy ------- -# # First create the extra policy document -# data "aws_iam_policy_document" "cdp_extra_xaccount_policy_doc" { -# count = var.create_extra_xaccount_policy ? 1 : 0 - -# version = "2012-10-17" - -# statement { -# sid = "AllowIAMTagRole" - -# actions = ["iam:TagRole"] -# effect = "Allow" - -# resources = [ -# "*", -# ] -# } -# } - -# # Then create the inline policy using the document -# resource "aws_iam_role_policy" "cdp_extra_xaccount_policy" { -# count = var.create_extra_xaccount_policy ? 1 : 0 - -# name = "${var.env_prefix}-cross-account-extra" -# role = aws_iam_role.cdp_xaccount_role.id - -# policy = data.aws_iam_policy_document.cdp_extra_xaccount_policy_doc[0].json -# } diff --git a/modules/terraform-aws-permissions/main.tf b/modules/terraform-aws-permissions/main.tf index 65a0a88..5f1aee0 100644 --- a/modules/terraform-aws-permissions/main.tf +++ b/modules/terraform-aws-permissions/main.tf @@ -1,7 +1,7 @@ # ------- CDP IDBroker Assume Role policy ------- resource "aws_iam_policy" "cdp_idbroker_policy" { - name = var.idbroker_policy_name - # description = "CDP IDBroker Assume Role policy for ${var.env_prefix}" + name = var.idbroker_policy_name + description = "CDP IDBroker Assume Role policy" tags = merge(var.tags, { Name = var.idbroker_policy_name }) @@ -10,8 +10,8 @@ resource "aws_iam_policy" "cdp_idbroker_policy" { # ------- CDP Data Access Policies - Log ------- resource "aws_iam_policy" "cdp_log_data_access_policy" { - name = var.log_data_access_policy_name - # description = "CDP Log Location Access policy for ${var.env_prefix}" + name = var.log_data_access_policy_name + description = "CDP Log Location Access policy" tags = merge(var.tags, { Name = var.log_data_access_policy_name }) @@ -20,8 +20,8 @@ resource "aws_iam_policy" "cdp_log_data_access_policy" { } # ------- CDP Data Access Policies - ranger_audit_s3 ------- resource "aws_iam_policy" "cdp_ranger_audit_s3_data_access_policy" { - name = var.ranger_audit_s3_policy_name - # description = "CDP Ranger Audit S3 Access policy for ${var.env_prefix}" + name = var.ranger_audit_s3_policy_name + description = "CDP Ranger Audit S3 Access policy" tags = merge(var.tags, { Name = var.ranger_audit_s3_policy_name }) @@ -30,8 +30,8 @@ resource "aws_iam_policy" "cdp_ranger_audit_s3_data_access_policy" { # ------- CDP Data Access Policies - datalake_admin_s3 ------- resource "aws_iam_policy" "cdp_datalake_admin_s3_data_access_policy" { - name = var.datalake_admin_s3_policy_name - # description = "CDP Datalake Admin S3 Access policy for ${var.env_prefix}" + name = var.datalake_admin_s3_policy_name + description = "CDP Datalake Admin S3 Access policy" tags = merge(var.tags, { Name = var.datalake_admin_s3_policy_name }) @@ -42,8 +42,8 @@ resource "aws_iam_policy" "cdp_datalake_admin_s3_data_access_policy" { # ------- CDP Data Access Policies - bucket_access ------- # Policy for Data bucket resource "aws_iam_policy" "cdp_data_bucket_data_access_policy" { - name = var.data_bucket_access_policy_name - # description = "CDP Data Bucket S3 Access policy for ${var.env_prefix}" + name = var.data_bucket_access_policy_name + description = "CDP Data Bucket S3 Access policy" tags = merge(var.tags, { Name = var.data_bucket_access_policy_name }) @@ -52,14 +52,8 @@ resource "aws_iam_policy" "cdp_data_bucket_data_access_policy" { # Policy for Log bucket...Only required if log bucket different from data bucket resource "aws_iam_policy" "cdp_log_bucket_data_access_policy" { - - # count = (( - # var.log_storage_bucket != var.data_storage_bucket) ? - # 1 : 0 - # ) - - name = var.log_bucket_access_policy_name - # description = "CDP Log Bucket S3 Access policy for ${var.env_prefix}" + name = var.log_bucket_access_policy_name + description = "CDP Log Bucket S3 Access policy" tags = merge(var.tags, { Name = var.log_bucket_access_policy_name }) @@ -67,16 +61,8 @@ resource "aws_iam_policy" "cdp_log_bucket_data_access_policy" { } # Policy for backup bucket...requied only if different from backup and log bucket resource "aws_iam_policy" "cdp_backup_bucket_data_access_policy" { - - # count = (( - # var.backup_storage_bucket != var.data_storage_bucket - # || - # var.backup_storage_bucket != var.log_storage_bucket) ? - # 1 : 0 - # ) - - name = var.backup_bucket_access_policy_name - # description = "CDP Backup Bucket S3 Access policy for ${var.env_prefix}" + name = var.backup_bucket_access_policy_name + description = "CDP Backup Bucket S3 Access policy" tags = merge(var.tags, { Name = var.backup_bucket_access_policy_name }) @@ -85,8 +71,8 @@ resource "aws_iam_policy" "cdp_backup_bucket_data_access_policy" { # ------- CDP Data Access Policies - datalake_backup_policy ------- resource "aws_iam_policy" "cdp_datalake_backup_policy" { - name = var.datalake_backup_policy_name - # description = "CDP Datalake Backup policy for ${var.env_prefix}" + name = var.datalake_backup_policy_name + description = "CDP Datalake Backup policy" tags = merge(var.tags, { Name = var.datalake_backup_policy_name }) @@ -95,8 +81,8 @@ resource "aws_iam_policy" "cdp_datalake_backup_policy" { # ------- CDP Data Access Policies - datalake_restore_policy ------- resource "aws_iam_policy" "cdp_datalake_restore_policy" { - name = var.datalake_restore_policy_name - # description = "CDP Datalake Restore policy for ${var.env_prefix}" + name = var.datalake_restore_policy_name + description = "CDP Datalake Restore policy" tags = merge(var.tags, { Name = var.datalake_restore_policy_name }) @@ -121,8 +107,8 @@ data "aws_iam_policy_document" "cdp_idbroker_role_policy_doc" { # Create the IAM role that uses the above assume_role_policy document resource "aws_iam_role" "cdp_idbroker_role" { - name = var.idbroker_role_name - # description = "CDP IDBroker role for ${var.env_prefix}" + name = var.idbroker_role_name + description = "CDP IDBroker role" assume_role_policy = data.aws_iam_policy_document.cdp_idbroker_role_policy_doc.json @@ -143,7 +129,6 @@ resource "aws_iam_role_policy_attachment" "cdp_idbroker_role_attach1" { # Attach AWS Log Location Policy to the Role resource "aws_iam_role_policy_attachment" "cdp_idbroker_role_attach2" { - role = aws_iam_role.cdp_idbroker_role.name policy_arn = aws_iam_policy.cdp_log_data_access_policy.arn } @@ -166,8 +151,8 @@ data "aws_iam_policy_document" "cdp_log_role_policy_doc" { # Create the IAM role that uses the above assume_role_policy document resource "aws_iam_role" "cdp_log_role" { - name = var.log_role_name - # description = "CDP Log role for ${var.env_prefix}" + name = var.log_role_name + description = "CDP Log role" assume_role_policy = data.aws_iam_policy_document.cdp_log_role_policy_doc.json @@ -182,7 +167,6 @@ resource "aws_iam_instance_profile" "cdp_log_role_instance_profile" { # Attach AWS Log Location Policy to the Role resource "aws_iam_role_policy_attachment" "cdp_log_role_attach1" { - role = aws_iam_role.cdp_log_role.name policy_arn = aws_iam_policy.cdp_log_data_access_policy.arn } @@ -196,7 +180,6 @@ resource "aws_iam_role_policy_attachment" "cdp_log_role_attach2" { # Attach AWS Datalake Backup Policy to the Role resource "aws_iam_role_policy_attachment" "cdp_log_role_attach3" { - role = aws_iam_role.cdp_log_role.name policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn } @@ -219,8 +202,8 @@ data "aws_iam_policy_document" "cdp_datalake_admin_role_policy_doc" { # Create the IAM role that uses the above assume_role_policy document resource "aws_iam_role" "cdp_datalake_admin_role" { - name = var.datalake_admin_role_name - # description = "CDP Datalake Admin role for ${var.env_prefix}" + name = var.datalake_admin_role_name + description = "CDP Datalake Admin role" assume_role_policy = data.aws_iam_policy_document.cdp_datalake_admin_role_policy_doc.json @@ -235,7 +218,6 @@ resource "aws_iam_instance_profile" "cdp_datalake_admin_role_instance_profile" { # Attach AWS Datalake Admin S3 Policy to the Role resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach1" { - role = aws_iam_role.cdp_datalake_admin_role.name policy_arn = aws_iam_policy.cdp_datalake_admin_s3_data_access_policy.arn } @@ -243,49 +225,30 @@ resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach1" { # Attach AWS Bucket Access Policy to the Role # ..data bucket policy resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach2" { - role = aws_iam_role.cdp_datalake_admin_role.name policy_arn = aws_iam_policy.cdp_data_bucket_data_access_policy.arn } # ..log bucket policy, if required resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach3" { - - # count = (( - # var.log_storage_bucket != var.data_storage_bucket) ? - # 1 : 0 - # ) - - role = aws_iam_role.cdp_datalake_admin_role.name - # policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy[0].arn + role = aws_iam_role.cdp_datalake_admin_role.name policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy.arn } # ..backup bucket policy, if required resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach4" { - - # count = (( - # var.backup_storage_bucket != var.data_storage_bucket - # || - # var.backup_storage_bucket != var.log_storage_bucket) ? - # 1 : 0 - # ) - - role = aws_iam_role.cdp_datalake_admin_role.name - # policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy[0].arn + role = aws_iam_role.cdp_datalake_admin_role.name policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy.arn } # Attach AWS Datalake Backup Policy to the Role resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach5" { - role = aws_iam_role.cdp_datalake_admin_role.name policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn } # Attach AWS Datalake Restore Policy to the Role resource "aws_iam_role_policy_attachment" "cdp_datalake_admin_role_attach6" { - role = aws_iam_role.cdp_datalake_admin_role.name policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn } @@ -309,8 +272,8 @@ data "aws_iam_policy_document" "cdp_ranger_audit_role_policy_doc" { # Create the IAM role that uses the above assume_role_policy document resource "aws_iam_role" "cdp_ranger_audit_role" { - name = var.ranger_audit_role_name - # description = "CDP Ranger Audit role for ${var.env_prefix}" + name = var.ranger_audit_role_name + description = "CDP Ranger Audit role" assume_role_policy = data.aws_iam_policy_document.cdp_ranger_audit_role_policy_doc.json @@ -325,7 +288,6 @@ resource "aws_iam_instance_profile" "cdp_ranger_audit_role_instance_profile" { # Attach AWS Ranger Audit S3 Policy to the Role resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach1" { - role = aws_iam_role.cdp_ranger_audit_role.name policy_arn = aws_iam_policy.cdp_ranger_audit_s3_data_access_policy.arn } @@ -333,50 +295,30 @@ resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach1" { # Attach AWS Bucket Access Policies to the Role # ..data bucket policy resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach2" { - role = aws_iam_role.cdp_ranger_audit_role.name policy_arn = aws_iam_policy.cdp_data_bucket_data_access_policy.arn } # ..log bucket policy, if required resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach3" { - - # count = (( - # var.log_storage_bucket != var.data_storage_bucket) ? - # 1 : 0 - # ) - - role = aws_iam_role.cdp_ranger_audit_role.name - # policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy[0].arn + role = aws_iam_role.cdp_ranger_audit_role.name policy_arn = aws_iam_policy.cdp_log_bucket_data_access_policy.arn } # ..backup bucket policy, if required resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach4" { - - # count = (( - # var.backup_storage_bucket != var.data_storage_bucket - # || - # var.backup_storage_bucket != var.log_storage_bucket) ? - # 1 : 0 - # ) - - role = aws_iam_role.cdp_ranger_audit_role.name - # policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy[0].arn + role = aws_iam_role.cdp_ranger_audit_role.name policy_arn = aws_iam_policy.cdp_backup_bucket_data_access_policy.arn } # Attach AWS Datalake Backup Policy to the Role resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach5" { - role = aws_iam_role.cdp_ranger_audit_role.name policy_arn = aws_iam_policy.cdp_datalake_backup_policy.arn } # Attach AWS Datalake Restore Policy to the Role resource "aws_iam_role_policy_attachment" "cdp_ranger_audit_role_attach6" { - role = aws_iam_role.cdp_ranger_audit_role.name policy_arn = aws_iam_policy.cdp_datalake_restore_policy.arn } -