From a6b5a20ed618ff311903909ff251e6b990fd7120 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Sat, 5 Aug 2023 02:23:23 +0100 Subject: [PATCH] Updates for use of CDP Terraform provider (#21) Signed-off-by: Jim Enright Co-authored-by: Balazs Gaspar --- .tflint.hcl | 6 + README.md | 35 +- modules/terraform-cdp-aws-pre-reqs/README.md | 128 ++++--- modules/terraform-cdp-aws-pre-reqs/data.tf | 13 - .../terraform-cdp-aws-pre-reqs/defaults.tf | 15 +- .../examples/ex01-minimal_inputs/main.tf | 16 + .../examples/ex02-existing-vpc/main.tf | 16 + .../examples/ex03-create-keypair/main.tf | 16 + modules/terraform-cdp-aws-pre-reqs/main.tf | 39 ++- .../modules/vpc/defaults.tf | 4 +- .../modules/vpc/main.tf | 7 +- .../modules/vpc/provider.tf | 4 +- .../modules/vpc/variables.tf | 7 + modules/terraform-cdp-aws-pre-reqs/outputs.tf | 31 +- .../terraform-cdp-aws-pre-reqs/provider.tf | 12 +- .../run_cdp_get_cred_prereqs.sh | 51 --- .../terraform-cdp-aws-pre-reqs/variables.tf | 38 +-- .../terraform-cdp-azure-pre-reqs/README.md | 14 +- modules/terraform-cdp-azure-pre-reqs/data.tf | 7 + .../terraform-cdp-azure-pre-reqs/defaults.tf | 3 + modules/terraform-cdp-azure-pre-reqs/main.tf | 66 ++-- .../modules/vnet/provider.tf | 2 +- .../terraform-cdp-azure-pre-reqs/outputs.tf | 38 ++- .../terraform-cdp-azure-pre-reqs/provider.tf | 2 +- .../terraform-cdp-azure-pre-reqs/variables.tf | 11 +- modules/terraform-cdp-deploy/README.md | 16 +- modules/terraform-cdp-deploy/defaults.tf | 11 +- .../examples/ex01-aws-basic/main.tf | 29 +- .../ex01-aws-basic/terraform.tfvars.sample | 12 + .../examples/ex01-aws-basic/variables.tf | 46 ++- .../examples/ex02-azure-basic/main.tf | 10 +- .../ex02-azure-basic/terraform.tfvars.sample | 12 +- .../examples/ex02-azure-basic/variables.tf | 37 +++ modules/terraform-cdp-deploy/main.tf | 24 +- .../terraform-cdp-deploy/modules/aws/main.tf | 166 ++++++---- .../modules/aws/outputs.tf | 25 ++ .../modules/aws/playbook_setup_cdp.yml | 307 ----------------- .../modules/aws/playbook_teardown_cdp.yml | 58 ---- .../modules/aws/provider.tf | 12 +- .../modules/aws/templates/cdp_config.yml.tpl | 57 ---- .../modules/aws/variables.tf | 32 +- .../modules/azure/main.tf | 181 ++++++---- .../modules/azure/outputs.tf | 12 + .../modules/azure/playbook_setup_cdp.yml | 313 ------------------ .../modules/azure/playbook_teardown_cdp.yml | 58 ---- .../modules/azure/provider.tf | 12 +- .../azure/templates/cdp_config.yml.tpl | 64 ---- .../modules/azure/variables.tf | 24 +- modules/terraform-cdp-deploy/outputs.tf | 18 + modules/terraform-cdp-deploy/provider.tf | 12 +- modules/terraform-cdp-deploy/variables.tf | 45 ++- 51 files changed, 787 insertions(+), 1387 deletions(-) delete mode 100755 modules/terraform-cdp-aws-pre-reqs/run_cdp_get_cred_prereqs.sh delete mode 100644 modules/terraform-cdp-deploy/modules/aws/playbook_setup_cdp.yml delete mode 100644 modules/terraform-cdp-deploy/modules/aws/playbook_teardown_cdp.yml delete mode 100644 modules/terraform-cdp-deploy/modules/aws/templates/cdp_config.yml.tpl delete mode 100644 modules/terraform-cdp-deploy/modules/azure/playbook_setup_cdp.yml delete mode 100644 modules/terraform-cdp-deploy/modules/azure/playbook_teardown_cdp.yml delete mode 100644 modules/terraform-cdp-deploy/modules/azure/templates/cdp_config.yml.tpl diff --git a/.tflint.hcl b/.tflint.hcl index f1919e4..54ec7ce 100644 --- a/.tflint.hcl +++ b/.tflint.hcl @@ -24,6 +24,12 @@ plugin "aws" { source = "github.com/terraform-linters/tflint-ruleset-aws" } +plugin "azurerm" { + enabled = true + version = "0.24.0" + source = "github.com/terraform-linters/tflint-ruleset-azurerm" +} + config { module = true force = false diff --git a/README.md b/README.md index 7ea8cdd..67b60b3 100644 --- a/README.md +++ b/README.md @@ -54,34 +54,17 @@ If you no longer need the infrastructure that’s provisioned by the Terraform m terraform destroy ``` -## External dependencies +## Dependencies -The module includes the option to discover the cross account Ids and to run the CDP deployment using external tools. +To set up CDP via deployment automation using this guide, the following dependencies must be installed in your local environment: -To utilize these options extra requirements are needed - Python 3, Ansible 2.12, the CDP CLI, the [jq utility](https://stedolan.github.io/jq/download/) and a number of support Python libraries and Ansible collections. +* Terraform can be installed by following the instructions at https://developer.hashicorp.com/terraform/downloads -A summary of the install and configuration steps for these additional requirements is given below. -We recommend these steps be performed within an Python virtual environment. +Configure Terraform Provider for AWS or Azure -```bash -# Install jq as per instructions at https://stedolan.github.io/jq/download/ -# Example for MacOS using homebew shown below -brew install jq - -# Install the Ansible core Python package -pip install ansible-core==2.12.10 jmespath==1.0.1 - -# Install cdpy, a Pythonic wrapper for Cloudera CDP CLI. This in turn installs the CDP CLI. -pip install git+https://github.com/cloudera-labs/cdpy@main#egg=cdpy - -# Install the cloudera.cloud Ansible Collection -ansible-galaxy collection install git+https://github.com/cloudera-labs/cloudera.cloud.git,devel - -# Install the community.general Ansible Collection -ansible-galaxy collection install community.general:==5.5.0 - -# Configure cdp with CDP API access key ID and private key -cdp configure -``` +* Configure the Terraform Provider for CDP with access key ID and private key by dowloading or creating a CDP configuation file. + * See the [CDP documentation for steps to Generate the API access key](https://docs.cloudera.com/cdp-public-cloud/cloud/cli/topics/mc-cli-generating-an-api-access-key.html). -NOTE - See the [CDP documentation for steps to Generate the API access key](https://docs.cloudera.com/cdp-public-cloud/cloud/cli/topics/mc-cli-generating-an-api-access-key.html) required in the `cdp configure` command above. +* To create resources in the Cloud Provider, access credentials or service account are needed for authentication. + * For **AWS** access keys are required to be able to create the Cloud resources via the Terraform aws provider. See the [AWS Terraform Provider Documentation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration). + * For **Azure**, authentication with the Azure subscription is required. There are a number of ways to do this outlined in the [Azure Terraform Provider Documentation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs#authenticating-to-azure). diff --git a/modules/terraform-cdp-aws-pre-reqs/README.md b/modules/terraform-cdp-aws-pre-reqs/README.md index e665da1..4c28b26 100644 --- a/modules/terraform-cdp-aws-pre-reqs/README.md +++ b/modules/terraform-cdp-aws-pre-reqs/README.md @@ -19,20 +19,20 @@ In each directory an example `terraform.tfvars.sample` values file is included t | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | > 1.3.0 | -| [aws](#requirement\_aws) | ~> 4.0 | -| [external](#requirement\_external) | 2.3.1 | +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [aws](#requirement\_aws) | 4.67.0 | | [http](#requirement\_http) | 3.2.1 | | [random](#requirement\_random) | 3.4.3 | +| [time](#requirement\_time) | 0.9.1 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | ~> 4.0 | -| [external](#provider\_external) | 2.3.1 | +| [aws](#provider\_aws) | 4.67.0 | | [http](#provider\_http) | 3.2.1 | | [random](#provider\_random) | 3.4.3 | +| [time](#provider\_time) | 0.9.1 | ## Modules @@ -44,60 +44,59 @@ In each directory an example `terraform.tfvars.sample` values file is included t | Name | Type | |------|------| -| [aws_iam_instance_profile.cdp_datalake_admin_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | -| [aws_iam_instance_profile.cdp_idbroker_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | -| [aws_iam_instance_profile.cdp_log_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | -| [aws_iam_instance_profile.cdp_ranger_audit_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | -| [aws_iam_policy.cdp_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_datalake_admin_s3_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_datalake_backup_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_datalake_restore_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_idbroker_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_log_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_ranger_audit_s3_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.cdp_xaccount_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_role.cdp_datalake_admin_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role.cdp_idbroker_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role.cdp_log_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role.cdp_ranger_audit_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role.cdp_xaccount_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach4](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_idbroker_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_idbroker_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_log_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_log_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_log_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach4](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.cdp_xaccount_role_attach](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_s3_bucket.cdp_storage_locations](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | -| [aws_s3_object.cdp_backup_storage_object](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource | -| [aws_s3_object.cdp_data_storage_object](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource | -| [aws_s3_object.cdp_log_storage_object](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object) | resource | -| [aws_security_group.cdp_default_sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [aws_security_group.cdp_knox_sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | -| [aws_security_group_rule.cdp_default_sg_egress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.cdp_default_sg_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.cdp_default_sg_ingress_self](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.cdp_knox_sg_egress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.cdp_knox_sg_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.cdp_knox_sg_ingress_self](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_iam_instance_profile.cdp_datalake_admin_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_instance_profile) | resource | +| [aws_iam_instance_profile.cdp_idbroker_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_instance_profile) | resource | +| [aws_iam_instance_profile.cdp_log_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_instance_profile) | resource | +| [aws_iam_instance_profile.cdp_ranger_audit_role_instance_profile](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_instance_profile) | resource | +| [aws_iam_policy.cdp_bucket_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_datalake_admin_s3_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_datalake_backup_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_datalake_restore_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_idbroker_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_log_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_ranger_audit_s3_data_access_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | +| [aws_iam_policy.cdp_xaccount_policy](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_policy) | resource | +| [aws_iam_role.cdp_datalake_admin_role](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role) | resource | +| [aws_iam_role.cdp_idbroker_role](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role) | resource | +| [aws_iam_role.cdp_log_role](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role) | resource | +| [aws_iam_role.cdp_ranger_audit_role](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role) | resource | +| [aws_iam_role.cdp_xaccount_role](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach4](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_idbroker_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_idbroker_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_log_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_log_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_log_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach1](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach2](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach3](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach4](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.cdp_xaccount_role_attach](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/iam_role_policy_attachment) | resource | +| [aws_s3_bucket.cdp_storage_locations](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/s3_bucket) | resource | +| [aws_s3_object.cdp_backup_storage_object](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/s3_object) | resource | +| [aws_s3_object.cdp_log_storage_object](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/s3_object) | resource | +| [aws_security_group.cdp_default_sg](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/security_group) | resource | +| [aws_security_group.cdp_knox_sg](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/security_group) | resource | +| [aws_security_group_rule.cdp_default_sg_egress](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.cdp_default_sg_ingress](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.cdp_default_sg_ingress_self](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.cdp_knox_sg_egress](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.cdp_knox_sg_ingress](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.cdp_knox_sg_ingress_self](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/resources/security_group_rule) | resource | | [random_id.bucket_suffix](https://registry.terraform.io/providers/hashicorp/random/3.4.3/docs/resources/id) | resource | -| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_iam_policy_document.cdp_datalake_admin_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cdp_idbroker_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cdp_idbroker_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cdp_log_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cdp_ranger_audit_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.cdp_xaccount_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_subnets.vpc_subnets](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnets) | data source | -| [aws_vpc.cdp_vpc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source | -| [external_external.cdpcli](https://registry.terraform.io/providers/hashicorp/external/2.3.1/docs/data-sources/external) | data source | +| [time_sleep.iam_propagation](https://registry.terraform.io/providers/hashicorp/time/0.9.1/docs/resources/sleep) | resource | +| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/data-sources/caller_identity) | data source | +| [aws_iam_policy_document.cdp_datalake_admin_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cdp_idbroker_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cdp_idbroker_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cdp_log_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cdp_ranger_audit_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cdp_xaccount_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/data-sources/iam_policy_document) | data source | +| [aws_subnets.vpc_subnets](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/data-sources/subnets) | data source | +| [aws_vpc.cdp_vpc](https://registry.terraform.io/providers/hashicorp/aws/4.67.0/docs/data-sources/vpc) | data source | | [http_http.bucket_access_policy_doc](https://registry.terraform.io/providers/hashicorp/http/3.2.1/docs/data-sources/http) | data source | | [http_http.datalake_admin_s3_policy_doc](https://registry.terraform.io/providers/hashicorp/http/3.2.1/docs/data-sources/http) | data source | | [http_http.datalake_backup_policy_doc](https://registry.terraform.io/providers/hashicorp/http/3.2.1/docs/data-sources/http) | data source | @@ -112,16 +111,16 @@ In each directory an example `terraform.tfvars.sample` values file is included t |------|-------------|------|---------|:--------:| | [deployment\_template](#input\_deployment\_template) | Deployment Pattern to use for Cloud resources and CDP | `string` | n/a | yes | | [env\_prefix](#input\_env\_prefix) | Shorthand name for the environment. Used in resource descriptions | `string` | n/a | yes | +| [xaccount\_account\_id](#input\_xaccount\_account\_id) | Account ID of the cross account | `string` | n/a | yes | +| [xaccount\_external\_id](#input\_xaccount\_external\_id) | External ID of the cross account | `string` | n/a | yes | | [agent\_source\_tag](#input\_agent\_source\_tag) | Tag to identify deployment source | `map(any)` | 
{
  "agent_source": "tf-cdp-module"
}
| no | | [aws\_region](#input\_aws\_region) | Region which Cloud resources will be created | `string` | `null` | no | | [backup\_storage](#input\_backup\_storage) | Optional Backup location for CDP environment. If not provided follow the data\_storage variable | 
object({
    backup_storage_bucket = string
    backup_storage_object = string
  })
| `null` | no | | [bucket\_access\_policy\_doc](#input\_bucket\_access\_policy\_doc) | Bucket Access Data Access Policy | `string` | `null` | no | | [bucket\_access\_policy\_name](#input\_bucket\_access\_policy\_name) | Bucket Access Data Access Policy Name | `string` | `null` | no | -| [cdp\_control\_plane\_region](#input\_cdp\_control\_plane\_region) | CDP Control Plane Region | `string` | `"us-west-1"` | no | | [cdp\_default\_sg\_egress\_cidrs](#input\_cdp\_default\_sg\_egress\_cidrs) | List of egress CIDR blocks for CDP Default Security Group Egress rule | `list(string)` | 
[
  "0.0.0.0/0"
]
| no | | [cdp\_knox\_sg\_egress\_cidrs](#input\_cdp\_knox\_sg\_egress\_cidrs) | List of egress CIDR blocks for CDP Knox Security Group Egress rule | `list(string)` | 
[
  "0.0.0.0/0"
]
| no | | [cdp\_private\_subnet\_ids](#input\_cdp\_private\_subnet\_ids) | List of private subnet ids. Required if create\_vpc is false. | `list(any)` | `null` | no | -| [cdp\_profile](#input\_cdp\_profile) | Profile for CDP credentials | `string` | `"default"` | no | | [cdp\_public\_subnet\_ids](#input\_cdp\_public\_subnet\_ids) | List of public subnet ids. Required if create\_vpc is false. | `list(any)` | `null` | no | | [cdp\_vpc\_id](#input\_cdp\_vpc\_id) | VPC ID for CDP environment. Required if create\_vpc is false. | `string` | `null` | no | | [create\_vpc](#input\_create\_vpc) | Flag to specify if the VPC should be created | `bool` | `true` | no | @@ -142,7 +141,7 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [log\_data\_access\_policy\_name](#input\_log\_data\_access\_policy\_name) | Log Data Access Policy Name | `string` | `null` | no | | [log\_role\_name](#input\_log\_role\_name) | Log service role Name | `string` | `null` | no | | [log\_storage](#input\_log\_storage) | Optional log locations for CDP environment. If not provided follow the data\_storage variable | 
object({
    log_storage_bucket = string
    log_storage_object = string
  })
| `null` | no | -| [lookup\_cdp\_account\_ids](#input\_lookup\_cdp\_account\_ids) | Auto lookup CDP Account and External ID using CDP CLI commands. If false then the xaccount\_account\_id and xaccount\_external\_id input variables need to be specified | `bool` | `true` | no | +| [private\_network\_extensions](#input\_private\_network\_extensions) | Enable creation of resources for connectivity to CDP Control Plane (public subnet and NAT Gateway) for Private Deployment. Only relevant for private deployment template | `bool` | `true` | no | | [random\_id\_for\_bucket](#input\_random\_id\_for\_bucket) | Create a random suffix for the bucket names | `bool` | `true` | no | | [ranger\_audit\_role\_name](#input\_ranger\_audit\_role\_name) | Ranger Audit role Name | `string` | `null` | no | | [ranger\_audit\_s3\_policy\_doc](#input\_ranger\_audit\_s3\_policy\_doc) | Location or Contents of Ranger S3 Audit Data Access Policy | `string` | `null` | no | @@ -150,9 +149,7 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [security\_group\_default\_name](#input\_security\_group\_default\_name) | Default Security Group for CDP environment | `string` | `null` | no | | [security\_group\_knox\_name](#input\_security\_group\_knox\_name) | Knox Security Group for CDP environment | `string` | `null` | no | | [vpc\_cidr](#input\_vpc\_cidr) | VPC CIDR Block | `string` | `"10.10.0.0/16"` | no | -| [xaccount\_account\_id](#input\_xaccount\_account\_id) | Account ID of the cross account | `string` | `null` | no | | [xaccount\_account\_policy\_doc](#input\_xaccount\_account\_policy\_doc) | Location of cross acount policy document | `string` | `null` | no | -| [xaccount\_external\_id](#input\_xaccount\_external\_id) | External ID of the cross account | `string` | `null` | no | | [xaccount\_policy\_name](#input\_xaccount\_policy\_name) | Cross Account Policy name | `string` | `null` | no | | [xaccount\_role\_name](#input\_xaccount\_role\_name) | Cross account Assume role Name | `string` | `null` | no | @@ -160,12 +157,15 @@ In each directory an example `terraform.tfvars.sample` values file is included t | Name | Description | |------|-------------| +| [aws\_backup\_storage\_bucket](#output\_aws\_backup\_storage\_bucket) | AWS backup storage bucket | | [aws\_backup\_storage\_location](#output\_aws\_backup\_storage\_location) | AWS backup storage location | +| [aws\_data\_storage\_bucket](#output\_aws\_data\_storage\_bucket) | AWS data storage bucket | | [aws\_data\_storage\_location](#output\_aws\_data\_storage\_location) | AWS data storage location | | [aws\_datalake\_admin\_role\_arn](#output\_aws\_datalake\_admin\_role\_arn) | Datalake Admin role ARN | | [aws\_default\_route\_table\_id](#output\_aws\_default\_route\_table\_id) | AWS default route table ID | | [aws\_idbroker\_instance\_profile\_arn](#output\_aws\_idbroker\_instance\_profile\_arn) | IDBroker instance profile ARN | | [aws\_log\_instance\_profile\_arn](#output\_aws\_log\_instance\_profile\_arn) | Log instance profile ARN | +| [aws\_log\_storage\_bucket](#output\_aws\_log\_storage\_bucket) | AWS log storage bucket | | [aws\_log\_storage\_location](#output\_aws\_log\_storage\_location) | AWS log storage location | | [aws\_private\_route\_table\_ids](#output\_aws\_private\_route\_table\_ids) | AWS private route table IDs | | [aws\_private\_subnet\_ids](#output\_aws\_private\_subnet\_ids) | AWS private subnet IDs | @@ -178,7 +178,5 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [aws\_vpc\_id](#output\_aws\_vpc\_id) | AWS VPC ID | | [aws\_vpc\_subnets](#output\_aws\_vpc\_subnets) | List of subnets associated with the CDP VPC | | [aws\_xaccount\_role\_arn](#output\_aws\_xaccount\_role\_arn) | Cross Account role ARN | -| [cdp\_control\_plane\_region](#output\_cdp\_control\_plane\_region) | CDP Control Plane region | -| [cdp\_profile](#output\_cdp\_profile) | Profile for CDP credentials | | [tags](#output\_tags) | Tags associated with the environment and its resources | \ No newline at end of file diff --git a/modules/terraform-cdp-aws-pre-reqs/data.tf b/modules/terraform-cdp-aws-pre-reqs/data.tf index 9588cd0..5378414 100644 --- a/modules/terraform-cdp-aws-pre-reqs/data.tf +++ b/modules/terraform-cdp-aws-pre-reqs/data.tf @@ -70,16 +70,3 @@ data "http" "datalake_backup_policy_doc" { data "http" "datalake_restore_policy_doc" { url = "https://raw.githubusercontent.com/hortonworks/cloudbreak/master/cloud-aws-cloudformation/src/main/resources/definitions/aws-datalake-restore-policy.json" } - -# Use the cdp cli to determin the -data "external" "cdpcli" { - - count = var.lookup_cdp_account_ids == true ? 1 : 0 - - program = ["bash", "${path.module}/run_cdp_get_cred_prereqs.sh"] - query = { - infra_type = var.infra_type - cdp_profile = var.cdp_profile - cdp_region = var.cdp_control_plane_region - } -} diff --git a/modules/terraform-cdp-aws-pre-reqs/defaults.tf b/modules/terraform-cdp-aws-pre-reqs/defaults.tf index 03de691..3426b55 100644 --- a/modules/terraform-cdp-aws-pre-reqs/defaults.tf +++ b/modules/terraform-cdp-aws-pre-reqs/defaults.tf @@ -30,8 +30,9 @@ locals { public_route_table_ids = (var.create_vpc ? module.aws_cdp_vpc[0].public_route_tables : null) private_route_table_ids = (var.create_vpc ? module.aws_cdp_vpc[0].private_route_tables : null) + # If we create the vpc and have private deployment template public subnets are always empty public_subnet_ids = (var.create_vpc ? - module.aws_cdp_vpc[0].public_subnets : var.cdp_public_subnet_ids) + (var.deployment_template == "private" ? [] : module.aws_cdp_vpc[0].public_subnets) : var.cdp_public_subnet_ids) private_subnet_ids = (var.create_vpc ? module.aws_cdp_vpc[0].private_subnets : var.cdp_private_subnet_ids @@ -111,7 +112,7 @@ locals { replace( replace( data.http.ranger_audit_s3_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), - "$${STORAGE_LOCATION_BASE}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}"), + "$${STORAGE_LOCATION_BASE}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}/${replace(local.data_storage.data_storage_object, "/", "")}"), "$${DATALAKE_BUCKET}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}") # ...then assign either input or downloaded policy doc to var used in resource @@ -125,7 +126,7 @@ locals { datalake_admin_s3_policy_doc_processed = replace( replace( data.http.datalake_admin_s3_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), - "$${STORAGE_LOCATION_BASE}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}") + "$${STORAGE_LOCATION_BASE}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}/${replace(local.data_storage.data_storage_object, "/", "")}") # ...then assign either input or downloaded policy doc to var used in resource datalake_admin_s3_policy_doc = coalesce(var.datalake_admin_s3_policy_doc, local.datalake_admin_s3_policy_doc_processed) @@ -151,7 +152,7 @@ locals { datalake_backup_policy_doc_processed = replace( replace( data.http.datalake_backup_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), - "$${BACKUP_LOCATION_BASE}", "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}") + "$${BACKUP_LOCATION_BASE}", "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${replace(local.backup_storage.backup_storage_object, "/", "")}") # ...then assign either input or downloaded policy doc to var used in resource datalake_backup_policy_doc = coalesce(var.datalake_backup_policy_doc, local.datalake_backup_policy_doc_processed) @@ -164,7 +165,7 @@ locals { datalake_restore_policy_doc_processed = replace( replace( data.http.datalake_restore_policy_doc.response_body, "$${ARN_PARTITION}", "aws"), - "$${BACKUP_LOCATION_BASE}", "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}") + "$${BACKUP_LOCATION_BASE}", "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${replace(local.backup_storage.backup_storage_object, "/", "")}") # ...then assign either input or downloaded policy doc to var used in resource datalake_restore_policy_doc = coalesce(var.datalake_restore_policy_doc, local.datalake_restore_policy_doc_processed) @@ -172,10 +173,6 @@ locals { # ------- Roles ------- xaccount_role_name = coalesce(var.xaccount_role_name, "${var.env_prefix}-xaccount-role") - xaccount_account_id = coalesce(var.xaccount_account_id, var.lookup_cdp_account_ids ? data.external.cdpcli[0].result.account_id : null) - - xaccount_external_id = coalesce(var.xaccount_external_id, var.lookup_cdp_account_ids ? data.external.cdpcli[0].result.external_id : null) - idbroker_role_name = coalesce(var.idbroker_role_name, "${var.env_prefix}-idbroker-role") log_role_name = coalesce(var.log_role_name, "${var.env_prefix}-logs-role") diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf index a58be05..54e1446 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf @@ -27,4 +27,20 @@ module "ex01_minimal_inputs" { ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports + # Using CDP TF Provider cred pre-reqs data source for values of xaccount account_id and external_id + xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id + xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + +} + +# Use the CDP Terraform Provider to find the xaccount account and external ids +terraform { + required_providers { + cdp = { + source = "cloudera/cdp" + version = "0.1.4-pre" + } + } } + +data "cdp_environments_aws_credential_prerequisites" "cdp_prereqs" {} diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf index 115c7c9..eee0fbd 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf @@ -27,6 +27,10 @@ module "ex02_existing_vpc" { ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports + # Using CDP TF Provider cred pre-reqs data source for values of xaccount account_id and external_id + xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id + xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + create_vpc = var.create_vpc cdp_vpc_id = aws_vpc.cdp_vpc.id cdp_public_subnet_ids = values(aws_subnet.cdp_public_subnets)[*].id @@ -41,3 +45,15 @@ module "ex02_existing_vpc" { ] } + +# Use the CDP Terraform Provider to find the xaccount account and external ids +terraform { + required_providers { + cdp = { + source = "cloudera/cdp" + version = "0.1.4-pre" + } + } +} + +data "cdp_environments_aws_credential_prerequisites" "cdp_prereqs" {} diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf index 13c4506..750f177 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf @@ -47,4 +47,20 @@ module "ex01_create_keypair" { ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports + # Using CDP TF Provider cred pre-reqs data source for values of xaccount account_id and external_id + xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id + xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + +} + +# Use the CDP Terraform Provider to find the xaccount account and external ids +terraform { + required_providers { + cdp = { + source = "cloudera/cdp" + version = "0.1.4-pre" + } + } } + +data "cdp_environments_aws_credential_prerequisites" "cdp_prereqs" {} diff --git a/modules/terraform-cdp-aws-pre-reqs/main.tf b/modules/terraform-cdp-aws-pre-reqs/main.tf index f5bd572..043e494 100644 --- a/modules/terraform-cdp-aws-pre-reqs/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/main.tf @@ -20,10 +20,11 @@ module "aws_cdp_vpc" { source = "./modules/vpc" - deployment_template = var.deployment_template - vpc_cidr = var.vpc_cidr - env_prefix = var.env_prefix - tags = local.env_tags + deployment_template = var.deployment_template + vpc_cidr = var.vpc_cidr + private_network_extensions = var.private_network_extensions + env_prefix = var.env_prefix + tags = local.env_tags } @@ -135,18 +136,19 @@ resource "aws_s3_bucket" "cdp_storage_locations" { } # ------- AWS Buckets directory structures ------- -# Data Storage Objects -resource "aws_s3_object" "cdp_data_storage_object" { +# # Data Storage Objects +# NOTE: Removing creation of the data storage object because CDP overrides this +# resource "aws_s3_object" "cdp_data_storage_object" { - bucket = "${local.data_storage.data_storage_bucket}${local.storage_suffix}" +# bucket = "${local.data_storage.data_storage_bucket}${local.storage_suffix}" - key = local.data_storage.data_storage_object - content_type = "application/x-directory" +# key = local.data_storage.data_storage_object +# content_type = "application/x-directory" - depends_on = [ - aws_s3_bucket.cdp_storage_locations - ] -} +# depends_on = [ +# aws_s3_bucket.cdp_storage_locations +# ] +# } # Log Storage Objects resource "aws_s3_object" "cdp_log_storage_object" { @@ -282,14 +284,14 @@ data "aws_iam_policy_document" "cdp_xaccount_role_policy_doc" { principals { type = "AWS" - identifiers = ["arn:aws:iam::${local.xaccount_account_id}:root"] + identifiers = ["arn:aws:iam::${var.xaccount_account_id}:root"] } condition { test = "StringEquals" variable = "sts:ExternalId" - values = [local.xaccount_external_id] + values = [var.xaccount_external_id] } } } @@ -310,6 +312,13 @@ resource "aws_iam_role_policy_attachment" "cdp_xaccount_role_attach" { policy_arn = aws_iam_policy.cdp_xaccount_policy.arn } +# Wait for propagation of IAM xaccount role. +# Required for CDP credential +resource "time_sleep" "iam_propagation" { + depends_on = [aws_iam_role.cdp_xaccount_role] + create_duration = "45s" +} + # ------- AWS Service Roles - CDP IDBroker ------- # First create the Assume role policy document data "aws_iam_policy_document" "cdp_idbroker_role_policy_doc" { diff --git a/modules/terraform-cdp-aws-pre-reqs/modules/vpc/defaults.tf b/modules/terraform-cdp-aws-pre-reqs/modules/vpc/defaults.tf index d8de543..30201f0 100644 --- a/modules/terraform-cdp-aws-pre-reqs/modules/vpc/defaults.tf +++ b/modules/terraform-cdp-aws-pre-reqs/modules/vpc/defaults.tf @@ -21,8 +21,8 @@ locals { # ------- Determine subnet details from inputs ------- subnets_required = { - total = (var.deployment_template == "public") ? length(local.zones_in_region) : 2 * length(local.zones_in_region) - public = length(local.zones_in_region) + total = contains(["public", "private"], var.deployment_template) ? length(local.zones_in_region) : 2 * length(local.zones_in_region) + public = (var.deployment_template == "private") ? (var.private_network_extensions ? 1 : 0) : length(local.zones_in_region) private = (var.deployment_template == "public") ? 0 : length(local.zones_in_region) } } \ No newline at end of file diff --git a/modules/terraform-cdp-aws-pre-reqs/modules/vpc/main.tf b/modules/terraform-cdp-aws-pre-reqs/modules/vpc/main.tf index 4061937..4e13f5e 100644 --- a/modules/terraform-cdp-aws-pre-reqs/modules/vpc/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/modules/vpc/main.tf @@ -23,7 +23,7 @@ module "cdp_vpc" { private_subnets = (local.subnets_required.private == 0 ? [] : [ - for k, v in local.zones_in_region : cidrsubnet(var.vpc_cidr, ceil(log(local.subnets_required.total, 2)), local.subnets_required.public + k) + for i in range(local.subnets_required.private) : cidrsubnet(var.vpc_cidr, ceil(log(local.subnets_required.total, 2)), local.subnets_required.public + i) ] ) private_subnet_tags = { @@ -33,7 +33,7 @@ module "cdp_vpc" { public_subnets = (local.subnets_required.public == 0 ? [] : [ - for k, v in local.zones_in_region : cidrsubnet(var.vpc_cidr, ceil(log(local.subnets_required.total, 2)), k) + for i in range(local.subnets_required.public) : cidrsubnet(var.vpc_cidr, ceil(log(local.subnets_required.total, 2)), i) ] ) @@ -41,7 +41,8 @@ module "cdp_vpc" { "kubernetes.io/role/elb" = "1" } - enable_nat_gateway = true + enable_nat_gateway = (var.deployment_template == "private") ? (var.private_network_extensions ? true : false) : true + single_nat_gateway = (var.deployment_template == "private") ? (var.private_network_extensions ? true : false) : false enable_dns_support = true enable_dns_hostnames = true diff --git a/modules/terraform-cdp-aws-pre-reqs/modules/vpc/provider.tf b/modules/terraform-cdp-aws-pre-reqs/modules/vpc/provider.tf index d45eadd..a90ee3e 100644 --- a/modules/terraform-cdp-aws-pre-reqs/modules/vpc/provider.tf +++ b/modules/terraform-cdp-aws-pre-reqs/modules/vpc/provider.tf @@ -16,9 +16,9 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.0" + version = "4.67.0" } } - required_version = "> 1.3.0" + required_version = ">= 1.3.0" } diff --git a/modules/terraform-cdp-aws-pre-reqs/modules/vpc/variables.tf b/modules/terraform-cdp-aws-pre-reqs/modules/vpc/variables.tf index 8564173..2ffdbb0 100644 --- a/modules/terraform-cdp-aws-pre-reqs/modules/vpc/variables.tf +++ b/modules/terraform-cdp-aws-pre-reqs/modules/vpc/variables.tf @@ -39,3 +39,10 @@ variable "deployment_template" { error_message = "Valid values for var: deployment_template are (public, semi-private, private)." } } + +variable "private_network_extensions" { + type = bool + + description = "Enable creation of resources for connectivity to CDP Control Plane (public subnet and NAT Gateway) for Private Deployment. Only relevant for private deployment template." + +} \ No newline at end of file diff --git a/modules/terraform-cdp-aws-pre-reqs/outputs.tf b/modules/terraform-cdp-aws-pre-reqs/outputs.tf index 57dd253..4f7b7fe 100644 --- a/modules/terraform-cdp-aws-pre-reqs/outputs.tf +++ b/modules/terraform-cdp-aws-pre-reqs/outputs.tf @@ -18,19 +18,6 @@ output "tags" { description = "Tags associated with the environment and its resources" } -# CDP settings -output "cdp_profile" { - value = var.cdp_profile - - description = "Profile for CDP credentials" -} - -output "cdp_control_plane_region" { - value = var.cdp_control_plane_region - - description = "CDP Control Plane region" -} - # CSP settings output "aws_region" { value = var.aws_region @@ -81,18 +68,36 @@ output "aws_vpc_subnets" { description = "List of subnets associated with the CDP VPC" } +output "aws_data_storage_bucket" { + value = "${local.data_storage.data_storage_bucket}${local.storage_suffix}" + + description = "AWS data storage bucket" +} + output "aws_data_storage_location" { value = "s3a://${local.data_storage.data_storage_bucket}${local.storage_suffix}/${local.data_storage.data_storage_object}" description = "AWS data storage location" } +output "aws_log_storage_bucket" { + value = "${local.log_storage.log_storage_bucket}${local.storage_suffix}" + + description = "AWS log storage bucket" +} + output "aws_log_storage_location" { value = "s3a://${local.log_storage.log_storage_bucket}${local.storage_suffix}/${local.log_storage.log_storage_object}" description = "AWS log storage location" } +output "aws_backup_storage_bucket" { + value = "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}" + + description = "AWS backup storage bucket" +} + output "aws_backup_storage_location" { value = "s3a://${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${local.backup_storage.backup_storage_object}" diff --git a/modules/terraform-cdp-aws-pre-reqs/provider.tf b/modules/terraform-cdp-aws-pre-reqs/provider.tf index 002dbbb..48e44a9 100644 --- a/modules/terraform-cdp-aws-pre-reqs/provider.tf +++ b/modules/terraform-cdp-aws-pre-reqs/provider.tf @@ -16,21 +16,21 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = "~> 4.0" + version = "4.67.0" } http = { source = "hashicorp/http" version = "3.2.1" } - external = { - source = "hashicorp/external" - version = "2.3.1" - } random = { source = "hashicorp/random" version = "3.4.3" } + time = { + source = "hashicorp/time" + version = "0.9.1" + } } - required_version = "> 1.3.0" + required_version = ">= 1.3.0" } diff --git a/modules/terraform-cdp-aws-pre-reqs/run_cdp_get_cred_prereqs.sh b/modules/terraform-cdp-aws-pre-reqs/run_cdp_get_cred_prereqs.sh deleted file mode 100755 index 2de6fd2..0000000 --- a/modules/terraform-cdp-aws-pre-reqs/run_cdp_get_cred_prereqs.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -################################################# -# Bash script to extract the account id and -# external id of the CDP Public Cloud control plane. -# -# Accepts the Cloud Provider type as a dictionary input -# and uses the command -# 'cdp environments get-credential-prerequisites' -# to then determine the ids. These are then returned as a -# JSON object for use in the TF pre-reqs module. -############################# - -# Step 1 - Parse the inputs and get upper and lower case version of infra_type -eval "$(jq -r '@sh "infra_type=\(.infra_type) cdp_profile=\(.cdp_profile) cdp_region=\(.cdp_region)"')" - -# Lower case, suitable for bash <4 -infra_type_lower=$(echo "$infra_type" | tr '[:upper:]' '[:lower:]') -# Upper case, suitable for bash <4 -infra_type_upper=$(echo "$infra_type" | tr '[:lower:]' '[:upper:]') - -# Step 2 - Run the cdpcli command -export CDP_OUTPUT=$(cdp environments get-credential-prerequisites --cloud-platform ${infra_type_upper} --profile ${cdp_profile} --cdp-region ${cdp_region} --output json) - -# Step 3 - Parse required outputs into variables -accountId=$(echo $CDP_OUTPUT | jq --raw-output '.accountId') -externalId=$(echo $CDP_OUTPUT | jq --arg infra_type "$infra_type_lower" --raw-output '.[$infra_type].externalId') - -# Step 4 - Output in JSON format -jq -n --arg accountId $accountId \ - --arg externalId $externalId \ - --arg infra_type "$infra_type_lower" \ - '{"infra_type":$infra_type, "account_id":$accountId, "external_id":$externalId}' - -# Step 3-4 - All-in-one alternative -# echo $CDP_OUTPUT | jq --arg infra_type "$infra_type_lower" '{"infra_type":$infra_type, "accountId":.accountId, "externalId":.[$infra_type].externalId}' - \ No newline at end of file diff --git a/modules/terraform-cdp-aws-pre-reqs/variables.tf b/modules/terraform-cdp-aws-pre-reqs/variables.tf index 0d67cb1..5676875 100644 --- a/modules/terraform-cdp-aws-pre-reqs/variables.tf +++ b/modules/terraform-cdp-aws-pre-reqs/variables.tf @@ -52,21 +52,13 @@ variable "env_prefix" { } # ------- CDP Environment Deployment ------- -variable "cdp_profile" { - type = string - description = "Profile for CDP credentials" - - # Profile is default unless explicitly specified - default = "default" -} +# variable "cdp_control_plane_region" { +# type = string +# description = "CDP Control Plane Region" -variable "cdp_control_plane_region" { - type = string - description = "CDP Control Plane Region" - - # Region is us-west-1 unless explicitly specified - default = "us-west-1" -} +# # Region is us-west-1 unless explicitly specified +# default = "us-west-1" +# } variable "deployment_template" { type = string @@ -79,14 +71,6 @@ variable "deployment_template" { } } -variable "lookup_cdp_account_ids" { - type = bool - - description = "Auto lookup CDP Account and External ID using CDP CLI commands. If false then the xaccount_account_id and xaccount_external_id input variables need to be specified" - - default = true -} - # variable "enable_raz" { # type = bool @@ -111,6 +95,14 @@ variable "vpc_cidr" { default = "10.10.0.0/16" } +variable "private_network_extensions" { + type = bool + + description = "Enable creation of resources for connectivity to CDP Control Plane (public subnet and NAT Gateway) for Private Deployment. Only relevant for private deployment template" + + default = true +} + variable "cdp_vpc_id" { type = string description = "VPC ID for CDP environment. Required if create_vpc is false." @@ -348,14 +340,12 @@ variable "xaccount_account_id" { type = string description = "Account ID of the cross account" - default = null } variable "xaccount_external_id" { type = string description = "External ID of the cross account" - default = null } # IDBroker service role diff --git a/modules/terraform-cdp-azure-pre-reqs/README.md b/modules/terraform-cdp-azure-pre-reqs/README.md index fe8949f..47b889e 100644 --- a/modules/terraform-cdp-azure-pre-reqs/README.md +++ b/modules/terraform-cdp-azure-pre-reqs/README.md @@ -15,7 +15,7 @@ In each directory an example `terraform.tfvars.sample` values file is included t | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | > 1.3.0 | +| [terraform](#requirement\_terraform) | >= 1.3.0 | | [azuread](#requirement\_azuread) | 2.39.0 | | [azurerm](#requirement\_azurerm) | 3.45.0 | | [random](#requirement\_random) | 3.4.3 | @@ -68,6 +68,7 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [azurerm_user_assigned_identity.cdp_raz](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/user_assigned_identity) | resource | | [random_id.bucket_suffix](https://registry.terraform.io/providers/hashicorp/random/3.4.3/docs/resources/id) | resource | | [azuread_client_config.current](https://registry.terraform.io/providers/hashicorp/azuread/2.39.0/docs/data-sources/client_config) | data source | +| [azurerm_resource_group.cdp_rmgp](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/data-sources/resource_group) | data source | | [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/data-sources/subscription) | data source | ## Inputs @@ -79,8 +80,9 @@ In each directory an example `terraform.tfvars.sample` values file is included t | [agent\_source\_tag](#input\_agent\_source\_tag) | Tag to identify deployment source | `map(any)` | 
{
  "agent_source": "tf-cdp-module"
}
| no | | [azure\_region](#input\_azure\_region) | Region which Cloud resources will be created | `string` | `null` | no | | [backup\_storage](#input\_backup\_storage) | Optional Backup location for CDP environment. If not provided follow the data\_storage variable | 
object({
    backup_storage_bucket = string
    backup_storage_object = string
  })
| `null` | no | -| [cdp\_gw\_subnet\_names](#input\_cdp\_gw\_subnet\_names) | List of subnet names for CDP Gateway. Required if create\_vpc is false. | `list(any)` | `null` | no | -| [cdp\_subnet\_names](#input\_cdp\_subnet\_names) | List of subnet names for CDP Resources. Required if create\_vpc is false. | `list(any)` | `null` | no | +| [cdp\_gw\_subnet\_names](#input\_cdp\_gw\_subnet\_names) | List of subnet names for CDP Gateway. Required if create\_vnet is false. | `list(any)` | `null` | no | +| [cdp\_resourcegroup\_name](#input\_cdp\_resourcegroup\_name) | Pre-existing Resource Group for CDP environment. Required if create\_vnet is false. | `string` | `null` | no | +| [cdp\_subnet\_names](#input\_cdp\_subnet\_names) | List of subnet names for CDP Resources. Required if create\_vnet is false. | `list(any)` | `null` | no | | [cdp\_vnet\_name](#input\_cdp\_vnet\_name) | Pre-existing VNet Name for CDP environment. Required if create\_vnet is false. | `string` | `null` | no | | [create\_vnet](#input\_create\_vnet) | Flag to specify if the VNet should be created | `bool` | `true` | no | | [data\_storage](#input\_data\_storage) | Data storage locations for CDP environment | 
object({
    data_storage_bucket = string
    data_storage_object = string
  })
| `null` | no | @@ -115,13 +117,19 @@ In each directory an example `terraform.tfvars.sample` values file is included t | Name | Description | |------|-------------| +| [azure\_backup\_storage\_account](#output\_azure\_backup\_storage\_account) | Azure backup storage account name | +| [azure\_backup\_storage\_container](#output\_azure\_backup\_storage\_container) | Azure backup storage container name | | [azure\_backup\_storage\_location](#output\_azure\_backup\_storage\_location) | Azure backup storage location | | [azure\_cdp\_gateway\_subnet\_names](#output\_azure\_cdp\_gateway\_subnet\_names) | Azure Virtual Subnet Names for CDP Endpoint Access Gateway | | [azure\_cdp\_subnet\_names](#output\_azure\_cdp\_subnet\_names) | Azure Virtual Subnet Names for CDP Resources | +| [azure\_data\_storage\_account](#output\_azure\_data\_storage\_account) | Azure data storage account name | +| [azure\_data\_storage\_container](#output\_azure\_data\_storage\_container) | Azure data storage container name | | [azure\_data\_storage\_location](#output\_azure\_data\_storage\_location) | Azure data storage location | | [azure\_datalakeadmin\_identity\_id](#output\_azure\_datalakeadmin\_identity\_id) | Datalake Admin Managed Identity ID | | [azure\_idbroker\_identity\_id](#output\_azure\_idbroker\_identity\_id) | IDBroker Managed Identity ID | | [azure\_log\_identity\_id](#output\_azure\_log\_identity\_id) | Log Data Access Managed Identity ID | +| [azure\_log\_storage\_account](#output\_azure\_log\_storage\_account) | Azure log storage account name | +| [azure\_log\_storage\_container](#output\_azure\_log\_storage\_container) | Azure log storage container name | | [azure\_log\_storage\_location](#output\_azure\_log\_storage\_location) | Azure log storage location | | [azure\_ranger\_audit\_identity\_id](#output\_azure\_ranger\_audit\_identity\_id) | Ranger Audit Managed Identity ID | | [azure\_raz\_identity\_id](#output\_azure\_raz\_identity\_id) | RAZ Managed Identity ID. Value returned if RAZ is enabled | diff --git a/modules/terraform-cdp-azure-pre-reqs/data.tf b/modules/terraform-cdp-azure-pre-reqs/data.tf index c02b15e..3a774fa 100644 --- a/modules/terraform-cdp-azure-pre-reqs/data.tf +++ b/modules/terraform-cdp-azure-pre-reqs/data.tf @@ -17,3 +17,10 @@ data "azurerm_subscription" "current" {} # Get the configuration of the AzureAD provider data "azuread_client_config" "current" {} + +# Find details of the Azure Resource group +data "azurerm_resource_group" "cdp_rmgp" { + name = local.cdp_resourcegroup_name + + depends_on = [azurerm_resource_group.cdp_rmgp] +} diff --git a/modules/terraform-cdp-azure-pre-reqs/defaults.tf b/modules/terraform-cdp-azure-pre-reqs/defaults.tf index 25e2a11..adef4a8 100644 --- a/modules/terraform-cdp-azure-pre-reqs/defaults.tf +++ b/modules/terraform-cdp-azure-pre-reqs/defaults.tf @@ -30,6 +30,9 @@ locals { security_group_knox_name = coalesce(var.security_group_knox_name, "${var.env_prefix}-knox-sg") + cdp_resourcegroup_name = (var.create_vnet ? + azurerm_resource_group.cdp_rmgp[0].name : var.cdp_resourcegroup_name) + cdp_vnet_name = (var.create_vnet ? module.azure_cdp_vnet[0].vnet_name : var.cdp_vnet_name) diff --git a/modules/terraform-cdp-azure-pre-reqs/main.tf b/modules/terraform-cdp-azure-pre-reqs/main.tf index 07ee2b6..f378622 100644 --- a/modules/terraform-cdp-azure-pre-reqs/main.tf +++ b/modules/terraform-cdp-azure-pre-reqs/main.tf @@ -14,6 +14,9 @@ # ------- Azure Resource Group ------- resource "azurerm_resource_group" "cdp_rmgp" { + + count = var.create_vnet ? 1 : 0 + name = local.resourcegroup_name location = var.azure_region @@ -28,7 +31,7 @@ module "azure_cdp_vnet" { source = "./modules/vnet" deployment_template = var.deployment_template - resourcegroup_name = azurerm_resource_group.cdp_rmgp.name + resourcegroup_name = local.cdp_resourcegroup_name vnet_name = local.vnet_name vnet_cidr = var.vnet_cidr vnet_region = var.azure_region @@ -43,8 +46,8 @@ module "azure_cdp_vnet" { # Default SG resource "azurerm_network_security_group" "cdp_default_sg" { name = local.security_group_default_name - location = azurerm_resource_group.cdp_rmgp.location - resource_group_name = azurerm_resource_group.cdp_rmgp.name + location = data.azurerm_resource_group.cdp_rmgp.location + resource_group_name = local.cdp_resourcegroup_name tags = merge(local.env_tags, { Name = local.security_group_default_name }) @@ -62,15 +65,15 @@ resource "azurerm_network_security_rule" "cdp_default_sg_ingress_extra_access" { destination_address_prefix = "*" source_port_range = "*" destination_port_ranges = var.ingress_extra_cidrs_and_ports.ports - resource_group_name = azurerm_resource_group.cdp_rmgp.name + resource_group_name = local.cdp_resourcegroup_name network_security_group_name = azurerm_network_security_group.cdp_default_sg.name } # Knox SG resource "azurerm_network_security_group" "cdp_knox_sg" { name = local.security_group_knox_name - location = azurerm_resource_group.cdp_rmgp.location - resource_group_name = azurerm_resource_group.cdp_rmgp.name + location = data.azurerm_resource_group.cdp_rmgp.location + resource_group_name = local.cdp_resourcegroup_name tags = merge(local.env_tags, { Name = local.security_group_knox_name }) @@ -89,7 +92,7 @@ resource "azurerm_network_security_rule" "cdp_knox_sg_ingress_extra_access" { destination_address_prefix = "*" source_port_range = "*" destination_port_ranges = var.ingress_extra_cidrs_and_ports.ports - resource_group_name = azurerm_resource_group.cdp_rmgp.name + resource_group_name = local.cdp_resourcegroup_name network_security_group_name = azurerm_network_security_group.cdp_knox_sg.name } @@ -106,8 +109,8 @@ resource "azurerm_storage_account" "cdp_storage_locations" { for_each = toset(concat([local.data_storage.data_storage_bucket], [local.log_storage.log_storage_bucket], [local.backup_storage.backup_storage_bucket])) name = "${each.value}${local.storage_suffix}" - resource_group_name = azurerm_resource_group.cdp_rmgp.name - location = azurerm_resource_group.cdp_rmgp.location + resource_group_name = local.cdp_resourcegroup_name + location = data.azurerm_resource_group.cdp_rmgp.location # TODO: Review and parameterize these options account_kind = "StorageV2" @@ -155,31 +158,6 @@ resource "azurerm_storage_container" "cdp_backup_storage" { ] } -# NOTE: I don't think below is needed by default - it's a customization -# ------- Azure Cross Account Role ------- - -# resource "azurerm_role_definition" "cdp_xaccount_role" { -# # TODO: -# name = local.xaccount_role_name -# # TODO: -# scope = data.azurerm_subscription.primary.id -# description = "CDP Cross Account role for ${var.env_prefix}" - -# # TODO: -# permissions { -# actions = ["*"] -# data_actions = [] -# not_actions = [] -# not_data_actions = -# } - -# # TODO: -# assignable_scopes = [ -# data.azurerm_subscription.primary.id, # /subscriptions/00000000-0000-0000-0000-000000000000 -# ] -# } - - # ------- Azure Cross Account App ------- # Create Azure AD Application @@ -213,9 +191,9 @@ resource "azuread_application_password" "cdp_xaccount_app_password" { # Create Azure Managed Identity resource "azurerm_user_assigned_identity" "cdp_idbroker" { - location = azurerm_resource_group.cdp_rmgp.location + location = data.azurerm_resource_group.cdp_rmgp.location name = local.idbroker_managed_identity_name - resource_group_name = azurerm_resource_group.cdp_rmgp.name + resource_group_name = local.cdp_resourcegroup_name tags = merge(local.env_tags, { Name = local.idbroker_managed_identity_name }) } @@ -236,9 +214,9 @@ resource "azurerm_role_assignment" "cdp_idbroker_assign" { # Create Azure Managed Identity resource "azurerm_user_assigned_identity" "cdp_datalake_admin" { - location = azurerm_resource_group.cdp_rmgp.location + location = data.azurerm_resource_group.cdp_rmgp.location name = local.datalake_admin_managed_identity_name - resource_group_name = azurerm_resource_group.cdp_rmgp.name + resource_group_name = local.cdp_resourcegroup_name tags = merge(local.env_tags, { Name = local.datalake_admin_managed_identity_name }) } @@ -283,9 +261,9 @@ resource "azurerm_role_assignment" "cdp_datalake_admin_backup_container_assign" # Create Azure Managed Identity resource "azurerm_user_assigned_identity" "cdp_log_data_access" { - location = azurerm_resource_group.cdp_rmgp.location + location = data.azurerm_resource_group.cdp_rmgp.location name = local.log_data_access_managed_identity_name - resource_group_name = azurerm_resource_group.cdp_rmgp.name + resource_group_name = local.cdp_resourcegroup_name tags = merge(local.env_tags, { Name = local.log_data_access_managed_identity_name }) } @@ -317,9 +295,9 @@ resource "azurerm_role_assignment" "cdp_log_data_access_backup_container_assign" # Create Azure Managed Identity resource "azurerm_user_assigned_identity" "cdp_ranger_audit_data_access" { - location = azurerm_resource_group.cdp_rmgp.location + location = data.azurerm_resource_group.cdp_rmgp.location name = local.ranger_audit_data_access_managed_identity_name - resource_group_name = azurerm_resource_group.cdp_rmgp.name + resource_group_name = local.cdp_resourcegroup_name tags = merge(local.env_tags, { Name = local.ranger_audit_data_access_managed_identity_name }) } @@ -365,9 +343,9 @@ resource "azurerm_user_assigned_identity" "cdp_raz" { count = var.enable_raz ? 1 : 0 - location = azurerm_resource_group.cdp_rmgp.location + location = data.azurerm_resource_group.cdp_rmgp.location name = local.raz_managed_identity_name - resource_group_name = azurerm_resource_group.cdp_rmgp.name + resource_group_name = local.cdp_resourcegroup_name tags = merge(local.env_tags, { Name = local.raz_managed_identity_name }) } diff --git a/modules/terraform-cdp-azure-pre-reqs/modules/vnet/provider.tf b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/provider.tf index da05167..cf05981 100644 --- a/modules/terraform-cdp-azure-pre-reqs/modules/vnet/provider.tf +++ b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/provider.tf @@ -20,5 +20,5 @@ terraform { } } - required_version = "> 1.3.0" + required_version = ">= 1.3.0" } diff --git a/modules/terraform-cdp-azure-pre-reqs/outputs.tf b/modules/terraform-cdp-azure-pre-reqs/outputs.tf index 303c491..9d842dd 100644 --- a/modules/terraform-cdp-azure-pre-reqs/outputs.tf +++ b/modules/terraform-cdp-azure-pre-reqs/outputs.tf @@ -25,7 +25,7 @@ output "azure_tenant_id" { } output "azure_resource_group_name" { - value = azurerm_resource_group.cdp_rmgp.name + value = local.cdp_resourcegroup_name description = "Azure Resource Group Name" } @@ -60,6 +60,18 @@ output "azure_security_group_knox_uri" { description = "Azure Knox Security Group URI" } +output "azure_data_storage_account" { + value = azurerm_storage_container.cdp_data_storage.storage_account_name + + description = "Azure data storage account name" +} + +output "azure_data_storage_container" { + value = azurerm_storage_container.cdp_data_storage.name + + description = "Azure data storage container name" +} + output "azure_data_storage_location" { value = "abfs://${azurerm_storage_container.cdp_data_storage.name}@${azurerm_storage_container.cdp_data_storage.storage_account_name}.dfs.core.windows.net" @@ -67,6 +79,18 @@ output "azure_data_storage_location" { } +output "azure_log_storage_account" { + value = azurerm_storage_container.cdp_log_storage.storage_account_name + + description = "Azure log storage account name" +} + +output "azure_log_storage_container" { + value = azurerm_storage_container.cdp_log_storage.name + + description = "Azure log storage container name" +} + output "azure_log_storage_location" { value = "abfs://${azurerm_storage_container.cdp_log_storage.name}@${azurerm_storage_container.cdp_log_storage.storage_account_name}.dfs.core.windows.net" @@ -74,6 +98,18 @@ output "azure_log_storage_location" { } +output "azure_backup_storage_account" { + value = azurerm_storage_container.cdp_backup_storage.storage_account_name + + description = "Azure backup storage account name" +} + +output "azure_backup_storage_container" { + value = azurerm_storage_container.cdp_backup_storage.name + + description = "Azure backup storage container name" +} + output "azure_backup_storage_location" { value = "abfs://${azurerm_storage_container.cdp_backup_storage.name}@${azurerm_storage_container.cdp_backup_storage.storage_account_name}.dfs.core.windows.net" diff --git a/modules/terraform-cdp-azure-pre-reqs/provider.tf b/modules/terraform-cdp-azure-pre-reqs/provider.tf index 889877c..30f3ff3 100644 --- a/modules/terraform-cdp-azure-pre-reqs/provider.tf +++ b/modules/terraform-cdp-azure-pre-reqs/provider.tf @@ -28,5 +28,5 @@ terraform { } } - required_version = "> 1.3.0" + required_version = ">= 1.3.0" } diff --git a/modules/terraform-cdp-azure-pre-reqs/variables.tf b/modules/terraform-cdp-azure-pre-reqs/variables.tf index 18e4744..70d4f6a 100644 --- a/modules/terraform-cdp-azure-pre-reqs/variables.tf +++ b/modules/terraform-cdp-azure-pre-reqs/variables.tf @@ -94,6 +94,13 @@ variable "vnet_cidr" { default = "10.10.0.0/16" } +variable "cdp_resourcegroup_name" { + type = string + description = "Pre-existing Resource Group for CDP environment. Required if create_vnet is false." + + default = null +} + variable "cdp_vnet_name" { type = string description = "Pre-existing VNet Name for CDP environment. Required if create_vnet is false." @@ -103,14 +110,14 @@ variable "cdp_vnet_name" { variable "cdp_subnet_names" { type = list(any) - description = "List of subnet names for CDP Resources. Required if create_vpc is false." + description = "List of subnet names for CDP Resources. Required if create_vnet is false." default = null } variable "cdp_gw_subnet_names" { type = list(any) - description = "List of subnet names for CDP Gateway. Required if create_vpc is false." + description = "List of subnet names for CDP Gateway. Required if create_vnet is false." default = null } diff --git a/modules/terraform-cdp-deploy/README.md b/modules/terraform-cdp-deploy/README.md index 3682b7c..a60619f 100644 --- a/modules/terraform-cdp-deploy/README.md +++ b/modules/terraform-cdp-deploy/README.md @@ -17,9 +17,8 @@ In each directory an example `terraform.tfvars.sample` values file is included t | Name | Version | |------|---------| -| [terraform](#requirement\_terraform) | > 1.3.0 | -| [local](#requirement\_local) | 2.2.3 | -| [null](#requirement\_null) | 3.2.1 | +| [terraform](#requirement\_terraform) | >= 1.3.0 | +| [cdp](#requirement\_cdp) | 0.1.4-pre | ## Providers @@ -46,7 +45,6 @@ No resources. | [infra\_type](#input\_infra\_type) | Cloud Provider to deploy CDP. | `string` | n/a | yes | | [log\_storage\_location](#input\_log\_storage\_location) | Log storage location. The location has to be in uri format for the cloud provider - i.e. s3a:// for AWS, abfs:// for Azure, gs:// | `string` | n/a | yes | | [region](#input\_region) | Region which cloud resources will be created | `string` | n/a | yes | -| [agent\_source\_tag](#input\_agent\_source\_tag) | Tag to identify deployment source | `map(any)` | 
{
  "agent_source": "tf-cdp-module"
}
| no | | [aws\_datalake\_admin\_role\_arn](#input\_aws\_datalake\_admin\_role\_arn) | Datalake Admin Role ARN. Required for CDP deployment on AWS. | `string` | `null` | no | | [aws\_idbroker\_instance\_profile\_arn](#input\_aws\_idbroker\_instance\_profile\_arn) | IDBroker Instance Profile ARN. Required for CDP deployment on AWS. | `string` | `null` | no | | [aws\_log\_instance\_profile\_arn](#input\_aws\_log\_instance\_profile\_arn) | Log Instance Profile ARN. Required for CDP deployment on AWS. | `string` | `null` | no | @@ -73,18 +71,15 @@ No resources. | [azure\_xaccount\_app\_pword](#input\_azure\_xaccount\_app\_pword) | Password for the Azure AD Cross Account Application. Required for CDP deployment on Azure. | `string` | `null` | no | | [azure\_xaccount\_app\_uuid](#input\_azure\_xaccount\_app\_uuid) | UUID for the Azure AD Cross Account Application. Required for CDP deployment on Azure. | `string` | `null` | no | | [cdp\_admin\_group\_name](#input\_cdp\_admin\_group\_name) | Name of the CDP IAM Admin Group associated with the environment. Defaults to '-cdp-admin-group' if not specified. | `string` | `null` | no | -| [cdp\_control\_plane\_region](#input\_cdp\_control\_plane\_region) | CDP Control Plane Region | `string` | `"us-west-1"` | no | -| [cdp\_profile](#input\_cdp\_profile) | Profile for CDP credentials | `string` | `"default"` | no | | [cdp\_user\_group\_name](#input\_cdp\_user\_group\_name) | Name of the CDP IAM User Group associated with the environment. Defaults to '-cdp-user-group' if not specified. | `string` | `null` | no | | [cdp\_xacccount\_credential\_name](#input\_cdp\_xacccount\_credential\_name) | Name of the CDP Cross Account Credential. Defaults to '-xaccount-cred' if not specified. | `string` | `null` | no | | [datalake\_name](#input\_datalake\_name) | Name of the CDP datalake. Defaults to '--dl' if not specified. | `string` | `null` | no | | [datalake\_scale](#input\_datalake\_scale) | The scale of the datalake. Valid values are LIGHT\_DUTY, MEDIUM\_DUTY\_HA. | `string` | `null` | no | -| [datalake\_version](#input\_datalake\_version) | The Datalake Runtime version. Valid values are semantic versions, e.g. 7.2.16 | `string` | `"7.2.16"` | no | +| [datalake\_version](#input\_datalake\_version) | The Datalake Runtime version. Valid values are semantic versions, e.g. 7.2.16 | `string` | `"7.2.17"` | no | | [enable\_ccm\_tunnel](#input\_enable\_ccm\_tunnel) | Flag to enable Cluster Connectivity Manager tunnel. If false then access from Cloud to CDP Control Plane CIDRs is required from via SG ingress | `bool` | `true` | no | | [enable\_raz](#input\_enable\_raz) | Flag to enable Ranger Authorization Service (RAZ) | `bool` | `true` | no | | [endpoint\_access\_scheme](#input\_endpoint\_access\_scheme) | The scheme for the workload endpoint gateway. PUBLIC creates an external endpoint that can be accessed over the Internet. PRIVATE which restricts the traffic to be internal to the VPC / Vnet. Relevant in Private Networks. | `string` | `null` | no | | [env\_prefix](#input\_env\_prefix) | Shorthand name for the environment. Used in CDP resource descriptions. This will be used to construct the value of where any of the CDP resource variables (e.g. environment\_name, cdp\_iam\_admin\_group\_name) are not defined. | `string` | `null` | no | -| [env\_tags](#input\_env\_tags) | Tags applied to provisioned resources | `map(any)` | `null` | no | | [environment\_name](#input\_environment\_name) | Name of the CDP environment. Defaults to '-cdp-env' if not specified. | `string` | `null` | no | | [freeipa\_instances](#input\_freeipa\_instances) | The number of FreeIPA instances to create in the environment | `number` | `3` | no | | [keypair\_name](#input\_keypair\_name) | SSH Keypair name in Cloud Service Provider. Required for CDP deployment on AWS. | `string` | `null` | no | @@ -96,5 +91,8 @@ No resources. ## Outputs -No outputs. +| Name | Description | +|------|-------------| +| [cdp\_environment\_crn](#output\_cdp\_environment\_crn) | CDP Environment CRN | +| [cdp\_environment\_name](#output\_cdp\_environment\_name) | CDP Environment Name | \ No newline at end of file diff --git a/modules/terraform-cdp-deploy/defaults.tf b/modules/terraform-cdp-deploy/defaults.tf index bdd3528..5c53a41 100644 --- a/modules/terraform-cdp-deploy/defaults.tf +++ b/modules/terraform-cdp-deploy/defaults.tf @@ -14,11 +14,12 @@ locals { # ------- Global settings ------- - env_tags = merge(var.agent_source_tag, ( - coalesce(var.env_tags, - { env_prefix = var.env_prefix } - )) - ) + # NOTE: Waiting on provider fix + # env_tags = merge(var.agent_source_tag, ( + # coalesce(var.env_tags, + # { env_prefix = var.env_prefix } + # )) + # ) # ------- CDP Environment Deployment ------- environment_name = coalesce(var.environment_name, diff --git a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf index a2db217..0587a83 100644 --- a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf +++ b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf @@ -23,10 +23,22 @@ module "cdp_aws_prereqs" { env_prefix = var.env_prefix aws_region = var.aws_region - deployment_template = var.deployment_template - cdp_profile = var.cdp_profile - + deployment_template = var.deployment_template ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports + + # Using CDP TF Provider cred pre-reqs data source for values of xaccount account_id and external_id + xaccount_account_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id + xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id + + # Inputs for BYO-VPC + create_vpc = var.create_vpc + cdp_vpc_id = var.cdp_vpc_id + cdp_public_subnet_ids = var.cdp_public_subnet_ids + cdp_private_subnet_ids = var.cdp_private_subnet_ids + + # Inputs for Control Plane Connectivity in fully private + private_network_extensions = var.private_network_extensions + } module "cdp_deploy" { @@ -61,3 +73,14 @@ module "cdp_deploy" { module.cdp_aws_prereqs ] } + +# Use the CDP Terraform Provider to find the xaccount account and external ids +terraform { + required_providers { + cdp = { + source = "cloudera/cdp" + version = "0.1.4-pre" + } + } +} +data "cdp_environments_aws_credential_prerequisites" "cdp_prereqs" {} diff --git a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/terraform.tfvars.sample b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/terraform.tfvars.sample index 796b406..e527195 100644 --- a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/terraform.tfvars.sample +++ b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/terraform.tfvars.sample @@ -28,3 +28,15 @@ ingress_extra_cidrs_and_ports = { cidrs = ["/32", "/32"], ports = [443, 22] } + +# ------- Optional inputs for BYO-VPC ------- +# **NOTE: Uncomment below settings if required + +# create_vpc=true # Set to false to use pre-existing VPC + +# cdp_vpc_id="" # VPC ID for CDP environment. Required if create_vpc is false +# cdp_public_subnet_ids=["",""] # List of pre-existing public subnet ids. Required if create_vpc is false +# cdp_private_subnet_ids=[","] # List of pre-existing private subnet ids. Required if create_vpc is false + +# ------- Optional inputs for Control Plane Connectivity in fully private environment ------- +# private_network_extensions=true # Set to false if external networking connectivity to CDP Control Plane exists diff --git a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/variables.tf b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/variables.tf index dd3d6b4..7ceaf8f 100644 --- a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/variables.tf +++ b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/variables.tf @@ -45,13 +45,6 @@ variable "deployment_template" { description = "Deployment Pattern to use for Cloud resources and CDP" } -variable "cdp_profile" { - type = string - description = "Profile for CDP credentials" - - # Profile is default unless explicitly specified - default = "default" -} # ------- Network Resources ------- variable "ingress_extra_cidrs_and_ports" { type = object({ @@ -60,3 +53,42 @@ variable "ingress_extra_cidrs_and_ports" { }) description = "List of extra CIDR blocks and ports to include in Security Group Ingress rules" } + +# ------- Optional inputs for BYO-VPC ------- +variable "create_vpc" { + type = bool + + description = "Flag to specify if the VPC should be created" + + default = true +} + +variable "cdp_vpc_id" { + type = string + description = "VPC ID for CDP environment. Required if create_vpc is false." + + default = null +} + +variable "cdp_public_subnet_ids" { + type = list(any) + description = "List of public subnet ids. Required if create_vpc is false." + + default = null +} + +variable "cdp_private_subnet_ids" { + type = list(any) + description = "List of private subnet ids. Required if create_vpc is false." + + default = null +} + +# ------- Optional inputs for Control Plane Connectivity in fully private environment ------- +variable "private_network_extensions" { + type = bool + + description = "Enable creation of resources for connectivity to CDP Control Plane (public subnet and NAT Gateway) for Private Deployment. Only relevant for private deployment template" + + default = true +} \ No newline at end of file diff --git a/modules/terraform-cdp-deploy/examples/ex02-azure-basic/main.tf b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/main.tf index a1832d7..1de73e8 100755 --- a/modules/terraform-cdp-deploy/examples/ex02-azure-basic/main.tf +++ b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/main.tf @@ -30,10 +30,16 @@ module "cdp_azure_prereqs" { env_prefix = var.env_prefix azure_region = var.azure_region - deployment_template = var.deployment_template - + deployment_template = var.deployment_template ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports + # Inputs for BYO-VNet + create_vnet = var.create_vnet + cdp_resourcegroup_name = var.cdp_resourcegroup_name + cdp_vnet_name = var.cdp_vnet_name + cdp_subnet_names = var.cdp_subnet_names + cdp_gw_subnet_names = var.cdp_gw_subnet_names + } module "cdp_deploy" { diff --git a/modules/terraform-cdp-deploy/examples/ex02-azure-basic/terraform.tfvars.sample b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/terraform.tfvars.sample index 6562502..25d9c4f 100644 --- a/modules/terraform-cdp-deploy/examples/ex02-azure-basic/terraform.tfvars.sample +++ b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/terraform.tfvars.sample @@ -16,7 +16,7 @@ env_prefix = "" # Required name prefix for cloud and CDP resources, e.g. cldr1 # ------- Cloud Settings ------- -azure_region = "" # Change this to specify Cloud Provider region, e.g. westeurpoe +azure_region = "" # Change this to specify Cloud Provider region, e.g. eastus public_key_text = "" # Change this with the SSH public key text, e.g. ssh-rsa AAA.... @@ -29,3 +29,13 @@ ingress_extra_cidrs_and_ports = { cidrs = ["/32", "/32"], ports = [443, 22] } + +# ------- Optional inputs for BYO-VNet ------- +# **NOTE: Uncomment below settings if required + +# create_vnet=true # Set to false to use pre-existing VNet + +# cdp_resourcegroup_name="" # Name of pre-existing Resource Group for CDP environment. Required if create_vnet is false +# cdp_vnet_name="" # Name of pre-existing VNet for CDP environment. Required if create_vnet is false +# cdp_subnet_names=["",""] # List of pre-existing subnet names for CDP resources. Required if create_vnet is false +# cdp_gw_subnet_names=[""] # List of pre-existing subnet names for CDP Gateway. Required if create_vnet is false \ No newline at end of file diff --git a/modules/terraform-cdp-deploy/examples/ex02-azure-basic/variables.tf b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/variables.tf index d1ca021..9d2d625 100755 --- a/modules/terraform-cdp-deploy/examples/ex02-azure-basic/variables.tf +++ b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/variables.tf @@ -44,3 +44,40 @@ variable "ingress_extra_cidrs_and_ports" { }) description = "List of extra CIDR blocks and ports to include in Security Group Ingress rules" } + +# ------- Optional inputs for BYO-VNet ------- +variable "create_vnet" { + type = bool + + description = "Flag to specify if the VNet should be created" + + default = true +} + +variable "cdp_resourcegroup_name" { + type = string + description = "Pre-existing Resource Group for CDP environment. Required if create_vnet is false." + + default = null +} + +variable "cdp_vnet_name" { + type = string + description = "Pre-existing VNet Name for CDP environment. Required if create_vnet is false." + + default = null +} + +variable "cdp_subnet_names" { + type = list(any) + description = "List of subnet names for CDP Resources. Required if create_vnet is false." + + default = null +} + +variable "cdp_gw_subnet_names" { + type = list(any) + description = "List of subnet names for CDP Gateway. Required if create_vnet is false." + + default = null +} \ No newline at end of file diff --git a/modules/terraform-cdp-deploy/main.tf b/modules/terraform-cdp-deploy/main.tf index 3dacda8..4ee57f6 100644 --- a/modules/terraform-cdp-deploy/main.tf +++ b/modules/terraform-cdp-deploy/main.tf @@ -18,7 +18,7 @@ module "cdp_on_aws" { source = "./modules/aws" - tags = local.env_tags + # tags = local.env_tags # NOTE: Waiting on provider fix environment_name = local.environment_name datalake_name = local.datalake_name @@ -38,15 +38,15 @@ module "cdp_on_aws" { workload_analytics = var.workload_analytics endpoint_access_scheme = local.endpoint_access_scheme - cdp_profile = var.cdp_profile - cdp_control_plane_region = var.cdp_control_plane_region + # TODO: Will be re-introducted once provider supports other regions + # cdp_control_plane_region = var.cdp_control_plane_region - region = var.region - vpc_id = var.aws_vpc_id - public_subnet_ids = var.aws_public_subnet_ids - private_subnet_ids = var.aws_private_subnet_ids - subnets_for_cdp = local.aws_subnets_for_cdp - keypair_name = var.keypair_name + region = var.region + vpc_id = var.aws_vpc_id + public_subnet_ids = var.aws_public_subnet_ids + # private_subnet_ids = var.aws_private_subnet_ids + subnets_for_cdp = local.aws_subnets_for_cdp + keypair_name = var.keypair_name data_storage_location = var.data_storage_location log_storage_location = var.log_storage_location @@ -67,7 +67,7 @@ module "cdp_on_azure" { source = "./modules/azure" - tags = local.env_tags + # tags = local.env_tags # NOTE: Waiting on provider fix environment_name = local.environment_name datalake_name = local.datalake_name @@ -86,8 +86,8 @@ module "cdp_on_azure" { workload_analytics = var.workload_analytics endpoint_access_scheme = local.endpoint_access_scheme - cdp_profile = var.cdp_profile - cdp_control_plane_region = var.cdp_control_plane_region + # TODO: Will be re-introducted once provider supports other regions + # cdp_control_plane_region = var.cdp_control_plane_region use_single_resource_group = var.use_single_resource_group use_public_ips = local.use_public_ips diff --git a/modules/terraform-cdp-deploy/modules/aws/main.tf b/modules/terraform-cdp-deploy/modules/aws/main.tf index 030c896..f31ba8a 100644 --- a/modules/terraform-cdp-deploy/modules/aws/main.tf +++ b/modules/terraform-cdp-deploy/modules/aws/main.tf @@ -12,75 +12,117 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Deployment and creation of CDP resources using Ansible Playbook called by TF local-exec - -# ------- Create Configuration file for CDP Deployment via Ansible ------- -resource "local_file" "cdp_deployment_template" { - - content = templatefile("${path.module}/templates/cdp_config.yml.tpl", { - # CDP environment & DL settings - plat__env_name = var.environment_name - plat__datalake_name = var.datalake_name - plat__datalake_scale = var.datalake_scale - plat__datalake_version = var.datalake_version - plat__xacccount_credential_name = var.cdp_xacccount_credential_name - plat__cdp_iam_admin_group_name = var.cdp_admin_group_name - plat__cdp_iam_user_group_name = var.cdp_user_group_name - plat__tunnel = var.enable_ccm_tunnel - plat__endpoint_access_scheme = var.endpoint_access_scheme - plat__enable_raz = var.enable_raz - plat__env_multiaz = var.multiaz - plat__env_freeipa_instances = var.freeipa_instances - plat__workload_analytics = var.workload_analytics - plat__tags = jsonencode(var.tags) - - # CDP settings - plat__cdp_profile = var.cdp_profile - plat__cdp_control_plane_region = var.cdp_control_plane_region - - # CSP settings - plat__infra_type = "aws" - plat__region = var.region - - plat__aws_vpc_id = var.vpc_id - plat__aws_public_subnet_ids = jsonencode(var.public_subnet_ids) - plat__aws_private_subnet_ids = jsonencode(var.private_subnet_ids) - plat__aws_subnets_for_cdp = jsonencode(var.subnets_for_cdp) - - plat__aws_storage_location = var.data_storage_location - plat__aws_log_location = var.log_storage_location - plat__aws_backup_location = var.backup_storage_location - - plat__public_key_id = var.keypair_name - plat__aws_security_group_default_id = var.security_group_default_id - plat__aws_security_group_knox_id = var.security_group_knox_id - - plat__aws_datalake_admin_role_arn = var.datalake_admin_role_arn - plat__aws_ranger_audit_role_arn = var.ranger_audit_role_arn - plat__aws_xaccount_role_arn = var.xaccount_role_arn - - plat__aws_log_instance_profile_arn = var.log_instance_profile_arn - plat__aws_idbroker_instance_profile_arn = var.idbroker_instance_profile_arn - } - ) - filename = "cdp_config.yml" +# ------- CDP Credential ------- +resource "cdp_environments_aws_credential" "cdp_cred" { + credential_name = var.cdp_xacccount_credential_name + role_arn = var.xaccount_role_arn + description = "AWS Cross Account Credential for AWS env ${var.environment_name}" } -# ------- Create CDP Deployment ------- -resource "null_resource" "cdp_deployment" { +# ------- CDP Environment ------- +resource "cdp_environments_aws_environment" "cdp_env" { + environment_name = var.environment_name + credential_name = cdp_environments_aws_credential.cdp_cred.credential_name + region = var.region + + security_access = { + default_security_group_id = var.security_group_default_id + security_group_id_for_knox = var.security_group_knox_id + } + + log_storage = { + storage_location_base = var.log_storage_location + backup_storage_location_base = var.backup_storage_location + instance_profile = var.log_instance_profile_arn + } - # Setup of CDP environment using playbook_setup_cdp.yml.yml Ansible Playbook - provisioner "local-exec" { - command = "ansible-playbook -vvv -e '@cdp_config.yml' ${path.module}/playbook_setup_cdp.yml" + authentication = { + public_key_id = var.keypair_name } - # Deletion of CDP environment using playbook_teardown_cdp.yml Ansible Playbook - provisioner "local-exec" { - when = destroy - command = "ansible-playbook -vvv -e '@cdp_config.yml' ${path.module}/playbook_teardown_cdp.yml" + vpc_id = var.vpc_id + subnet_ids = var.subnets_for_cdp + endpoint_access_gateway_scheme = var.endpoint_access_scheme + endpoint_access_gateway_subnet_ids = (length(var.public_subnet_ids) > 0) ? var.public_subnet_ids : null + + freeipa = { + instance_count_by_group = var.freeipa_instances + multi_az = var.multiaz } + workload_analytics = var.workload_analytics + enable_tunnel = var.enable_ccm_tunnel + # tags = var.tags # NOTE: Waiting on provider fix + + depends_on = [ + cdp_environments_aws_credential.cdp_cred + ] +} + +# ------- CDP Admin Group ------- +# Create group +resource "cdp_iam_group" "cdp_admin_group" { + group_name = var.cdp_admin_group_name + sync_membership_on_user_login = false +} + +# TODO: Assign roles and resource roles to the group + +# TODO: Assign users to the group + +# ------- CDP User Group ------- +# Create group +resource "cdp_iam_group" "cdp_user_group" { + group_name = var.cdp_user_group_name + sync_membership_on_user_login = false +} + +# TODO: Assign roles and resource roles to the group + +# TODO: Assign users to the group + +# ------- IdBroker Mappings ------- +resource "cdp_environments_id_broker_mappings" "cdp_idbroker" { + environment_name = cdp_environments_aws_environment.cdp_env.environment_name + environment_crn = cdp_environments_aws_environment.cdp_env.crn + + ranger_audit_role = var.ranger_audit_role_arn + data_access_role = var.datalake_admin_role_arn + ranger_cloud_access_authorizer_role = var.enable_raz ? var.datalake_admin_role_arn : null + + mappings = [{ + accessor_crn = cdp_iam_group.cdp_admin_group.crn + role = var.datalake_admin_role_arn + }, + { + accessor_crn = cdp_iam_group.cdp_user_group.crn + role = var.datalake_admin_role_arn + } + ] + + depends_on = [ + cdp_environments_aws_environment.cdp_env + ] +} + +# ------- CDP Datalake ------- +resource "cdp_datalake_aws_datalake" "cdp_datalake" { + datalake_name = var.datalake_name + environment_name = cdp_environments_aws_environment.cdp_env.environment_name + + instance_profile = var.idbroker_instance_profile_arn + storage_bucket_location = var.data_storage_location + + runtime = var.datalake_version + scale = var.datalake_scale + enable_ranger_raz = var.enable_raz + multi_az = var.multiaz + + # tags = var.tags # NOTE: Waiting on provider fix + depends_on = [ - local_file.cdp_deployment_template, + cdp_environments_aws_credential.cdp_cred, + cdp_environments_aws_environment.cdp_env, + cdp_environments_id_broker_mappings.cdp_idbroker ] } diff --git a/modules/terraform-cdp-deploy/modules/aws/outputs.tf b/modules/terraform-cdp-deploy/modules/aws/outputs.tf index e69de29..92e960a 100644 --- a/modules/terraform-cdp-deploy/modules/aws/outputs.tf +++ b/modules/terraform-cdp-deploy/modules/aws/outputs.tf @@ -0,0 +1,25 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "cdp_environment_name" { + value = cdp_environments_aws_environment.cdp_env.environment_name + + description = "CDP Environment Name" +} + +output "cdp_environment_crn" { + value = cdp_environments_aws_environment.cdp_env.crn + + description = "CDP Environment CRN" +} diff --git a/modules/terraform-cdp-deploy/modules/aws/playbook_setup_cdp.yml b/modules/terraform-cdp-deploy/modules/aws/playbook_setup_cdp.yml deleted file mode 100644 index 1f2c977..0000000 --- a/modules/terraform-cdp-deploy/modules/aws/playbook_setup_cdp.yml +++ /dev/null @@ -1,307 +0,0 @@ ---- - -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -################################################# -# Ansible Playbook to create CDP resources based -# on output generated from TF pre-reqs module. -# Uses the cloudera.cloud Ansible collection -# -# Tasks performed: -# * Create Cross Account credential -# * Set up Environment Deployment -# * Creates user and admin groups and CDP IDBroker Mappings -# * Set up Datalake -############################# -- name: Create CDP Resources - hosts: localhost - environment: - CDP_PROFILE: "{{ plat__cdp_profile }}" - CDP_REGION: "{{ plat__cdp_control_plane_region }}" - vars: - plat__cdp_iam_identities: - role_suffix: role - resource_role_suffix: resourceRole - env_admin: EnvironmentAdmin - env_user: EnvironmentUser - dw_admin: DWAdmin - dw_user: DWUser - ml_admin: MLAdmin - ml_user: MLUser - df_admin: DFAdmin - df_flow_admin: DFFlowAdmin - df_flow_user: DFFlowUser - de_admin: DEAdmin - de_user: DEUser - plat__cdp_iam_admin_group_resource_roles_default: - - "{{ plat__cdp_iam_identities.env_admin }}" - - "{{ plat__cdp_iam_identities.env_user }}" - - "{{ plat__cdp_iam_identities.dw_admin }}" - - "{{ plat__cdp_iam_identities.dw_user }}" - - "{{ plat__cdp_iam_identities.ml_admin }}" - - "{{ plat__cdp_iam_identities.ml_user }}" - - "{{ plat__cdp_iam_identities.df_admin }}" - - "{{ plat__cdp_iam_identities.df_flow_admin }}" - - "{{ plat__cdp_iam_identities.df_flow_user }}" - - "{{ plat__cdp_iam_identities.de_admin }}" - - "{{ plat__cdp_iam_identities.de_user }}" - plat__cdp_iam_user_group_resource_roles_default: - - "{{ plat__cdp_iam_identities.env_user }}" - - "{{ plat__cdp_iam_identities.dw_user }}" - - "{{ plat__cdp_iam_identities.ml_user }}" - - "{{ plat__cdp_iam_identities.df_flow_admin }}" - - "{{ plat__cdp_iam_identities.df_flow_user }}" - - "{{ plat__cdp_iam_identities.de_user }}" - plat__cdp_iam_admin_group_roles: - - "{{ plat__cdp_iam_identities.env_admin }}" - plat__cdp_iam_admin_group_resource_roles: "{{ plat__cdp_iam_admin_group_resource_roles_default }}" - plat__cdp_iam_user_group_roles: - - "{{ plat__cdp_iam_identities.env_user }}" - plat__cdp_iam_user_group_resource_roles: "{{ plat__cdp_iam_user_group_resource_roles_default }}" - common__cdp_control_planes: - us-west-1: "crn:altus:iam:us-west-1:altus" - eu-1: "crn:altus:iam:eu-1:altus" - ap-1: "crn:altus:iam:ap-1:altus" - common__cdp_control_plane_region: "{{ plat__cdp_control_plane_region | default('us-west-1') }}" - common__cdp_control_plane_crn: "{{ common__cdp_control_planes[common__cdp_control_plane_region] }}" - plat__cdp_control_plane_crn: "{{ common__cdp_control_plane_crn }}" - plat__cdp_iam_role_suffix: "{{ plat__cdp_iam_identities.role_suffix }}" - plat__cdp_iam_resource_suffix: "{{ plat__cdp_iam_identities.resource_role_suffix }}" - tasks: - -############# CDP Credential ############# - - name: Create CDP Cross Account Credential for AWS - cloudera.cloud.env_cred: - cloud: "{{ plat__infra_type }}" - name: "{{ plat__xacccount_credential_name }}" - role: "{{ plat__aws_xaccount_role_arn }}" - state: present - -############# CDP Environment ############# - - name: Start set up of CDP Environment Deployment on AWS - block: - - name: Call cloudera.cloud.env to register CDP environment - cloudera.cloud.env: - name: "{{ plat__env_name }}" - state: started - credential: "{{ plat__xacccount_credential_name }}" - cloud: "{{ plat__infra_type }}" - region: "{{ plat__region }}" - default_sg: "{{ plat__aws_security_group_default_id }}" - knox_sg: "{{ plat__aws_security_group_knox_id }}" - log_location: "{{ plat__aws_log_location }}" - log_identity: "{{ plat__aws_log_instance_profile_arn }}" - backup_location: "{{ plat__aws_backup_location }}" - public_key_id: "{{ plat__public_key_id }}" - workload_analytics: "{{ plat__workload_analytics }}" - vpc_id: "{{ plat__aws_vpc_id }}" - subnet_ids: "{{ plat__aws_subnets_for_cdp }}" - tags: "{{ plat__tags }}" - tunnel: "{{ plat__tunnel }}" - endpoint_access_scheme: "{{ plat__endpoint_access_scheme | default(omit) }}" - endpoint_access_subnets: "{{ plat__aws_public_subnet_ids | default(omit) }}" - freeipa: - instanceCountByGroup: "{{ plat__env_freeipa_instances }}" - multiAz: "{{ plat__env_multiaz }}" - async: 3600 # 1 hour timeout - poll: 0 - register: __cdp_env_setup - - - name: Wait for CDP environment setup to start - ansible.builtin.pause: - seconds: 60 - - - name: Update CDP Environment information - cloudera.cloud.env_info: - name: "{{ plat__env_name }}" - register: plat__cdp_env_info - failed_when: plat__cdp_env_info.environments | length == 0 - rescue: - # Fallback to synchronous if environment not found - - name: Wait for CDP Environment setup to complete - when: __cdp_env_setup.ansible_job_id is defined - ansible.builtin.async_status: - jid: "{{ __cdp_env_setup.ansible_job_id }}" - register: __cdp_env_setup_async - until: __cdp_env_setup_async.finished - retries: 120 - delay: 30 - - - name: Attempt to update CDP Environment information again - cloudera.cloud.env_info: - name: "{{ plat__env_name }}" - register: plat__cdp_env_info - failed_when: plat__cdp_env_info.environments | length == 0 - - # Resume normal service - - name: Set fact for CDP Environment CRN - ansible.builtin.set_fact: - plat__cdp_env_crn: "{{ plat__cdp_env_info.environments[0].crn | default('Unknown') }}" - - - name: Print Environment CRN - debug: - msg: - - plat__cdp_env_crn is {{ plat__cdp_env_crn }} - -############# CDP Groups ############# - - name: Query CDP Caller to confirm access to Endpoints - cloudera.cloud.iam_user_info: - current_user: yes - register: __cdp_iam_current_user_info - - - name: Set facts for CDP Caller Workload Username and CRN - ansible.builtin.set_fact: - plat__cdp_workload_username: "{{ __cdp_iam_current_user_info.users[0].workloadUsername }}" - plat__cdp_user_crn: "{{ __cdp_iam_current_user_info.users[0].crn }}" - - - name: Set fact for CDP Environment Admin Group Role CRNs - ansible.builtin.set_fact: - plat__cdp_pub_admin_group_role_crns: "{{ plat__cdp_pub_admin_group_role_crns | default([]) | union([role]) }}" - vars: - role: "{{ [plat__cdp_control_plane_crn, plat__cdp_iam_role_suffix, __cdp_pub_admin_group_role_item] | join(':') }}" - loop_control: - loop_var: __cdp_pub_admin_group_role_item - loop: "{{ plat__cdp_iam_admin_group_roles }}" - - - name: Set fact for CDP Environment Admin Group Resource Role CRNs - ansible.builtin.set_fact: - plat__cdp_pub_admin_group_resource_role_crns: "{{ plat__cdp_pub_admin_group_resource_role_crns | default([]) | union([resource_role]) }}" - vars: - resource_role: "{{ [plat__cdp_control_plane_crn, plat__cdp_iam_resource_suffix, __cdp_env_admin_group_resource_role_item] | join(':') }}" - loop_control: - loop_var: __cdp_env_admin_group_resource_role_item - loop: "{{ plat__cdp_iam_admin_group_resource_roles }}" - - - name: Set fact for CDP Environment User Group Role CRNs - ansible.builtin.set_fact: - plat__cdp_pub_user_group_role_crns: "{{ plat__cdp_pub_user_group_role_crns | default([]) | union([role]) }}" - vars: - role: "{{ [plat__cdp_control_plane_crn, plat__cdp_iam_role_suffix, __cdp_pub_user_group_role_item] | join(':') }}" - loop_control: - loop_var: __cdp_pub_user_group_role_item - loop: "{{ plat__cdp_iam_user_group_roles }}" - - - name: Set fact for CDP Environment User Group Resource Role CRNs - ansible.builtin.set_fact: - plat__cdp_pub_user_group_resource_role_crns: "{{ plat__cdp_pub_user_group_resource_role_crns | default([]) | union([resource_role]) }}" - vars: - resource_role: "{{ [plat__cdp_control_plane_crn, plat__cdp_iam_resource_suffix, __cdp_pub_user_group_resource_role_item] | join(':') }}" - loop_control: - loop_var: __cdp_pub_user_group_resource_role_item - loop: "{{ plat__cdp_iam_user_group_resource_roles }}" - - - name: Set fact for CDP Admin Group Resource Role assignments - ansible.builtin.set_fact: - plat__cdp_env_admin_group_resource_role_assignments: "{{ plat__cdp_env_admin_group_resource_role_assignments | default([]) | union([resource_role_assignment]) }}" - vars: - resource_role_assignment: - resource: "{{ plat__cdp_env_crn }}" - role: "{{ __cdp_resource_role_item }}" - loop_control: - loop_var: __cdp_resource_role_item - loop: "{{ plat__cdp_pub_admin_group_resource_role_crns }}" - - - name: Create CDP Admin group - cloudera.cloud.iam_group: - name: "{{ plat__cdp_iam_admin_group_name }}" - state: present - sync: false - roles: "{{ plat__cdp_pub_admin_group_role_crns }}" - resource_roles: "{{ plat__cdp_env_admin_group_resource_role_assignments }}" - users: - - "{{ plat__cdp_user_crn }}" - register: __cdp_pub_admin_group_info - - - name: Set fact for CDP Admin Group CRN - ansible.builtin.set_fact: - plat__cdp_pub_admin_group_crn: "{{ __cdp_pub_admin_group_info | community.general.json_query('group.crn') }}" - - - name: Set fact for CDP User Group Resource Role assignments - ansible.builtin.set_fact: - plat__cdp_pub_user_group_resource_role_assignments: "{{ plat__cdp_pub_user_group_resource_role_assignments | default([]) | union([resource_role_assignment]) }}" - vars: - resource_role_assignment: - resource: "{{ plat__cdp_env_crn }}" - role: "{{ __cdp_resource_role_item }}" - loop_control: - loop_var: __cdp_resource_role_item - loop: "{{ plat__cdp_pub_user_group_resource_role_crns }}" - - - name: Create CDP User group - cloudera.cloud.iam_group: - name: "{{ plat__cdp_iam_user_group_name }}" - state: present - sync: false - roles: "{{ plat__cdp_pub_user_group_role_crns }}" - resource_roles: "{{ plat__cdp_pub_user_group_resource_role_assignments }}" - register: __cdp_pub_user_group_info - - - name: Set fact for CDP User Group CRN - ansible.builtin.set_fact: - plat__cdp_pub_user_group_crn: "{{ __cdp_pub_user_group_info | community.general.json_query('group.crn') }}" - -############# IDBroker Mappings ############# - - name: Set CDP IDBroker Mappings for AWS - cloudera.cloud.env_idbroker: - name: "{{ plat__env_name }}" - sync: no - data_access: "{{ plat__aws_datalake_admin_role_arn }}" - ranger_audit: "{{ plat__aws_ranger_audit_role_arn }}" - ranger_cloud_access: "{{ (plat__enable_raz | bool) | ternary(plat__aws_datalake_admin_role_arn, omit) }}" # TODO: Review this setting - mappings: - - accessor: "{{ plat__cdp_pub_admin_group_crn }}" - role: "{{ plat__aws_datalake_admin_role_arn }}" - - accessor: "{{ plat__cdp_pub_user_group_crn }}" - role: "{{ plat__aws_datalake_admin_role_arn }}" - -############# CDP Datalake ############# - - name: Start set up of CDP Datalake for AWS - cloudera.cloud.datalake: - name: "{{ plat__datalake_name }}" - env: "{{ plat__env_name }}" - instance_profile: "{{ plat__aws_idbroker_instance_profile_arn }}" - storage: "{{ plat__aws_storage_location }}" - runtime: "{{ plat__datalake_version | default(omit) }}" - scale: "{{ plat__datalake_scale | default(omit) }}" - tags: "{{ plat__tags }}" - raz: "{{ plat__enable_raz }}" - multi_az: "{{ plat__env_multiaz }}" - state: present - async: 3600 # 1 hour timeout - poll: 0 - register: __cdp_dl_setup - -############# Wait for Completion of Async tasks ############# - - - name: Wait for CDP Environment setup to complete - when: - - __cdp_env_setup_async is not defined - - __cdp_env_setup_async.finished is not defined - - __cdp_env_setup.ansible_job_id is defined - ansible.builtin.async_status: - jid: "{{ __cdp_env_setup.ansible_job_id }}" - register: __cdp_env_setup_async - until: __cdp_env_setup_async.finished - retries: 120 - delay: 30 - - - name: Wait for CDP Datalake setup to complete - when: __cdp_dl_setup.ansible_job_id is defined - ansible.builtin.async_status: - jid: "{{ __cdp_dl_setup.ansible_job_id }}" - register: __cdp_dl_setup_async - until: __cdp_dl_setup_async.finished - retries: 120 - delay: 30 diff --git a/modules/terraform-cdp-deploy/modules/aws/playbook_teardown_cdp.yml b/modules/terraform-cdp-deploy/modules/aws/playbook_teardown_cdp.yml deleted file mode 100644 index 3807257..0000000 --- a/modules/terraform-cdp-deploy/modules/aws/playbook_teardown_cdp.yml +++ /dev/null @@ -1,58 +0,0 @@ ---- - -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -################################################# -# Ansible Playbook to remove CDP resources based -# on output generated from TF pre-reqs module. -# Uses the cloudera.cloud Ansible collection -# -# Tasks performed: -# * Cascading delete to remove Environment, DL, DataHubs and Data Services -# * Delete user and admin groups -# * Delete Cross Account credential -############################# - -- name: Remove CDP Resources - hosts: localhost - environment: - CDP_PROFILE: "{{ plat__cdp_profile }}" - vars: - tasks: - -############# Cascading Delete of CDP Environment ############# - - name: Delete CDP Environment Deployment on AWS - cloudera.cloud.env: - name: "{{ plat__env_name }}" - cascading: True - state: absent - -############# CDP Groups ############# - - name: Remove CDP Admin group - cloudera.cloud.iam_group: - name: "{{ plat__cdp_iam_admin_group_name }}" - state: absent - - - name: Remove CDP User group - cloudera.cloud.iam_group: - name: "{{ plat__cdp_iam_user_group_name }}" - state: absent - -############# CDP Credential ############# - - name: Remove CDP Cross Account Credential for AWS - cloudera.cloud.env_cred: - name: "{{ plat__xacccount_credential_name }}" - state: absent - diff --git a/modules/terraform-cdp-deploy/modules/aws/provider.tf b/modules/terraform-cdp-deploy/modules/aws/provider.tf index b4c3f07..44b2880 100644 --- a/modules/terraform-cdp-deploy/modules/aws/provider.tf +++ b/modules/terraform-cdp-deploy/modules/aws/provider.tf @@ -14,15 +14,11 @@ terraform { required_providers { - local = { - source = "hashicorp/local" - version = "2.2.3" - } - null = { - source = "hashicorp/null" - version = "3.2.1" + cdp = { + source = "cloudera/cdp" + version = "0.1.4-pre" } } - required_version = "> 1.3.0" + required_version = ">= 1.3.0" } diff --git a/modules/terraform-cdp-deploy/modules/aws/templates/cdp_config.yml.tpl b/modules/terraform-cdp-deploy/modules/aws/templates/cdp_config.yml.tpl deleted file mode 100644 index 8ee2a71..0000000 --- a/modules/terraform-cdp-deploy/modules/aws/templates/cdp_config.yml.tpl +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# CDP environment & DL settings -plat__env_name: ${plat__env_name} -plat__datalake_name: ${plat__datalake_name} -plat__xacccount_credential_name: ${plat__xacccount_credential_name} -plat__cdp_iam_admin_group_name: ${plat__cdp_iam_admin_group_name} -plat__cdp_iam_user_group_name: ${plat__cdp_iam_user_group_name} -plat__datalake_scale: ${plat__datalake_scale} -plat__datalake_version: ${plat__datalake_version} -plat__tunnel: ${plat__tunnel} -plat__endpoint_access_scheme: ${plat__endpoint_access_scheme} -plat__enable_raz: ${plat__enable_raz} -plat__env_multiaz: ${plat__env_multiaz} -plat__env_freeipa_instances: ${plat__env_freeipa_instances} -plat__workload_analytics: ${plat__workload_analytics} -plat__tags: ${plat__tags} - -# CDP settings -plat__cdp_profile: ${plat__cdp_profile} -plat__cdp_control_plane_region: ${plat__cdp_control_plane_region} - -# CSP settings -plat__infra_type: ${plat__infra_type} -plat__region: ${plat__region} - -plat__aws_vpc_id: ${plat__aws_vpc_id} -plat__aws_public_subnet_ids: ${plat__aws_public_subnet_ids} -plat__aws_private_subnet_ids: ${plat__aws_private_subnet_ids} -plat__aws_subnets_for_cdp: ${plat__aws_subnets_for_cdp} - -plat__aws_storage_location: ${plat__aws_storage_location} -plat__aws_log_location: ${plat__aws_log_location} -plat__aws_backup_location: ${plat__aws_backup_location} - -plat__public_key_id: ${plat__public_key_id} -plat__aws_security_group_default_id: ${plat__aws_security_group_default_id} -plat__aws_security_group_knox_id: ${plat__aws_security_group_knox_id} - -plat__aws_datalake_admin_role_arn: ${plat__aws_datalake_admin_role_arn} -plat__aws_ranger_audit_role_arn: ${plat__aws_ranger_audit_role_arn} -plat__aws_xaccount_role_arn: ${plat__aws_xaccount_role_arn} - -plat__aws_log_instance_profile_arn: ${plat__aws_log_instance_profile_arn} -plat__aws_idbroker_instance_profile_arn: ${plat__aws_idbroker_instance_profile_arn} diff --git a/modules/terraform-cdp-deploy/modules/aws/variables.tf b/modules/terraform-cdp-deploy/modules/aws/variables.tf index 57c9060..2af4a0e 100644 --- a/modules/terraform-cdp-deploy/modules/aws/variables.tf +++ b/modules/terraform-cdp-deploy/modules/aws/variables.tf @@ -13,11 +13,12 @@ # limitations under the License. # ------- Global settings ------- -variable "tags" { - type = map(any) - description = "Tags applied to provisioned resources" +# NOTE: Waiting on provider fix +# variable "tags" { +# type = map(any) +# description = "Tags applied to provisioned resources" -} +# } # ------- CDP Environment Deployment ------- variable "environment_name" { @@ -50,17 +51,12 @@ variable "cdp_user_group_name" { } -variable "cdp_profile" { - type = string - description = "Profile for CDP credentials" +# TODO: Will be re-introducted once provider supports other regions +# variable "cdp_control_plane_region" { +# type = string +# description = "CDP Control Plane Region" -} - -variable "cdp_control_plane_region" { - type = string - description = "CDP Control Plane Region" - -} +# } variable "enable_ccm_tunnel" { type = bool @@ -152,11 +148,11 @@ variable "public_subnet_ids" { description = "List of public subnet ids." } -variable "private_subnet_ids" { - type = list(string) - description = "List of private subnet ids." +# variable "private_subnet_ids" { +# type = list(string) +# description = "List of private subnet ids." -} +# } variable "subnets_for_cdp" { type = list(string) diff --git a/modules/terraform-cdp-deploy/modules/azure/main.tf b/modules/terraform-cdp-deploy/modules/azure/main.tf index aceccd5..d908fe1 100644 --- a/modules/terraform-cdp-deploy/modules/azure/main.tf +++ b/modules/terraform-cdp-deploy/modules/azure/main.tf @@ -12,80 +12,127 @@ # See the License for the specific language governing permissions and # limitations under the License. -# ------- Create Configuration file for CDP Deployment via Ansible ------- -resource "local_file" "cdp_deployment_template" { - - content = templatefile("${path.module}/templates/cdp_config.yml.tpl", { - # CDP environment & DL settings - plat__env_name = var.environment_name - plat__datalake_name = var.datalake_name - plat__datalake_scale = var.datalake_scale - plat__datalake_version = var.datalake_version - plat__xacccount_credential_name = var.cdp_xacccount_credential_name - plat__cdp_iam_admin_group_name = var.cdp_admin_group_name - plat__cdp_iam_user_group_name = var.cdp_user_group_name - plat__tunnel = var.enable_ccm_tunnel - plat__endpoint_access_scheme = var.endpoint_access_scheme - plat__enable_raz = var.enable_raz - plat__use_single_resource_group = var.use_single_resource_group - plat__use_public_ips = var.use_public_ips - plat__env_freeipa_instances = var.freeipa_instances - plat__workload_analytics = var.workload_analytics - plat__tags = jsonencode(var.tags) - - # CDP settings - plat__cdp_profile = var.cdp_profile - plat__cdp_control_plane_region = var.cdp_control_plane_region - - # # CSP settings - plat__infra_type = "azure" - plat__region = var.region - - plat__azure_subscription_id = var.subscription_id - plat__azure_tenant_id = var.tenant_id - - plat__azure_resourcegroup_name = var.resource_group_name - plat__azure_vnet_name = var.vnet_name - plat__azure_subnet_names_for_cdp = jsonencode(var.cdp_subnet_names) - plat__azure_subnet_names_for_gateway = jsonencode(var.cdp_gateway_subnet_names) - - plat__azure_storage_location = var.data_storage_location - plat__azure_log_location = var.log_storage_location - plat__azure_backup_location = var.backup_storage_location - - plat__public_key_text = var.public_key_text - plat__azure_security_group_default_uri = var.security_group_default_uri - plat__azure_security_group_knox_uri = var.security_group_knox_uri - - plat__azure_xaccount_app_uuid = var.xaccount_app_uuid - plat__azure_xaccount_app_pword = var.xaccount_app_pword - - plat__azure_idbroker_identity_id = var.idbroker_identity_id - plat__azure_datalakeadmin_identity_id = var.datalakeadmin_identity_id - plat__azure_ranger_audit_identity_id = var.ranger_audit_identity_id - plat__azure_log_identity_id = var.log_identity_id - plat__azure_raz_identity_id = var.raz_identity_id - - } - ) - filename = "cdp_config.yml" +# ------- CDP Credential ------- +resource "cdp_environments_azure_credential" "cdp_cred" { + credential_name = var.cdp_xacccount_credential_name + subscription_id = var.subscription_id + tenant_id = var.tenant_id + app_based = { + application_id = var.xaccount_app_uuid + secret_key = var.xaccount_app_pword + } + description = "Azure Cross Account Credential for Azure env ${var.environment_name}" } -# ------- Create CDP Deployment ------- -resource "null_resource" "cdp_deployment" { +# ------- CDP Environment ------- +resource "cdp_environments_azure_environment" "cdp_env" { + environment_name = var.environment_name + credential_name = cdp_environments_azure_credential.cdp_cred.credential_name + region = var.region + + security_access = { + default_security_group_id = var.security_group_default_uri + security_group_id_for_knox = var.security_group_knox_uri + } + + log_storage = { + storage_location_base = var.log_storage_location + backup_storage_location_base = var.backup_storage_location + managed_identity = var.log_identity_id + } + + public_key = var.public_key_text - # Setup of CDP environment using playbook_setup_cdp.yml Ansible Playbook - provisioner "local-exec" { - command = "ansible-playbook -vvv -e '@cdp_config.yml' ${path.module}/playbook_setup_cdp.yml" + use_public_ip = var.use_public_ips + existing_network_params = { + resource_group_name = var.resource_group_name + network_id = var.vnet_name + subnet_ids = var.cdp_subnet_names } - # Deletion of CDP environment using playbook_teardown_cdp.yml Ansible Playbook - provisioner "local-exec" { - when = destroy - command = "ansible-playbook -vvv -e '@cdp_config.yml' ${path.module}/playbook_teardown_cdp.yml" + endpoint_access_gateway_scheme = var.endpoint_access_scheme + endpoint_access_gateway_subnet_ids = (length(var.cdp_gateway_subnet_names) > 0) ? var.cdp_gateway_subnet_names : null + + # Set this parameter to deploy all resources into a single resource group + resource_group_name = var.use_single_resource_group ? var.resource_group_name : null + + freeipa = { + instance_count_by_group = var.freeipa_instances } + workload_analytics = var.workload_analytics + enable_tunnel = var.enable_ccm_tunnel + # tags = var.tags # NOTE: Waiting on provider fix + + depends_on = [ + cdp_environments_azure_credential.cdp_cred + ] + +} + +# ------- CDP Admin Group ------- +# Create group +resource "cdp_iam_group" "cdp_admin_group" { + group_name = var.cdp_admin_group_name + sync_membership_on_user_login = false +} + +# TODO: Assign roles and resource roles to the group + +# TODO: Assign users to the group + +# ------- CDP User Group ------- +# Create group +resource "cdp_iam_group" "cdp_user_group" { + group_name = var.cdp_user_group_name + sync_membership_on_user_login = false +} + +# TODO: Assign roles and resource roles to the group + +# TODO: Assign users to the group + +# ------- IdBroker Mappings ------- +resource "cdp_environments_id_broker_mappings" "cdp_idbroker" { + environment_name = cdp_environments_azure_environment.cdp_env.environment_name + environment_crn = cdp_environments_azure_environment.cdp_env.crn + + ranger_audit_role = var.ranger_audit_identity_id + data_access_role = var.datalakeadmin_identity_id + ranger_cloud_access_authorizer_role = var.enable_raz ? var.raz_identity_id : null + + mappings = [{ + accessor_crn = cdp_iam_group.cdp_admin_group.crn + role = var.datalakeadmin_identity_id + }, + { + accessor_crn = cdp_iam_group.cdp_user_group.crn + role = var.datalakeadmin_identity_id + } + ] + + depends_on = [ + cdp_environments_azure_environment.cdp_env + ] +} + +# ------- CDP Datalake ------- +resource "cdp_datalake_azure_datalake" "cdp_datalake" { + datalake_name = var.datalake_name + environment_name = cdp_environments_azure_environment.cdp_env.environment_name + + managed_identity = var.idbroker_identity_id + storage_location = var.data_storage_location + + runtime = var.datalake_version + scale = var.datalake_scale + enable_ranger_raz = var.enable_raz + + # tags = var.tags # NOTE: Waiting on provider fix + depends_on = [ - local_file.cdp_deployment_template, + cdp_environments_azure_credential.cdp_cred, + cdp_environments_azure_environment.cdp_env, + cdp_environments_id_broker_mappings.cdp_idbroker ] } diff --git a/modules/terraform-cdp-deploy/modules/azure/outputs.tf b/modules/terraform-cdp-deploy/modules/azure/outputs.tf index e3cf504..0671c70 100644 --- a/modules/terraform-cdp-deploy/modules/azure/outputs.tf +++ b/modules/terraform-cdp-deploy/modules/azure/outputs.tf @@ -11,3 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +output "cdp_environment_name" { + value = cdp_environments_azure_environment.cdp_env.environment_name + + description = "CDP Environment Name" +} + +output "cdp_environment_crn" { + value = cdp_environments_azure_environment.cdp_env.crn + + description = "CDP Environment CRN" +} diff --git a/modules/terraform-cdp-deploy/modules/azure/playbook_setup_cdp.yml b/modules/terraform-cdp-deploy/modules/azure/playbook_setup_cdp.yml deleted file mode 100644 index 87a4823..0000000 --- a/modules/terraform-cdp-deploy/modules/azure/playbook_setup_cdp.yml +++ /dev/null @@ -1,313 +0,0 @@ ---- - -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -################################################# -# Ansible Playbook to create CDP resources based -# on output generated from TF pre-reqs module. -# Uses the cloudera.cloud Ansible collection -# -# Tasks performed: -# * Create Cross Account credential -# * Set up Environment Deployment -# * Creates user and admin groups and CDP IDBroker Mappings -# * Set up Datalake -############################# -- name: Create CDP Resources - hosts: localhost - environment: - CDP_PROFILE: "{{ plat__cdp_profile }}" - CDP_REGION: "{{ plat__cdp_control_plane_region }}" - vars: - plat__cdp_iam_identities: - role_suffix: role - resource_role_suffix: resourceRole - env_admin: EnvironmentAdmin - env_user: EnvironmentUser - dw_admin: DWAdmin - dw_user: DWUser - ml_admin: MLAdmin - ml_user: MLUser - df_admin: DFAdmin - df_flow_admin: DFFlowAdmin - df_flow_user: DFFlowUser - de_admin: DEAdmin - de_user: DEUser - plat__cdp_iam_admin_group_resource_roles_default: - - "{{ plat__cdp_iam_identities.env_admin }}" - - "{{ plat__cdp_iam_identities.env_user }}" - - "{{ plat__cdp_iam_identities.dw_admin }}" - - "{{ plat__cdp_iam_identities.dw_user }}" - - "{{ plat__cdp_iam_identities.ml_admin }}" - - "{{ plat__cdp_iam_identities.ml_user }}" - - "{{ plat__cdp_iam_identities.df_admin }}" - - "{{ plat__cdp_iam_identities.df_flow_admin }}" - - "{{ plat__cdp_iam_identities.df_flow_user }}" - - "{{ plat__cdp_iam_identities.de_admin }}" - - "{{ plat__cdp_iam_identities.de_user }}" - plat__cdp_iam_user_group_resource_roles_default: - - "{{ plat__cdp_iam_identities.env_user }}" - - "{{ plat__cdp_iam_identities.dw_user }}" - - "{{ plat__cdp_iam_identities.ml_user }}" - - "{{ plat__cdp_iam_identities.df_flow_admin }}" - - "{{ plat__cdp_iam_identities.df_flow_user }}" - - "{{ plat__cdp_iam_identities.de_user }}" - plat__cdp_iam_admin_group_roles: - - "{{ plat__cdp_iam_identities.env_admin }}" - plat__cdp_iam_admin_group_resource_roles: "{{ plat__cdp_iam_admin_group_resource_roles_default }}" - plat__cdp_iam_user_group_roles: - - "{{ plat__cdp_iam_identities.env_user }}" - plat__cdp_iam_user_group_resource_roles: "{{ plat__cdp_iam_user_group_resource_roles_default }}" - common__cdp_control_planes: - us-west-1: "crn:altus:iam:us-west-1:altus" - eu-1: "crn:altus:iam:eu-1:altus" - ap-1: "crn:altus:iam:ap-1:altus" - common__cdp_control_plane_region: "{{ plat__cdp_control_plane_region | default('us-west-1') }}" - common__cdp_control_plane_crn: "{{ common__cdp_control_planes[common__cdp_control_plane_region] }}" - plat__cdp_control_plane_crn: "{{ common__cdp_control_plane_crn }}" - plat__cdp_iam_role_suffix: "{{ plat__cdp_iam_identities.role_suffix }}" - plat__cdp_iam_resource_suffix: "{{ plat__cdp_iam_identities.resource_role_suffix }}" - tasks: - -############# CDP Credential ############# - - name: Create CDP Cross Account Credential for Azure - cloudera.cloud.env_cred: - cloud: "{{ plat__infra_type }}" - name: "{{ plat__xacccount_credential_name }}" - subscription: "{{ plat__azure_subscription_id }}" - tenant: "{{ plat__azure_tenant_id }}" - application: "{{ plat__azure_xaccount_app_uuid }}" - secret: "{{ plat__azure_xaccount_app_pword }}" - state: present - -############# CDP Environment ############# - - name: Start set up of CDP Environment Deployment on Azure - block: - - name: Call cloudera.cloud.env to register CDP environment - cloudera.cloud.env: - name: "{{ plat__env_name }}" - state: started - credential: "{{ plat__xacccount_credential_name }}" - cloud: "{{ plat__infra_type }}" - region: "{{ plat__region }}" - default_sg: "{{ plat__azure_security_group_default_uri }}" - knox_sg: "{{ plat__azure_security_group_knox_uri }}" - log_location: "{{ plat__azure_log_location }}" - log_identity: "{{ plat__azure_log_identity_id }}" - backup_location: "{{ plat__azure_backup_location }}" - public_key_text: "{{ plat__public_key_text }}" - workload_analytics: "{{ plat__workload_analytics }}" - network: "{{ plat__azure_vnet_name }}" - subnet_ids: "{{ plat__azure_subnet_names_for_cdp }}" - tags: "{{ plat__tags }}" - tunnel: "{{ plat__tunnel }}" - # endpoint_access_scheme: "{{ plat__endpoint_access_scheme | default(omit) }}" - # endpoint_access_subnets: "{{ plat__azure_subnet_names_for_gateway | default(omit) }}" - resource_gp: "{{ plat__azure_resourcegroup_name }}" - use_single_resource_group: "{{ plat__use_single_resource_group }}" - public_ip: "{{ plat__use_public_ips }}" - freeipa: - instanceCountByGroup: "{{ plat__env_freeipa_instances }}" - async: 3600 # 1 hour timeout - poll: 0 - register: __cdp_env_setup - - - name: Wait for CDP environment setup to start - ansible.builtin.pause: - seconds: 60 - - - name: Update CDP Environment information - cloudera.cloud.env_info: - name: "{{ plat__env_name }}" - register: plat__cdp_env_info - failed_when: plat__cdp_env_info.environments | length == 0 - rescue: - # Fallback to synchronous if environment not found - - name: Wait for CDP Environment setup to complete - when: __cdp_env_setup.ansible_job_id is defined - ansible.builtin.async_status: - jid: "{{ __cdp_env_setup.ansible_job_id }}" - register: __cdp_env_setup_async - until: __cdp_env_setup_async.finished - retries: 120 - delay: 30 - - - name: Attempt to update CDP Environment information again - cloudera.cloud.env_info: - name: "{{ plat__env_name }}" - register: plat__cdp_env_info - failed_when: plat__cdp_env_info.environments | length == 0 - - # Resume normal service - - name: Set fact for CDP Environment CRN - ansible.builtin.set_fact: - plat__cdp_env_crn: "{{ plat__cdp_env_info.environments[0].crn | default('Unknown') }}" - - - name: Print Environment CRN - debug: - msg: - - plat__cdp_env_crn is {{ plat__cdp_env_crn }} - -############# CDP Groups ############# - - name: Query CDP Caller to confirm access to Endpoints - cloudera.cloud.iam_user_info: - current_user: yes - register: __cdp_iam_current_user_info - - - name: Set facts for CDP Caller Workload Username and CRN - ansible.builtin.set_fact: - plat__cdp_workload_username: "{{ __cdp_iam_current_user_info.users[0].workloadUsername }}" - plat__cdp_user_crn: "{{ __cdp_iam_current_user_info.users[0].crn }}" - - - name: Set fact for CDP Environment Admin Group Role CRNs - ansible.builtin.set_fact: - plat__cdp_pub_admin_group_role_crns: "{{ plat__cdp_pub_admin_group_role_crns | default([]) | union([role]) }}" - vars: - role: "{{ [plat__cdp_control_plane_crn, plat__cdp_iam_role_suffix, __cdp_pub_admin_group_role_item] | join(':') }}" - loop_control: - loop_var: __cdp_pub_admin_group_role_item - loop: "{{ plat__cdp_iam_admin_group_roles }}" - - - name: Set fact for CDP Environment Admin Group Resource Role CRNs - ansible.builtin.set_fact: - plat__cdp_pub_admin_group_resource_role_crns: "{{ plat__cdp_pub_admin_group_resource_role_crns | default([]) | union([resource_role]) }}" - vars: - resource_role: "{{ [plat__cdp_control_plane_crn, plat__cdp_iam_resource_suffix, __cdp_env_admin_group_resource_role_item] | join(':') }}" - loop_control: - loop_var: __cdp_env_admin_group_resource_role_item - loop: "{{ plat__cdp_iam_admin_group_resource_roles }}" - - - name: Set fact for CDP Environment User Group Role CRNs - ansible.builtin.set_fact: - plat__cdp_pub_user_group_role_crns: "{{ plat__cdp_pub_user_group_role_crns | default([]) | union([role]) }}" - vars: - role: "{{ [plat__cdp_control_plane_crn, plat__cdp_iam_role_suffix, __cdp_pub_user_group_role_item] | join(':') }}" - loop_control: - loop_var: __cdp_pub_user_group_role_item - loop: "{{ plat__cdp_iam_user_group_roles }}" - - - name: Set fact for CDP Environment User Group Resource Role CRNs - ansible.builtin.set_fact: - plat__cdp_pub_user_group_resource_role_crns: "{{ plat__cdp_pub_user_group_resource_role_crns | default([]) | union([resource_role]) }}" - vars: - resource_role: "{{ [plat__cdp_control_plane_crn, plat__cdp_iam_resource_suffix, __cdp_pub_user_group_resource_role_item] | join(':') }}" - loop_control: - loop_var: __cdp_pub_user_group_resource_role_item - loop: "{{ plat__cdp_iam_user_group_resource_roles }}" - - - name: Set fact for CDP Admin Group Resource Role assignments - ansible.builtin.set_fact: - plat__cdp_env_admin_group_resource_role_assignments: "{{ plat__cdp_env_admin_group_resource_role_assignments | default([]) | union([resource_role_assignment]) }}" - vars: - resource_role_assignment: - resource: "{{ plat__cdp_env_crn }}" - role: "{{ __cdp_resource_role_item }}" - loop_control: - loop_var: __cdp_resource_role_item - loop: "{{ plat__cdp_pub_admin_group_resource_role_crns }}" - - - name: Create CDP Admin group - cloudera.cloud.iam_group: - name: "{{ plat__cdp_iam_admin_group_name }}" - state: present - sync: false - roles: "{{ plat__cdp_pub_admin_group_role_crns }}" - resource_roles: "{{ plat__cdp_env_admin_group_resource_role_assignments }}" - users: - - "{{ plat__cdp_user_crn }}" - register: __cdp_pub_admin_group_info - - - name: Set fact for CDP Admin Group CRN - ansible.builtin.set_fact: - plat__cdp_pub_admin_group_crn: "{{ __cdp_pub_admin_group_info | community.general.json_query('group.crn') }}" - - - name: Set fact for CDP User Group Resource Role assignments - ansible.builtin.set_fact: - plat__cdp_pub_user_group_resource_role_assignments: "{{ plat__cdp_pub_user_group_resource_role_assignments | default([]) | union([resource_role_assignment]) }}" - vars: - resource_role_assignment: - resource: "{{ plat__cdp_env_crn }}" - role: "{{ __cdp_resource_role_item }}" - loop_control: - loop_var: __cdp_resource_role_item - loop: "{{ plat__cdp_pub_user_group_resource_role_crns }}" - - - name: Create CDP User group - cloudera.cloud.iam_group: - name: "{{ plat__cdp_iam_user_group_name }}" - state: present - sync: false - roles: "{{ plat__cdp_pub_user_group_role_crns }}" - resource_roles: "{{ plat__cdp_pub_user_group_resource_role_assignments }}" - register: __cdp_pub_user_group_info - - - name: Set fact for CDP User Group CRN - ansible.builtin.set_fact: - plat__cdp_pub_user_group_crn: "{{ __cdp_pub_user_group_info | community.general.json_query('group.crn') }}" - -############# IDBroker Mappings ############# - - - name: Set IDBroker Mappings for CDP on Azure - cloudera.cloud.env_idbroker: - name: "{{ plat__env_name }}" - sync: no - data_access: "{{ plat__azure_datalakeadmin_identity_id }}" - ranger_audit: "{{ plat__azure_ranger_audit_identity_id }}" - ranger_cloud_access: "{{ (plat__enable_raz | bool) | ternary(plat__azure_raz_identity_id, omit) }}" - mappings: - - accessor: "{{ plat__cdp_pub_admin_group_crn }}" - role: "{{ plat__azure_datalakeadmin_identity_id }}" - - accessor: "{{ plat__cdp_pub_user_group_crn }}" - role: "{{ plat__azure_datalakeadmin_identity_id }}" - -############# CDP Datalake ############# - - - name: Start set up of CDP Datalake for Azure - cloudera.cloud.datalake: - name: "{{ plat__datalake_name }}" - env: "{{ plat__env_name }}" - managed_identity: "{{ plat__azure_idbroker_identity_id }}" - storage: "{{ plat__azure_storage_location }}" - runtime: "{{ plat__datalake_version | default(omit) }}" - scale: "{{ plat__datalake_scale | default(omit) }}" - tags: "{{ plat__tags }}" - raz: "{{ plat__enable_raz }}" - state: present - async: 10000 - poll: 0 - register: __cdp_dl_setup - -############# Wait for Completion of Async tasks ############# - - - name: Wait for CDP Environment setup to complete - when: - - __cdp_env_setup_async is not defined - - __cdp_env_setup_async.finished is not defined - - __cdp_env_setup.ansible_job_id is defined - ansible.builtin.async_status: - jid: "{{ __cdp_env_setup.ansible_job_id }}" - register: __cdp_env_setup_async - until: __cdp_env_setup_async.finished - retries: 120 - delay: 30 - - - name: Wait for CDP Datalake setup to complete - when: __cdp_dl_setup.ansible_job_id is defined - ansible.builtin.async_status: - jid: "{{ __cdp_dl_setup.ansible_job_id }}" - register: __cdp_dl_setup_async - until: __cdp_dl_setup_async.finished - retries: 240 - delay: 30 diff --git a/modules/terraform-cdp-deploy/modules/azure/playbook_teardown_cdp.yml b/modules/terraform-cdp-deploy/modules/azure/playbook_teardown_cdp.yml deleted file mode 100644 index 3807257..0000000 --- a/modules/terraform-cdp-deploy/modules/azure/playbook_teardown_cdp.yml +++ /dev/null @@ -1,58 +0,0 @@ ---- - -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -################################################# -# Ansible Playbook to remove CDP resources based -# on output generated from TF pre-reqs module. -# Uses the cloudera.cloud Ansible collection -# -# Tasks performed: -# * Cascading delete to remove Environment, DL, DataHubs and Data Services -# * Delete user and admin groups -# * Delete Cross Account credential -############################# - -- name: Remove CDP Resources - hosts: localhost - environment: - CDP_PROFILE: "{{ plat__cdp_profile }}" - vars: - tasks: - -############# Cascading Delete of CDP Environment ############# - - name: Delete CDP Environment Deployment on AWS - cloudera.cloud.env: - name: "{{ plat__env_name }}" - cascading: True - state: absent - -############# CDP Groups ############# - - name: Remove CDP Admin group - cloudera.cloud.iam_group: - name: "{{ plat__cdp_iam_admin_group_name }}" - state: absent - - - name: Remove CDP User group - cloudera.cloud.iam_group: - name: "{{ plat__cdp_iam_user_group_name }}" - state: absent - -############# CDP Credential ############# - - name: Remove CDP Cross Account Credential for AWS - cloudera.cloud.env_cred: - name: "{{ plat__xacccount_credential_name }}" - state: absent - diff --git a/modules/terraform-cdp-deploy/modules/azure/provider.tf b/modules/terraform-cdp-deploy/modules/azure/provider.tf index b4c3f07..44b2880 100644 --- a/modules/terraform-cdp-deploy/modules/azure/provider.tf +++ b/modules/terraform-cdp-deploy/modules/azure/provider.tf @@ -14,15 +14,11 @@ terraform { required_providers { - local = { - source = "hashicorp/local" - version = "2.2.3" - } - null = { - source = "hashicorp/null" - version = "3.2.1" + cdp = { + source = "cloudera/cdp" + version = "0.1.4-pre" } } - required_version = "> 1.3.0" + required_version = ">= 1.3.0" } diff --git a/modules/terraform-cdp-deploy/modules/azure/templates/cdp_config.yml.tpl b/modules/terraform-cdp-deploy/modules/azure/templates/cdp_config.yml.tpl deleted file mode 100644 index 113cae2..0000000 --- a/modules/terraform-cdp-deploy/modules/azure/templates/cdp_config.yml.tpl +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# CDP environment & DL settings -plat__env_name: ${plat__env_name} -plat__datalake_name: ${plat__datalake_name} -plat__xacccount_credential_name: ${plat__xacccount_credential_name} -plat__cdp_iam_admin_group_name: ${plat__cdp_iam_admin_group_name} -plat__cdp_iam_user_group_name: ${plat__cdp_iam_user_group_name} - -plat__datalake_scale: ${plat__datalake_scale} -plat__datalake_version: ${plat__datalake_version} -plat__tunnel: ${plat__tunnel} -plat__endpoint_access_scheme: ${plat__endpoint_access_scheme} -plat__use_public_ips: ${plat__use_public_ips} -plat__enable_raz: ${plat__enable_raz} -plat__use_single_resource_group: ${plat__use_single_resource_group} -plat__workload_analytics: ${plat__workload_analytics} -plat__env_freeipa_instances: ${plat__env_freeipa_instances} -plat__tags: ${plat__tags} - -# CDP settings -plat__cdp_profile: ${plat__cdp_profile} -plat__cdp_control_plane_region: ${plat__cdp_control_plane_region} - -# CSP settings -plat__infra_type: ${plat__infra_type} -plat__region: ${plat__region} - -plat__azure_subscription_id: ${plat__azure_subscription_id} -plat__azure_tenant_id: ${plat__azure_tenant_id} - -plat__azure_vnet_name: ${plat__azure_vnet_name} -plat__azure_resourcegroup_name: ${plat__azure_resourcegroup_name} -plat__azure_subnet_names_for_cdp: ${plat__azure_subnet_names_for_cdp} -plat__azure_subnet_names_for_gateway: ${plat__azure_subnet_names_for_gateway} - -plat__azure_storage_location: ${plat__azure_storage_location} -plat__azure_log_location: ${plat__azure_log_location} -plat__azure_backup_location: ${plat__azure_backup_location} - -plat__public_key_text: ${plat__public_key_text} -plat__azure_security_group_default_uri: ${plat__azure_security_group_default_uri} -plat__azure_security_group_knox_uri: ${plat__azure_security_group_knox_uri} - -plat__azure_xaccount_app_uuid: ${plat__azure_xaccount_app_uuid} -plat__azure_xaccount_app_pword: ${plat__azure_xaccount_app_pword} - -plat__azure_idbroker_identity_id: ${plat__azure_idbroker_identity_id} -plat__azure_datalakeadmin_identity_id: ${plat__azure_datalakeadmin_identity_id} -plat__azure_ranger_audit_identity_id: ${plat__azure_ranger_audit_identity_id} -plat__azure_log_identity_id: ${plat__azure_log_identity_id} -plat__azure_raz_identity_id: ${plat__azure_raz_identity_id} diff --git a/modules/terraform-cdp-deploy/modules/azure/variables.tf b/modules/terraform-cdp-deploy/modules/azure/variables.tf index 10ec1d1..2263494 100644 --- a/modules/terraform-cdp-deploy/modules/azure/variables.tf +++ b/modules/terraform-cdp-deploy/modules/azure/variables.tf @@ -13,11 +13,12 @@ # limitations under the License. # ------- Global settings ------- -variable "tags" { - type = map(any) - description = "Tags applied to provisioned resources" +# NOTE: Waiting on provider fix +# variable "tags" { +# type = map(any) +# description = "Tags applied to provisioned resources" -} +# } # ------- CDP Environment Deployment ------- variable "environment_name" { @@ -50,17 +51,12 @@ variable "cdp_user_group_name" { } -variable "cdp_profile" { - type = string - description = "Profile for CDP credentials" - -} +# TODO: Will be re-introducted once provider supports other regions +# variable "cdp_control_plane_region" { +# type = string +# description = "CDP Control Plane Region" -variable "cdp_control_plane_region" { - type = string - description = "CDP Control Plane Region" - -} +# } variable "enable_ccm_tunnel" { type = bool diff --git a/modules/terraform-cdp-deploy/outputs.tf b/modules/terraform-cdp-deploy/outputs.tf index e3cf504..834a705 100644 --- a/modules/terraform-cdp-deploy/outputs.tf +++ b/modules/terraform-cdp-deploy/outputs.tf @@ -11,3 +11,21 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + +output "cdp_environment_name" { + value = try( + module.cdp_on_aws[0].cdp_environment_name, + module.cdp_on_azure[0].cdp_environment_name + ) + + description = "CDP Environment Name" +} + +output "cdp_environment_crn" { + value = try( + module.cdp_on_aws[0].cdp_environment_crn, + module.cdp_on_azure[0].cdp_environment_crn + ) + + description = "CDP Environment CRN" +} diff --git a/modules/terraform-cdp-deploy/provider.tf b/modules/terraform-cdp-deploy/provider.tf index b4c3f07..44b2880 100644 --- a/modules/terraform-cdp-deploy/provider.tf +++ b/modules/terraform-cdp-deploy/provider.tf @@ -14,15 +14,11 @@ terraform { required_providers { - local = { - source = "hashicorp/local" - version = "2.2.3" - } - null = { - source = "hashicorp/null" - version = "3.2.1" + cdp = { + source = "cloudera/cdp" + version = "0.1.4-pre" } } - required_version = "> 1.3.0" + required_version = ">= 1.3.0" } diff --git a/modules/terraform-cdp-deploy/variables.tf b/modules/terraform-cdp-deploy/variables.tf index 627dc82..1516e8f 100644 --- a/modules/terraform-cdp-deploy/variables.tf +++ b/modules/terraform-cdp-deploy/variables.tf @@ -23,19 +23,21 @@ variable "infra_type" { } } -variable "env_tags" { - type = map(any) - description = "Tags applied to provisioned resources" +# NOTE: Waiting on provider fix +# variable "env_tags" { +# type = map(any) +# description = "Tags applied to provisioned resources" - default = null -} +# default = null +# } -variable "agent_source_tag" { - type = map(any) - description = "Tag to identify deployment source" +# NOTE: Waiting on provider fix +# variable "agent_source_tag" { +# type = map(any) +# description = "Tag to identify deployment source" - default = { agent_source = "tf-cdp-module" } -} +# default = { agent_source = "tf-cdp-module" } +# } variable "env_prefix" { type = string @@ -80,21 +82,14 @@ variable "cdp_user_group_name" { default = null } -variable "cdp_profile" { - type = string - description = "Profile for CDP credentials" +# TODO: Will be re-introducted once provider supports other regions +# variable "cdp_control_plane_region" { +# type = string +# description = "CDP Control Plane Region" - # Profile is default unless explicitly specified - default = "default" -} - -variable "cdp_control_plane_region" { - type = string - description = "CDP Control Plane Region" - - # Region is us-west-1 unless explicitly specified - default = "us-west-1" -} +# # Region is us-west-1 unless explicitly specified +# default = "us-west-1" +# } variable "deployment_template" { type = string @@ -171,7 +166,7 @@ variable "datalake_version" { error_message = "Valid values for var: datalake_version must match semantic versioning conventions." } - default = "7.2.16" + default = "7.2.17" } variable "endpoint_access_scheme" {