Updates for use of CDP Terraform provider (#21)

Signed-off-by: Jim Enright <[email protected]>
Co-authored-by: Balazs Gaspar <[email protected]>

.tflint.hcl

@@ -24,6 +24,12 @@ plugin "aws" {
   source  = "github.com/terraform-linters/tflint-ruleset-aws"
 }
 
+plugin "azurerm" {
+  enabled = true
+  version = "0.24.0"
+  source  = "github.com/terraform-linters/tflint-ruleset-azurerm"
+}
+
 config {
   module = true
   force  = false

README.md

@@ -54,34 +54,17 @@ If you no longer need the infrastructure that’s provisioned by the Terraform m
 terraform destroy
 ```
 
-## External dependencies
+## Dependencies
 
-The module includes the option to discover the cross account Ids and to run the CDP deployment using external tools.
+To set up CDP via deployment automation using this guide, the following dependencies must be installed in your local environment:
 
-To utilize these options extra requirements are needed - Python 3, Ansible 2.12, the CDP CLI, the [jq utility](https://stedolan.github.io/jq/download/) and a number of support Python libraries and Ansible collections.
+* Terraform can be installed by following the instructions at https://developer.hashicorp.com/terraform/downloads
 
-A summary of the install and configuration steps for these additional requirements is given below.
-We recommend these steps be performed within an Python virtual environment.
+Configure Terraform Provider for AWS or Azure
 
-```bash
-# Install jq as per instructions at https://stedolan.github.io/jq/download/
-# Example for MacOS using homebew shown below
-brew install jq
-
-# Install the Ansible core Python package
-pip install ansible-core==2.12.10 jmespath==1.0.1
-
-# Install cdpy, a Pythonic wrapper for Cloudera CDP CLI. This in turn installs the CDP CLI.
-pip install git+https://github.com/cloudera-labs/cdpy@main#egg=cdpy
-
-# Install the cloudera.cloud Ansible Collection
-ansible-galaxy collection install git+https://github.com/cloudera-labs/cloudera.cloud.git,devel
-
-# Install the community.general Ansible Collection
-ansible-galaxy collection install community.general:==5.5.0
-
-# Configure cdp with CDP API access key ID and private key
-cdp configure
-```
+* Configure the Terraform Provider for CDP with access key ID and private key by dowloading or creating a CDP configuation file.
+  * See the [CDP documentation for steps to Generate the API access key](https://docs.cloudera.com/cdp-public-cloud/cloud/cli/topics/mc-cli-generating-an-api-access-key.html).
 
-NOTE - See the [CDP documentation for steps to Generate the API access key](https://docs.cloudera.com/cdp-public-cloud/cloud/cli/topics/mc-cli-generating-an-api-access-key.html) required in the `cdp configure` command above.
+* To create resources in the Cloud Provider, access credentials or service account are needed for authentication.
+  * For **AWS** access keys are required to be able to create the Cloud resources via the Terraform aws provider. See the [AWS Terraform Provider Documentation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs#authentication-and-configuration).
+  * For **Azure**, authentication with the Azure subscription is required. There are a number of ways to do this outlined in the [Azure Terraform Provider Documentation](https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs#authenticating-to-azure).

modules/terraform-cdp-aws-pre-reqs/data.tf

@@ -70,16 +70,3 @@ data "http" "datalake_backup_policy_doc" {
 data "http" "datalake_restore_policy_doc" {
   url = "https://raw.githubusercontent.com/hortonworks/cloudbreak/master/cloud-aws-cloudformation/src/main/resources/definitions/aws-datalake-restore-policy.json"
 }
-
-# Use the cdp cli to determin the 
-data "external" "cdpcli" {
-
-  count = var.lookup_cdp_account_ids == true ? 1 : 0
-
-  program = ["bash", "${path.module}/run_cdp_get_cred_prereqs.sh"]
-  query = {
-    infra_type  = var.infra_type
-    cdp_profile = var.cdp_profile
-    cdp_region  = var.cdp_control_plane_region
-  }
-}

modules/terraform-cdp-aws-pre-reqs/defaults.tf

@@ -30,8 +30,9 @@ locals {
   public_route_table_ids  = (var.create_vpc ? module.aws_cdp_vpc[0].public_route_tables : null)
   private_route_table_ids = (var.create_vpc ? module.aws_cdp_vpc[0].private_route_tables : null)
 
+  # If we create the vpc and have private deployment template public subnets are always empty
   public_subnet_ids = (var.create_vpc ?
-  module.aws_cdp_vpc[0].public_subnets : var.cdp_public_subnet_ids)
+  (var.deployment_template == "private" ? [] : module.aws_cdp_vpc[0].public_subnets) : var.cdp_public_subnet_ids)
 
   private_subnet_ids = (var.create_vpc ?
     module.aws_cdp_vpc[0].private_subnets : var.cdp_private_subnet_ids
@@ -111,7 +112,7 @@ locals {
     replace(
       replace(
       data.http.ranger_audit_s3_policy_doc.response_body, "$${ARN_PARTITION}", "aws"),
-    "$${STORAGE_LOCATION_BASE}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}"),
+    "$${STORAGE_LOCATION_BASE}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}/${replace(local.data_storage.data_storage_object, "/", "")}"),
   "$${DATALAKE_BUCKET}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}")
 
   # ...then assign either input or downloaded policy doc to var used in resource
@@ -125,7 +126,7 @@ locals {
   datalake_admin_s3_policy_doc_processed = replace(
     replace(
     data.http.datalake_admin_s3_policy_doc.response_body, "$${ARN_PARTITION}", "aws"),
-  "$${STORAGE_LOCATION_BASE}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}")
+  "$${STORAGE_LOCATION_BASE}", "${local.data_storage.data_storage_bucket}${local.storage_suffix}/${replace(local.data_storage.data_storage_object, "/", "")}")
 
   # ...then assign either input or downloaded policy doc to var used in resource
   datalake_admin_s3_policy_doc = coalesce(var.datalake_admin_s3_policy_doc, local.datalake_admin_s3_policy_doc_processed)
@@ -151,7 +152,7 @@ locals {
   datalake_backup_policy_doc_processed = replace(
     replace(
     data.http.datalake_backup_policy_doc.response_body, "$${ARN_PARTITION}", "aws"),
-  "$${BACKUP_LOCATION_BASE}", "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}")
+  "$${BACKUP_LOCATION_BASE}", "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${replace(local.backup_storage.backup_storage_object, "/", "")}")
 
   # ...then assign either input or downloaded policy doc to var used in resource
   datalake_backup_policy_doc = coalesce(var.datalake_backup_policy_doc, local.datalake_backup_policy_doc_processed)
@@ -164,18 +165,14 @@ locals {
   datalake_restore_policy_doc_processed = replace(
     replace(
     data.http.datalake_restore_policy_doc.response_body, "$${ARN_PARTITION}", "aws"),
-  "$${BACKUP_LOCATION_BASE}", "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}")
+  "$${BACKUP_LOCATION_BASE}", "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${replace(local.backup_storage.backup_storage_object, "/", "")}")
 
   # ...then assign either input or downloaded policy doc to var used in resource
   datalake_restore_policy_doc = coalesce(var.datalake_restore_policy_doc, local.datalake_restore_policy_doc_processed)
 
   # ------- Roles -------
   xaccount_role_name = coalesce(var.xaccount_role_name, "${var.env_prefix}-xaccount-role")
 
-  xaccount_account_id = coalesce(var.xaccount_account_id, var.lookup_cdp_account_ids ? data.external.cdpcli[0].result.account_id : null)
-
-  xaccount_external_id = coalesce(var.xaccount_external_id, var.lookup_cdp_account_ids ? data.external.cdpcli[0].result.external_id : null)
-
   idbroker_role_name = coalesce(var.idbroker_role_name, "${var.env_prefix}-idbroker-role")
 
   log_role_name = coalesce(var.log_role_name, "${var.env_prefix}-logs-role")

modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf

@@ -27,4 +27,20 @@ module "ex01_minimal_inputs" {
 
   ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports
 
+  # Using CDP TF Provider cred pre-reqs data source for values of xaccount account_id and external_id
+  xaccount_account_id  = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id
+  xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id
+
+}
+
+# Use the CDP Terraform Provider to find the xaccount account and external ids
+terraform {
+  required_providers {
+    cdp = {
+      source  = "cloudera/cdp"
+      version = "0.1.4-pre"
+    }
+  }
 }
+
+data "cdp_environments_aws_credential_prerequisites" "cdp_prereqs" {}

modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf

@@ -27,6 +27,10 @@ module "ex02_existing_vpc" {
 
   ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports
 
+  # Using CDP TF Provider cred pre-reqs data source for values of xaccount account_id and external_id
+  xaccount_account_id  = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id
+  xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id
+
   create_vpc             = var.create_vpc
   cdp_vpc_id             = aws_vpc.cdp_vpc.id
   cdp_public_subnet_ids  = values(aws_subnet.cdp_public_subnets)[*].id
@@ -41,3 +45,15 @@ module "ex02_existing_vpc" {
   ]
 
 }
+
+# Use the CDP Terraform Provider to find the xaccount account and external ids
+terraform {
+  required_providers {
+    cdp = {
+      source  = "cloudera/cdp"
+      version = "0.1.4-pre"
+    }
+  }
+}
+
+data "cdp_environments_aws_credential_prerequisites" "cdp_prereqs" {}

modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf

@@ -47,4 +47,20 @@ module "ex01_create_keypair" {
 
   ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports
 
+  # Using CDP TF Provider cred pre-reqs data source for values of xaccount account_id and external_id
+  xaccount_account_id  = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.account_id
+  xaccount_external_id = data.cdp_environments_aws_credential_prerequisites.cdp_prereqs.external_id
+
+}
+
+# Use the CDP Terraform Provider to find the xaccount account and external ids
+terraform {
+  required_providers {
+    cdp = {
+      source  = "cloudera/cdp"
+      version = "0.1.4-pre"
+    }
+  }
 }
+
+data "cdp_environments_aws_credential_prerequisites" "cdp_prereqs" {}

modules/terraform-cdp-aws-pre-reqs/main.tf

@@ -20,10 +20,11 @@ module "aws_cdp_vpc" {
 
   source = "./modules/vpc"
 
-  deployment_template = var.deployment_template
-  vpc_cidr            = var.vpc_cidr
-  env_prefix          = var.env_prefix
-  tags                = local.env_tags
+  deployment_template        = var.deployment_template
+  vpc_cidr                   = var.vpc_cidr
+  private_network_extensions = var.private_network_extensions
+  env_prefix                 = var.env_prefix
+  tags                       = local.env_tags
 
 }
 
@@ -135,18 +136,19 @@ resource "aws_s3_bucket" "cdp_storage_locations" {
 }
 
 # ------- AWS Buckets directory structures -------
-# Data Storage Objects
-resource "aws_s3_object" "cdp_data_storage_object" {
+# # Data Storage Objects
+# NOTE: Removing creation of the data storage object because CDP overrides this
+# resource "aws_s3_object" "cdp_data_storage_object" {
 
-  bucket = "${local.data_storage.data_storage_bucket}${local.storage_suffix}"
+#   bucket = "${local.data_storage.data_storage_bucket}${local.storage_suffix}"
 
-  key          = local.data_storage.data_storage_object
-  content_type = "application/x-directory"
+#   key          = local.data_storage.data_storage_object
+#   content_type = "application/x-directory"
 
-  depends_on = [
-    aws_s3_bucket.cdp_storage_locations
-  ]
-}
+#   depends_on = [
+#     aws_s3_bucket.cdp_storage_locations
+#   ]
+# }
 
 # Log Storage Objects
 resource "aws_s3_object" "cdp_log_storage_object" {
@@ -282,14 +284,14 @@ data "aws_iam_policy_document" "cdp_xaccount_role_policy_doc" {
 
     principals {
       type        = "AWS"
-      identifiers = ["arn:aws:iam::${local.xaccount_account_id}:root"]
+      identifiers = ["arn:aws:iam::${var.xaccount_account_id}:root"]
     }
 
     condition {
       test     = "StringEquals"
       variable = "sts:ExternalId"
 
-      values = [local.xaccount_external_id]
+      values = [var.xaccount_external_id]
     }
   }
 }
@@ -310,6 +312,13 @@ resource "aws_iam_role_policy_attachment" "cdp_xaccount_role_attach" {
   policy_arn = aws_iam_policy.cdp_xaccount_policy.arn
 }
 
+# Wait for propagation of IAM xaccount role.
+# Required for CDP credential
+resource "time_sleep" "iam_propagation" {
+  depends_on      = [aws_iam_role.cdp_xaccount_role]
+  create_duration = "45s"
+}
+
 # ------- AWS Service Roles - CDP IDBroker -------
 # First create the Assume role policy document
 data "aws_iam_policy_document" "cdp_idbroker_role_policy_doc" {

modules/terraform-cdp-aws-pre-reqs/modules/vpc/defaults.tf

@@ -21,8 +21,8 @@ locals {
 
   # ------- Determine subnet details from inputs -------
   subnets_required = {
-    total   = (var.deployment_template == "public") ? length(local.zones_in_region) : 2 * length(local.zones_in_region)
-    public  = length(local.zones_in_region)
+    total   = contains(["public", "private"], var.deployment_template) ? length(local.zones_in_region) : 2 * length(local.zones_in_region)
+    public  = (var.deployment_template == "private") ? (var.private_network_extensions ? 1 : 0) : length(local.zones_in_region)
     private = (var.deployment_template == "public") ? 0 : length(local.zones_in_region)
   }
 }

modules/terraform-cdp-aws-pre-reqs/modules/vpc/main.tf

@@ -23,7 +23,7 @@ module "cdp_vpc" {
   private_subnets = (local.subnets_required.private == 0 ?
     [] :
     [
-      for k, v in local.zones_in_region : cidrsubnet(var.vpc_cidr, ceil(log(local.subnets_required.total, 2)), local.subnets_required.public + k)
+      for i in range(local.subnets_required.private) : cidrsubnet(var.vpc_cidr, ceil(log(local.subnets_required.total, 2)), local.subnets_required.public + i)
     ]
   )
   private_subnet_tags = {
@@ -33,15 +33,16 @@ module "cdp_vpc" {
   public_subnets = (local.subnets_required.public == 0 ?
     [] :
     [
-      for k, v in local.zones_in_region : cidrsubnet(var.vpc_cidr, ceil(log(local.subnets_required.total, 2)), k)
+      for i in range(local.subnets_required.public) : cidrsubnet(var.vpc_cidr, ceil(log(local.subnets_required.total, 2)), i)
     ]
   )
 
   public_subnet_tags = {
     "kubernetes.io/role/elb" = "1"
   }
 
-  enable_nat_gateway   = true
+  enable_nat_gateway   = (var.deployment_template == "private") ? (var.private_network_extensions ? true : false) : true
+  single_nat_gateway   = (var.deployment_template == "private") ? (var.private_network_extensions ? true : false) : false
   enable_dns_support   = true
   enable_dns_hostnames = true
 

modules/terraform-cdp-aws-pre-reqs/modules/vpc/provider.tf

@@ -16,9 +16,9 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 4.0"
+      version = "4.67.0"
     }
   }
 
-  required_version = "> 1.3.0"
+  required_version = ">= 1.3.0"
 }

modules/terraform-cdp-aws-pre-reqs/modules/vpc/variables.tf

@@ -39,3 +39,10 @@ variable "deployment_template" {
     error_message = "Valid values for var: deployment_template are (public, semi-private, private)."
   }
 }
+
+variable "private_network_extensions" {
+  type = bool
+
+  description = "Enable creation of resources for connectivity to CDP Control Plane (public subnet and NAT Gateway) for Private Deployment. Only relevant for private deployment template."
+
+}

modules/terraform-cdp-aws-pre-reqs/outputs.tf

@@ -18,19 +18,6 @@ output "tags" {
   description = "Tags associated with the environment and its resources"
 }
 
-# CDP settings
-output "cdp_profile" {
-  value = var.cdp_profile
-
-  description = "Profile for CDP credentials"
-}
-
-output "cdp_control_plane_region" {
-  value = var.cdp_control_plane_region
-
-  description = "CDP Control Plane region"
-}
-
 # CSP settings
 output "aws_region" {
   value = var.aws_region
@@ -81,18 +68,36 @@ output "aws_vpc_subnets" {
   description = "List of subnets associated with the CDP VPC"
 }
 
+output "aws_data_storage_bucket" {
+  value = "${local.data_storage.data_storage_bucket}${local.storage_suffix}"
+
+  description = "AWS data storage bucket"
+}
+
 output "aws_data_storage_location" {
   value = "s3a://${local.data_storage.data_storage_bucket}${local.storage_suffix}/${local.data_storage.data_storage_object}"
 
   description = "AWS data storage location"
 }
 
+output "aws_log_storage_bucket" {
+  value = "${local.log_storage.log_storage_bucket}${local.storage_suffix}"
+
+  description = "AWS log storage bucket"
+}
+
 output "aws_log_storage_location" {
   value = "s3a://${local.log_storage.log_storage_bucket}${local.storage_suffix}/${local.log_storage.log_storage_object}"
 
   description = "AWS log storage location"
 }
 
+output "aws_backup_storage_bucket" {
+  value = "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}"
+
+  description = "AWS backup storage bucket"
+}
+
 output "aws_backup_storage_location" {
   value = "s3a://${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${local.backup_storage.backup_storage_object}"