diff --git a/README.md b/README.md index 05e16f9..7ea8cdd 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,27 @@ # Terraform Modules for CDP Prerequisites -This module contains resource files and example variable definition files for creation of the pre-requisite Cloud resources on AWS and optional deployment of Cloudera Data Platform (CDP) Public Cloud. +This repository contains a number of Terraform modules for creation of the pre-requisite Cloud resources on AWS and Azure and the deployment of Cloudera Data Platform (CDP) Public Cloud. -## Usage - -The [examples](./modules/terraform-cdp-aws-pre-reqs/examples) directory has example AWS Cloud Service Provider deployments for different scenarios: +## Modules -* `ex01-minimal-inputs` uses the minimum set of inputs for the module. +* For AWS pre-requisite Cloud resources see [terraform-cdp-aws-prereqs](modules/terraform-cdp-aws-pre-reqs/README.md). +* For Azure pre-requisite Cloud resources see [terraform-cdp-azure-prereqs](modules/terraform-cdp-azure-pre-reqs/README.md). +* For deployment of CDP on Azure or AWS see [terraform-cdp-deploy](modules/terraform-cdp-deploy/README.md). -* `ex02-existing-vpc` creates a VPC and subnets outside of the module and passes this as an additional input. CDP deployment then uses these network assets. +Each module contains Terraform resource configuration and example variable definition files. -* `ex03-create-keypair` creates the AWS EC2 Keypair in the module caller and passes this as an additional input. +## Usage -* TODO: `ex04-all_inputs_specified` contains an example with all input parameters for the module. +The [cdp-tf-quickstarts](https://github.com/cloudera-labs/cdp-tf-quickstarts) repository demonstrates how to use the modules together to deploy CDP on different cloud environments. -In each directory an example `terraform.tfvars.sample` values file is included to show input variable values. +Each module also has a set of examples to show different configuration options for that module. ## Deployment ### Create infrastructure +Note that the instructions below give the steps to create pre-requisite resources and the CDP deployment all together. The modules can be used on their own to allow further customization. + 1. Clone this repository using the following commands: ```bash @@ -27,10 +29,10 @@ git clone https://github.com/cloudera-labs/terraform-cdp-modules.git cd terraform-cdp-modules ``` -2. Choose one of the deployment types in the [examples](./modules/terraform-cdp-aws-pre-reqs/examples) directory and change to this directory. +2. To create cloud pre-requisite resources and the CDP deployment all together, change to the [terraform-cdp-deploy directory](./modules/terraform-cdp-deploy) and select one of the cloud providers. ```bash -cd modules/terraform-cdp-aws-pre-reqs/examples/ex/ +cd modules/terraform-cdp-deploy/examples/ex/ ``` 3. Create a `terraform.tfvars` file with variable definitions to run the module. Reference the `terraform.tfvars.sample` file in each example folder to create this file. @@ -73,7 +75,7 @@ pip install ansible-core==2.12.10 jmespath==1.0.1 pip install git+https://github.com/cloudera-labs/cdpy@main#egg=cdpy # Install the cloudera.cloud Ansible Collection -ansible-galaxy collection install git+https://github.com/cloudera-labs/cloudera.cloud.git +ansible-galaxy collection install git+https://github.com/cloudera-labs/cloudera.cloud.git,devel # Install the community.general Ansible Collection ansible-galaxy collection install community.general:==5.5.0 @@ -83,7 +85,3 @@ cdp configure ``` NOTE - See the [CDP documentation for steps to Generate the API access key](https://docs.cloudera.com/cdp-public-cloud/cloud/cli/topics/mc-cli-generating-an-api-access-key.html) required in the `cdp configure` command above. - -## Modules - -See [terraform-cdp-aws-prereqs](modules/terraform-cdp-aws-pre-reqs/README.md). diff --git a/modules/terraform-cdp-aws-pre-reqs/README.md b/modules/terraform-cdp-aws-pre-reqs/README.md index 9281e6e..e665da1 100644 --- a/modules/terraform-cdp-aws-pre-reqs/README.md +++ b/modules/terraform-cdp-aws-pre-reqs/README.md @@ -1,5 +1,19 @@ -# Terraform Module for CDP Prerequisites +# Terraform Module for CDP Prerequisites on AWS + +This module contains resource files and example variable definition files for creation of the pre-requisite AWS cloud resources required for Cloudera Data Platform (CDP) Public Cloud. + +## Usage + +The [examples](./examples) directory has example AWS Cloud Service Provider deployments for different scenarios: + +* `ex01-minimal-inputs` uses the minimum set of inputs for the module. + +* `ex02-existing-vpc` creates a VPC and subnets outside of the module and passes this as an additional input. CDP deployment then uses these network assets. + +* `ex03-create-keypair` creates the AWS EC2 Keypair in the module caller and passes this as an additional input. + +In each directory an example `terraform.tfvars.sample` values file is included to show input variable values. ## Requirements @@ -9,19 +23,15 @@ | [aws](#requirement\_aws) | ~> 4.0 | | [external](#requirement\_external) | 2.3.1 | | [http](#requirement\_http) | 3.2.1 | -| [local](#requirement\_local) | 2.2.3 | -| [null](#requirement\_null) | 3.2.1 | | [random](#requirement\_random) | 3.4.3 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | 4.67.0 | +| [aws](#provider\_aws) | ~> 4.0 | | [external](#provider\_external) | 2.3.1 | | [http](#provider\_http) | 3.2.1 | -| [local](#provider\_local) | 2.2.3 | -| [null](#provider\_null) | 3.2.1 | | [random](#provider\_random) | 3.4.3 | ## Modules @@ -77,8 +87,6 @@ | [aws_security_group_rule.cdp_knox_sg_egress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.cdp_knox_sg_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.cdp_knox_sg_ingress_self](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [local_file.cdp_deployment_template](https://registry.terraform.io/providers/hashicorp/local/2.2.3/docs/resources/file) | resource | -| [null_resource.cdp_deployment](https://registry.terraform.io/providers/hashicorp/null/3.2.1/docs/resources/resource) | resource | | [random_id.bucket_suffix](https://registry.terraform.io/providers/hashicorp/random/3.4.3/docs/resources/id) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.cdp_datalake_admin_role_policy_doc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | @@ -102,7 +110,6 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [aws\_key\_pair](#input\_aws\_key\_pair) | Name of the Public SSH key for the CDP environment | `string` | n/a | yes | | [deployment\_template](#input\_deployment\_template) | Deployment Pattern to use for Cloud resources and CDP | `string` | n/a | yes | | [env\_prefix](#input\_env\_prefix) | Shorthand name for the environment. Used in resource descriptions | `string` | n/a | yes | | [agent\_source\_tag](#input\_agent\_source\_tag) | Tag to identify deployment source | `map(any)` | 
{
  "agent_source": "tf-cdp-module"
}
| no | @@ -126,13 +133,7 @@ | [datalake\_backup\_policy\_name](#input\_datalake\_backup\_policy\_name) | Datalake backup Data Access Policy Name | `string` | `null` | no | | [datalake\_restore\_policy\_doc](#input\_datalake\_restore\_policy\_doc) | Location of Datalake Restore Data Access Policy | `string` | `null` | no | | [datalake\_restore\_policy\_name](#input\_datalake\_restore\_policy\_name) | Datalake restore Data Access Policy Name | `string` | `null` | no | -| [datalake\_scale](#input\_datalake\_scale) | The scale of the datalake. Valid values are LIGHT\_DUTY, MEDIUM\_DUTY\_HA. | `string` | `null` | no | -| [datalake\_version](#input\_datalake\_version) | The Datalake Runtime version. Valid values are semantic versions, e.g. 7.2.16 | `string` | `"7.2.16"` | no | -| [deploy\_cdp](#input\_deploy\_cdp) | Deploy the CDP environment as part of Terraform | `bool` | `true` | no | -| [enable\_ccm\_tunnel](#input\_enable\_ccm\_tunnel) | Flag to enable Cluster Connectivity Manager tunnel. If false then access from Cloud to CDP Control Plane CIDRs is required from via SG ingress | `bool` | `true` | no | -| [enable\_raz](#input\_enable\_raz) | Flag to enable Ranger Authorization Service (RAZ) | `bool` | `true` | no | | [env\_tags](#input\_env\_tags) | Tags applied to provised resources | `map(any)` | `null` | no | -| [freeipa\_instances](#input\_freeipa\_instances) | The number of FreeIPA instances to create in the environment | `number` | `2` | no | | [idbroker\_policy\_name](#input\_idbroker\_policy\_name) | IDBroker Policy name | `string` | `null` | no | | [idbroker\_role\_name](#input\_idbroker\_role\_name) | IDBroker service role Name | `string` | `null` | no | | [infra\_type](#input\_infra\_type) | Cloud Provider to deploy CDP. | `string` | `"aws"` | no | @@ -142,7 +143,6 @@ | [log\_role\_name](#input\_log\_role\_name) | Log service role Name | `string` | `null` | no | | [log\_storage](#input\_log\_storage) | Optional log locations for CDP environment. If not provided follow the data\_storage variable | 
object({
    log_storage_bucket = string
    log_storage_object = string
  })
| `null` | no | | [lookup\_cdp\_account\_ids](#input\_lookup\_cdp\_account\_ids) | Auto lookup CDP Account and External ID using CDP CLI commands. If false then the xaccount\_account\_id and xaccount\_external\_id input variables need to be specified | `bool` | `true` | no | -| [multiaz](#input\_multiaz) | Flag to specify that the FreeIPA and DataLake instances will be deployed across multi-availability zones | `bool` | `true` | no | | [random\_id\_for\_bucket](#input\_random\_id\_for\_bucket) | Create a random suffix for the bucket names | `bool` | `true` | no | | [ranger\_audit\_role\_name](#input\_ranger\_audit\_role\_name) | Ranger Audit role Name | `string` | `null` | no | | [ranger\_audit\_s3\_policy\_doc](#input\_ranger\_audit\_s3\_policy\_doc) | Location or Contents of Ranger S3 Audit Data Access Policy | `string` | `null` | no | @@ -150,7 +150,6 @@ | [security\_group\_default\_name](#input\_security\_group\_default\_name) | Default Security Group for CDP environment | `string` | `null` | no | | [security\_group\_knox\_name](#input\_security\_group\_knox\_name) | Knox Security Group for CDP environment | `string` | `null` | no | | [vpc\_cidr](#input\_vpc\_cidr) | VPC CIDR Block | `string` | `"10.10.0.0/16"` | no | -| [workload\_analytics](#input\_workload\_analytics) | Flag to specify if workload analytics should be enabled for the CDP environment | `bool` | `true` | no | | [xaccount\_account\_id](#input\_xaccount\_account\_id) | Account ID of the cross account | `string` | `null` | no | | [xaccount\_account\_policy\_doc](#input\_xaccount\_account\_policy\_doc) | Location of cross acount policy document | `string` | `null` | no | | [xaccount\_external\_id](#input\_xaccount\_external\_id) | External ID of the cross account | `string` | `null` | no | @@ -161,12 +160,13 @@ | Name | Description | |------|-------------| -| [aws\_backup\_location](#output\_aws\_backup\_location) | AWS backup storage location | +| [aws\_backup\_storage\_location](#output\_aws\_backup\_storage\_location) | AWS backup storage location | +| [aws\_data\_storage\_location](#output\_aws\_data\_storage\_location) | AWS data storage location | | [aws\_datalake\_admin\_role\_arn](#output\_aws\_datalake\_admin\_role\_arn) | Datalake Admin role ARN | | [aws\_default\_route\_table\_id](#output\_aws\_default\_route\_table\_id) | AWS default route table ID | | [aws\_idbroker\_instance\_profile\_arn](#output\_aws\_idbroker\_instance\_profile\_arn) | IDBroker instance profile ARN | | [aws\_log\_instance\_profile\_arn](#output\_aws\_log\_instance\_profile\_arn) | Log instance profile ARN | -| [aws\_log\_location](#output\_aws\_log\_location) | AWS log storage location | +| [aws\_log\_storage\_location](#output\_aws\_log\_storage\_location) | AWS log storage location | | [aws\_private\_route\_table\_ids](#output\_aws\_private\_route\_table\_ids) | AWS private route table IDs | | [aws\_private\_subnet\_ids](#output\_aws\_private\_subnet\_ids) | AWS private subnet IDs | | [aws\_public\_route\_table\_ids](#output\_aws\_public\_route\_table\_ids) | AWS public route table IDs | @@ -175,24 +175,10 @@ | [aws\_region](#output\_aws\_region) | Cloud provider region of the Environment | | [aws\_security\_group\_default\_id](#output\_aws\_security\_group\_default\_id) | AWS security group id for default CDP SG | | [aws\_security\_group\_knox\_id](#output\_aws\_security\_group\_knox\_id) | AWS security group id for Knox CDP SG | -| [aws\_storage\_location](#output\_aws\_storage\_location) | AWS data storage location | | [aws\_vpc\_id](#output\_aws\_vpc\_id) | AWS VPC ID | | [aws\_vpc\_subnets](#output\_aws\_vpc\_subnets) | List of subnets associated with the CDP VPC | | [aws\_xaccount\_role\_arn](#output\_aws\_xaccount\_role\_arn) | Cross Account role ARN | | [cdp\_control\_plane\_region](#output\_cdp\_control\_plane\_region) | CDP Control Plane region | -| [cdp\_datalake\_name](#output\_cdp\_datalake\_name) | CDP Datalake name | -| [cdp\_enable\_multiaz](#output\_cdp\_enable\_multiaz) | Flag to specify if multi-AZ deployment is enabled for the CDP environment | -| [cdp\_enable\_raz](#output\_cdp\_enable\_raz) | Flag to enable Ranger Authorization Service (RAZ) for the CDP environment | -| [cdp\_endpoint\_access\_scheme](#output\_cdp\_endpoint\_access\_scheme) | The scheme for the workload endpoint gateway. `PUBLIC` creates an external endpoint that can be accessed over the Internet. `PRIVATE` restricts the traffic to be internal to the VPC / Vnet. Relevant in Private Networks. | -| [cdp\_env\_name](#output\_cdp\_env\_name) | CDP environment name | -| [cdp\_freeipa\_instances](#output\_cdp\_freeipa\_instances) | Number of instances for the FreeIPA service of the environment | -| [cdp\_iam\_admin\_group\_name](#output\_cdp\_iam\_admin\_group\_name) | CDP IAM admin group name | -| [cdp\_iam\_user\_group\_name](#output\_cdp\_iam\_user\_group\_name) | CDP IAM user group name | | [cdp\_profile](#output\_cdp\_profile) | Profile for CDP credentials | -| [cdp\_tunnel\_enabled](#output\_cdp\_tunnel\_enabled) | Flag to enable SSH tunnelling for the CDP environment | -| [cdp\_workload\_analytics](#output\_cdp\_workload\_analytics) | Flag to enable Workload Analytics | -| [cdp\_xacccount\_credential\_name](#output\_cdp\_xacccount\_credential\_name) | Cross Account credential name | -| [infra\_type](#output\_infra\_type) | Cloud Service Provider type | -| [public\_key\_id](#output\_public\_key\_id) | Keypair name in Cloud Service Provider | | [tags](#output\_tags) | Tags associated with the environment and its resources | \ No newline at end of file diff --git a/modules/terraform-cdp-aws-pre-reqs/cdp_deploy.tf b/modules/terraform-cdp-aws-pre-reqs/cdp_deploy.tf deleted file mode 100755 index 680e4bc..0000000 --- a/modules/terraform-cdp-aws-pre-reqs/cdp_deploy.tf +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2023 Cloudera, Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Deployment and creation of CDP resources using Ansible Playbook called by TF local-exec - -# ------- Create Configuration file for CDP Deployment via Ansible ------- -resource "local_file" "cdp_deployment_template" { - - count = var.deploy_cdp == true ? 1 : 0 - - content = templatefile("${path.module}/templates/cdp_config.yml.tpl", { - # CDP environment & DL settings - plat__env_name = "${var.env_prefix}-cdp-env" - plat__datalake_name = "${var.env_prefix}-aws-dl" - plat__datalake_scale = local.datalake_scale - plat__datalake_version = var.datalake_version - plat__xacccount_credential_name = "${var.env_prefix}-xaccount-cred" - plat__cdp_iam_admin_group_name = "${var.env_prefix}-cdp-admin-group" - plat__cdp_iam_user_group_name = "${var.env_prefix}-cdp-user-group" - plat__tunnel = var.enable_ccm_tunnel - plat__endpoint_access_scheme = (var.deployment_template == "semi-private") ? "PUBLIC" : "PRIVATE" - plat__enable_raz = var.enable_raz - plat__env_multiaz = var.multiaz - plat__env_freeipa_instances = var.freeipa_instances - plat__workload_analytics = var.workload_analytics - plat__tags = jsonencode(local.env_tags) - - # CDP settings - plat__cdp_profile = var.cdp_profile - plat__cdp_control_plane_region = var.cdp_control_plane_region - - # CSP settings - plat__infra_type = var.infra_type - plat__region = var.aws_region - - plat__aws_vpc_id = local.vpc_id - plat__aws_public_subnet_ids = jsonencode(local.public_subnet_ids) - plat__aws_private_subnet_ids = jsonencode(local.private_subnet_ids) - plat__aws_subnets_for_cdp = (var.deployment_template == "public") ? jsonencode(concat(local.public_subnet_ids, local.private_subnet_ids)) : jsonencode(local.private_subnet_ids) - - plat__aws_storage_location = "s3a://${local.data_storage.data_storage_bucket}${local.storage_suffix}/${local.data_storage.data_storage_object}" - plat__aws_log_location = "s3a://${local.log_storage.log_storage_bucket}${local.storage_suffix}/${local.log_storage.log_storage_object}" - plat__aws_backup_location = "s3a://${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${local.backup_storage.backup_storage_object}" - - plat__public_key_id = var.aws_key_pair - plat__aws_security_group_default_id = aws_security_group.cdp_default_sg.id - plat__aws_security_group_knox_id = aws_security_group.cdp_knox_sg.id - - plat__aws_datalake_admin_role_arn = aws_iam_role.cdp_datalake_admin_role.arn - plat__aws_ranger_audit_role_arn = aws_iam_role.cdp_ranger_audit_role.arn - plat__aws_xaccount_role_arn = aws_iam_role.cdp_xaccount_role.arn - - plat__aws_log_instance_profile_arn = aws_iam_instance_profile.cdp_log_role_instance_profile.arn - plat__aws_idbroker_instance_profile_arn = aws_iam_instance_profile.cdp_idbroker_role_instance_profile.arn - } - ) - filename = "cdp_config.yml" -} - -# ------- Create CDP Deployment ------- -resource "null_resource" "cdp_deployment" { - - count = var.deploy_cdp == true ? 1 : 0 - - # Setup of CDP environment using playbook_setup_cdp.yml.yml Ansible Playbook - provisioner "local-exec" { - command = "ansible-playbook -vvv -e '@cdp_config.yml' ${path.module}/playbook_setup_cdp.yml" - } - - # Deletion of CDP environment using playbook_teardown_cdp.yml Ansible Playbook - provisioner "local-exec" { - when = destroy - command = "ansible-playbook -vvv -e '@cdp_config.yml' ${path.module}/playbook_teardown_cdp.yml" - } - - # Depends on * resources to ensure CDP environment is setup/deleted after/before all pre-reqs. - # TODO: Need to investigate further to see if this list can be trimmed. - depends_on = [ - local_file.cdp_deployment_template, - module.aws_cdp_vpc, - aws_security_group.cdp_default_sg, - aws_security_group.cdp_knox_sg, - random_id.bucket_suffix, - aws_s3_bucket.cdp_storage_locations, - aws_s3_object.cdp_data_storage_object, - aws_s3_object.cdp_log_storage_object, - aws_iam_policy.cdp_xaccount_policy, - data.aws_iam_policy_document.cdp_idbroker_policy_doc, - aws_iam_policy.cdp_idbroker_policy, - aws_iam_policy.cdp_log_data_access_policy, - aws_iam_policy.cdp_ranger_audit_s3_data_access_policy, - aws_iam_policy.cdp_datalake_admin_s3_data_access_policy, - aws_iam_policy.cdp_bucket_data_access_policy, - data.aws_iam_policy_document.cdp_xaccount_role_policy_doc, - aws_iam_role.cdp_xaccount_role, - aws_iam_role_policy_attachment.cdp_xaccount_role_attach, - data.aws_iam_policy_document.cdp_idbroker_role_policy_doc, - aws_iam_role.cdp_idbroker_role, - aws_iam_instance_profile.cdp_idbroker_role_instance_profile, - aws_iam_role_policy_attachment.cdp_idbroker_role_attach1, - aws_iam_role_policy_attachment.cdp_idbroker_role_attach2, - data.aws_iam_policy_document.cdp_log_role_policy_doc, - aws_iam_role.cdp_log_role, - aws_iam_instance_profile.cdp_log_role_instance_profile, - aws_iam_role_policy_attachment.cdp_log_role_attach1, - aws_iam_role_policy_attachment.cdp_log_role_attach2, - data.aws_iam_policy_document.cdp_datalake_admin_role_policy_doc, - aws_iam_role.cdp_datalake_admin_role, - aws_iam_instance_profile.cdp_datalake_admin_role_instance_profile, - aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach1, - aws_iam_role_policy_attachment.cdp_datalake_admin_role_attach2, - data.aws_iam_policy_document.cdp_ranger_audit_role_policy_doc, - aws_iam_role.cdp_ranger_audit_role, - aws_iam_instance_profile.cdp_ranger_audit_role_instance_profile, - aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach1, - aws_iam_role_policy_attachment.cdp_ranger_audit_role_attach2 - ] -} diff --git a/modules/terraform-cdp-aws-pre-reqs/defaults.tf b/modules/terraform-cdp-aws-pre-reqs/defaults.tf index 6a07d12..03de691 100644 --- a/modules/terraform-cdp-aws-pre-reqs/defaults.tf +++ b/modules/terraform-cdp-aws-pre-reqs/defaults.tf @@ -22,14 +22,6 @@ locals { caller_account_id = data.aws_caller_identity.current.account_id - # ------- CDP Environment Deployment ------- - datalake_scale = coalesce( - var.datalake_scale, - (var.deployment_template == "public" ? - "LIGHT_DUTY" : "MEDIUM_DUTY_HA" - ) - ) - # ------- Network Resources ------- vpc_id = (var.create_vpc ? module.aws_cdp_vpc[0].vpc_id : var.cdp_vpc_id) diff --git a/modules/terraform-cdp-aws-pre-reqs/doc_fragments/header.md b/modules/terraform-cdp-aws-pre-reqs/doc_fragments/header.md index 76ca68c..a797ec4 100644 --- a/modules/terraform-cdp-aws-pre-reqs/doc_fragments/header.md +++ b/modules/terraform-cdp-aws-pre-reqs/doc_fragments/header.md @@ -1,2 +1,15 @@ -# Terraform Module for CDP Prerequisites +# Terraform Module for CDP Prerequisites on AWS +This module contains resource files and example variable definition files for creation of the pre-requisite AWS cloud resources required for Cloudera Data Platform (CDP) Public Cloud. + +## Usage + +The [examples](./examples) directory has example AWS Cloud Service Provider deployments for different scenarios: + +* `ex01-minimal-inputs` uses the minimum set of inputs for the module. + +* `ex02-existing-vpc` creates a VPC and subnets outside of the module and passes this as an additional input. CDP deployment then uses these network assets. + +* `ex03-create-keypair` creates the AWS EC2 Keypair in the module caller and passes this as an additional input. + +In each directory an example `terraform.tfvars.sample` values file is included to show input variable values. diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf index 69e6a95..a58be05 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf @@ -23,8 +23,6 @@ module "ex01_minimal_inputs" { env_prefix = var.env_prefix aws_region = var.aws_region - aws_key_pair = var.aws_key_pair - deployment_template = var.deployment_template ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/terraform.tfvars.sample b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/terraform.tfvars.sample index e63853d..01ef607 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/terraform.tfvars.sample +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/terraform.tfvars.sample @@ -17,7 +17,6 @@ env_prefix = "" # Required name prefix for cloud and CDP resources, # ------- Cloud Settings ------- aws_region = "" # Change this to specify Cloud Provider region, e.g. eu-west-1 -aws_key_pair = "" # Change this with the name of a pre-existing AWS keypair, e.g. my-keypair # ------- CDP Environment Deployment ------- deployment_template = "" # Specify the deployment pattern below. Options are public, semi-private or private diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/variables.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/variables.tf index 17e3fcc..aa021db 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/variables.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/variables.tf @@ -31,13 +31,6 @@ variable "env_prefix" { description = "Shorthand name for the environment. Used in resource descriptions" } -variable "aws_key_pair" { - type = string - - description = "Name of the Public SSH key for the CDP environment" - -} - # ------- CDP Environment Deployment ------- variable "deployment_template" { type = string diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf index 481ed66..115c7c9 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf @@ -23,8 +23,6 @@ module "ex02_existing_vpc" { env_prefix = var.env_prefix aws_region = var.aws_region - aws_key_pair = var.aws_key_pair - deployment_template = var.deployment_template ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/terraform.tfvars.sample b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/terraform.tfvars.sample index 44c5c35..aad9f4a 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/terraform.tfvars.sample +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/terraform.tfvars.sample @@ -28,7 +28,6 @@ env_prefix = "" # Required name prefix for cloud and CDP resources, # ------- Cloud Settings ------- aws_region = "" # Change this to specify Cloud Provider region, e.g. eu-west-1 -aws_key_pair = "" # Change this with the name of a pre-existing AWS keypair, e.g. my-keypair # ------- CDP Environment Deployment ------- deployment_template = "" # Specify the deployment pattern below. Options are public, semi-private or private diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/variables.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/variables.tf index 8aeb4b7..ed27ab5 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/variables.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/variables.tf @@ -31,13 +31,6 @@ variable "env_prefix" { description = "Shorthand name for the environment. Used in resource descriptions" } -variable "aws_key_pair" { - type = string - - description = "Name of the Public SSH key for the CDP environment" - -} - # ------- CDP Environment Deployment ------- variable "deployment_template" { type = string diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf index 508c494..13c4506 100644 --- a/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf +++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf @@ -43,8 +43,6 @@ module "ex01_create_keypair" { env_prefix = var.env_prefix aws_region = var.aws_region - aws_key_pair = aws_key_pair.cdp_keypair.key_name - deployment_template = var.deployment_template ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports diff --git a/modules/terraform-cdp-aws-pre-reqs/outputs.tf b/modules/terraform-cdp-aws-pre-reqs/outputs.tf index fd6f394..57dd253 100644 --- a/modules/terraform-cdp-aws-pre-reqs/outputs.tf +++ b/modules/terraform-cdp-aws-pre-reqs/outputs.tf @@ -12,74 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# CDP environment & DL settings -output "cdp_env_name" { - value = "${var.env_prefix}-cdp-env" - - description = "CDP environment name" -} - -output "cdp_datalake_name" { - value = "${var.env_prefix}-aws-dl" - - description = "CDP Datalake name" -} - -output "cdp_xacccount_credential_name" { - value = "${var.env_prefix}-xaccount-cred" - - description = "Cross Account credential name" -} - - -output "cdp_iam_admin_group_name" { - value = "${var.env_prefix}-cdp-admin-group" - - description = "CDP IAM admin group name" -} - -output "cdp_iam_user_group_name" { - value = "${var.env_prefix}-cdp-user-group" - - description = "CDP IAM user group name" -} - -output "cdp_tunnel_enabled" { - value = (var.deployment_template == "public") ? "false" : "true" - - description = "Flag to enable SSH tunnelling for the CDP environment" -} - -output "cdp_endpoint_access_scheme" { - value = (var.deployment_template == "semi-private") ? "PUBLIC" : "PRIVATE" - - description = "The scheme for the workload endpoint gateway. `PUBLIC` creates an external endpoint that can be accessed over the Internet. `PRIVATE` restricts the traffic to be internal to the VPC / Vnet. Relevant in Private Networks." -} - -output "cdp_enable_raz" { - value = var.enable_raz - - description = "Flag to enable Ranger Authorization Service (RAZ) for the CDP environment" -} - -output "cdp_enable_multiaz" { - value = var.multiaz - - description = "Flag to specify if multi-AZ deployment is enabled for the CDP environment" -} - -output "cdp_freeipa_instances" { - value = var.freeipa_instances - - description = "Number of instances for the FreeIPA service of the environment" -} - -output "cdp_workload_analytics" { - value = var.workload_analytics - - description = "Flag to enable Workload Analytics" -} - output "tags" { value = local.env_tags @@ -100,12 +32,6 @@ output "cdp_control_plane_region" { } # CSP settings -output "infra_type" { - value = var.infra_type - - description = "Cloud Service Provider type" -} - output "aws_region" { value = var.aws_region @@ -155,30 +81,24 @@ output "aws_vpc_subnets" { description = "List of subnets associated with the CDP VPC" } -output "aws_storage_location" { +output "aws_data_storage_location" { value = "s3a://${local.data_storage.data_storage_bucket}${local.storage_suffix}/${local.data_storage.data_storage_object}" description = "AWS data storage location" } -output "aws_log_location" { +output "aws_log_storage_location" { value = "s3a://${local.log_storage.log_storage_bucket}${local.storage_suffix}/${local.log_storage.log_storage_object}" description = "AWS log storage location" } -output "aws_backup_location" { +output "aws_backup_storage_location" { value = "s3a://${local.backup_storage.backup_storage_bucket}${local.storage_suffix}/${local.backup_storage.backup_storage_object}" description = "AWS backup storage location" } -output "public_key_id" { - value = var.aws_key_pair - - description = "Keypair name in Cloud Service Provider" -} - output "aws_security_group_default_id" { value = aws_security_group.cdp_default_sg.id diff --git a/modules/terraform-cdp-aws-pre-reqs/provider.tf b/modules/terraform-cdp-aws-pre-reqs/provider.tf index cc4d539..002dbbb 100644 --- a/modules/terraform-cdp-aws-pre-reqs/provider.tf +++ b/modules/terraform-cdp-aws-pre-reqs/provider.tf @@ -18,15 +18,6 @@ terraform { source = "hashicorp/aws" version = "~> 4.0" } - local = { - source = "hashicorp/local" - version = "2.2.3" - } - null = { - source = "hashicorp/null" - version = "3.2.1" - } - http = { source = "hashicorp/http" version = "3.2.1" diff --git a/modules/terraform-cdp-aws-pre-reqs/variables.tf b/modules/terraform-cdp-aws-pre-reqs/variables.tf index 21a4534..0d67cb1 100644 --- a/modules/terraform-cdp-aws-pre-reqs/variables.tf +++ b/modules/terraform-cdp-aws-pre-reqs/variables.tf @@ -51,12 +51,6 @@ variable "env_prefix" { description = "Shorthand name for the environment. Used in resource descriptions" } -variable "aws_key_pair" { - type = string - - description = "Name of the Public SSH key for the CDP environment" -} - # ------- CDP Environment Deployment ------- variable "cdp_profile" { type = string @@ -84,13 +78,6 @@ variable "deployment_template" { error_message = "Valid values for var: deployment_template are (public, semi-private, private)." } } -variable "deploy_cdp" { - type = bool - - description = "Deploy the CDP environment as part of Terraform" - - default = true -} variable "lookup_cdp_account_ids" { type = bool @@ -100,72 +87,14 @@ variable "lookup_cdp_account_ids" { default = true } -variable "enable_ccm_tunnel" { - type = bool - - description = "Flag to enable Cluster Connectivity Manager tunnel. If false then access from Cloud to CDP Control Plane CIDRs is required from via SG ingress" - - default = true -} - -variable "enable_raz" { - type = bool - - description = "Flag to enable Ranger Authorization Service (RAZ)" - - default = true -} - -variable "multiaz" { - type = bool - - description = "Flag to specify that the FreeIPA and DataLake instances will be deployed across multi-availability zones" - - default = true -} - -variable "freeipa_instances" { - type = number - - description = "The number of FreeIPA instances to create in the environment" - - default = 2 -} - -variable "workload_analytics" { - type = bool - - description = "Flag to specify if workload analytics should be enabled for the CDP environment" - - default = true -} - -variable "datalake_scale" { - type = string - - description = "The scale of the datalake. Valid values are LIGHT_DUTY, MEDIUM_DUTY_HA." - - # NOTE: Unable to have validation when we want a default behaviour depending on deployment_template - # validation { - # condition = contains(["LIGHT_DUTY", "MEDIUM_DUTY_HA"], var.datalake_scale) - # error_message = "Valid values for var: datalake_scale are (LIGHT_DUTY, MEDIUM_DUTY_HA)." - # } - - default = null -} - -variable "datalake_version" { - type = string +# variable "enable_raz" { +# type = bool - description = "The Datalake Runtime version. Valid values are semantic versions, e.g. 7.2.16" +# description = "Flag to enable Ranger Authorization Service (RAZ)" - validation { - condition = (var.datalake_version == null ? true : length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0) - error_message = "Valid values for var: datalake_version must match semantic versioning conventions." - } +# default = true +# } - default = "7.2.16" -} # ------- Network Resources ------- variable "create_vpc" { type = bool diff --git a/modules/terraform-cdp-azure-pre-reqs/.terraform-docs.yaml b/modules/terraform-cdp-azure-pre-reqs/.terraform-docs.yaml new file mode 100644 index 0000000..0936036 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/.terraform-docs.yaml @@ -0,0 +1,21 @@ +formatter: markdown +header-from: doc_fragments/header.md +settings: + anchor: true + color: true + default: true + escape: true + html: true + indent: 2 + required: true + sensitive: true + type: true + + +sort: + enabled: true + by: required + +output: + file: README.md + mode: replace \ No newline at end of file diff --git a/modules/terraform-cdp-azure-pre-reqs/README.md b/modules/terraform-cdp-azure-pre-reqs/README.md new file mode 100644 index 0000000..fe8949f --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/README.md @@ -0,0 +1,136 @@ + +# Terraform Module for CDP Prerequisites on Azure + +This module contains resource files and example variable definition files for creation of the pre-requisite Azure cloud resources required for Cloudera Data Platform (CDP) Public Cloud. + +## Usage + +The [examples](./examples) directory has example Azure Cloud Service Provider deployments for different scenarios: + +* `ex01-minimal-inputs` uses the minimum set of inputs for the module. + +In each directory an example `terraform.tfvars.sample` values file is included to show input variable values. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | > 1.3.0 | +| [azuread](#requirement\_azuread) | 2.39.0 | +| [azurerm](#requirement\_azurerm) | 3.45.0 | +| [random](#requirement\_random) | 3.4.3 | + +## Providers + +| Name | Version | +|------|---------| +| [azuread](#provider\_azuread) | 2.39.0 | +| [azurerm](#provider\_azurerm) | 3.45.0 | +| [random](#provider\_random) | 3.4.3 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [azure\_cdp\_vnet](#module\_azure\_cdp\_vnet) | ./modules/vnet | n/a | + +## Resources + +| Name | Type | +|------|------| +| [azuread_application.cdp_xaccount_app](https://registry.terraform.io/providers/hashicorp/azuread/2.39.0/docs/resources/application) | resource | +| [azuread_application_password.cdp_xaccount_app_password](https://registry.terraform.io/providers/hashicorp/azuread/2.39.0/docs/resources/application_password) | resource | +| [azuread_service_principal.cdp_xaccount_app_sp](https://registry.terraform.io/providers/hashicorp/azuread/2.39.0/docs/resources/service_principal) | resource | +| [azurerm_network_security_group.cdp_default_sg](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/network_security_group) | resource | +| [azurerm_network_security_group.cdp_knox_sg](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/network_security_group) | resource | +| [azurerm_network_security_rule.cdp_default_sg_ingress_extra_access](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/network_security_rule) | resource | +| [azurerm_network_security_rule.cdp_knox_sg_ingress_extra_access](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/network_security_rule) | resource | +| [azurerm_resource_group.cdp_rmgp](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/resource_group) | resource | +| [azurerm_role_assignment.cdp_datalake_admin_backup_container_assign](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.cdp_datalake_admin_data_container_assign](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.cdp_datalake_admin_log_container_assign](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.cdp_idbroker_assign](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.cdp_log_data_access_backup_container_assign](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.cdp_log_data_access_log_container_assign](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.cdp_ranger_audit_backup_container_assign](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.cdp_ranger_audit_data_container_assign](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.cdp_ranger_audit_log_container_assign](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.cdp_raz_assign](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/role_assignment) | resource | +| [azurerm_role_assignment.cdp_xaccount_role](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/role_assignment) | resource | +| [azurerm_storage_account.cdp_storage_locations](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/storage_account) | resource | +| [azurerm_storage_container.cdp_backup_storage](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/storage_container) | resource | +| [azurerm_storage_container.cdp_data_storage](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/storage_container) | resource | +| [azurerm_storage_container.cdp_log_storage](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/storage_container) | resource | +| [azurerm_user_assigned_identity.cdp_datalake_admin](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/user_assigned_identity) | resource | +| [azurerm_user_assigned_identity.cdp_idbroker](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/user_assigned_identity) | resource | +| [azurerm_user_assigned_identity.cdp_log_data_access](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/user_assigned_identity) | resource | +| [azurerm_user_assigned_identity.cdp_ranger_audit_data_access](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/user_assigned_identity) | resource | +| [azurerm_user_assigned_identity.cdp_raz](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/resources/user_assigned_identity) | resource | +| [random_id.bucket_suffix](https://registry.terraform.io/providers/hashicorp/random/3.4.3/docs/resources/id) | resource | +| [azuread_client_config.current](https://registry.terraform.io/providers/hashicorp/azuread/2.39.0/docs/data-sources/client_config) | data source | +| [azurerm_subscription.current](https://registry.terraform.io/providers/hashicorp/azurerm/3.45.0/docs/data-sources/subscription) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deployment\_template](#input\_deployment\_template) | Deployment Pattern to use for Cloud resources and CDP | `string` | n/a | yes | +| [env\_prefix](#input\_env\_prefix) | Shorthand name for the environment. Used in resource descriptions | `string` | n/a | yes | +| [agent\_source\_tag](#input\_agent\_source\_tag) | Tag to identify deployment source | `map(any)` | 
{
  "agent_source": "tf-cdp-module"
}
| no | +| [azure\_region](#input\_azure\_region) | Region which Cloud resources will be created | `string` | `null` | no | +| [backup\_storage](#input\_backup\_storage) | Optional Backup location for CDP environment. If not provided follow the data\_storage variable | 
object({
    backup_storage_bucket = string
    backup_storage_object = string
  })
| `null` | no | +| [cdp\_gw\_subnet\_names](#input\_cdp\_gw\_subnet\_names) | List of subnet names for CDP Gateway. Required if create\_vpc is false. | `list(any)` | `null` | no | +| [cdp\_subnet\_names](#input\_cdp\_subnet\_names) | List of subnet names for CDP Resources. Required if create\_vpc is false. | `list(any)` | `null` | no | +| [cdp\_vnet\_name](#input\_cdp\_vnet\_name) | Pre-existing VNet Name for CDP environment. Required if create\_vnet is false. | `string` | `null` | no | +| [create\_vnet](#input\_create\_vnet) | Flag to specify if the VNet should be created | `bool` | `true` | no | +| [data\_storage](#input\_data\_storage) | Data storage locations for CDP environment | 
object({
    data_storage_bucket = string
    data_storage_object = string
  })
| `null` | no | +| [datalake\_admin\_backup\_container\_role\_assignments](#input\_datalake\_admin\_backup\_container\_role\_assignments) | List of Role Assignments for the Datalake Admin Managed Identity assigned to the Backup Storage Container. | 
list(object({
    role        = string
    description = string
    })
  )
| 
[
  {
    "description": "Assign Storage Blob Data Owner Role to Data Lake Admin Identity at Backup Container Level",
    "role": "Storage Blob Data Owner"
  }
]
| no | +| [datalake\_admin\_data\_container\_role\_assignments](#input\_datalake\_admin\_data\_container\_role\_assignments) | List of Role Assignments for the Datalake Admin Managed Identity assigned to the Data Storage Container. | 
list(object({
    role        = string
    description = string
    })
  )
| 
[
  {
    "description": "Assign Storage Blob Data Owner Role to Data Lake Admin Identity at Data Container Level",
    "role": "Storage Blob Data Owner"
  }
]
| no | +| [datalake\_admin\_log\_container\_role\_assignments](#input\_datalake\_admin\_log\_container\_role\_assignments) | List of Role Assignments for the Datalake Admin Managed Identity assigned to the Logs Storage Container. | 
list(object({
    role        = string
    description = string
    })
  )
| 
[
  {
    "description": "Assign Storage Blob Data Owner Role to Data Lake Admin Identity at Logs Container Level",
    "role": "Storage Blob Data Owner"
  }
]
| no | +| [datalake\_admin\_managed\_identity\_name](#input\_datalake\_admin\_managed\_identity\_name) | Datalake Admin Managed Identity name | `string` | `null` | no | +| [enable\_raz](#input\_enable\_raz) | Flag to enable Ranger Authorization Service (RAZ) | `bool` | `true` | no | +| [env\_tags](#input\_env\_tags) | Tags applied to provisioned resources | `map(any)` | `null` | no | +| [idbroker\_managed\_identity\_name](#input\_idbroker\_managed\_identity\_name) | IDBroker Managed Identity name | `string` | `null` | no | +| [idbroker\_role\_assignments](#input\_idbroker\_role\_assignments) | List of Role Assignments for the IDBroker Managed Identity | 
list(object({
    role        = string
    description = string
    })
  )
| 
[
  {
    "description": "Assign VM Contributor Role to IDBroker Identity at Subscription Level",
    "role": "Virtual Machine Contributor"
  },
  {
    "description": "Assign Managed Identity Operator Role to IDBroker Identity at Subscription Level",
    "role": "Managed Identity Operator"
  }
]
| no | +| [ingress\_extra\_cidrs\_and\_ports](#input\_ingress\_extra\_cidrs\_and\_ports) | List of extra CIDR blocks and ports to include in Security Group Ingress rules | 
object({
    cidrs = list(string)
    ports = list(number)
  })
| 
{
  "cidrs": [],
  "ports": []
}
| no | +| [log\_data\_access\_managed\_identity\_name](#input\_log\_data\_access\_managed\_identity\_name) | Log Data Access Managed Identity name | `string` | `null` | no | +| [log\_data\_access\_role\_assignments](#input\_log\_data\_access\_role\_assignments) | List of Role Assignments for the Log Data Access Managed Identity. | 
list(object({
    role        = string
    description = string
    })
  )
| 
[
  {
    "description": "Assign Storage Blob Data Contributor Role to Log Role at Logs and Backup Container level",
    "role": "Storage Blob Data Contributor"
  }
]
| no | +| [log\_storage](#input\_log\_storage) | Optional log locations for CDP environment. If not provided follow the data\_storage variable | 
object({
    log_storage_bucket = string
    log_storage_object = string
  })
| `null` | no | +| [random\_id\_for\_bucket](#input\_random\_id\_for\_bucket) | Create a random suffix for the Storage Account names | `bool` | `true` | no | +| [ranger\_audit\_backup\_container\_role\_assignments](#input\_ranger\_audit\_backup\_container\_role\_assignments) | List of Role Assignments for the Ranger Audit Managed Identity assigned to the Backup Storage Container. | 
list(object({
    role        = string
    description = string
    })
  )
| 
[
  {
    "description": "Assign Storage Blob Data Contributor Role to Ranger Audit Role at Backup Container level",
    "role": "Storage Blob Data Contributor"
  }
]
| no | +| [ranger\_audit\_data\_access\_managed\_identity\_name](#input\_ranger\_audit\_data\_access\_managed\_identity\_name) | Ranger Audit Managed Identity name | `string` | `null` | no | +| [ranger\_audit\_data\_container\_role\_assignments](#input\_ranger\_audit\_data\_container\_role\_assignments) | List of Role Assignments for the Ranger Audit Managed Identity assigned to the Data Storage Container. | 
list(object({
    role        = string
    description = string
    })
  )
| 
[
  {
    "description": "Assign Storage Blob Data Contributor Role to Ranger Audit Role at Data Container level",
    "role": "Storage Blob Data Contributor"
  }
]
| no | +| [ranger\_audit\_log\_container\_role\_assignments](#input\_ranger\_audit\_log\_container\_role\_assignments) | List of Role Assignments for the Ranger Audit Managed Identity assigned to the Log Storage Container. | 
list(object({
    role        = string
    description = string
    })
  )
| 
[
  {
    "description": "Assign Storage Blob Data Contributor Role to Ranger Audit Role at Logs Container level",
    "role": "Storage Blob Data Contributor"
  }
]
| no | +| [raz\_managed\_identity\_name](#input\_raz\_managed\_identity\_name) | RAZ Managed Identity name | `string` | `null` | no | +| [raz\_storage\_role\_assignments](#input\_raz\_storage\_role\_assignments) | List of Role Assignments for the Ranger Audit Managed Identity assigned to the Log Storage Container. | 
list(object({
    role        = string
    description = string
    })
  )
| 
[
  {
    "description": "Assign Storage Blob Delegator Role to RAZ Identity at Storage Account level",
    "role": "Storage Blob Delegator"
  },
  {
    "description": "Assign Storage Blob Data Owner Role to RAZ Identity at Storage Account level",
    "role": "Storage Blob Data Owner"
  }
]
| no | +| [resourcegroup\_name](#input\_resourcegroup\_name) | Resource Group name | `string` | `null` | no | +| [security\_group\_default\_name](#input\_security\_group\_default\_name) | Default Security Group for CDP environment | `string` | `null` | no | +| [security\_group\_knox\_name](#input\_security\_group\_knox\_name) | Knox Security Group for CDP environment | `string` | `null` | no | +| [subnet\_count](#input\_subnet\_count) | Number of Subnets Required | `string` | `"3"` | no | +| [vnet\_cidr](#input\_vnet\_cidr) | VNet CIDR Block | `string` | `"10.10.0.0/16"` | no | +| [vnet\_name](#input\_vnet\_name) | VNet name | `string` | `null` | no | +| [xaccount\_app\_name](#input\_xaccount\_app\_name) | Cross account application name within Azure Active Directory | `string` | `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [azure\_backup\_storage\_location](#output\_azure\_backup\_storage\_location) | Azure backup storage location | +| [azure\_cdp\_gateway\_subnet\_names](#output\_azure\_cdp\_gateway\_subnet\_names) | Azure Virtual Subnet Names for CDP Endpoint Access Gateway | +| [azure\_cdp\_subnet\_names](#output\_azure\_cdp\_subnet\_names) | Azure Virtual Subnet Names for CDP Resources | +| [azure\_data\_storage\_location](#output\_azure\_data\_storage\_location) | Azure data storage location | +| [azure\_datalakeadmin\_identity\_id](#output\_azure\_datalakeadmin\_identity\_id) | Datalake Admin Managed Identity ID | +| [azure\_idbroker\_identity\_id](#output\_azure\_idbroker\_identity\_id) | IDBroker Managed Identity ID | +| [azure\_log\_identity\_id](#output\_azure\_log\_identity\_id) | Log Data Access Managed Identity ID | +| [azure\_log\_storage\_location](#output\_azure\_log\_storage\_location) | Azure log storage location | +| [azure\_ranger\_audit\_identity\_id](#output\_azure\_ranger\_audit\_identity\_id) | Ranger Audit Managed Identity ID | +| [azure\_raz\_identity\_id](#output\_azure\_raz\_identity\_id) | RAZ Managed Identity ID. Value returned if RAZ is enabled | +| [azure\_resource\_group\_name](#output\_azure\_resource\_group\_name) | Azure Resource Group Name | +| [azure\_security\_group\_default\_uri](#output\_azure\_security\_group\_default\_uri) | Azure Default Security Group URI | +| [azure\_security\_group\_knox\_uri](#output\_azure\_security\_group\_knox\_uri) | Azure Knox Security Group URI | +| [azure\_subscription\_id](#output\_azure\_subscription\_id) | Subscription ID where the Azure pre-reqs are created | +| [azure\_tenant\_id](#output\_azure\_tenant\_id) | Tenant ID where the Azure pre-reqs are created | +| [azure\_vnet\_name](#output\_azure\_vnet\_name) | Azure Virtual Network Name | +| [azure\_xaccount\_app\_pword](#output\_azure\_xaccount\_app\_pword) | Password for the Azure AD Cross Account Application | +| [azure\_xaccount\_app\_uuid](#output\_azure\_xaccount\_app\_uuid) | UUID for the Azure AD Cross Account Application | + \ No newline at end of file diff --git a/modules/terraform-cdp-azure-pre-reqs/data.tf b/modules/terraform-cdp-azure-pre-reqs/data.tf new file mode 100644 index 0000000..c02b15e --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/data.tf @@ -0,0 +1,19 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Access information about Azure Subscription +data "azurerm_subscription" "current" {} + +# Get the configuration of the AzureAD provider +data "azuread_client_config" "current" {} diff --git a/modules/terraform-cdp-azure-pre-reqs/defaults.tf b/modules/terraform-cdp-azure-pre-reqs/defaults.tf new file mode 100644 index 0000000..25e2a11 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/defaults.tf @@ -0,0 +1,88 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # ------- Global settings ------- + env_tags = merge(var.agent_source_tag, ( + coalesce(var.env_tags, + { env_prefix = var.env_prefix } + )) + ) + + # ------- Network Resources ------- + resourcegroup_name = coalesce(var.resourcegroup_name, "${var.env_prefix}-rmgp") + + vnet_name = coalesce(var.vnet_name, "${var.env_prefix}-net") + + # Security Groups + security_group_default_name = coalesce(var.security_group_default_name, "${var.env_prefix}-default-sg") + + security_group_knox_name = coalesce(var.security_group_knox_name, "${var.env_prefix}-knox-sg") + + cdp_vnet_name = (var.create_vnet ? + module.azure_cdp_vnet[0].vnet_name : var.cdp_vnet_name) + + cdp_subnet_names = (var.create_vnet ? + module.azure_cdp_vnet[0].vnet_cdp_subnet_names : var.cdp_subnet_names) + + cdp_gateway_subnet_names = (var.create_vnet ? + module.azure_cdp_vnet[0].vnet_gateway_subnet_names : var.cdp_gw_subnet_names) + + # ------- Storage Resources ------- + storage_suffix = var.random_id_for_bucket ? one(random_id.bucket_suffix).hex : "" + + # Azure storage containers can only contain numbers and letters + default_data_storage_name = "${replace(var.env_prefix, "/[-_]/", "")}stor" + + data_storage = { + data_storage_bucket = try(var.data_storage.data_storage_bucket, local.default_data_storage_name) + data_storage_object = try(var.data_storage.data_storage_object, "data") + } + log_storage = { + log_storage_bucket = try(var.log_storage.log_storage_bucket, local.data_storage.data_storage_bucket) + log_storage_object = try(var.log_storage.log_storage_object, "logs") + } + + backup_storage = { + backup_storage_bucket = try(var.backup_storage.backup_storage_bucket, local.data_storage.data_storage_bucket) + backup_storage_object = try(var.backup_storage.backup_storage_object, "backups") + } + + # ------- Authz Resources ------- + + # xaccount app + xaccount_app_name = coalesce(var.xaccount_app_name, "${var.env_prefix}-xaccount-app") + + # Managed Identities + datalake_admin_managed_identity_name = coalesce(var.datalake_admin_managed_identity_name, "${var.env_prefix}-dladmin-identity") + + idbroker_managed_identity_name = coalesce(var.idbroker_managed_identity_name, "${var.env_prefix}-idbroker-identity") + + log_data_access_managed_identity_name = coalesce(var.log_data_access_managed_identity_name, "${var.env_prefix}-logs-identity") + + ranger_audit_data_access_managed_identity_name = coalesce(var.ranger_audit_data_access_managed_identity_name, "${var.env_prefix}-audit-identity") + + raz_managed_identity_name = coalesce(var.raz_managed_identity_name, "${var.env_prefix}-raz-identity") + + raz_storage_role_assignments = flatten([ + + for k, v in azurerm_storage_account.cdp_storage_locations : [ + for role_assign in var.raz_storage_role_assignments : { + scope = v.id + role = role_assign.role, + description = role_assign.description + } + ] + ]) +} diff --git a/modules/terraform-cdp-azure-pre-reqs/doc_fragments/header.md b/modules/terraform-cdp-azure-pre-reqs/doc_fragments/header.md new file mode 100644 index 0000000..bc711e0 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/doc_fragments/header.md @@ -0,0 +1,11 @@ +# Terraform Module for CDP Prerequisites on Azure + +This module contains resource files and example variable definition files for creation of the pre-requisite Azure cloud resources required for Cloudera Data Platform (CDP) Public Cloud. + +## Usage + +The [examples](./examples) directory has example Azure Cloud Service Provider deployments for different scenarios: + +* `ex01-minimal-inputs` uses the minimum set of inputs for the module. + +In each directory an example `terraform.tfvars.sample` values file is included to show input variable values. diff --git a/modules/terraform-cdp-azure-pre-reqs/examples/ex01-minimal_inputs/main.tf b/modules/terraform-cdp-azure-pre-reqs/examples/ex01-minimal_inputs/main.tf new file mode 100644 index 0000000..4635a57 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/examples/ex01-minimal_inputs/main.tf @@ -0,0 +1,38 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +provider "azurerm" { + features { + resource_group { + prevent_deletion_if_contains_resources = false + } + } + +} + +provider "azuread" { + +} + +module "ex01_minimal_inputs" { + source = "../.." + + env_prefix = var.env_prefix + azure_region = var.azure_region + + deployment_template = var.deployment_template + + ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports + +} diff --git a/modules/terraform-cdp-azure-pre-reqs/examples/ex01-minimal_inputs/terraform.tfvars.sample b/modules/terraform-cdp-azure-pre-reqs/examples/ex01-minimal_inputs/terraform.tfvars.sample new file mode 100644 index 0000000..c3d5f25 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/examples/ex01-minimal_inputs/terraform.tfvars.sample @@ -0,0 +1,29 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +env_prefix = "" # Required name prefix for cloud and CDP resources, e.g. cldr1 + +# ------- Cloud Settings ------- +azure_region = "" # Change this to specify Cloud Provider region, e.g. eastus + +# ------- CDP Environment Deployment ------- +deployment_template = "" # Specify the deployment pattern below. Options are public, semi-private or private + +# ------- Network Settings ------- +# **NOTE: If required change the values below any additional CIDRs to add the the AWS Security Groups** +ingress_extra_cidrs_and_ports = { + cidrs = ["/32", "/32"], + ports = [443, 22] +} \ No newline at end of file diff --git a/modules/terraform-cdp-azure-pre-reqs/examples/ex01-minimal_inputs/variables.tf b/modules/terraform-cdp-azure-pre-reqs/examples/ex01-minimal_inputs/variables.tf new file mode 100644 index 0000000..618c4ff --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/examples/ex01-minimal_inputs/variables.tf @@ -0,0 +1,40 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "env_prefix" { + type = string + description = "Shorthand name for the environment. Used in resource descriptions" +} + +variable "azure_region" { + type = string + description = "Region which Cloud resources will be created" +} + +# ------- CDP Environment Deployment ------- +variable "deployment_template" { + type = string + + description = "Deployment Pattern to use for Cloud resources and CDP" +} + +# ------- Network Resources ------- +variable "ingress_extra_cidrs_and_ports" { + type = object({ + cidrs = list(string) + ports = list(number) + }) + description = "List of extra CIDR blocks and ports to include in Security Group Ingress rules" +} diff --git a/modules/terraform-cdp-azure-pre-reqs/main.tf b/modules/terraform-cdp-azure-pre-reqs/main.tf new file mode 100644 index 0000000..07ee2b6 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/main.tf @@ -0,0 +1,386 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Azure Resource Group ------- +resource "azurerm_resource_group" "cdp_rmgp" { + name = local.resourcegroup_name + location = var.azure_region + + tags = merge(local.env_tags, { Name = local.resourcegroup_name }) +} + +# ------- VNet ------- +# Create the VNet & subnets if required +module "azure_cdp_vnet" { + count = var.create_vnet ? 1 : 0 + + source = "./modules/vnet" + + deployment_template = var.deployment_template + resourcegroup_name = azurerm_resource_group.cdp_rmgp.name + vnet_name = local.vnet_name + vnet_cidr = var.vnet_cidr + vnet_region = var.azure_region + subnet_count = var.subnet_count + + env_prefix = var.env_prefix + tags = local.env_tags +} + + +# ------- Security Groups ------- +# Default SG +resource "azurerm_network_security_group" "cdp_default_sg" { + name = local.security_group_default_name + location = azurerm_resource_group.cdp_rmgp.location + resource_group_name = azurerm_resource_group.cdp_rmgp.name + + tags = merge(local.env_tags, { Name = local.security_group_default_name }) + +} + +# Create security group rules for extra list of ingress rules +# TODO: How to handle the case where ingress_extra_cidrs_and_ports is [] +resource "azurerm_network_security_rule" "cdp_default_sg_ingress_extra_access" { + name = "AllowAccessForExtraCidrsAndPorts" + priority = 201 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_address_prefixes = var.ingress_extra_cidrs_and_ports.cidrs + destination_address_prefix = "*" + source_port_range = "*" + destination_port_ranges = var.ingress_extra_cidrs_and_ports.ports + resource_group_name = azurerm_resource_group.cdp_rmgp.name + network_security_group_name = azurerm_network_security_group.cdp_default_sg.name +} + +# Knox SG +resource "azurerm_network_security_group" "cdp_knox_sg" { + name = local.security_group_knox_name + location = azurerm_resource_group.cdp_rmgp.location + resource_group_name = azurerm_resource_group.cdp_rmgp.name + + tags = merge(local.env_tags, { Name = local.security_group_knox_name }) + +} + + +# Create security group rules for extra list of ingress rules +# TODO: How to handle the case where ingress_extra_cidrs_and_ports is [] +resource "azurerm_network_security_rule" "cdp_knox_sg_ingress_extra_access" { + name = "AllowAccessForExtraCidrsAndPorts" + priority = 201 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_address_prefixes = var.ingress_extra_cidrs_and_ports.cidrs + destination_address_prefix = "*" + source_port_range = "*" + destination_port_ranges = var.ingress_extra_cidrs_and_ports.ports + resource_group_name = azurerm_resource_group.cdp_rmgp.name + network_security_group_name = azurerm_network_security_group.cdp_knox_sg.name +} + + +# ------- Azure Storage Account ------- +resource "random_id" "bucket_suffix" { + count = var.random_id_for_bucket ? 1 : 0 + + byte_length = 4 +} + +resource "azurerm_storage_account" "cdp_storage_locations" { + # Create buckets for the unique list of buckets in data and log storage + for_each = toset(concat([local.data_storage.data_storage_bucket], [local.log_storage.log_storage_bucket], [local.backup_storage.backup_storage_bucket])) + + name = "${each.value}${local.storage_suffix}" + resource_group_name = azurerm_resource_group.cdp_rmgp.name + location = azurerm_resource_group.cdp_rmgp.location + + # TODO: Review and parameterize these options + account_kind = "StorageV2" + account_tier = "Standard" + account_replication_type = "LRS" + is_hns_enabled = true + + tags = merge(local.env_tags, { Name = "${each.value}${local.storage_suffix}" }) +} + +# ------- Azure Storage Containers ------- +# Data Storage Objects +resource "azurerm_storage_container" "cdp_data_storage" { + + name = local.data_storage.data_storage_object + storage_account_name = "${local.data_storage.data_storage_bucket}${local.storage_suffix}" + container_access_type = "private" + + depends_on = [ + azurerm_storage_account.cdp_storage_locations + ] +} + +# Log Storage Objects +resource "azurerm_storage_container" "cdp_log_storage" { + + name = local.log_storage.log_storage_object + storage_account_name = "${local.log_storage.log_storage_bucket}${local.storage_suffix}" + container_access_type = "private" + + depends_on = [ + azurerm_storage_account.cdp_storage_locations + ] +} + +# Backup Storage Object +resource "azurerm_storage_container" "cdp_backup_storage" { + + name = local.backup_storage.backup_storage_object + storage_account_name = "${local.backup_storage.backup_storage_bucket}${local.storage_suffix}" + container_access_type = "private" + + depends_on = [ + azurerm_storage_account.cdp_storage_locations + ] +} + +# NOTE: I don't think below is needed by default - it's a customization +# ------- Azure Cross Account Role ------- + +# resource "azurerm_role_definition" "cdp_xaccount_role" { +# # TODO: +# name = local.xaccount_role_name +# # TODO: +# scope = data.azurerm_subscription.primary.id +# description = "CDP Cross Account role for ${var.env_prefix}" + +# # TODO: +# permissions { +# actions = ["*"] +# data_actions = [] +# not_actions = [] +# not_data_actions = +# } + +# # TODO: +# assignable_scopes = [ +# data.azurerm_subscription.primary.id, # /subscriptions/00000000-0000-0000-0000-000000000000 +# ] +# } + + +# ------- Azure Cross Account App ------- + +# Create Azure AD Application +resource "azuread_application" "cdp_xaccount_app" { + display_name = local.xaccount_app_name + + owners = [data.azuread_client_config.current.object_id] +} + +# Create Service Principal associated with the Azure AD App +resource "azuread_service_principal" "cdp_xaccount_app_sp" { + application_id = azuread_application.cdp_xaccount_app.application_id + + owners = [data.azuread_client_config.current.object_id] +} + +# Create role assignment for Service Principal +resource "azurerm_role_assignment" "cdp_xaccount_role" { + scope = data.azurerm_subscription.current.id + role_definition_name = "Contributor" + principal_id = azuread_service_principal.cdp_xaccount_app_sp.id +} + +# Create Application password (client secret) +resource "azuread_application_password" "cdp_xaccount_app_password" { + application_object_id = azuread_application.cdp_xaccount_app.object_id + end_date_relative = "17520h" #expire in 2 years # TODO: Review and parameterize +} + +# ------- Azure Managed Identities & Role Asignment - IDBroker ------- + +# Create Azure Managed Identity +resource "azurerm_user_assigned_identity" "cdp_idbroker" { + location = azurerm_resource_group.cdp_rmgp.location + name = local.idbroker_managed_identity_name + resource_group_name = azurerm_resource_group.cdp_rmgp.name + + tags = merge(local.env_tags, { Name = local.idbroker_managed_identity_name }) +} + +# Assign the required roles to the managed identity +resource "azurerm_role_assignment" "cdp_idbroker_assign" { + + for_each = { for idx, role in var.idbroker_role_assignments : idx => role } + + scope = data.azurerm_subscription.current.id + role_definition_name = each.value.role + principal_id = azurerm_user_assigned_identity.cdp_idbroker.principal_id + + description = each.value.description +} + +# ------- Azure Managed Identities & Role Asignment - Datalake Admin ------- + +# Create Azure Managed Identity +resource "azurerm_user_assigned_identity" "cdp_datalake_admin" { + location = azurerm_resource_group.cdp_rmgp.location + name = local.datalake_admin_managed_identity_name + resource_group_name = azurerm_resource_group.cdp_rmgp.name + + tags = merge(local.env_tags, { Name = local.datalake_admin_managed_identity_name }) +} + +# Assign the required roles to the managed identity for Data storage container +resource "azurerm_role_assignment" "cdp_datalake_admin_data_container_assign" { + + for_each = { for idx, role in var.datalake_admin_data_container_role_assignments : idx => role } + + scope = azurerm_storage_container.cdp_data_storage.resource_manager_id + role_definition_name = each.value.role + principal_id = azurerm_user_assigned_identity.cdp_datalake_admin.principal_id + + description = each.value.description +} + +# Assign the required roles to the managed identity for Log storage container +resource "azurerm_role_assignment" "cdp_datalake_admin_log_container_assign" { + + for_each = { for idx, role in var.datalake_admin_log_container_role_assignments : idx => role } + + scope = azurerm_storage_container.cdp_log_storage.resource_manager_id + role_definition_name = each.value.role + principal_id = azurerm_user_assigned_identity.cdp_datalake_admin.principal_id + + description = each.value.description +} + +# Assign the required roles to the managed identity for Backup storage container +resource "azurerm_role_assignment" "cdp_datalake_admin_backup_container_assign" { + + for_each = { for idx, role in var.datalake_admin_backup_container_role_assignments : idx => role } + + scope = azurerm_storage_container.cdp_backup_storage.resource_manager_id + role_definition_name = each.value.role + principal_id = azurerm_user_assigned_identity.cdp_datalake_admin.principal_id + + description = each.value.description +} + +# ------- Azure Managed Identities & Role Asignment - Log Data Access ------- + +# Create Azure Managed Identity +resource "azurerm_user_assigned_identity" "cdp_log_data_access" { + location = azurerm_resource_group.cdp_rmgp.location + name = local.log_data_access_managed_identity_name + resource_group_name = azurerm_resource_group.cdp_rmgp.name + + tags = merge(local.env_tags, { Name = local.log_data_access_managed_identity_name }) +} + +# Assign the required roles to the managed identity +resource "azurerm_role_assignment" "cdp_log_data_access_log_container_assign" { + + for_each = { for idx, role in var.log_data_access_role_assignments : idx => role } + + scope = azurerm_storage_container.cdp_log_storage.resource_manager_id + role_definition_name = each.value.role + principal_id = azurerm_user_assigned_identity.cdp_log_data_access.principal_id + + description = each.value.description +} + +resource "azurerm_role_assignment" "cdp_log_data_access_backup_container_assign" { + + for_each = { for idx, role in var.log_data_access_role_assignments : idx => role } + + scope = azurerm_storage_container.cdp_backup_storage.resource_manager_id + role_definition_name = each.value.role + principal_id = azurerm_user_assigned_identity.cdp_log_data_access.principal_id + + description = each.value.description +} + +# ------- Azure Managed Identities & Role Asignment - Ranger Audit ------- + +# Create Azure Managed Identity +resource "azurerm_user_assigned_identity" "cdp_ranger_audit_data_access" { + location = azurerm_resource_group.cdp_rmgp.location + name = local.ranger_audit_data_access_managed_identity_name + resource_group_name = azurerm_resource_group.cdp_rmgp.name + + tags = merge(local.env_tags, { Name = local.ranger_audit_data_access_managed_identity_name }) +} + +# Assign the required roles to the managed identity +resource "azurerm_role_assignment" "cdp_ranger_audit_data_container_assign" { + + for_each = { for idx, role in var.ranger_audit_data_container_role_assignments : idx => role } + + scope = azurerm_storage_container.cdp_data_storage.resource_manager_id + role_definition_name = each.value.role + principal_id = azurerm_user_assigned_identity.cdp_ranger_audit_data_access.principal_id + + description = each.value.description +} + +resource "azurerm_role_assignment" "cdp_ranger_audit_log_container_assign" { + + for_each = { for idx, role in var.ranger_audit_log_container_role_assignments : idx => role } + + scope = azurerm_storage_container.cdp_log_storage.resource_manager_id + role_definition_name = each.value.role + principal_id = azurerm_user_assigned_identity.cdp_ranger_audit_data_access.principal_id + + description = each.value.description +} + +resource "azurerm_role_assignment" "cdp_ranger_audit_backup_container_assign" { + + for_each = { for idx, role in var.ranger_audit_backup_container_role_assignments : idx => role } + + scope = azurerm_storage_container.cdp_backup_storage.resource_manager_id + role_definition_name = each.value.role + principal_id = azurerm_user_assigned_identity.cdp_ranger_audit_data_access.principal_id + + description = each.value.description +} + +# ------- Azure Managed Identities & Role Asignment - RAZ ------- + +# Create Azure Managed Identity +resource "azurerm_user_assigned_identity" "cdp_raz" { + + count = var.enable_raz ? 1 : 0 + + location = azurerm_resource_group.cdp_rmgp.location + name = local.raz_managed_identity_name + resource_group_name = azurerm_resource_group.cdp_rmgp.name + + tags = merge(local.env_tags, { Name = local.raz_managed_identity_name }) +} + +# Assign the required roles to the managed identity +resource "azurerm_role_assignment" "cdp_raz_assign" { + + for_each = { for idx, item in local.raz_storage_role_assignments : idx => item + if var.enable_raz == true } + + scope = each.value.scope + role_definition_name = each.value.role + principal_id = azurerm_user_assigned_identity.cdp_raz[0].principal_id + + description = each.value.description +} diff --git a/modules/terraform-cdp-azure-pre-reqs/modules/vnet/data.tf b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/data.tf new file mode 100644 index 0000000..d17d895 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/data.tf @@ -0,0 +1,14 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/modules/terraform-cdp-azure-pre-reqs/modules/vnet/defaults.tf b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/defaults.tf new file mode 100644 index 0000000..8428b63 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/defaults.tf @@ -0,0 +1,44 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + + # Calculate subnets CIDR and names + subnets_required = { + total = (var.deployment_template == "semi-private") ? var.subnet_count + 1 : var.subnet_count + cdp_subnets = var.subnet_count + gateway_subnets = (var.deployment_template == "semi-private") ? 1 : 0 + # name = "${var.env_prefix}-sbnt-${format("%02d", idx + 1)}" + # cidr = cidrsubnet(var.vnet_cidr, ceil(log(var.subnet_count, 2)), idx) + } + + # Network infrastructure for CDP resources + cdp_subnets = [ + for idx in range(local.subnets_required.cdp_subnets) : + { + name = "${var.env_prefix}-sbnt-${format("%02d", idx + 1)}" + cidr = cidrsubnet(var.vnet_cidr, ceil(log(local.subnets_required.total, 2)), idx) + } + ] + + # Network infrastructure for CDP resources + gw_subnets = [ + for idx in range(local.subnets_required.gateway_subnets) : + { + name = "${var.env_prefix}-gw-sbnt-${format("%02d", idx + 1)}" + cidr = cidrsubnet(var.vnet_cidr, ceil(log(local.subnets_required.total, 2)), local.subnets_required.cdp_subnets + idx) + } + ] + +} \ No newline at end of file diff --git a/modules/terraform-cdp-azure-pre-reqs/modules/vnet/main.tf b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/main.tf new file mode 100644 index 0000000..0c345f5 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/main.tf @@ -0,0 +1,55 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# ------- VNet ------- +resource "azurerm_virtual_network" "cdp_vnet" { + name = var.vnet_name + location = var.vnet_region + resource_group_name = var.resourcegroup_name + address_space = [var.vnet_cidr] + dns_servers = [] + + tags = merge(var.tags, { Name = "${var.env_prefix}-net" }) +} + +# ------- Subnets ------- +# Azure VNet Public Subnets +resource "azurerm_subnet" "cdp_subnets" { + + for_each = { for idx, subnet in local.cdp_subnets : idx => subnet } + + virtual_network_name = azurerm_virtual_network.cdp_vnet.name + resource_group_name = var.resourcegroup_name + name = each.value.name + address_prefixes = [each.value.cidr] + + service_endpoints = ["Microsoft.Sql", "Microsoft.Storage"] + private_endpoint_network_policies_enabled = true + +} + +resource "azurerm_subnet" "gateway_subnets" { + + for_each = { for idx, subnet in local.gw_subnets : idx => subnet } + + virtual_network_name = azurerm_virtual_network.cdp_vnet.name + resource_group_name = var.resourcegroup_name + name = each.value.name + address_prefixes = [each.value.cidr] + + service_endpoints = ["Microsoft.Sql", "Microsoft.Storage"] + private_endpoint_network_policies_enabled = true + +} diff --git a/modules/terraform-cdp-azure-pre-reqs/modules/vnet/outputs.tf b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/outputs.tf new file mode 100644 index 0000000..b7e8b56 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/outputs.tf @@ -0,0 +1,43 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "vnet_id" { + description = "The ID of the VNet" + value = azurerm_virtual_network.cdp_vnet.id +} + +output "vnet_name" { + description = "The ID of the VNet" + value = azurerm_virtual_network.cdp_vnet.name +} + +output "vnet_cdp_subnet_ids" { + description = "List of IDs of subnets for CDP Resources" + value = values(azurerm_subnet.cdp_subnets)[*].id +} + +output "vnet_cdp_subnet_names" { + description = "Names of the subnets for CDP Resources" + value = values(azurerm_subnet.cdp_subnets)[*].name +} + +output "vnet_gateway_subnet_ids" { + description = "List of IDs of subnets for CDP Gateway" + value = values(azurerm_subnet.gateway_subnets)[*].id +} + +output "vnet_gateway_subnet_names" { + description = "Names of the subnets for CDP Gateway" + value = values(azurerm_subnet.gateway_subnets)[*].name +} \ No newline at end of file diff --git a/modules/terraform-cdp-azure-pre-reqs/modules/vnet/provider.tf b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/provider.tf new file mode 100644 index 0000000..da05167 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/provider.tf @@ -0,0 +1,24 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">=3.11.0, <4.0" + } + } + + required_version = "> 1.3.0" +} diff --git a/modules/terraform-cdp-azure-pre-reqs/modules/vnet/variables.tf b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/variables.tf new file mode 100644 index 0000000..f1903d6 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/modules/vnet/variables.tf @@ -0,0 +1,64 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +variable "deployment_template" { + type = string + + description = "Deployment Pattern to use for Cloud resources and CDP" + + validation { + condition = contains(["public", "semi-private", "private"], var.deployment_template) + error_message = "Valid values for var: deployment_template are (public, semi-private, private)." + } +} + +variable "resourcegroup_name" { + type = string + description = "Resource Group Name" +} + +variable "vnet_name" { + type = string + description = "VNet name" + +} + +variable "vnet_cidr" { + type = string + description = "VNet CIDR Block" + +} + +variable "vnet_region" { + type = string + description = "Region which VNet will be created" + +} + +variable "subnet_count" { + type = string + description = "Number of Subnets Required" + +} + +variable "tags" { + type = map(any) + description = "Tags applied to provised resources" + +} + +variable "env_prefix" { + type = string + description = "Shorthand name for the environment. Used in resource descriptions" +} diff --git a/modules/terraform-cdp-azure-pre-reqs/outputs.tf b/modules/terraform-cdp-azure-pre-reqs/outputs.tf new file mode 100644 index 0000000..303c491 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/outputs.tf @@ -0,0 +1,128 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "azure_subscription_id" { + value = data.azurerm_subscription.current.subscription_id + + description = "Subscription ID where the Azure pre-reqs are created" +} + +output "azure_tenant_id" { + value = data.azurerm_subscription.current.tenant_id + + description = "Tenant ID where the Azure pre-reqs are created" +} + +output "azure_resource_group_name" { + value = azurerm_resource_group.cdp_rmgp.name + + description = "Azure Resource Group Name" +} + +output "azure_vnet_name" { + value = local.cdp_vnet_name + + description = "Azure Virtual Network Name" +} + +output "azure_cdp_subnet_names" { + value = local.cdp_subnet_names + + description = "Azure Virtual Subnet Names for CDP Resources" +} + +output "azure_cdp_gateway_subnet_names" { + value = local.cdp_gateway_subnet_names + + description = "Azure Virtual Subnet Names for CDP Endpoint Access Gateway" +} + +output "azure_security_group_default_uri" { + value = azurerm_network_security_group.cdp_default_sg.id + + description = "Azure Default Security Group URI" +} + +output "azure_security_group_knox_uri" { + value = azurerm_network_security_group.cdp_knox_sg.id + + description = "Azure Knox Security Group URI" +} + +output "azure_data_storage_location" { + value = "abfs://${azurerm_storage_container.cdp_data_storage.name}@${azurerm_storage_container.cdp_data_storage.storage_account_name}.dfs.core.windows.net" + + description = "Azure data storage location" + +} + +output "azure_log_storage_location" { + value = "abfs://${azurerm_storage_container.cdp_log_storage.name}@${azurerm_storage_container.cdp_log_storage.storage_account_name}.dfs.core.windows.net" + + description = "Azure log storage location" + +} + +output "azure_backup_storage_location" { + value = "abfs://${azurerm_storage_container.cdp_backup_storage.name}@${azurerm_storage_container.cdp_backup_storage.storage_account_name}.dfs.core.windows.net" + + description = "Azure backup storage location" + +} + +output "azure_xaccount_app_uuid" { + value = azuread_application.cdp_xaccount_app.application_id + + description = "UUID for the Azure AD Cross Account Application" + +} + +output "azure_xaccount_app_pword" { + value = azuread_application_password.cdp_xaccount_app_password.value + + description = "Password for the Azure AD Cross Account Application" + + sensitive = true +} + +output "azure_idbroker_identity_id" { + value = azurerm_user_assigned_identity.cdp_idbroker.id + + description = "IDBroker Managed Identity ID" +} + +output "azure_datalakeadmin_identity_id" { + value = azurerm_user_assigned_identity.cdp_datalake_admin.id + + description = "Datalake Admin Managed Identity ID" +} + +output "azure_ranger_audit_identity_id" { + value = azurerm_user_assigned_identity.cdp_ranger_audit_data_access.id + + description = "Ranger Audit Managed Identity ID" +} + +output "azure_log_identity_id" { + value = azurerm_user_assigned_identity.cdp_log_data_access.id + + description = "Log Data Access Managed Identity ID" + +} + +output "azure_raz_identity_id" { + value = (var.enable_raz) ? azurerm_user_assigned_identity.cdp_raz[0].id : "" + + description = "RAZ Managed Identity ID. Value returned if RAZ is enabled" +} diff --git a/modules/terraform-cdp-azure-pre-reqs/provider.tf b/modules/terraform-cdp-azure-pre-reqs/provider.tf new file mode 100644 index 0000000..889877c --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/provider.tf @@ -0,0 +1,32 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = "3.45.0" + } + azuread = { + source = "hashicorp/azuread" + version = "2.39.0" + } + random = { + source = "hashicorp/random" + version = "3.4.3" + } + } + + required_version = "> 1.3.0" +} diff --git a/modules/terraform-cdp-azure-pre-reqs/variables.tf b/modules/terraform-cdp-azure-pre-reqs/variables.tf new file mode 100644 index 0000000..18e4744 --- /dev/null +++ b/modules/terraform-cdp-azure-pre-reqs/variables.tf @@ -0,0 +1,422 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "azure_region" { + type = string + description = "Region which Cloud resources will be created" + + default = null +} + +variable "env_tags" { + type = map(any) + description = "Tags applied to provisioned resources" + + default = null +} + +variable "agent_source_tag" { + type = map(any) + description = "Tag to identify deployment source" + + default = { agent_source = "tf-cdp-module" } +} + +variable "env_prefix" { + type = string + description = "Shorthand name for the environment. Used in resource descriptions" +} + +# variable "public_key_text" { +# type = string + +# description = "SSH Public key string for the nodes of the CDP environment" +# } +# ------- CDP Environment Deployment ------- +variable "deployment_template" { + type = string + + description = "Deployment Pattern to use for Cloud resources and CDP" + + validation { + condition = contains(["public", "semi-private", "private"], var.deployment_template) + error_message = "Valid values for var: deployment_template are (public, semi-private, private)." + } +} + +variable "enable_raz" { + type = bool + + description = "Flag to enable Ranger Authorization Service (RAZ)" + + default = true +} + +# ------- Network Resources ------- +variable "resourcegroup_name" { + type = string + description = "Resource Group name" + + default = null +} + +variable "create_vnet" { + type = bool + + description = "Flag to specify if the VNet should be created" + + default = true +} + +variable "vnet_name" { + type = string + description = "VNet name" + + default = null +} + +variable "vnet_cidr" { + type = string + description = "VNet CIDR Block" + + default = "10.10.0.0/16" +} + +variable "cdp_vnet_name" { + type = string + description = "Pre-existing VNet Name for CDP environment. Required if create_vnet is false." + + default = null +} + +variable "cdp_subnet_names" { + type = list(any) + description = "List of subnet names for CDP Resources. Required if create_vpc is false." + + default = null +} + +variable "cdp_gw_subnet_names" { + type = list(any) + description = "List of subnet names for CDP Gateway. Required if create_vpc is false." + + default = null +} + +variable "subnet_count" { + type = string + description = "Number of Subnets Required" + + default = "3" +} + +# Security Groups +variable "security_group_default_name" { + type = string + + description = "Default Security Group for CDP environment" + + default = null +} + +variable "security_group_knox_name" { + type = string + + description = "Knox Security Group for CDP environment" + + default = null +} + +variable "ingress_extra_cidrs_and_ports" { + type = object({ + cidrs = list(string) + ports = list(number) + }) + description = "List of extra CIDR blocks and ports to include in Security Group Ingress rules" + + default = { + cidrs = [], + ports = [] + } +} + +# ------- Storage Resources ------- +variable "random_id_for_bucket" { + type = bool + + description = "Create a random suffix for the Storage Account names" + + default = true + +} + +variable "data_storage" { + type = object({ + data_storage_bucket = string + data_storage_object = string + }) + + description = "Data storage locations for CDP environment" + + default = null +} + +variable "log_storage" { + type = object({ + log_storage_bucket = string + log_storage_object = string + }) + + description = "Optional log locations for CDP environment. If not provided follow the data_storage variable" + + default = null +} + +variable "backup_storage" { + type = object({ + backup_storage_bucket = string + backup_storage_object = string + }) + + description = "Optional Backup location for CDP environment. If not provided follow the data_storage variable" + + default = null +} + +# ------- Authz Resources ------- +# Cross Account Application +variable "xaccount_app_name" { + type = string + + description = " Cross account application name within Azure Active Directory" + + default = null +} + +# Managed Identities +variable "datalake_admin_managed_identity_name" { + type = string + + description = "Datalake Admin Managed Identity name" + + default = null + +} + +variable "idbroker_managed_identity_name" { + type = string + + description = "IDBroker Managed Identity name" + + default = null + +} + +variable "log_data_access_managed_identity_name" { + type = string + + description = "Log Data Access Managed Identity name" + + default = null + +} + +variable "ranger_audit_data_access_managed_identity_name" { + type = string + + description = " Ranger Audit Managed Identity name" + + default = null + +} + +variable "raz_managed_identity_name" { + type = string + + description = "RAZ Managed Identity name" + + default = null + +} + +# Role Assignments to Manage Identifies +variable "idbroker_role_assignments" { + type = list(object({ + role = string + description = string + }) + ) + + description = " List of Role Assignments for the IDBroker Managed Identity" + + default = [ + { + "description" : "Assign VM Contributor Role to IDBroker Identity at Subscription Level", + "role" : "Virtual Machine Contributor" + }, + { + "description" : "Assign Managed Identity Operator Role to IDBroker Identity at Subscription Level", + "role" : "Managed Identity Operator" + } + ] + +} + +variable "datalake_admin_data_container_role_assignments" { + type = list(object({ + role = string + description = string + }) + ) + + description = "List of Role Assignments for the Datalake Admin Managed Identity assigned to the Data Storage Container." + + default = [ + { + "description" : "Assign Storage Blob Data Owner Role to Data Lake Admin Identity at Data Container Level", + "role" : "Storage Blob Data Owner" + } + ] + +} + +variable "datalake_admin_log_container_role_assignments" { + type = list(object({ + role = string + description = string + }) + ) + + description = "List of Role Assignments for the Datalake Admin Managed Identity assigned to the Logs Storage Container." + + default = [ + { + "description" : "Assign Storage Blob Data Owner Role to Data Lake Admin Identity at Logs Container Level", + "role" : "Storage Blob Data Owner" + } + ] + +} + +variable "datalake_admin_backup_container_role_assignments" { + type = list(object({ + role = string + description = string + }) + ) + + description = "List of Role Assignments for the Datalake Admin Managed Identity assigned to the Backup Storage Container." + + default = [ + { + "description" : "Assign Storage Blob Data Owner Role to Data Lake Admin Identity at Backup Container Level", + "role" : "Storage Blob Data Owner" + } + ] + +} + +variable "log_data_access_role_assignments" { + type = list(object({ + role = string + description = string + }) + ) + + description = "List of Role Assignments for the Log Data Access Managed Identity." + + default = [ + { + "description" : "Assign Storage Blob Data Contributor Role to Log Role at Logs and Backup Container level", + "role" : "Storage Blob Data Contributor" + } + ] + +} + +variable "ranger_audit_data_container_role_assignments" { + type = list(object({ + role = string + description = string + }) + ) + + description = "List of Role Assignments for the Ranger Audit Managed Identity assigned to the Data Storage Container." + + default = [ + { + "description" : "Assign Storage Blob Data Contributor Role to Ranger Audit Role at Data Container level", + "role" : "Storage Blob Data Contributor" + } + ] + +} + +variable "ranger_audit_log_container_role_assignments" { + type = list(object({ + role = string + description = string + }) + ) + + description = "List of Role Assignments for the Ranger Audit Managed Identity assigned to the Log Storage Container." + + default = [ + { + "description" : "Assign Storage Blob Data Contributor Role to Ranger Audit Role at Logs Container level", + "role" : "Storage Blob Data Contributor" + } + ] + +} + +variable "ranger_audit_backup_container_role_assignments" { + type = list(object({ + role = string + description = string + }) + ) + + description = "List of Role Assignments for the Ranger Audit Managed Identity assigned to the Backup Storage Container." + + default = [ + { + "description" : "Assign Storage Blob Data Contributor Role to Ranger Audit Role at Backup Container level", + "role" : "Storage Blob Data Contributor" + } + ] + +} +variable "raz_storage_role_assignments" { + type = list(object({ + role = string + description = string + }) + ) + + description = "List of Role Assignments for the Ranger Audit Managed Identity assigned to the Log Storage Container." + + default = [ + { + "description" : "Assign Storage Blob Delegator Role to RAZ Identity at Storage Account level", + "role" : "Storage Blob Delegator" + }, + { + "description" : "Assign Storage Blob Data Owner Role to RAZ Identity at Storage Account level", + "role" : "Storage Blob Data Owner" + } + ] + +} \ No newline at end of file diff --git a/modules/terraform-cdp-deploy/.terraform-docs.yaml b/modules/terraform-cdp-deploy/.terraform-docs.yaml new file mode 100644 index 0000000..0936036 --- /dev/null +++ b/modules/terraform-cdp-deploy/.terraform-docs.yaml @@ -0,0 +1,21 @@ +formatter: markdown +header-from: doc_fragments/header.md +settings: + anchor: true + color: true + default: true + escape: true + html: true + indent: 2 + required: true + sensitive: true + type: true + + +sort: + enabled: true + by: required + +output: + file: README.md + mode: replace \ No newline at end of file diff --git a/modules/terraform-cdp-deploy/README.md b/modules/terraform-cdp-deploy/README.md new file mode 100644 index 0000000..3682b7c --- /dev/null +++ b/modules/terraform-cdp-deploy/README.md @@ -0,0 +1,100 @@ + +# Terraform Module for CDP Deployment + +This module contains resource files and example variable definition files for deployment of Cloudera Data Platform (CDP) Public Cloud environment and Datalake creation on AWS or Azure. + +## Usage + +The [examples](./examples) directory has example CDP deployments: + +* `ex01-aws-basic` creates a basic CDP deployment on AWS. This example makes use of the [terraform-cdp-aws-pre-reqs module](../terraform-cdp-aws-pre-reqs) to create the required cloud resources. + +* `ex02-azure-basic` creates a basic CDP deployment on Azure. This example makes use of the [terraform-cdp-azure-pre-reqs module](../terraform-cdp-azure-pre-reqs) to create the required cloud resources. + +In each directory an example `terraform.tfvars.sample` values file is included to show input variable values. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | > 1.3.0 | +| [local](#requirement\_local) | 2.2.3 | +| [null](#requirement\_null) | 3.2.1 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [cdp\_on\_aws](#module\_cdp\_on\_aws) | ./modules/aws | n/a | +| [cdp\_on\_azure](#module\_cdp\_on\_azure) | ./modules/azure | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [backup\_storage\_location](#input\_backup\_storage\_location) | Backup storage location. The location has to be in uri format for the cloud provider - i.e. s3a:// for AWS, abfs:// for Azure, gs:// | `string` | n/a | yes | +| [data\_storage\_location](#input\_data\_storage\_location) | Data storage location. The location has to be in uri format for the cloud provider - i.e. s3a:// for AWS, abfs:// for Azure, gs:// | `string` | n/a | yes | +| [deployment\_template](#input\_deployment\_template) | Deployment Pattern to use for Cloud resources and CDP | `string` | n/a | yes | +| [infra\_type](#input\_infra\_type) | Cloud Provider to deploy CDP. | `string` | n/a | yes | +| [log\_storage\_location](#input\_log\_storage\_location) | Log storage location. The location has to be in uri format for the cloud provider - i.e. s3a:// for AWS, abfs:// for Azure, gs:// | `string` | n/a | yes | +| [region](#input\_region) | Region which cloud resources will be created | `string` | n/a | yes | +| [agent\_source\_tag](#input\_agent\_source\_tag) | Tag to identify deployment source | `map(any)` | 
{
  "agent_source": "tf-cdp-module"
}
| no | +| [aws\_datalake\_admin\_role\_arn](#input\_aws\_datalake\_admin\_role\_arn) | Datalake Admin Role ARN. Required for CDP deployment on AWS. | `string` | `null` | no | +| [aws\_idbroker\_instance\_profile\_arn](#input\_aws\_idbroker\_instance\_profile\_arn) | IDBroker Instance Profile ARN. Required for CDP deployment on AWS. | `string` | `null` | no | +| [aws\_log\_instance\_profile\_arn](#input\_aws\_log\_instance\_profile\_arn) | Log Instance Profile ARN. Required for CDP deployment on AWS. | `string` | `null` | no | +| [aws\_private\_subnet\_ids](#input\_aws\_private\_subnet\_ids) | List of private subnet ids. Required for CDP deployment on AWS. | `list(string)` | `null` | no | +| [aws\_public\_subnet\_ids](#input\_aws\_public\_subnet\_ids) | List of public subnet ids. Required for CDP deployment on AWS. | `list(string)` | `null` | no | +| [aws\_ranger\_audit\_role\_arn](#input\_aws\_ranger\_audit\_role\_arn) | Ranger Audit Role ARN. Required for CDP deployment on AWS. | `string` | `null` | no | +| [aws\_security\_group\_default\_id](#input\_aws\_security\_group\_default\_id) | ID of the Default Security Group for CDP environment. Required for CDP deployment on AWS. | `string` | `null` | no | +| [aws\_security\_group\_knox\_id](#input\_aws\_security\_group\_knox\_id) | ID of the Knox Security Group for CDP environment. Required for CDP deployment on AWS. | `string` | `null` | no | +| [aws\_vpc\_id](#input\_aws\_vpc\_id) | AWS Virtual Private Network ID. Required for CDP deployment on AWS. | `string` | `null` | no | +| [aws\_xaccount\_role\_arn](#input\_aws\_xaccount\_role\_arn) | Cross Account Role ARN. Required for CDP deployment on AWS. | `string` | `null` | no | +| [azure\_cdp\_gateway\_subnet\_names](#input\_azure\_cdp\_gateway\_subnet\_names) | List of Azure Subnet Names CDP Endpoint Access Gateway. Required for CDP deployment on Azure. | `list(any)` | `null` | no | +| [azure\_cdp\_subnet\_names](#input\_azure\_cdp\_subnet\_names) | List of Azure Subnet Names for CDP Resources. Required for CDP deployment on Azure. | `list(any)` | `null` | no | +| [azure\_datalakeadmin\_identity\_id](#input\_azure\_datalakeadmin\_identity\_id) | Datalake Admin Managed Identity ID. Required for CDP deployment on Azure. | `string` | `null` | no | +| [azure\_idbroker\_identity\_id](#input\_azure\_idbroker\_identity\_id) | IDBroker Managed Identity ID. Required for CDP deployment on Azure. | `string` | `null` | no | +| [azure\_log\_identity\_id](#input\_azure\_log\_identity\_id) | Log Data Access Managed Identity ID. Required for CDP deployment on Azure. | `string` | `null` | no | +| [azure\_ranger\_audit\_identity\_id](#input\_azure\_ranger\_audit\_identity\_id) | Ranger Audit Managed Identity ID. Required for CDP deployment on Azure. | `string` | `null` | no | +| [azure\_raz\_identity\_id](#input\_azure\_raz\_identity\_id) | RAZ Managed Identity ID. Required for CDP deployment on Azure. | `string` | `null` | no | +| [azure\_resource\_group\_name](#input\_azure\_resource\_group\_name) | Azure Resource Group name. Required for CDP deployment on Azure. | `string` | `null` | no | +| [azure\_security\_group\_default\_uri](#input\_azure\_security\_group\_default\_uri) | Azure Default Security Group URI. Required for CDP deployment on Azure. | `string` | `null` | no | +| [azure\_security\_group\_knox\_uri](#input\_azure\_security\_group\_knox\_uri) | Azure Knox Security Group URI. Required for CDP deployment on Azure. | `string` | `null` | no | +| [azure\_subscription\_id](#input\_azure\_subscription\_id) | Subscription ID where the Azure pre-reqs are created. Required for CDP deployment on Azure. | `string` | `null` | no | +| [azure\_tenant\_id](#input\_azure\_tenant\_id) | Tenant ID where the Azure pre-reqs are created. Required for CDP deployment on Azure. | `string` | `null` | no | +| [azure\_vnet\_name](#input\_azure\_vnet\_name) | Azure Virtual Network ID. Required for CDP deployment on Azure. | `string` | `null` | no | +| [azure\_xaccount\_app\_pword](#input\_azure\_xaccount\_app\_pword) | Password for the Azure AD Cross Account Application. Required for CDP deployment on Azure. | `string` | `null` | no | +| [azure\_xaccount\_app\_uuid](#input\_azure\_xaccount\_app\_uuid) | UUID for the Azure AD Cross Account Application. Required for CDP deployment on Azure. | `string` | `null` | no | +| [cdp\_admin\_group\_name](#input\_cdp\_admin\_group\_name) | Name of the CDP IAM Admin Group associated with the environment. Defaults to '-cdp-admin-group' if not specified. | `string` | `null` | no | +| [cdp\_control\_plane\_region](#input\_cdp\_control\_plane\_region) | CDP Control Plane Region | `string` | `"us-west-1"` | no | +| [cdp\_profile](#input\_cdp\_profile) | Profile for CDP credentials | `string` | `"default"` | no | +| [cdp\_user\_group\_name](#input\_cdp\_user\_group\_name) | Name of the CDP IAM User Group associated with the environment. Defaults to '-cdp-user-group' if not specified. | `string` | `null` | no | +| [cdp\_xacccount\_credential\_name](#input\_cdp\_xacccount\_credential\_name) | Name of the CDP Cross Account Credential. Defaults to '-xaccount-cred' if not specified. | `string` | `null` | no | +| [datalake\_name](#input\_datalake\_name) | Name of the CDP datalake. Defaults to '--dl' if not specified. | `string` | `null` | no | +| [datalake\_scale](#input\_datalake\_scale) | The scale of the datalake. Valid values are LIGHT\_DUTY, MEDIUM\_DUTY\_HA. | `string` | `null` | no | +| [datalake\_version](#input\_datalake\_version) | The Datalake Runtime version. Valid values are semantic versions, e.g. 7.2.16 | `string` | `"7.2.16"` | no | +| [enable\_ccm\_tunnel](#input\_enable\_ccm\_tunnel) | Flag to enable Cluster Connectivity Manager tunnel. If false then access from Cloud to CDP Control Plane CIDRs is required from via SG ingress | `bool` | `true` | no | +| [enable\_raz](#input\_enable\_raz) | Flag to enable Ranger Authorization Service (RAZ) | `bool` | `true` | no | +| [endpoint\_access\_scheme](#input\_endpoint\_access\_scheme) | The scheme for the workload endpoint gateway. PUBLIC creates an external endpoint that can be accessed over the Internet. PRIVATE which restricts the traffic to be internal to the VPC / Vnet. Relevant in Private Networks. | `string` | `null` | no | +| [env\_prefix](#input\_env\_prefix) | Shorthand name for the environment. Used in CDP resource descriptions. This will be used to construct the value of where any of the CDP resource variables (e.g. environment\_name, cdp\_iam\_admin\_group\_name) are not defined. | `string` | `null` | no | +| [env\_tags](#input\_env\_tags) | Tags applied to provisioned resources | `map(any)` | `null` | no | +| [environment\_name](#input\_environment\_name) | Name of the CDP environment. Defaults to '-cdp-env' if not specified. | `string` | `null` | no | +| [freeipa\_instances](#input\_freeipa\_instances) | The number of FreeIPA instances to create in the environment | `number` | `3` | no | +| [keypair\_name](#input\_keypair\_name) | SSH Keypair name in Cloud Service Provider. Required for CDP deployment on AWS. | `string` | `null` | no | +| [multiaz](#input\_multiaz) | Flag to specify that the FreeIPA and DataLake instances will be deployed across multi-availability zones. | `bool` | `true` | no | +| [public\_key\_text](#input\_public\_key\_text) | SSH Public key string for the nodes of the CDP environment. Required for CDP deployment on Azure. | `string` | `null` | no | +| [use\_public\_ips](#input\_use\_public\_ips) | Use public ip's for the CDP resources created within the Azure network. Required for CDP deployment on Azure. | `bool` | `null` | no | +| [use\_single\_resource\_group](#input\_use\_single\_resource\_group) | Use a single resource group for all provisioned CDP resources. Required for CDP deployment on Azure. | `bool` | `true` | no | +| [workload\_analytics](#input\_workload\_analytics) | Flag to specify if workload analytics should be enabled for the CDP environment | `bool` | `true` | no | + +## Outputs + +No outputs. + \ No newline at end of file diff --git a/modules/terraform-cdp-deploy/data.tf b/modules/terraform-cdp-deploy/data.tf new file mode 100644 index 0000000..e3cf504 --- /dev/null +++ b/modules/terraform-cdp-deploy/data.tf @@ -0,0 +1,13 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/modules/terraform-cdp-deploy/defaults.tf b/modules/terraform-cdp-deploy/defaults.tf new file mode 100644 index 0000000..bdd3528 --- /dev/null +++ b/modules/terraform-cdp-deploy/defaults.tf @@ -0,0 +1,68 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + # ------- Global settings ------- + env_tags = merge(var.agent_source_tag, ( + coalesce(var.env_tags, + { env_prefix = var.env_prefix } + )) + ) + + # ------- CDP Environment Deployment ------- + environment_name = coalesce(var.environment_name, + "${var.env_prefix}-cdp-env") + + datalake_name = coalesce(var.datalake_name, + "${var.env_prefix}-${local.cloud_shorthand[var.infra_type]}-dl") + + cdp_xacccount_credential_name = coalesce(var.cdp_xacccount_credential_name, + "${var.env_prefix}-xaccount-cred") + + cdp_admin_group_name = coalesce(var.cdp_admin_group_name, + "${var.env_prefix}-cdp-admin-group") + + cdp_user_group_name = coalesce(var.cdp_user_group_name, + "${var.env_prefix}-cdp-user-group") + + datalake_scale = coalesce( + var.datalake_scale, + (var.deployment_template == "public" ? + "LIGHT_DUTY" : "MEDIUM_DUTY_HA" + ) + ) + + endpoint_access_scheme = coalesce( + var.endpoint_access_scheme, + (var.deployment_template == "semi-private") ? "PUBLIC" : "PRIVATE" + ) + + # ------- Cloud Provider Settings - General ------- + cloud_shorthand = { + azure = "az" + aws = "aw" + gcp = "gc" + } + + # ------- Cloud Service Provider Settings - AWS specific ------- + aws_subnets_for_cdp = ( + var.infra_type == "aws" && var.deployment_template == "public") ? (concat(var.aws_public_subnet_ids, var.aws_private_subnet_ids)) : (var.aws_private_subnet_ids) + + # ------- Cloud Service Provider Settings - Azure specific ------- + use_public_ips = coalesce( + var.use_public_ips, + (var.deployment_template == "public") + ) + +} diff --git a/modules/terraform-cdp-deploy/doc_fragments/header.md b/modules/terraform-cdp-deploy/doc_fragments/header.md new file mode 100644 index 0000000..6f1bf0b --- /dev/null +++ b/modules/terraform-cdp-deploy/doc_fragments/header.md @@ -0,0 +1,13 @@ +# Terraform Module for CDP Deployment + +This module contains resource files and example variable definition files for deployment of Cloudera Data Platform (CDP) Public Cloud environment and Datalake creation on AWS or Azure. + +## Usage + +The [examples](./examples) directory has example CDP deployments: + +* `ex01-aws-basic` creates a basic CDP deployment on AWS. This example makes use of the [terraform-cdp-aws-pre-reqs module](../terraform-cdp-aws-pre-reqs) to create the required cloud resources. + +* `ex02-azure-basic` creates a basic CDP deployment on Azure. This example makes use of the [terraform-cdp-azure-pre-reqs module](../terraform-cdp-azure-pre-reqs) to create the required cloud resources. + +In each directory an example `terraform.tfvars.sample` values file is included to show input variable values. diff --git a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf new file mode 100644 index 0000000..a2db217 --- /dev/null +++ b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf @@ -0,0 +1,63 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +provider "aws" { + profile = var.aws_profile + region = var.aws_region +} + +module "cdp_aws_prereqs" { + source = "../../../terraform-cdp-aws-pre-reqs" + + env_prefix = var.env_prefix + aws_region = var.aws_region + + deployment_template = var.deployment_template + cdp_profile = var.cdp_profile + + ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports +} + +module "cdp_deploy" { + source = "../.." + + env_prefix = var.env_prefix + infra_type = "aws" + region = var.aws_region + keypair_name = var.aws_key_pair + deployment_template = var.deployment_template + + # From pre-reqs module output + aws_vpc_id = module.cdp_aws_prereqs.aws_vpc_id + aws_public_subnet_ids = module.cdp_aws_prereqs.aws_public_subnet_ids + aws_private_subnet_ids = module.cdp_aws_prereqs.aws_private_subnet_ids + + aws_security_group_default_id = module.cdp_aws_prereqs.aws_security_group_default_id + aws_security_group_knox_id = module.cdp_aws_prereqs.aws_security_group_knox_id + + data_storage_location = module.cdp_aws_prereqs.aws_data_storage_location + log_storage_location = module.cdp_aws_prereqs.aws_log_storage_location + backup_storage_location = module.cdp_aws_prereqs.aws_backup_storage_location + + aws_xaccount_role_arn = module.cdp_aws_prereqs.aws_xaccount_role_arn + aws_datalake_admin_role_arn = module.cdp_aws_prereqs.aws_datalake_admin_role_arn + aws_ranger_audit_role_arn = module.cdp_aws_prereqs.aws_ranger_audit_role_arn + + aws_log_instance_profile_arn = module.cdp_aws_prereqs.aws_log_instance_profile_arn + aws_idbroker_instance_profile_arn = module.cdp_aws_prereqs.aws_idbroker_instance_profile_arn + + depends_on = [ + module.cdp_aws_prereqs + ] +} diff --git a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/terraform.tfvars.sample b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/terraform.tfvars.sample new file mode 100644 index 0000000..796b406 --- /dev/null +++ b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/terraform.tfvars.sample @@ -0,0 +1,30 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +env_prefix = "" # Required name prefix for cloud and CDP resources, e.g. cldr1 + +# ------- Cloud Settings ------- +aws_region = "" # Change this to specify Cloud Provider region, e.g. eu-west-1 +aws_key_pair = "" # Change this with the name of a pre-existing AWS keypair, e.g. my-keypair + +# ------- CDP Environment Deployment ------- +deployment_template = "" # Specify the deployment pattern below. Options are public, semi-private or private + +# ------- Network Settings ------- +# **NOTE: If required change the values below any additional CIDRs to add the the AWS Security Groups** +ingress_extra_cidrs_and_ports = { + cidrs = ["/32", "/32"], + ports = [443, 22] +} diff --git a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/variables.tf b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/variables.tf new file mode 100644 index 0000000..dd3d6b4 --- /dev/null +++ b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/variables.tf @@ -0,0 +1,62 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "aws_profile" { + type = string + description = "Profile for AWS cloud credentials" + + # Profile is default unless explicitly specified + default = "default" +} + +variable "aws_region" { + type = string + description = "Region which Cloud resources will be created" +} + +variable "env_prefix" { + type = string + description = "Shorthand name for the environment. Used in resource descriptions" +} + +variable "aws_key_pair" { + type = string + + description = "Name of the Public SSH key for the CDP environment" + +} + +# ------- CDP Environment Deployment ------- +variable "deployment_template" { + type = string + + description = "Deployment Pattern to use for Cloud resources and CDP" +} + +variable "cdp_profile" { + type = string + description = "Profile for CDP credentials" + + # Profile is default unless explicitly specified + default = "default" +} +# ------- Network Resources ------- +variable "ingress_extra_cidrs_and_ports" { + type = object({ + cidrs = list(string) + ports = list(number) + }) + description = "List of extra CIDR blocks and ports to include in Security Group Ingress rules" +} diff --git a/modules/terraform-cdp-deploy/examples/ex02-azure-basic/main.tf b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/main.tf new file mode 100755 index 0000000..a1832d7 --- /dev/null +++ b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/main.tf @@ -0,0 +1,76 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +provider "azurerm" { + features { + resource_group { + prevent_deletion_if_contains_resources = false + } + } + +} + +provider "azuread" { +} + +module "cdp_azure_prereqs" { + source = "../../../terraform-cdp-azure-pre-reqs" + + env_prefix = var.env_prefix + azure_region = var.azure_region + + deployment_template = var.deployment_template + + ingress_extra_cidrs_and_ports = var.ingress_extra_cidrs_and_ports + +} + +module "cdp_deploy" { + source = "../.." + + env_prefix = var.env_prefix + infra_type = "azure" + region = var.azure_region + public_key_text = var.public_key_text + deployment_template = var.deployment_template + + # From pre-reqs module output + azure_subscription_id = module.cdp_azure_prereqs.azure_subscription_id + azure_tenant_id = module.cdp_azure_prereqs.azure_tenant_id + + azure_resource_group_name = module.cdp_azure_prereqs.azure_resource_group_name + azure_vnet_name = module.cdp_azure_prereqs.azure_vnet_name + azure_cdp_subnet_names = module.cdp_azure_prereqs.azure_cdp_subnet_names + azure_cdp_gateway_subnet_names = module.cdp_azure_prereqs.azure_cdp_gateway_subnet_names + + azure_security_group_default_uri = module.cdp_azure_prereqs.azure_security_group_default_uri + azure_security_group_knox_uri = module.cdp_azure_prereqs.azure_security_group_knox_uri + + data_storage_location = module.cdp_azure_prereqs.azure_data_storage_location + log_storage_location = module.cdp_azure_prereqs.azure_log_storage_location + backup_storage_location = module.cdp_azure_prereqs.azure_backup_storage_location + + azure_xaccount_app_uuid = module.cdp_azure_prereqs.azure_xaccount_app_uuid + azure_xaccount_app_pword = module.cdp_azure_prereqs.azure_xaccount_app_pword + + azure_idbroker_identity_id = module.cdp_azure_prereqs.azure_idbroker_identity_id + azure_datalakeadmin_identity_id = module.cdp_azure_prereqs.azure_datalakeadmin_identity_id + azure_ranger_audit_identity_id = module.cdp_azure_prereqs.azure_ranger_audit_identity_id + azure_log_identity_id = module.cdp_azure_prereqs.azure_log_identity_id + azure_raz_identity_id = module.cdp_azure_prereqs.azure_raz_identity_id + + depends_on = [ + module.cdp_azure_prereqs + ] +} diff --git a/modules/terraform-cdp-deploy/examples/ex02-azure-basic/terraform.tfvars.sample b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/terraform.tfvars.sample new file mode 100644 index 0000000..6562502 --- /dev/null +++ b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/terraform.tfvars.sample @@ -0,0 +1,31 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +env_prefix = "" # Required name prefix for cloud and CDP resources, e.g. cldr1 + +# ------- Cloud Settings ------- +azure_region = "" # Change this to specify Cloud Provider region, e.g. westeurpoe + +public_key_text = "" # Change this with the SSH public key text, e.g. ssh-rsa AAA.... + +# ------- CDP Environment Deployment ------- +deployment_template = "" # Specify the deployment pattern below. Options are public, semi-private or private + +# ------- Network Settings ------- +# **NOTE: If required change the values below any additional CIDRs to add the the AWS Security Groups** +ingress_extra_cidrs_and_ports = { + cidrs = ["/32", "/32"], + ports = [443, 22] +} diff --git a/modules/terraform-cdp-deploy/examples/ex02-azure-basic/variables.tf b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/variables.tf new file mode 100755 index 0000000..d1ca021 --- /dev/null +++ b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/variables.tf @@ -0,0 +1,46 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "env_prefix" { + type = string + description = "Shorthand name for the environment. Used in resource descriptions" +} + +variable "azure_region" { + type = string + description = "Region which Cloud resources will be created" +} + +variable "public_key_text" { + type = string + + description = "SSH Public key string for the nodes of the CDP environment" +} + +# ------- CDP Environment Deployment ------- +variable "deployment_template" { + type = string + + description = "Deployment Pattern to use for Cloud resources and CDP" +} + +# ------- Network Resources ------- +variable "ingress_extra_cidrs_and_ports" { + type = object({ + cidrs = list(string) + ports = list(number) + }) + description = "List of extra CIDR blocks and ports to include in Security Group Ingress rules" +} diff --git a/modules/terraform-cdp-deploy/main.tf b/modules/terraform-cdp-deploy/main.tf new file mode 100644 index 0000000..3dacda8 --- /dev/null +++ b/modules/terraform-cdp-deploy/main.tf @@ -0,0 +1,117 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Call sub-module for AWS Deployment ------- +module "cdp_on_aws" { + count = (var.infra_type == "aws") ? 1 : 0 + + source = "./modules/aws" + + tags = local.env_tags + + environment_name = local.environment_name + datalake_name = local.datalake_name + cdp_xacccount_credential_name = local.cdp_xacccount_credential_name + cdp_admin_group_name = local.cdp_admin_group_name + cdp_user_group_name = local.cdp_user_group_name + + security_group_default_id = var.aws_security_group_default_id + security_group_knox_id = var.aws_security_group_knox_id + + datalake_scale = local.datalake_scale + datalake_version = var.datalake_version + enable_ccm_tunnel = var.enable_ccm_tunnel + enable_raz = var.enable_raz + multiaz = var.multiaz + freeipa_instances = var.freeipa_instances + workload_analytics = var.workload_analytics + endpoint_access_scheme = local.endpoint_access_scheme + + cdp_profile = var.cdp_profile + cdp_control_plane_region = var.cdp_control_plane_region + + region = var.region + vpc_id = var.aws_vpc_id + public_subnet_ids = var.aws_public_subnet_ids + private_subnet_ids = var.aws_private_subnet_ids + subnets_for_cdp = local.aws_subnets_for_cdp + keypair_name = var.keypair_name + + data_storage_location = var.data_storage_location + log_storage_location = var.log_storage_location + backup_storage_location = var.backup_storage_location + + xaccount_role_arn = var.aws_xaccount_role_arn + datalake_admin_role_arn = var.aws_datalake_admin_role_arn + ranger_audit_role_arn = var.aws_ranger_audit_role_arn + + idbroker_instance_profile_arn = var.aws_idbroker_instance_profile_arn + log_instance_profile_arn = var.aws_log_instance_profile_arn + +} + +# ------- Call sub-module for Azure Deployment ------- +module "cdp_on_azure" { + count = (var.infra_type == "azure") ? 1 : 0 + + source = "./modules/azure" + + tags = local.env_tags + + environment_name = local.environment_name + datalake_name = local.datalake_name + cdp_xacccount_credential_name = local.cdp_xacccount_credential_name + cdp_admin_group_name = local.cdp_admin_group_name + cdp_user_group_name = local.cdp_user_group_name + + security_group_default_uri = var.azure_security_group_default_uri + security_group_knox_uri = var.azure_security_group_knox_uri + + datalake_scale = local.datalake_scale + datalake_version = var.datalake_version + enable_ccm_tunnel = var.enable_ccm_tunnel + enable_raz = var.enable_raz + freeipa_instances = var.freeipa_instances + workload_analytics = var.workload_analytics + endpoint_access_scheme = local.endpoint_access_scheme + + cdp_profile = var.cdp_profile + cdp_control_plane_region = var.cdp_control_plane_region + + use_single_resource_group = var.use_single_resource_group + use_public_ips = local.use_public_ips + + subscription_id = var.azure_subscription_id + tenant_id = var.azure_tenant_id + + region = var.region + resource_group_name = var.azure_resource_group_name + vnet_name = var.azure_vnet_name + cdp_subnet_names = var.azure_cdp_subnet_names + cdp_gateway_subnet_names = var.azure_cdp_gateway_subnet_names + public_key_text = var.public_key_text + + data_storage_location = var.data_storage_location + log_storage_location = var.log_storage_location + backup_storage_location = var.backup_storage_location + + xaccount_app_uuid = var.azure_xaccount_app_uuid + xaccount_app_pword = var.azure_xaccount_app_pword + + idbroker_identity_id = var.azure_idbroker_identity_id + datalakeadmin_identity_id = var.azure_datalakeadmin_identity_id + ranger_audit_identity_id = var.azure_ranger_audit_identity_id + log_identity_id = var.azure_log_identity_id + raz_identity_id = var.azure_raz_identity_id +} diff --git a/modules/terraform-cdp-deploy/modules/aws/main.tf b/modules/terraform-cdp-deploy/modules/aws/main.tf new file mode 100644 index 0000000..030c896 --- /dev/null +++ b/modules/terraform-cdp-deploy/modules/aws/main.tf @@ -0,0 +1,86 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Deployment and creation of CDP resources using Ansible Playbook called by TF local-exec + +# ------- Create Configuration file for CDP Deployment via Ansible ------- +resource "local_file" "cdp_deployment_template" { + + content = templatefile("${path.module}/templates/cdp_config.yml.tpl", { + # CDP environment & DL settings + plat__env_name = var.environment_name + plat__datalake_name = var.datalake_name + plat__datalake_scale = var.datalake_scale + plat__datalake_version = var.datalake_version + plat__xacccount_credential_name = var.cdp_xacccount_credential_name + plat__cdp_iam_admin_group_name = var.cdp_admin_group_name + plat__cdp_iam_user_group_name = var.cdp_user_group_name + plat__tunnel = var.enable_ccm_tunnel + plat__endpoint_access_scheme = var.endpoint_access_scheme + plat__enable_raz = var.enable_raz + plat__env_multiaz = var.multiaz + plat__env_freeipa_instances = var.freeipa_instances + plat__workload_analytics = var.workload_analytics + plat__tags = jsonencode(var.tags) + + # CDP settings + plat__cdp_profile = var.cdp_profile + plat__cdp_control_plane_region = var.cdp_control_plane_region + + # CSP settings + plat__infra_type = "aws" + plat__region = var.region + + plat__aws_vpc_id = var.vpc_id + plat__aws_public_subnet_ids = jsonencode(var.public_subnet_ids) + plat__aws_private_subnet_ids = jsonencode(var.private_subnet_ids) + plat__aws_subnets_for_cdp = jsonencode(var.subnets_for_cdp) + + plat__aws_storage_location = var.data_storage_location + plat__aws_log_location = var.log_storage_location + plat__aws_backup_location = var.backup_storage_location + + plat__public_key_id = var.keypair_name + plat__aws_security_group_default_id = var.security_group_default_id + plat__aws_security_group_knox_id = var.security_group_knox_id + + plat__aws_datalake_admin_role_arn = var.datalake_admin_role_arn + plat__aws_ranger_audit_role_arn = var.ranger_audit_role_arn + plat__aws_xaccount_role_arn = var.xaccount_role_arn + + plat__aws_log_instance_profile_arn = var.log_instance_profile_arn + plat__aws_idbroker_instance_profile_arn = var.idbroker_instance_profile_arn + } + ) + filename = "cdp_config.yml" +} + +# ------- Create CDP Deployment ------- +resource "null_resource" "cdp_deployment" { + + # Setup of CDP environment using playbook_setup_cdp.yml.yml Ansible Playbook + provisioner "local-exec" { + command = "ansible-playbook -vvv -e '@cdp_config.yml' ${path.module}/playbook_setup_cdp.yml" + } + + # Deletion of CDP environment using playbook_teardown_cdp.yml Ansible Playbook + provisioner "local-exec" { + when = destroy + command = "ansible-playbook -vvv -e '@cdp_config.yml' ${path.module}/playbook_teardown_cdp.yml" + } + + depends_on = [ + local_file.cdp_deployment_template, + ] +} diff --git a/modules/terraform-cdp-deploy/modules/aws/outputs.tf b/modules/terraform-cdp-deploy/modules/aws/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/terraform-cdp-aws-pre-reqs/playbook_setup_cdp.yml b/modules/terraform-cdp-deploy/modules/aws/playbook_setup_cdp.yml similarity index 100% rename from modules/terraform-cdp-aws-pre-reqs/playbook_setup_cdp.yml rename to modules/terraform-cdp-deploy/modules/aws/playbook_setup_cdp.yml diff --git a/modules/terraform-cdp-aws-pre-reqs/playbook_teardown_cdp.yml b/modules/terraform-cdp-deploy/modules/aws/playbook_teardown_cdp.yml similarity index 100% rename from modules/terraform-cdp-aws-pre-reqs/playbook_teardown_cdp.yml rename to modules/terraform-cdp-deploy/modules/aws/playbook_teardown_cdp.yml diff --git a/modules/terraform-cdp-deploy/modules/aws/provider.tf b/modules/terraform-cdp-deploy/modules/aws/provider.tf new file mode 100644 index 0000000..b4c3f07 --- /dev/null +++ b/modules/terraform-cdp-deploy/modules/aws/provider.tf @@ -0,0 +1,28 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + local = { + source = "hashicorp/local" + version = "2.2.3" + } + null = { + source = "hashicorp/null" + version = "3.2.1" + } + } + + required_version = "> 1.3.0" +} diff --git a/modules/terraform-cdp-aws-pre-reqs/templates/cdp_config.yml.tpl b/modules/terraform-cdp-deploy/modules/aws/templates/cdp_config.yml.tpl similarity index 92% rename from modules/terraform-cdp-aws-pre-reqs/templates/cdp_config.yml.tpl rename to modules/terraform-cdp-deploy/modules/aws/templates/cdp_config.yml.tpl index db424da..8ee2a71 100644 --- a/modules/terraform-cdp-aws-pre-reqs/templates/cdp_config.yml.tpl +++ b/modules/terraform-cdp-deploy/modules/aws/templates/cdp_config.yml.tpl @@ -23,14 +23,14 @@ plat__datalake_version: ${plat__datalake_version} plat__tunnel: ${plat__tunnel} plat__endpoint_access_scheme: ${plat__endpoint_access_scheme} plat__enable_raz: ${plat__enable_raz} -plat__env_multiaz: ${plat__env_multiaz} +plat__env_multiaz: ${plat__env_multiaz} plat__env_freeipa_instances: ${plat__env_freeipa_instances} plat__workload_analytics: ${plat__workload_analytics} plat__tags: ${plat__tags} # CDP settings plat__cdp_profile: ${plat__cdp_profile} -plat__cdp_control_plane_region: ${plat__cdp_control_plane_region} +plat__cdp_control_plane_region: ${plat__cdp_control_plane_region} # CSP settings plat__infra_type: ${plat__infra_type} @@ -50,8 +50,8 @@ plat__aws_security_group_default_id: ${plat__aws_security_group_default_id} plat__aws_security_group_knox_id: ${plat__aws_security_group_knox_id} plat__aws_datalake_admin_role_arn: ${plat__aws_datalake_admin_role_arn} -plat__aws_ranger_audit_role_arn: ${plat__aws_ranger_audit_role_arn} -plat__aws_xaccount_role_arn: ${plat__aws_xaccount_role_arn} +plat__aws_ranger_audit_role_arn: ${plat__aws_ranger_audit_role_arn} +plat__aws_xaccount_role_arn: ${plat__aws_xaccount_role_arn} -plat__aws_log_instance_profile_arn: ${plat__aws_log_instance_profile_arn} +plat__aws_log_instance_profile_arn: ${plat__aws_log_instance_profile_arn} plat__aws_idbroker_instance_profile_arn: ${plat__aws_idbroker_instance_profile_arn} diff --git a/modules/terraform-cdp-deploy/modules/aws/variables.tf b/modules/terraform-cdp-deploy/modules/aws/variables.tf new file mode 100644 index 0000000..57c9060 --- /dev/null +++ b/modules/terraform-cdp-deploy/modules/aws/variables.tf @@ -0,0 +1,263 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "tags" { + type = map(any) + description = "Tags applied to provisioned resources" + +} + +# ------- CDP Environment Deployment ------- +variable "environment_name" { + type = string + description = "Name of the CDP environment." + +} + +variable "datalake_name" { + type = string + description = "Name of the CDP DataLake." + +} + +variable "cdp_xacccount_credential_name" { + type = string + description = "Name of the CDP Cross Account Credential." + +} + +variable "cdp_admin_group_name" { + type = string + description = "Name of the CDP IAM Admin Group associated with the environment." + +} + +variable "cdp_user_group_name" { + type = string + description = "Name of the CDP IAM User Group associated with the environment." + +} + +variable "cdp_profile" { + type = string + description = "Profile for CDP credentials" + +} + +variable "cdp_control_plane_region" { + type = string + description = "CDP Control Plane Region" + +} + +variable "enable_ccm_tunnel" { + type = bool + + description = "Flag to enable Cluster Connectivity Manager tunnel. If false then access from Cloud to CDP Control Plane CIDRs is required from via SG ingress" + +} + +variable "enable_raz" { + type = bool + + description = "Flag to enable Ranger Authorization Service (RAZ)" + +} + +variable "multiaz" { + type = bool + + description = "Flag to specify that the FreeIPA and DataLake instances will be deployed across multi-availability zones" + +} + +variable "freeipa_instances" { + type = number + + description = "The number of FreeIPA instances to create in the environment" + +} + +variable "workload_analytics" { + type = bool + + description = "Flag to specify if workload analytics should be enabled for the CDP environment" + +} + +variable "datalake_scale" { + type = string + + description = "The scale of the datalake. Valid values are LIGHT_DUTY, MEDIUM_DUTY_HA." + + validation { + condition = contains(["LIGHT_DUTY", "MEDIUM_DUTY_HA"], var.datalake_scale) + error_message = "Valid values for var: datalake_scale are (LIGHT_DUTY, MEDIUM_DUTY_HA)." + } + +} + +variable "datalake_version" { + type = string + + description = "The Datalake Runtime version. Valid values are semantic versions, e.g. 7.2.16" + + validation { + condition = length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0 + error_message = "Valid values for var: datalake_version must match semantic versioning conventions." + } + +} + +# ------- Cloud Service Provider Settings ------- +variable "region" { + type = string + description = "Region which Cloud resources will be created" + +} + +variable "keypair_name" { + type = string + + description = "SSH Keypair name in Cloud Service Provider. Required for CDP deployment on AWS." + + default = null +} + +variable "vpc_id" { + type = string + description = "AWS Virtual Private Network ID." + + validation { + condition = var.vpc_id != null + error_message = "Valid values for var: vpc_id must be a existing AWS VPC." + } + +} + +variable "public_subnet_ids" { + type = list(string) + description = "List of public subnet ids." +} + +variable "private_subnet_ids" { + type = list(string) + description = "List of private subnet ids." + +} + +variable "subnets_for_cdp" { + type = list(string) + description = "List of subnets to use for the CDP nodes. This is a subset of the public and private subnets and depending in the deployment template." + +} + +variable "security_group_default_id" { + type = string + + description = "ID of the Default Security Group for CDP environment" +} + +variable "security_group_knox_id" { + type = string + + description = "ID of the Knox Security Group for CDP environment" +} + +variable "endpoint_access_scheme" { + type = string + + description = "The scheme for the workload endpoint gateway. PUBLIC creates an external endpoint that can be accessed over the Internet. PRIVATE which restricts the traffic to be internal to the VPC / Vnet. Relevant in Private Networks." + + validation { + condition = contains(["PUBLIC", "PRIVATE"], var.endpoint_access_scheme) + error_message = "Valid values for var: endpoint_access_scheme are (PUBLIC, PRIVATE)." + } +} + +variable "data_storage_location" { + type = string + description = "Data storage location." +} + +variable "log_storage_location" { + type = string + description = "Log storage location." +} + +variable "backup_storage_location" { + type = string + description = "Backup storage location." +} + +variable "datalake_admin_role_arn" { + type = string + + description = "Datalake Admin Role ARN." + + validation { + condition = var.datalake_admin_role_arn != null + error_message = "Valid values for var: datalake_admin_role_arn must be a valid ARN for Datalake Admin Role." + } + +} + +variable "ranger_audit_role_arn" { + type = string + + description = "Ranger Audit Role ARN." + + validation { + condition = var.ranger_audit_role_arn != null + error_message = "Valid values for var: ranger_audit_role_arn must be a valid ARN for Ranger Audit Role." + } + +} + +variable "xaccount_role_arn" { + type = string + + description = "Cross Account Role ARN." + + validation { + condition = var.xaccount_role_arn != null + error_message = "Valid values for var: xaccount_role_arn must be a valid ARN for Cross Account Role." + } + +} + +variable "log_instance_profile_arn" { + type = string + + description = "Log Instance Profile ARN." + + validation { + condition = var.log_instance_profile_arn != null + error_message = "Valid values for var: log_instance_profile_arn must be a valid ARN for Log Instance Profile." + } + +} + +variable "idbroker_instance_profile_arn" { + type = string + + description = "IDBroker Instance Profile ARN." + + validation { + condition = var.idbroker_instance_profile_arn != null + error_message = "Valid values for var: idbroker_instance_profile_arn must be a valid ARN for IDBroker Instance Profile." + } + +} \ No newline at end of file diff --git a/modules/terraform-cdp-deploy/modules/azure/main.tf b/modules/terraform-cdp-deploy/modules/azure/main.tf new file mode 100644 index 0000000..aceccd5 --- /dev/null +++ b/modules/terraform-cdp-deploy/modules/azure/main.tf @@ -0,0 +1,91 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Create Configuration file for CDP Deployment via Ansible ------- +resource "local_file" "cdp_deployment_template" { + + content = templatefile("${path.module}/templates/cdp_config.yml.tpl", { + # CDP environment & DL settings + plat__env_name = var.environment_name + plat__datalake_name = var.datalake_name + plat__datalake_scale = var.datalake_scale + plat__datalake_version = var.datalake_version + plat__xacccount_credential_name = var.cdp_xacccount_credential_name + plat__cdp_iam_admin_group_name = var.cdp_admin_group_name + plat__cdp_iam_user_group_name = var.cdp_user_group_name + plat__tunnel = var.enable_ccm_tunnel + plat__endpoint_access_scheme = var.endpoint_access_scheme + plat__enable_raz = var.enable_raz + plat__use_single_resource_group = var.use_single_resource_group + plat__use_public_ips = var.use_public_ips + plat__env_freeipa_instances = var.freeipa_instances + plat__workload_analytics = var.workload_analytics + plat__tags = jsonencode(var.tags) + + # CDP settings + plat__cdp_profile = var.cdp_profile + plat__cdp_control_plane_region = var.cdp_control_plane_region + + # # CSP settings + plat__infra_type = "azure" + plat__region = var.region + + plat__azure_subscription_id = var.subscription_id + plat__azure_tenant_id = var.tenant_id + + plat__azure_resourcegroup_name = var.resource_group_name + plat__azure_vnet_name = var.vnet_name + plat__azure_subnet_names_for_cdp = jsonencode(var.cdp_subnet_names) + plat__azure_subnet_names_for_gateway = jsonencode(var.cdp_gateway_subnet_names) + + plat__azure_storage_location = var.data_storage_location + plat__azure_log_location = var.log_storage_location + plat__azure_backup_location = var.backup_storage_location + + plat__public_key_text = var.public_key_text + plat__azure_security_group_default_uri = var.security_group_default_uri + plat__azure_security_group_knox_uri = var.security_group_knox_uri + + plat__azure_xaccount_app_uuid = var.xaccount_app_uuid + plat__azure_xaccount_app_pword = var.xaccount_app_pword + + plat__azure_idbroker_identity_id = var.idbroker_identity_id + plat__azure_datalakeadmin_identity_id = var.datalakeadmin_identity_id + plat__azure_ranger_audit_identity_id = var.ranger_audit_identity_id + plat__azure_log_identity_id = var.log_identity_id + plat__azure_raz_identity_id = var.raz_identity_id + + } + ) + filename = "cdp_config.yml" +} + +# ------- Create CDP Deployment ------- +resource "null_resource" "cdp_deployment" { + + # Setup of CDP environment using playbook_setup_cdp.yml Ansible Playbook + provisioner "local-exec" { + command = "ansible-playbook -vvv -e '@cdp_config.yml' ${path.module}/playbook_setup_cdp.yml" + } + + # Deletion of CDP environment using playbook_teardown_cdp.yml Ansible Playbook + provisioner "local-exec" { + when = destroy + command = "ansible-playbook -vvv -e '@cdp_config.yml' ${path.module}/playbook_teardown_cdp.yml" + } + + depends_on = [ + local_file.cdp_deployment_template, + ] +} diff --git a/modules/terraform-cdp-deploy/modules/azure/outputs.tf b/modules/terraform-cdp-deploy/modules/azure/outputs.tf new file mode 100644 index 0000000..e3cf504 --- /dev/null +++ b/modules/terraform-cdp-deploy/modules/azure/outputs.tf @@ -0,0 +1,13 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/modules/terraform-cdp-deploy/modules/azure/playbook_setup_cdp.yml b/modules/terraform-cdp-deploy/modules/azure/playbook_setup_cdp.yml new file mode 100644 index 0000000..87a4823 --- /dev/null +++ b/modules/terraform-cdp-deploy/modules/azure/playbook_setup_cdp.yml @@ -0,0 +1,313 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################# +# Ansible Playbook to create CDP resources based +# on output generated from TF pre-reqs module. +# Uses the cloudera.cloud Ansible collection +# +# Tasks performed: +# * Create Cross Account credential +# * Set up Environment Deployment +# * Creates user and admin groups and CDP IDBroker Mappings +# * Set up Datalake +############################# +- name: Create CDP Resources + hosts: localhost + environment: + CDP_PROFILE: "{{ plat__cdp_profile }}" + CDP_REGION: "{{ plat__cdp_control_plane_region }}" + vars: + plat__cdp_iam_identities: + role_suffix: role + resource_role_suffix: resourceRole + env_admin: EnvironmentAdmin + env_user: EnvironmentUser + dw_admin: DWAdmin + dw_user: DWUser + ml_admin: MLAdmin + ml_user: MLUser + df_admin: DFAdmin + df_flow_admin: DFFlowAdmin + df_flow_user: DFFlowUser + de_admin: DEAdmin + de_user: DEUser + plat__cdp_iam_admin_group_resource_roles_default: + - "{{ plat__cdp_iam_identities.env_admin }}" + - "{{ plat__cdp_iam_identities.env_user }}" + - "{{ plat__cdp_iam_identities.dw_admin }}" + - "{{ plat__cdp_iam_identities.dw_user }}" + - "{{ plat__cdp_iam_identities.ml_admin }}" + - "{{ plat__cdp_iam_identities.ml_user }}" + - "{{ plat__cdp_iam_identities.df_admin }}" + - "{{ plat__cdp_iam_identities.df_flow_admin }}" + - "{{ plat__cdp_iam_identities.df_flow_user }}" + - "{{ plat__cdp_iam_identities.de_admin }}" + - "{{ plat__cdp_iam_identities.de_user }}" + plat__cdp_iam_user_group_resource_roles_default: + - "{{ plat__cdp_iam_identities.env_user }}" + - "{{ plat__cdp_iam_identities.dw_user }}" + - "{{ plat__cdp_iam_identities.ml_user }}" + - "{{ plat__cdp_iam_identities.df_flow_admin }}" + - "{{ plat__cdp_iam_identities.df_flow_user }}" + - "{{ plat__cdp_iam_identities.de_user }}" + plat__cdp_iam_admin_group_roles: + - "{{ plat__cdp_iam_identities.env_admin }}" + plat__cdp_iam_admin_group_resource_roles: "{{ plat__cdp_iam_admin_group_resource_roles_default }}" + plat__cdp_iam_user_group_roles: + - "{{ plat__cdp_iam_identities.env_user }}" + plat__cdp_iam_user_group_resource_roles: "{{ plat__cdp_iam_user_group_resource_roles_default }}" + common__cdp_control_planes: + us-west-1: "crn:altus:iam:us-west-1:altus" + eu-1: "crn:altus:iam:eu-1:altus" + ap-1: "crn:altus:iam:ap-1:altus" + common__cdp_control_plane_region: "{{ plat__cdp_control_plane_region | default('us-west-1') }}" + common__cdp_control_plane_crn: "{{ common__cdp_control_planes[common__cdp_control_plane_region] }}" + plat__cdp_control_plane_crn: "{{ common__cdp_control_plane_crn }}" + plat__cdp_iam_role_suffix: "{{ plat__cdp_iam_identities.role_suffix }}" + plat__cdp_iam_resource_suffix: "{{ plat__cdp_iam_identities.resource_role_suffix }}" + tasks: + +############# CDP Credential ############# + - name: Create CDP Cross Account Credential for Azure + cloudera.cloud.env_cred: + cloud: "{{ plat__infra_type }}" + name: "{{ plat__xacccount_credential_name }}" + subscription: "{{ plat__azure_subscription_id }}" + tenant: "{{ plat__azure_tenant_id }}" + application: "{{ plat__azure_xaccount_app_uuid }}" + secret: "{{ plat__azure_xaccount_app_pword }}" + state: present + +############# CDP Environment ############# + - name: Start set up of CDP Environment Deployment on Azure + block: + - name: Call cloudera.cloud.env to register CDP environment + cloudera.cloud.env: + name: "{{ plat__env_name }}" + state: started + credential: "{{ plat__xacccount_credential_name }}" + cloud: "{{ plat__infra_type }}" + region: "{{ plat__region }}" + default_sg: "{{ plat__azure_security_group_default_uri }}" + knox_sg: "{{ plat__azure_security_group_knox_uri }}" + log_location: "{{ plat__azure_log_location }}" + log_identity: "{{ plat__azure_log_identity_id }}" + backup_location: "{{ plat__azure_backup_location }}" + public_key_text: "{{ plat__public_key_text }}" + workload_analytics: "{{ plat__workload_analytics }}" + network: "{{ plat__azure_vnet_name }}" + subnet_ids: "{{ plat__azure_subnet_names_for_cdp }}" + tags: "{{ plat__tags }}" + tunnel: "{{ plat__tunnel }}" + # endpoint_access_scheme: "{{ plat__endpoint_access_scheme | default(omit) }}" + # endpoint_access_subnets: "{{ plat__azure_subnet_names_for_gateway | default(omit) }}" + resource_gp: "{{ plat__azure_resourcegroup_name }}" + use_single_resource_group: "{{ plat__use_single_resource_group }}" + public_ip: "{{ plat__use_public_ips }}" + freeipa: + instanceCountByGroup: "{{ plat__env_freeipa_instances }}" + async: 3600 # 1 hour timeout + poll: 0 + register: __cdp_env_setup + + - name: Wait for CDP environment setup to start + ansible.builtin.pause: + seconds: 60 + + - name: Update CDP Environment information + cloudera.cloud.env_info: + name: "{{ plat__env_name }}" + register: plat__cdp_env_info + failed_when: plat__cdp_env_info.environments | length == 0 + rescue: + # Fallback to synchronous if environment not found + - name: Wait for CDP Environment setup to complete + when: __cdp_env_setup.ansible_job_id is defined + ansible.builtin.async_status: + jid: "{{ __cdp_env_setup.ansible_job_id }}" + register: __cdp_env_setup_async + until: __cdp_env_setup_async.finished + retries: 120 + delay: 30 + + - name: Attempt to update CDP Environment information again + cloudera.cloud.env_info: + name: "{{ plat__env_name }}" + register: plat__cdp_env_info + failed_when: plat__cdp_env_info.environments | length == 0 + + # Resume normal service + - name: Set fact for CDP Environment CRN + ansible.builtin.set_fact: + plat__cdp_env_crn: "{{ plat__cdp_env_info.environments[0].crn | default('Unknown') }}" + + - name: Print Environment CRN + debug: + msg: + - plat__cdp_env_crn is {{ plat__cdp_env_crn }} + +############# CDP Groups ############# + - name: Query CDP Caller to confirm access to Endpoints + cloudera.cloud.iam_user_info: + current_user: yes + register: __cdp_iam_current_user_info + + - name: Set facts for CDP Caller Workload Username and CRN + ansible.builtin.set_fact: + plat__cdp_workload_username: "{{ __cdp_iam_current_user_info.users[0].workloadUsername }}" + plat__cdp_user_crn: "{{ __cdp_iam_current_user_info.users[0].crn }}" + + - name: Set fact for CDP Environment Admin Group Role CRNs + ansible.builtin.set_fact: + plat__cdp_pub_admin_group_role_crns: "{{ plat__cdp_pub_admin_group_role_crns | default([]) | union([role]) }}" + vars: + role: "{{ [plat__cdp_control_plane_crn, plat__cdp_iam_role_suffix, __cdp_pub_admin_group_role_item] | join(':') }}" + loop_control: + loop_var: __cdp_pub_admin_group_role_item + loop: "{{ plat__cdp_iam_admin_group_roles }}" + + - name: Set fact for CDP Environment Admin Group Resource Role CRNs + ansible.builtin.set_fact: + plat__cdp_pub_admin_group_resource_role_crns: "{{ plat__cdp_pub_admin_group_resource_role_crns | default([]) | union([resource_role]) }}" + vars: + resource_role: "{{ [plat__cdp_control_plane_crn, plat__cdp_iam_resource_suffix, __cdp_env_admin_group_resource_role_item] | join(':') }}" + loop_control: + loop_var: __cdp_env_admin_group_resource_role_item + loop: "{{ plat__cdp_iam_admin_group_resource_roles }}" + + - name: Set fact for CDP Environment User Group Role CRNs + ansible.builtin.set_fact: + plat__cdp_pub_user_group_role_crns: "{{ plat__cdp_pub_user_group_role_crns | default([]) | union([role]) }}" + vars: + role: "{{ [plat__cdp_control_plane_crn, plat__cdp_iam_role_suffix, __cdp_pub_user_group_role_item] | join(':') }}" + loop_control: + loop_var: __cdp_pub_user_group_role_item + loop: "{{ plat__cdp_iam_user_group_roles }}" + + - name: Set fact for CDP Environment User Group Resource Role CRNs + ansible.builtin.set_fact: + plat__cdp_pub_user_group_resource_role_crns: "{{ plat__cdp_pub_user_group_resource_role_crns | default([]) | union([resource_role]) }}" + vars: + resource_role: "{{ [plat__cdp_control_plane_crn, plat__cdp_iam_resource_suffix, __cdp_pub_user_group_resource_role_item] | join(':') }}" + loop_control: + loop_var: __cdp_pub_user_group_resource_role_item + loop: "{{ plat__cdp_iam_user_group_resource_roles }}" + + - name: Set fact for CDP Admin Group Resource Role assignments + ansible.builtin.set_fact: + plat__cdp_env_admin_group_resource_role_assignments: "{{ plat__cdp_env_admin_group_resource_role_assignments | default([]) | union([resource_role_assignment]) }}" + vars: + resource_role_assignment: + resource: "{{ plat__cdp_env_crn }}" + role: "{{ __cdp_resource_role_item }}" + loop_control: + loop_var: __cdp_resource_role_item + loop: "{{ plat__cdp_pub_admin_group_resource_role_crns }}" + + - name: Create CDP Admin group + cloudera.cloud.iam_group: + name: "{{ plat__cdp_iam_admin_group_name }}" + state: present + sync: false + roles: "{{ plat__cdp_pub_admin_group_role_crns }}" + resource_roles: "{{ plat__cdp_env_admin_group_resource_role_assignments }}" + users: + - "{{ plat__cdp_user_crn }}" + register: __cdp_pub_admin_group_info + + - name: Set fact for CDP Admin Group CRN + ansible.builtin.set_fact: + plat__cdp_pub_admin_group_crn: "{{ __cdp_pub_admin_group_info | community.general.json_query('group.crn') }}" + + - name: Set fact for CDP User Group Resource Role assignments + ansible.builtin.set_fact: + plat__cdp_pub_user_group_resource_role_assignments: "{{ plat__cdp_pub_user_group_resource_role_assignments | default([]) | union([resource_role_assignment]) }}" + vars: + resource_role_assignment: + resource: "{{ plat__cdp_env_crn }}" + role: "{{ __cdp_resource_role_item }}" + loop_control: + loop_var: __cdp_resource_role_item + loop: "{{ plat__cdp_pub_user_group_resource_role_crns }}" + + - name: Create CDP User group + cloudera.cloud.iam_group: + name: "{{ plat__cdp_iam_user_group_name }}" + state: present + sync: false + roles: "{{ plat__cdp_pub_user_group_role_crns }}" + resource_roles: "{{ plat__cdp_pub_user_group_resource_role_assignments }}" + register: __cdp_pub_user_group_info + + - name: Set fact for CDP User Group CRN + ansible.builtin.set_fact: + plat__cdp_pub_user_group_crn: "{{ __cdp_pub_user_group_info | community.general.json_query('group.crn') }}" + +############# IDBroker Mappings ############# + + - name: Set IDBroker Mappings for CDP on Azure + cloudera.cloud.env_idbroker: + name: "{{ plat__env_name }}" + sync: no + data_access: "{{ plat__azure_datalakeadmin_identity_id }}" + ranger_audit: "{{ plat__azure_ranger_audit_identity_id }}" + ranger_cloud_access: "{{ (plat__enable_raz | bool) | ternary(plat__azure_raz_identity_id, omit) }}" + mappings: + - accessor: "{{ plat__cdp_pub_admin_group_crn }}" + role: "{{ plat__azure_datalakeadmin_identity_id }}" + - accessor: "{{ plat__cdp_pub_user_group_crn }}" + role: "{{ plat__azure_datalakeadmin_identity_id }}" + +############# CDP Datalake ############# + + - name: Start set up of CDP Datalake for Azure + cloudera.cloud.datalake: + name: "{{ plat__datalake_name }}" + env: "{{ plat__env_name }}" + managed_identity: "{{ plat__azure_idbroker_identity_id }}" + storage: "{{ plat__azure_storage_location }}" + runtime: "{{ plat__datalake_version | default(omit) }}" + scale: "{{ plat__datalake_scale | default(omit) }}" + tags: "{{ plat__tags }}" + raz: "{{ plat__enable_raz }}" + state: present + async: 10000 + poll: 0 + register: __cdp_dl_setup + +############# Wait for Completion of Async tasks ############# + + - name: Wait for CDP Environment setup to complete + when: + - __cdp_env_setup_async is not defined + - __cdp_env_setup_async.finished is not defined + - __cdp_env_setup.ansible_job_id is defined + ansible.builtin.async_status: + jid: "{{ __cdp_env_setup.ansible_job_id }}" + register: __cdp_env_setup_async + until: __cdp_env_setup_async.finished + retries: 120 + delay: 30 + + - name: Wait for CDP Datalake setup to complete + when: __cdp_dl_setup.ansible_job_id is defined + ansible.builtin.async_status: + jid: "{{ __cdp_dl_setup.ansible_job_id }}" + register: __cdp_dl_setup_async + until: __cdp_dl_setup_async.finished + retries: 240 + delay: 30 diff --git a/modules/terraform-cdp-deploy/modules/azure/playbook_teardown_cdp.yml b/modules/terraform-cdp-deploy/modules/azure/playbook_teardown_cdp.yml new file mode 100644 index 0000000..3807257 --- /dev/null +++ b/modules/terraform-cdp-deploy/modules/azure/playbook_teardown_cdp.yml @@ -0,0 +1,58 @@ +--- + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +################################################# +# Ansible Playbook to remove CDP resources based +# on output generated from TF pre-reqs module. +# Uses the cloudera.cloud Ansible collection +# +# Tasks performed: +# * Cascading delete to remove Environment, DL, DataHubs and Data Services +# * Delete user and admin groups +# * Delete Cross Account credential +############################# + +- name: Remove CDP Resources + hosts: localhost + environment: + CDP_PROFILE: "{{ plat__cdp_profile }}" + vars: + tasks: + +############# Cascading Delete of CDP Environment ############# + - name: Delete CDP Environment Deployment on AWS + cloudera.cloud.env: + name: "{{ plat__env_name }}" + cascading: True + state: absent + +############# CDP Groups ############# + - name: Remove CDP Admin group + cloudera.cloud.iam_group: + name: "{{ plat__cdp_iam_admin_group_name }}" + state: absent + + - name: Remove CDP User group + cloudera.cloud.iam_group: + name: "{{ plat__cdp_iam_user_group_name }}" + state: absent + +############# CDP Credential ############# + - name: Remove CDP Cross Account Credential for AWS + cloudera.cloud.env_cred: + name: "{{ plat__xacccount_credential_name }}" + state: absent + diff --git a/modules/terraform-cdp-deploy/modules/azure/provider.tf b/modules/terraform-cdp-deploy/modules/azure/provider.tf new file mode 100644 index 0000000..b4c3f07 --- /dev/null +++ b/modules/terraform-cdp-deploy/modules/azure/provider.tf @@ -0,0 +1,28 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + local = { + source = "hashicorp/local" + version = "2.2.3" + } + null = { + source = "hashicorp/null" + version = "3.2.1" + } + } + + required_version = "> 1.3.0" +} diff --git a/modules/terraform-cdp-deploy/modules/azure/templates/cdp_config.yml.tpl b/modules/terraform-cdp-deploy/modules/azure/templates/cdp_config.yml.tpl new file mode 100644 index 0000000..113cae2 --- /dev/null +++ b/modules/terraform-cdp-deploy/modules/azure/templates/cdp_config.yml.tpl @@ -0,0 +1,64 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# CDP environment & DL settings +plat__env_name: ${plat__env_name} +plat__datalake_name: ${plat__datalake_name} +plat__xacccount_credential_name: ${plat__xacccount_credential_name} +plat__cdp_iam_admin_group_name: ${plat__cdp_iam_admin_group_name} +plat__cdp_iam_user_group_name: ${plat__cdp_iam_user_group_name} + +plat__datalake_scale: ${plat__datalake_scale} +plat__datalake_version: ${plat__datalake_version} +plat__tunnel: ${plat__tunnel} +plat__endpoint_access_scheme: ${plat__endpoint_access_scheme} +plat__use_public_ips: ${plat__use_public_ips} +plat__enable_raz: ${plat__enable_raz} +plat__use_single_resource_group: ${plat__use_single_resource_group} +plat__workload_analytics: ${plat__workload_analytics} +plat__env_freeipa_instances: ${plat__env_freeipa_instances} +plat__tags: ${plat__tags} + +# CDP settings +plat__cdp_profile: ${plat__cdp_profile} +plat__cdp_control_plane_region: ${plat__cdp_control_plane_region} + +# CSP settings +plat__infra_type: ${plat__infra_type} +plat__region: ${plat__region} + +plat__azure_subscription_id: ${plat__azure_subscription_id} +plat__azure_tenant_id: ${plat__azure_tenant_id} + +plat__azure_vnet_name: ${plat__azure_vnet_name} +plat__azure_resourcegroup_name: ${plat__azure_resourcegroup_name} +plat__azure_subnet_names_for_cdp: ${plat__azure_subnet_names_for_cdp} +plat__azure_subnet_names_for_gateway: ${plat__azure_subnet_names_for_gateway} + +plat__azure_storage_location: ${plat__azure_storage_location} +plat__azure_log_location: ${plat__azure_log_location} +plat__azure_backup_location: ${plat__azure_backup_location} + +plat__public_key_text: ${plat__public_key_text} +plat__azure_security_group_default_uri: ${plat__azure_security_group_default_uri} +plat__azure_security_group_knox_uri: ${plat__azure_security_group_knox_uri} + +plat__azure_xaccount_app_uuid: ${plat__azure_xaccount_app_uuid} +plat__azure_xaccount_app_pword: ${plat__azure_xaccount_app_pword} + +plat__azure_idbroker_identity_id: ${plat__azure_idbroker_identity_id} +plat__azure_datalakeadmin_identity_id: ${plat__azure_datalakeadmin_identity_id} +plat__azure_ranger_audit_identity_id: ${plat__azure_ranger_audit_identity_id} +plat__azure_log_identity_id: ${plat__azure_log_identity_id} +plat__azure_raz_identity_id: ${plat__azure_raz_identity_id} diff --git a/modules/terraform-cdp-deploy/modules/azure/variables.tf b/modules/terraform-cdp-deploy/modules/azure/variables.tf new file mode 100644 index 0000000..10ec1d1 --- /dev/null +++ b/modules/terraform-cdp-deploy/modules/azure/variables.tf @@ -0,0 +1,348 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "tags" { + type = map(any) + description = "Tags applied to provisioned resources" + +} + +# ------- CDP Environment Deployment ------- +variable "environment_name" { + type = string + description = "Name of the CDP environment." + +} + +variable "datalake_name" { + type = string + description = "Name of the CDP DataLake." + +} + +variable "cdp_xacccount_credential_name" { + type = string + description = "Name of the CDP Cross Account Credential." + +} + +variable "cdp_admin_group_name" { + type = string + description = "Name of the CDP IAM Admin Group associated with the environment." + +} + +variable "cdp_user_group_name" { + type = string + description = "Name of the CDP IAM User Group associated with the environment." + +} + +variable "cdp_profile" { + type = string + description = "Profile for CDP credentials" + +} + +variable "cdp_control_plane_region" { + type = string + description = "CDP Control Plane Region" + +} + +variable "enable_ccm_tunnel" { + type = bool + + description = "Flag to enable Cluster Connectivity Manager tunnel. If false then access from Cloud to CDP Control Plane CIDRs is required from via SG ingress" + +} + +variable "enable_raz" { + type = bool + + description = "Flag to enable Ranger Authorization Service (RAZ)" + +} + +variable "freeipa_instances" { + type = number + + description = "The number of FreeIPA instances to create in the environment" + +} + + +variable "workload_analytics" { + type = bool + + description = "Flag to specify if workload analytics should be enabled for the CDP environment" + +} + + +variable "datalake_scale" { + type = string + + description = "The scale of the datalake. Valid values are LIGHT_DUTY, MEDIUM_DUTY_HA." + + validation { + condition = contains(["LIGHT_DUTY", "MEDIUM_DUTY_HA"], var.datalake_scale) + error_message = "Valid values for var: datalake_scale are (LIGHT_DUTY, MEDIUM_DUTY_HA)." + } + +} + +variable "datalake_version" { + type = string + + description = "The Datalake Runtime version. Valid values are semantic versions, e.g. 7.2.16" + + validation { + condition = length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0 + error_message = "Valid values for var: datalake_version must match semantic versioning conventions." + } + +} + +# ------- Cloud Service Provider Settings ------- +variable "subscription_id" { + type = string + + description = "Subscription ID where the Azure pre-reqs are created" + + validation { + condition = var.subscription_id != null + error_message = "Valid values for var: subscription_id must be a existing Azure Subscription." + } + + +} + +variable "tenant_id" { + type = string + + description = "Tenant ID where the Azure pre-reqs are created" + + validation { + condition = var.tenant_id != null + error_message = "Valid values for var: tenant_id must be a existing Azure Tenant." + } + +} + +variable "region" { + type = string + description = "Region which Cloud resources will be created" + +} + +variable "resource_group_name" { + type = string + description = "Resource Group name" + + validation { + condition = var.resource_group_name != null + error_message = "Valid values for var: resource_group_name must be a existing Azure Resource group." + } + +} + +variable "vnet_name" { + type = string + description = "Azure Virtual Network ID." + + validation { + condition = var.vnet_name != null + error_message = "Valid values for var: vnet_name must be a existing Azure Virtual Network." + } + +} + +variable "cdp_subnet_names" { + type = list(any) + description = "Azure Subnet Names for CDP resources." + + validation { + condition = var.cdp_subnet_names != null + error_message = "Valid values for var: cdp_subnet_names must be a list of existing Azure Virtual Subnets." + } + +} + +variable "cdp_gateway_subnet_names" { + type = list(any) + description = "Azure Subnet Names for Endpoint Access Gateway." + + validation { + condition = var.cdp_gateway_subnet_names != null + error_message = "Valid values for var: cdp_gateway_subnet_names must be a list of existing Azure Virtual Subnets." + } + +} + +variable "security_group_default_uri" { + type = string + description = "Azure Default Security Group URI." + + validation { + condition = var.security_group_default_uri != null + error_message = "Valid values for var: security_group_default_uri must be a valid Azure SG Uri for the Default SG." + } + +} + +variable "security_group_knox_uri" { + type = string + description = "Azure Knox Security Group URI." + + validation { + condition = var.security_group_knox_uri != null + error_message = "Valid values for var: security_group_knox_uri must be a valid Azure SG Uri for the Knox SG." + } + +} + +variable "endpoint_access_scheme" { + type = string + + description = "The scheme for the workload endpoint gateway. PUBLIC creates an external endpoint that can be accessed over the Internet. PRIVATE which restricts the traffic to be internal to the VPC / Vnet. Relevant in Private Networks." + + validation { + condition = contains(["PUBLIC", "PRIVATE"], var.endpoint_access_scheme) + error_message = "Valid values for var: endpoint_access_scheme are (PUBLIC, PRIVATE)." + } +} + +variable "public_key_text" { + type = string + + description = "SSH Public key string for the nodes of the CDP environment" +} + +variable "use_single_resource_group" { + type = bool + + description = "Use a single resource group for all provisioned CDP resources" + +} + +variable "use_public_ips" { + type = bool + + description = "Use public ip's for the CDP resources created within the Azure network" + +} + +variable "data_storage_location" { + type = string + description = "Data storage location." +} + +variable "log_storage_location" { + type = string + description = "Log storage location." +} + +variable "backup_storage_location" { + type = string + description = "Backup storage location." +} + +variable "xaccount_app_uuid" { + type = string + + description = "UUID for the Azure AD Cross Account Application." + + validation { + condition = var.xaccount_app_uuid != null + error_message = "Valid values for var: xaccount_app_uuid must be a valid uuid for the Azure AD Cross Account Application." + } + +} + +variable "xaccount_app_pword" { + type = string + + description = "Password for the Azure AD Cross Account Application." + + sensitive = true + validation { + condition = var.xaccount_app_pword != null + error_message = "Valid values for var: xaccount_app_pword must be a valid password for the Azure AD Cross Account Application." + } + +} + +variable "idbroker_identity_id" { + type = string + + description = "IDBroker Managed Identity ID." + + validation { + condition = var.idbroker_identity_id != null + error_message = "Valid values for var: idbroker_identity_id must be a valid ID for IDBroker Managed Identity." + } + +} + +variable "datalakeadmin_identity_id" { + type = string + + description = "Datalake Admin Managed Identity ID." + + validation { + condition = var.datalakeadmin_identity_id != null + error_message = "Valid values for var: datalakeadmin_identity_id must be a valid ID for Datalake Admin Managed Identity." + } + +} + +variable "ranger_audit_identity_id" { + type = string + + description = "Ranger Audit Managed Identity ID." + + validation { + condition = var.ranger_audit_identity_id != null + error_message = "Valid values for var: ranger_audit_identity_id must be a valid ID for Ranger Audit Managed Identity." + } + + +} + +variable "log_identity_id" { + type = string + + description = "Log Data Access Managed Identity ID." + + validation { + condition = var.log_identity_id != null + error_message = "Valid values for var: log_identity_id must be a valid ID for Log Data Access Managed Identity." + } + + +} + +variable "raz_identity_id" { + type = string + + description = "RAZ Managed Identity ID." + + validation { + condition = var.raz_identity_id != null + error_message = "Valid values for var: raz_identity_id must be a valid ID for RAZ Managed Identity." + } + +} diff --git a/modules/terraform-cdp-deploy/outputs.tf b/modules/terraform-cdp-deploy/outputs.tf new file mode 100644 index 0000000..e3cf504 --- /dev/null +++ b/modules/terraform-cdp-deploy/outputs.tf @@ -0,0 +1,13 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/modules/terraform-cdp-deploy/provider.tf b/modules/terraform-cdp-deploy/provider.tf new file mode 100644 index 0000000..b4c3f07 --- /dev/null +++ b/modules/terraform-cdp-deploy/provider.tf @@ -0,0 +1,28 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + local = { + source = "hashicorp/local" + version = "2.2.3" + } + null = { + source = "hashicorp/null" + version = "3.2.1" + } + } + + required_version = "> 1.3.0" +} diff --git a/modules/terraform-cdp-deploy/variables.tf b/modules/terraform-cdp-deploy/variables.tf new file mode 100644 index 0000000..627dc82 --- /dev/null +++ b/modules/terraform-cdp-deploy/variables.tf @@ -0,0 +1,451 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "infra_type" { + type = string + description = "Cloud Provider to deploy CDP." + + validation { + condition = contains(["aws", "azure"], var.infra_type) + error_message = "Valid values for var: infra_type are (azure, aws)." + } +} + +variable "env_tags" { + type = map(any) + description = "Tags applied to provisioned resources" + + default = null +} + +variable "agent_source_tag" { + type = map(any) + description = "Tag to identify deployment source" + + default = { agent_source = "tf-cdp-module" } +} + +variable "env_prefix" { + type = string + description = "Shorthand name for the environment. Used in CDP resource descriptions. This will be used to construct the value of where any of the CDP resource variables (e.g. environment_name, cdp_iam_admin_group_name) are not defined." + + default = null +} + +# ------- CDP Environment Deployment ------- +variable "environment_name" { + type = string + description = "Name of the CDP environment. Defaults to '-cdp-env' if not specified." + + default = null +} + +variable "datalake_name" { + type = string + description = "Name of the CDP datalake. Defaults to '--dl' if not specified." + + default = null +} + +variable "cdp_xacccount_credential_name" { + type = string + description = "Name of the CDP Cross Account Credential. Defaults to '-xaccount-cred' if not specified." + + default = null +} + +variable "cdp_admin_group_name" { + type = string + description = "Name of the CDP IAM Admin Group associated with the environment. Defaults to '-cdp-admin-group' if not specified." + + default = null +} + +variable "cdp_user_group_name" { + type = string + description = "Name of the CDP IAM User Group associated with the environment. Defaults to '-cdp-user-group' if not specified." + + default = null +} + +variable "cdp_profile" { + type = string + description = "Profile for CDP credentials" + + # Profile is default unless explicitly specified + default = "default" +} + +variable "cdp_control_plane_region" { + type = string + description = "CDP Control Plane Region" + + # Region is us-west-1 unless explicitly specified + default = "us-west-1" +} + +variable "deployment_template" { + type = string + + description = "Deployment Pattern to use for Cloud resources and CDP" + + validation { + condition = contains(["public", "semi-private", "private"], var.deployment_template) + error_message = "Valid values for var: deployment_template are (public, semi-private, private)." + } +} + +variable "enable_ccm_tunnel" { + type = bool + + description = "Flag to enable Cluster Connectivity Manager tunnel. If false then access from Cloud to CDP Control Plane CIDRs is required from via SG ingress" + + default = true +} + +variable "enable_raz" { + type = bool + + description = "Flag to enable Ranger Authorization Service (RAZ)" + + default = true +} + +variable "multiaz" { + type = bool + + description = "Flag to specify that the FreeIPA and DataLake instances will be deployed across multi-availability zones. " + + default = true +} + +variable "freeipa_instances" { + type = number + + description = "The number of FreeIPA instances to create in the environment" + + default = 3 +} + +variable "workload_analytics" { + type = bool + + description = "Flag to specify if workload analytics should be enabled for the CDP environment" + + default = true +} + +variable "datalake_scale" { + type = string + + description = "The scale of the datalake. Valid values are LIGHT_DUTY, MEDIUM_DUTY_HA." + + validation { + condition = (var.datalake_scale == null ? true : contains(["LIGHT_DUTY", "MEDIUM_DUTY_HA"], var.datalake_scale)) + error_message = "Valid values for var: datalake_scale are (LIGHT_DUTY, MEDIUM_DUTY_HA)." + } + + default = null + +} + +variable "datalake_version" { + type = string + + description = "The Datalake Runtime version. Valid values are semantic versions, e.g. 7.2.16" + + validation { + condition = (var.datalake_version == null ? true : length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0) + error_message = "Valid values for var: datalake_version must match semantic versioning conventions." + } + + default = "7.2.16" +} + +variable "endpoint_access_scheme" { + type = string + + description = "The scheme for the workload endpoint gateway. PUBLIC creates an external endpoint that can be accessed over the Internet. PRIVATE which restricts the traffic to be internal to the VPC / Vnet. Relevant in Private Networks." + + validation { + condition = (var.endpoint_access_scheme == null ? true : contains(["PUBLIC", "PRIVATE"], var.endpoint_access_scheme)) + error_message = "Valid values for var: endpoint_access_scheme are (PUBLIC, PRIVATE)." + } + + default = null + +} +# ------- Cloud Service Provider Settings - General ------- +variable "region" { + type = string + description = "Region which cloud resources will be created" + +} + +variable "data_storage_location" { + type = string + description = "Data storage location. The location has to be in uri format for the cloud provider - i.e. s3a:// for AWS, abfs:// for Azure, gs://" +} + +variable "log_storage_location" { + type = string + description = "Log storage location. The location has to be in uri format for the cloud provider - i.e. s3a:// for AWS, abfs:// for Azure, gs://" +} + +variable "backup_storage_location" { + type = string + description = "Backup storage location. The location has to be in uri format for the cloud provider - i.e. s3a:// for AWS, abfs:// for Azure, gs://" +} + +# ------- Cloud Service Provider Settings - AWS specific ------- + +variable "aws_vpc_id" { + type = string + description = "AWS Virtual Private Network ID. Required for CDP deployment on AWS." + + default = null +} + +variable "aws_public_subnet_ids" { + type = list(string) + description = "List of public subnet ids. Required for CDP deployment on AWS." + + default = null +} + +variable "aws_private_subnet_ids" { + type = list(string) + description = "List of private subnet ids. Required for CDP deployment on AWS." + + default = null +} + +variable "aws_security_group_default_id" { + type = string + + description = "ID of the Default Security Group for CDP environment. Required for CDP deployment on AWS." + + default = null +} + +variable "aws_security_group_knox_id" { + type = string + + description = "ID of the Knox Security Group for CDP environment. Required for CDP deployment on AWS." + + default = null +} + +variable "keypair_name" { + type = string + + description = "SSH Keypair name in Cloud Service Provider. Required for CDP deployment on AWS." + + default = null +} + +variable "aws_datalake_admin_role_arn" { + type = string + + description = "Datalake Admin Role ARN. Required for CDP deployment on AWS." + + default = null + +} + +variable "aws_ranger_audit_role_arn" { + type = string + + description = "Ranger Audit Role ARN. Required for CDP deployment on AWS." + + default = null + +} + +variable "aws_xaccount_role_arn" { + type = string + + description = "Cross Account Role ARN. Required for CDP deployment on AWS." + + default = null + +} + +variable "aws_log_instance_profile_arn" { + type = string + + description = "Log Instance Profile ARN. Required for CDP deployment on AWS." + + default = null + +} + +variable "aws_idbroker_instance_profile_arn" { + type = string + + description = "IDBroker Instance Profile ARN. Required for CDP deployment on AWS." + + default = null +} + +# ------- Cloud Service Provider Settings - Azure specific ------- +variable "azure_subscription_id" { + type = string + + description = "Subscription ID where the Azure pre-reqs are created. Required for CDP deployment on Azure." + + default = null +} + +variable "azure_tenant_id" { + type = string + + description = "Tenant ID where the Azure pre-reqs are created. Required for CDP deployment on Azure." + + default = null +} + +variable "azure_resource_group_name" { + type = string + description = "Azure Resource Group name. Required for CDP deployment on Azure." + + default = null +} + +variable "azure_vnet_name" { + type = string + description = "Azure Virtual Network ID. Required for CDP deployment on Azure." + + default = null + +} + +variable "azure_cdp_subnet_names" { + type = list(any) + description = "List of Azure Subnet Names for CDP Resources. Required for CDP deployment on Azure." + + default = null + +} + +variable "azure_cdp_gateway_subnet_names" { + type = list(any) + description = "List of Azure Subnet Names CDP Endpoint Access Gateway. Required for CDP deployment on Azure." + + default = null + +} + +variable "azure_security_group_default_uri" { + type = string + description = "Azure Default Security Group URI. Required for CDP deployment on Azure." + + default = null + +} + +variable "azure_security_group_knox_uri" { + type = string + description = "Azure Knox Security Group URI. Required for CDP deployment on Azure." + + default = null + +} + +variable "public_key_text" { + type = string + + description = "SSH Public key string for the nodes of the CDP environment. Required for CDP deployment on Azure." + + default = null +} + +variable "use_public_ips" { + type = bool + + description = "Use public ip's for the CDP resources created within the Azure network. Required for CDP deployment on Azure." + + default = null +} + +variable "use_single_resource_group" { + type = bool + + description = "Use a single resource group for all provisioned CDP resources. Required for CDP deployment on Azure." + + default = true +} + +variable "azure_xaccount_app_uuid" { + type = string + + description = "UUID for the Azure AD Cross Account Application. Required for CDP deployment on Azure." + + default = null +} + +variable "azure_xaccount_app_pword" { + type = string + + description = "Password for the Azure AD Cross Account Application. Required for CDP deployment on Azure." + + sensitive = true + default = null +} + +variable "azure_idbroker_identity_id" { + type = string + + description = "IDBroker Managed Identity ID. Required for CDP deployment on Azure." + + default = null + +} + +variable "azure_datalakeadmin_identity_id" { + type = string + + description = "Datalake Admin Managed Identity ID. Required for CDP deployment on Azure." + + default = null + +} + +variable "azure_ranger_audit_identity_id" { + type = string + + description = "Ranger Audit Managed Identity ID. Required for CDP deployment on Azure." + + default = null + +} + +variable "azure_log_identity_id" { + type = string + + description = "Log Data Access Managed Identity ID. Required for CDP deployment on Azure." + + default = null + +} + +variable "azure_raz_identity_id" { + type = string + + description = "RAZ Managed Identity ID. Required for CDP deployment on Azure." + + default = null + +}