From 184a162bca685c37539f30d7ae22ca6535330a54 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Mon, 21 Aug 2023 15:25:21 +0100 Subject: [PATCH 1/2] Expose all parameters for CDP provider datalake resource Signed-off-by: Jim Enright --- modules/terraform-cdp-deploy/README.md | 8 ++- modules/terraform-cdp-deploy/defaults.tf | 2 +- modules/terraform-cdp-deploy/main.tf | 10 ++++ .../terraform-cdp-deploy/modules/aws/main.tf | 6 +- .../modules/aws/variables.tf | 53 ++++++++++++++-- .../modules/azure/main.tf | 5 +- .../modules/azure/variables.tf | 42 ++++++++++--- modules/terraform-cdp-deploy/variables.tf | 60 ++++++++++++++++--- 8 files changed, 161 insertions(+), 25 deletions(-) diff --git a/modules/terraform-cdp-deploy/README.md b/modules/terraform-cdp-deploy/README.md index a60619f..06a920d 100644 --- a/modules/terraform-cdp-deploy/README.md +++ b/modules/terraform-cdp-deploy/README.md @@ -73,9 +73,13 @@ No resources. | [cdp\_admin\_group\_name](#input\_cdp\_admin\_group\_name) | Name of the CDP IAM Admin Group associated with the environment. Defaults to '-cdp-admin-group' if not specified. | `string` | `null` | no | | [cdp\_user\_group\_name](#input\_cdp\_user\_group\_name) | Name of the CDP IAM User Group associated with the environment. Defaults to '-cdp-user-group' if not specified. | `string` | `null` | no | | [cdp\_xacccount\_credential\_name](#input\_cdp\_xacccount\_credential\_name) | Name of the CDP Cross Account Credential. Defaults to '-xaccount-cred' if not specified. | `string` | `null` | no | +| [datalake\_custom\_instance\_groups](#input\_datalake\_custom\_instance\_groups) | A set of custom instance groups for the datalake. Only applicable for CDP deployment on AWS. | 
list(
    object({
      name          = string,
      instance_type = optional(string)
    })
  )
| `null` | no | +| [datalake\_image](#input\_datalake\_image) | The image to use for the datalake. Can only be used when the 'datalake\_version' parameter is set to null. You can use 'catalog' name and/or 'id' for selecting an image. | 
object({
    id      = optional(string)
    catalog = optional(string)
  })
| `null` | no | +| [datalake\_java\_version](#input\_datalake\_java\_version) | The Java major version to use on the datalake cluster. | `number` | `null` | no | | [datalake\_name](#input\_datalake\_name) | Name of the CDP datalake. Defaults to '--dl' if not specified. | `string` | `null` | no | -| [datalake\_scale](#input\_datalake\_scale) | The scale of the datalake. Valid values are LIGHT\_DUTY, MEDIUM\_DUTY\_HA. | `string` | `null` | no | -| [datalake\_version](#input\_datalake\_version) | The Datalake Runtime version. Valid values are semantic versions, e.g. 7.2.16 | `string` | `"7.2.17"` | no | +| [datalake\_recipes](#input\_datalake\_recipes) | Additional recipes that will be attached on the datalake instances | 
list(
    object({
      instance_group_name = string,
      recipe_names        = string
    })
  )
| `null` | no | +| [datalake\_scale](#input\_datalake\_scale) | The scale of the datalake. Valid values are LIGHT\_DUTY, ENTERPRISE. | `string` | `null` | no | +| [datalake\_version](#input\_datalake\_version) | The Datalake Runtime version. Valid values are latest or a semantic version, e.g. 7.2.17 | `string` | `"latest"` | no | | [enable\_ccm\_tunnel](#input\_enable\_ccm\_tunnel) | Flag to enable Cluster Connectivity Manager tunnel. If false then access from Cloud to CDP Control Plane CIDRs is required from via SG ingress | `bool` | `true` | no | | [enable\_raz](#input\_enable\_raz) | Flag to enable Ranger Authorization Service (RAZ) | `bool` | `true` | no | | [endpoint\_access\_scheme](#input\_endpoint\_access\_scheme) | The scheme for the workload endpoint gateway. PUBLIC creates an external endpoint that can be accessed over the Internet. PRIVATE which restricts the traffic to be internal to the VPC / Vnet. Relevant in Private Networks. | `string` | `null` | no | diff --git a/modules/terraform-cdp-deploy/defaults.tf b/modules/terraform-cdp-deploy/defaults.tf index 5c53a41..eb75204 100644 --- a/modules/terraform-cdp-deploy/defaults.tf +++ b/modules/terraform-cdp-deploy/defaults.tf @@ -40,7 +40,7 @@ locals { datalake_scale = coalesce( var.datalake_scale, (var.deployment_template == "public" ? - "LIGHT_DUTY" : "MEDIUM_DUTY_HA" + "LIGHT_DUTY" : "ENTERPRISE" ) ) diff --git a/modules/terraform-cdp-deploy/main.tf b/modules/terraform-cdp-deploy/main.tf index 4ee57f6..f50c441 100644 --- a/modules/terraform-cdp-deploy/main.tf +++ b/modules/terraform-cdp-deploy/main.tf @@ -59,6 +59,11 @@ module "cdp_on_aws" { idbroker_instance_profile_arn = var.aws_idbroker_instance_profile_arn log_instance_profile_arn = var.aws_log_instance_profile_arn + # Optional parameters defaulting to null + datalake_custom_instance_groups = var.datalake_custom_instance_groups + datalake_image = var.datalake_image + datalake_java_version = var.datalake_java_version + datalake_recipes = var.datalake_recipes } # ------- Call sub-module for Azure Deployment ------- @@ -114,4 +119,9 @@ module "cdp_on_azure" { ranger_audit_identity_id = var.azure_ranger_audit_identity_id log_identity_id = var.azure_log_identity_id raz_identity_id = var.azure_raz_identity_id + + # Optional parameters defaulting to null + datalake_image = var.datalake_image + datalake_java_version = var.datalake_java_version + datalake_recipes = var.datalake_recipes } diff --git a/modules/terraform-cdp-deploy/modules/aws/main.tf b/modules/terraform-cdp-deploy/modules/aws/main.tf index f31ba8a..72c0e96 100644 --- a/modules/terraform-cdp-deploy/modules/aws/main.tf +++ b/modules/terraform-cdp-deploy/modules/aws/main.tf @@ -113,11 +113,15 @@ resource "cdp_datalake_aws_datalake" "cdp_datalake" { instance_profile = var.idbroker_instance_profile_arn storage_bucket_location = var.data_storage_location - runtime = var.datalake_version + runtime = var.datalake_version == "latest" ? null : var.datalake_version scale = var.datalake_scale enable_ranger_raz = var.enable_raz multi_az = var.multiaz + custom_instance_groups = var.datalake_custom_instance_groups + image = var.datalake_image + java_version = var.datalake_java_version + recipes = var.datalake_recipes # tags = var.tags # NOTE: Waiting on provider fix depends_on = [ diff --git a/modules/terraform-cdp-deploy/modules/aws/variables.tf b/modules/terraform-cdp-deploy/modules/aws/variables.tf index 2af4a0e..797a32e 100644 --- a/modules/terraform-cdp-deploy/modules/aws/variables.tf +++ b/modules/terraform-cdp-deploy/modules/aws/variables.tf @@ -96,11 +96,11 @@ variable "workload_analytics" { variable "datalake_scale" { type = string - description = "The scale of the datalake. Valid values are LIGHT_DUTY, MEDIUM_DUTY_HA." + description = "The scale of the datalake. Valid values are LIGHT_DUTY, ENTERPRISE." validation { - condition = contains(["LIGHT_DUTY", "MEDIUM_DUTY_HA"], var.datalake_scale) - error_message = "Valid values for var: datalake_scale are (LIGHT_DUTY, MEDIUM_DUTY_HA)." + condition = contains(["LIGHT_DUTY", "ENTERPRISE", "MEDIUM_DUTY_HA"], var.datalake_scale) + error_message = "Valid values for var: datalake_scale are (LIGHT_DUTY, ENTERPRISE, MEDIUM_DUTY_HA)." } } @@ -108,15 +108,56 @@ variable "datalake_scale" { variable "datalake_version" { type = string - description = "The Datalake Runtime version. Valid values are semantic versions, e.g. 7.2.16" + description = "The Datalake Runtime version. Valid values are latest or a semantic version, e.g. 7.2.17" validation { - condition = length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0 - error_message = "Valid values for var: datalake_version must match semantic versioning conventions." + condition = (var.datalake_version == "latest" ? true : length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0) + error_message = "Valid values for var: datalake_version are 'latest' or a semantic versioning conventions." } + default = "latest" } +variable "datalake_custom_instance_groups" { + type = list( + object({ + name = string, + instance_type = optional(string) + }) + ) + + description = "A set of custom instance groups for the datalake." + +} + +variable "datalake_image" { + type = object({ + id = optional(string) + catalog = optional(string) + }) + + description = "The image to use for the datalake. Can only be used when the 'datalake_version' parameter is set to null. You can use 'catalog' name and/or 'id' for selecting an image." + +} + +variable "datalake_java_version" { + type = number + + description = "The Java major version to use on the datalake cluster." + +} + +variable "datalake_recipes" { + type = list( + object({ + instance_group_name = string, + recipe_names = string + }) + ) + + description = "Additional recipes that will be attached on the datalake instances" + +} # ------- Cloud Service Provider Settings ------- variable "region" { type = string diff --git a/modules/terraform-cdp-deploy/modules/azure/main.tf b/modules/terraform-cdp-deploy/modules/azure/main.tf index d908fe1..c82208e 100644 --- a/modules/terraform-cdp-deploy/modules/azure/main.tf +++ b/modules/terraform-cdp-deploy/modules/azure/main.tf @@ -124,10 +124,13 @@ resource "cdp_datalake_azure_datalake" "cdp_datalake" { managed_identity = var.idbroker_identity_id storage_location = var.data_storage_location - runtime = var.datalake_version + runtime = var.datalake_version == "latest" ? null : var.datalake_version scale = var.datalake_scale enable_ranger_raz = var.enable_raz + image = var.datalake_image + java_version = var.datalake_java_version + recipes = var.datalake_recipes # tags = var.tags # NOTE: Waiting on provider fix depends_on = [ diff --git a/modules/terraform-cdp-deploy/modules/azure/variables.tf b/modules/terraform-cdp-deploy/modules/azure/variables.tf index 2263494..120dc42 100644 --- a/modules/terraform-cdp-deploy/modules/azure/variables.tf +++ b/modules/terraform-cdp-deploy/modules/azure/variables.tf @@ -87,15 +87,14 @@ variable "workload_analytics" { } - variable "datalake_scale" { type = string - description = "The scale of the datalake. Valid values are LIGHT_DUTY, MEDIUM_DUTY_HA." + description = "The scale of the datalake. Valid values are LIGHT_DUTY, ENTERPRISE." validation { - condition = contains(["LIGHT_DUTY", "MEDIUM_DUTY_HA"], var.datalake_scale) - error_message = "Valid values for var: datalake_scale are (LIGHT_DUTY, MEDIUM_DUTY_HA)." + condition = contains(["LIGHT_DUTY", "ENTERPRISE", "MEDIUM_DUTY_HA"], var.datalake_scale) + error_message = "Valid values for var: datalake_scale are (LIGHT_DUTY, ENTERPRISE, MEDIUM_DUTY_HA)." } } @@ -103,15 +102,44 @@ variable "datalake_scale" { variable "datalake_version" { type = string - description = "The Datalake Runtime version. Valid values are semantic versions, e.g. 7.2.16" + description = "The Datalake Runtime version. Valid values are latest or a semantic version, e.g. 7.2.17" validation { - condition = length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0 - error_message = "Valid values for var: datalake_version must match semantic versioning conventions." + condition = (var.datalake_version == "latest" ? true : length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0) + error_message = "Valid values for var: datalake_version are 'latest' or a semantic versioning conventions." } + default = "latest" } +variable "datalake_image" { + type = object({ + id = optional(string) + catalog = optional(string) + }) + + description = "The image to use for the datalake. Can only be used when the 'datalake_version' parameter is set to null. You can use 'catalog' name and/or 'id' for selecting an image." + +} + +variable "datalake_java_version" { + type = number + + description = "The Java major version to use on the datalake cluster." + +} + +variable "datalake_recipes" { + type = list( + object({ + instance_group_name = string, + recipe_names = string + }) + ) + + description = "Additional recipes that will be attached on the datalake instances" + +} # ------- Cloud Service Provider Settings ------- variable "subscription_id" { type = string diff --git a/modules/terraform-cdp-deploy/variables.tf b/modules/terraform-cdp-deploy/variables.tf index 1516e8f..022924a 100644 --- a/modules/terraform-cdp-deploy/variables.tf +++ b/modules/terraform-cdp-deploy/variables.tf @@ -145,11 +145,11 @@ variable "workload_analytics" { variable "datalake_scale" { type = string - description = "The scale of the datalake. Valid values are LIGHT_DUTY, MEDIUM_DUTY_HA." + description = "The scale of the datalake. Valid values are LIGHT_DUTY, ENTERPRISE." validation { - condition = (var.datalake_scale == null ? true : contains(["LIGHT_DUTY", "MEDIUM_DUTY_HA"], var.datalake_scale)) - error_message = "Valid values for var: datalake_scale are (LIGHT_DUTY, MEDIUM_DUTY_HA)." + condition = (var.datalake_scale == null ? true : contains(["LIGHT_DUTY", "ENTERPRISE", "MEDIUM_DUTY_HA"], var.datalake_scale)) + error_message = "Valid values for var: datalake_scale are (LIGHT_DUTY, ENTERPRISE, MEDIUM_DUTY_HA)." } default = null @@ -159,14 +159,14 @@ variable "datalake_scale" { variable "datalake_version" { type = string - description = "The Datalake Runtime version. Valid values are semantic versions, e.g. 7.2.16" + description = "The Datalake Runtime version. Valid values are latest or a semantic version, e.g. 7.2.17" validation { - condition = (var.datalake_version == null ? true : length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0) - error_message = "Valid values for var: datalake_version must match semantic versioning conventions." + condition = (var.datalake_version == "latest" ? true : length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0) + error_message = "Valid values for var: datalake_version are 'latest' or a semantic versioning conventions." } - default = "7.2.17" + default = "latest" } variable "endpoint_access_scheme" { @@ -182,6 +182,52 @@ variable "endpoint_access_scheme" { default = null } + +variable "datalake_custom_instance_groups" { + type = list( + object({ + name = string, + instance_type = optional(string) + }) + ) + + description = "A set of custom instance groups for the datalake. Only applicable for CDP deployment on AWS." + + default = null +} + +variable "datalake_image" { + type = object({ + id = optional(string) + catalog = optional(string) + }) + + description = "The image to use for the datalake. Can only be used when the 'datalake_version' parameter is set to null. You can use 'catalog' name and/or 'id' for selecting an image." + + default = null +} + +variable "datalake_java_version" { + type = number + + description = "The Java major version to use on the datalake cluster." + + default = null +} + +variable "datalake_recipes" { + type = list( + object({ + instance_group_name = string, + recipe_names = string + }) + ) + + description = "Additional recipes that will be attached on the datalake instances" + + default = null +} + # ------- Cloud Service Provider Settings - General ------- variable "region" { type = string From f26e7f4971f9a066bae1ab21988f33bed9f5d22c Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Tue, 22 Aug 2023 17:10:49 +0100 Subject: [PATCH 2/2] Expose all parameters for CDP provider environment resource Signed-off-by: Jim Enright --- modules/terraform-cdp-deploy/README.md | 16 +- modules/terraform-cdp-deploy/main.tf | 27 +++ .../terraform-cdp-deploy/modules/aws/main.tf | 13 +- .../modules/aws/variables.tf | 61 ++++++- .../modules/azure/main.tf | 18 +- .../modules/azure/variables.tf | 69 ++++++++ modules/terraform-cdp-deploy/variables.tf | 158 ++++++++++++++---- 7 files changed, 324 insertions(+), 38 deletions(-) diff --git a/modules/terraform-cdp-deploy/README.md b/modules/terraform-cdp-deploy/README.md index 06a920d..a0c7581 100644 --- a/modules/terraform-cdp-deploy/README.md +++ b/modules/terraform-cdp-deploy/README.md @@ -55,8 +55,10 @@ No resources. | [aws\_security\_group\_knox\_id](#input\_aws\_security\_group\_knox\_id) | ID of the Knox Security Group for CDP environment. Required for CDP deployment on AWS. | `string` | `null` | no | | [aws\_vpc\_id](#input\_aws\_vpc\_id) | AWS Virtual Private Network ID. Required for CDP deployment on AWS. | `string` | `null` | no | | [aws\_xaccount\_role\_arn](#input\_aws\_xaccount\_role\_arn) | Cross Account Role ARN. Required for CDP deployment on AWS. | `string` | `null` | no | +| [azure\_aks\_private\_dns\_zone\_id](#input\_azure\_aks\_private\_dns\_zone\_id) | The ID of an existing private DNS zone used for the AKS. | `string` | `null` | no | | [azure\_cdp\_gateway\_subnet\_names](#input\_azure\_cdp\_gateway\_subnet\_names) | List of Azure Subnet Names CDP Endpoint Access Gateway. Required for CDP deployment on Azure. | `list(any)` | `null` | no | | [azure\_cdp\_subnet\_names](#input\_azure\_cdp\_subnet\_names) | List of Azure Subnet Names for CDP Resources. Required for CDP deployment on Azure. | `list(any)` | `null` | no | +| [azure\_database\_private\_dns\_zone\_id](#input\_azure\_database\_private\_dns\_zone\_id) | The ID of an existing private DNS zone used for the database. | `string` | `null` | no | | [azure\_datalakeadmin\_identity\_id](#input\_azure\_datalakeadmin\_identity\_id) | Datalake Admin Managed Identity ID. Required for CDP deployment on Azure. | `string` | `null` | no | | [azure\_idbroker\_identity\_id](#input\_azure\_idbroker\_identity\_id) | IDBroker Managed Identity ID. Required for CDP deployment on Azure. | `string` | `null` | no | | [azure\_log\_identity\_id](#input\_azure\_log\_identity\_id) | Log Data Access Managed Identity ID. Required for CDP deployment on Azure. | `string` | `null` | no | @@ -81,14 +83,24 @@ No resources. | [datalake\_scale](#input\_datalake\_scale) | The scale of the datalake. Valid values are LIGHT\_DUTY, ENTERPRISE. | `string` | `null` | no | | [datalake\_version](#input\_datalake\_version) | The Datalake Runtime version. Valid values are latest or a semantic version, e.g. 7.2.17 | `string` | `"latest"` | no | | [enable\_ccm\_tunnel](#input\_enable\_ccm\_tunnel) | Flag to enable Cluster Connectivity Manager tunnel. If false then access from Cloud to CDP Control Plane CIDRs is required from via SG ingress | `bool` | `true` | no | +| [enable\_outbound\_load\_balancer](#input\_enable\_outbound\_load\_balancer) | Create outbound load balancers for Azure environments. Only applicable for CDP deployment on Azure. | `bool` | `null` | no | | [enable\_raz](#input\_enable\_raz) | Flag to enable Ranger Authorization Service (RAZ) | `bool` | `true` | no | +| [encryption\_key\_arn](#input\_encryption\_key\_arn) | ARN of the AWS KMS CMK to use for the server-side encryption of AWS storage resources. Only applicable for CDP deployment on AWS. | `string` | `null` | no | +| [encryption\_key\_resource\_group\_name](#input\_encryption\_key\_resource\_group\_name) | Name of the existing Azure resource group hosting the Azure Key Vault containing customer managed key which will be used to encrypt the Azure Managed Disk. Only applicable for CDP deployment on Azure. | `string` | `null` | no | +| [encryption\_key\_url](#input\_encryption\_key\_url) | URL of the key which will be used to encrypt the Azure Managed Disks. Only applicable for CDP deployment on Azure. | `string` | `null` | no | | [endpoint\_access\_scheme](#input\_endpoint\_access\_scheme) | The scheme for the workload endpoint gateway. PUBLIC creates an external endpoint that can be accessed over the Internet. PRIVATE which restricts the traffic to be internal to the VPC / Vnet. Relevant in Private Networks. | `string` | `null` | no | | [env\_prefix](#input\_env\_prefix) | Shorthand name for the environment. Used in CDP resource descriptions. This will be used to construct the value of where any of the CDP resource variables (e.g. environment\_name, cdp\_iam\_admin\_group\_name) are not defined. | `string` | `null` | no | | [environment\_name](#input\_environment\_name) | Name of the CDP environment. Defaults to '-cdp-env' if not specified. | `string` | `null` | no | +| [freeipa\_catalog](#input\_freeipa\_catalog) | Image catalog to use for FreeIPA image selection | `string` | `null` | no | +| [freeipa\_image\_id](#input\_freeipa\_image\_id) | Image ID to use for creating FreeIPA instances | `string` | `null` | no | +| [freeipa\_instance\_type](#input\_freeipa\_instance\_type) | Instance Type to use for creating FreeIPA instances | `string` | `null` | no | | [freeipa\_instances](#input\_freeipa\_instances) | The number of FreeIPA instances to create in the environment | `number` | `3` | no | -| [keypair\_name](#input\_keypair\_name) | SSH Keypair name in Cloud Service Provider. Required for CDP deployment on AWS. | `string` | `null` | no | +| [freeipa\_recipes](#input\_freeipa\_recipes) | The recipes for the FreeIPA cluster | `set(string)` | `null` | no | +| [keypair\_name](#input\_keypair\_name) | SSH Keypair name in Cloud Service Provider. For CDP deployment on AWS, either 'keypair\_name' or 'public\_key\_text' needs to be set. | `string` | `null` | no | | [multiaz](#input\_multiaz) | Flag to specify that the FreeIPA and DataLake instances will be deployed across multi-availability zones. | `bool` | `true` | no | -| [public\_key\_text](#input\_public\_key\_text) | SSH Public key string for the nodes of the CDP environment. Required for CDP deployment on Azure. | `string` | `null` | no | +| [proxy\_config\_name](#input\_proxy\_config\_name) | Name of the proxy config to use for the environment. | `string` | `null` | no | +| [public\_key\_text](#input\_public\_key\_text) | SSH Public key string for the nodes of the CDP environment. Required for CDP deployment on Azure. For CDP deployment on AWS, either 'keypair\_name' or 'public\_key\_text' needs to be set. | `string` | `null` | no | +| [s3\_guard\_table\_name](#input\_s3\_guard\_table\_name) | Name for the DynamoDB table backing S3Guard. Only applicable for CDP deployment on AWS. | `string` | `null` | no | | [use\_public\_ips](#input\_use\_public\_ips) | Use public ip's for the CDP resources created within the Azure network. Required for CDP deployment on Azure. | `bool` | `null` | no | | [use\_single\_resource\_group](#input\_use\_single\_resource\_group) | Use a single resource group for all provisioned CDP resources. Required for CDP deployment on Azure. | `bool` | `true` | no | | [workload\_analytics](#input\_workload\_analytics) | Flag to specify if workload analytics should be enabled for the CDP environment | `bool` | `true` | no | diff --git a/modules/terraform-cdp-deploy/main.tf b/modules/terraform-cdp-deploy/main.tf index f50c441..f7a9c19 100644 --- a/modules/terraform-cdp-deploy/main.tf +++ b/modules/terraform-cdp-deploy/main.tf @@ -46,7 +46,9 @@ module "cdp_on_aws" { public_subnet_ids = var.aws_public_subnet_ids # private_subnet_ids = var.aws_private_subnet_ids subnets_for_cdp = local.aws_subnets_for_cdp + # One of key settings below need to be set keypair_name = var.keypair_name + public_key_text = var.public_key_text data_storage_location = var.data_storage_location log_storage_location = var.log_storage_location @@ -60,6 +62,16 @@ module "cdp_on_aws" { log_instance_profile_arn = var.aws_log_instance_profile_arn # Optional parameters defaulting to null + freeipa_catalog = var.freeipa_catalog + freeipa_image_id = var.freeipa_image_id + freeipa_instance_type = var.freeipa_instance_type + freeipa_recipes = var.freeipa_recipes + + encryption_key_arn = var.encryption_key_arn + + proxy_config_name = var.proxy_config_name + s3_guard_table_name = var.s3_guard_table_name + datalake_custom_instance_groups = var.datalake_custom_instance_groups datalake_image = var.datalake_image datalake_java_version = var.datalake_java_version @@ -121,6 +133,21 @@ module "cdp_on_azure" { raz_identity_id = var.azure_raz_identity_id # Optional parameters defaulting to null + freeipa_catalog = var.freeipa_catalog + freeipa_image_id = var.freeipa_image_id + freeipa_instance_type = var.freeipa_instance_type + freeipa_recipes = var.freeipa_recipes + + enable_outbound_load_balancer = var.enable_outbound_load_balancer + + encryption_key_resource_group_name = var.encryption_key_resource_group_name + encryption_key_url = var.encryption_key_url + + azure_aks_private_dns_zone_id = var.azure_aks_private_dns_zone_id + azure_database_private_dns_zone_id = var.azure_database_private_dns_zone_id + + proxy_config_name = var.proxy_config_name + datalake_image = var.datalake_image datalake_java_version = var.datalake_java_version datalake_recipes = var.datalake_recipes diff --git a/modules/terraform-cdp-deploy/modules/aws/main.tf b/modules/terraform-cdp-deploy/modules/aws/main.tf index 72c0e96..e8c933e 100644 --- a/modules/terraform-cdp-deploy/modules/aws/main.tf +++ b/modules/terraform-cdp-deploy/modules/aws/main.tf @@ -37,6 +37,7 @@ resource "cdp_environments_aws_environment" "cdp_env" { } authentication = { + public_key = var.public_key_text public_key_id = var.keypair_name } @@ -48,10 +49,18 @@ resource "cdp_environments_aws_environment" "cdp_env" { freeipa = { instance_count_by_group = var.freeipa_instances multi_az = var.multiaz + catalog = var.freeipa_catalog + image_id = var.freeipa_image_id + instance_type = var.freeipa_instance_type + recipes = var.freeipa_recipes } - workload_analytics = var.workload_analytics - enable_tunnel = var.enable_ccm_tunnel + proxy_config_name = var.proxy_config_name + s3_guard_table_name = var.s3_guard_table_name + workload_analytics = var.workload_analytics + enable_tunnel = var.enable_ccm_tunnel + + encryption_key_arn = var.encryption_key_arn # tags = var.tags # NOTE: Waiting on provider fix depends_on = [ diff --git a/modules/terraform-cdp-deploy/modules/aws/variables.tf b/modules/terraform-cdp-deploy/modules/aws/variables.tf index 797a32e..b9259c7 100644 --- a/modules/terraform-cdp-deploy/modules/aws/variables.tf +++ b/modules/terraform-cdp-deploy/modules/aws/variables.tf @@ -86,6 +86,48 @@ variable "freeipa_instances" { } +variable "freeipa_catalog" { + type = string + + description = "Image catalog to use for FreeIPA image selection" + +} + +variable "freeipa_image_id" { + type = string + + description = "Image ID to use for creating FreeIPA instances" + +} + +variable "freeipa_instance_type" { + type = string + + description = "Instance Type to use for creating FreeIPA instances" + +} + +variable "freeipa_recipes" { + type = set(string) + + description = "The recipes for the FreeIPA cluster" + +} + +variable "proxy_config_name" { + type = string + + description = "Name of the proxy config to use for the environment." + +} + +variable "s3_guard_table_name" { + type = string + + description = "Name for the DynamoDB table backing S3Guard. Only applicable for CDP deployment on AWS." + +} + variable "workload_analytics" { type = bool @@ -93,6 +135,7 @@ variable "workload_analytics" { } + variable "datalake_scale" { type = string @@ -168,7 +211,15 @@ variable "region" { variable "keypair_name" { type = string - description = "SSH Keypair name in Cloud Service Provider. Required for CDP deployment on AWS." + description = "SSH Keypair name in Cloud Service Provider. Either 'keypair_name' or 'public_key_text' needs to be set." + + default = null +} + +variable "public_key_text" { + type = string + + description = "SSH Public key string for the nodes of the CDP environment. Either 'keypair_name' or 'public_key_text' needs to be set." default = null } @@ -224,6 +275,14 @@ variable "endpoint_access_scheme" { } } +variable "encryption_key_arn" { + type = string + + description = "ARN of the AWS KMS CMK to use for the server-side encryption of AWS storage resources." + +} + + variable "data_storage_location" { type = string description = "Data storage location." diff --git a/modules/terraform-cdp-deploy/modules/azure/main.tf b/modules/terraform-cdp-deploy/modules/azure/main.tf index c82208e..1024622 100644 --- a/modules/terraform-cdp-deploy/modules/azure/main.tf +++ b/modules/terraform-cdp-deploy/modules/azure/main.tf @@ -45,9 +45,11 @@ resource "cdp_environments_azure_environment" "cdp_env" { use_public_ip = var.use_public_ips existing_network_params = { - resource_group_name = var.resource_group_name - network_id = var.vnet_name - subnet_ids = var.cdp_subnet_names + resource_group_name = var.resource_group_name + network_id = var.vnet_name + subnet_ids = var.cdp_subnet_names + aks_private_dns_zone_id = var.azure_aks_private_dns_zone_id + database_private_dns_zone_id = var.azure_database_private_dns_zone_id } endpoint_access_gateway_scheme = var.endpoint_access_scheme @@ -58,10 +60,20 @@ resource "cdp_environments_azure_environment" "cdp_env" { freeipa = { instance_count_by_group = var.freeipa_instances + catalog = var.freeipa_catalog + image_id = var.freeipa_image_id + instance_type = var.freeipa_instance_type + recipes = var.freeipa_recipes } + proxy_config_name = var.proxy_config_name workload_analytics = var.workload_analytics enable_tunnel = var.enable_ccm_tunnel + + enable_outbound_load_balancer = var.enable_outbound_load_balancer + encryption_key_resource_group_name = var.encryption_key_resource_group_name + encryption_key_url = var.encryption_key_url + # tags = var.tags # NOTE: Waiting on provider fix depends_on = [ diff --git a/modules/terraform-cdp-deploy/modules/azure/variables.tf b/modules/terraform-cdp-deploy/modules/azure/variables.tf index 120dc42..79ca88d 100644 --- a/modules/terraform-cdp-deploy/modules/azure/variables.tf +++ b/modules/terraform-cdp-deploy/modules/azure/variables.tf @@ -79,6 +79,33 @@ variable "freeipa_instances" { } +variable "freeipa_catalog" { + type = string + + description = "Image catalog to use for FreeIPA image selection" + +} + +variable "freeipa_image_id" { + type = string + + description = "Image ID to use for creating FreeIPA instances" + +} + +variable "freeipa_instance_type" { + type = string + + description = "Instance Type to use for creating FreeIPA instances" + +} + +variable "freeipa_recipes" { + type = set(string) + + description = "The recipes for the FreeIPA cluster" + +} variable "workload_analytics" { type = bool @@ -87,6 +114,36 @@ variable "workload_analytics" { } +variable "enable_outbound_load_balancer" { + type = bool + + description = "Create outbound load balancers for Azure environments." + + default = null +} + +variable "encryption_key_resource_group_name" { + type = string + + description = "Name of the existing Azure resource group hosting the Azure Key Vault containing customer managed key which will be used to encrypt the Azure Managed Disk." + +} + +variable "encryption_key_url" { + type = string + + description = "URL of the key which will be used to encrypt the Azure Managed Disks." + +} + +variable "proxy_config_name" { + type = string + + description = "Name of the proxy config to use for the environment." + +} + + variable "datalake_scale" { type = string @@ -205,6 +262,18 @@ variable "cdp_subnet_names" { } +variable "azure_aks_private_dns_zone_id" { + type = string + description = "The ID of an existing private DNS zone used for the AKS." + +} + +variable "azure_database_private_dns_zone_id" { + type = string + description = "The ID of an existing private DNS zone used for the database." + +} + variable "cdp_gateway_subnet_names" { type = list(any) description = "Azure Subnet Names for Endpoint Access Gateway." diff --git a/modules/terraform-cdp-deploy/variables.tf b/modules/terraform-cdp-deploy/variables.tf index 022924a..15750bd 100644 --- a/modules/terraform-cdp-deploy/variables.tf +++ b/modules/terraform-cdp-deploy/variables.tf @@ -46,7 +46,7 @@ variable "env_prefix" { default = null } -# ------- CDP Environment Deployment ------- +# ------- CDP Environment Deployment - General ------- variable "environment_name" { type = string description = "Name of the CDP environment. Defaults to '-cdp-env' if not specified." @@ -134,6 +134,46 @@ variable "freeipa_instances" { default = 3 } +variable "freeipa_catalog" { + type = string + + description = "Image catalog to use for FreeIPA image selection" + + default = null +} + +variable "freeipa_image_id" { + type = string + + description = "Image ID to use for creating FreeIPA instances" + + default = null +} + +variable "freeipa_instance_type" { + type = string + + description = "Instance Type to use for creating FreeIPA instances" + + default = null +} + +variable "freeipa_recipes" { + type = set(string) + + description = "The recipes for the FreeIPA cluster" + + default = null +} + +variable "proxy_config_name" { + type = string + + description = "Name of the proxy config to use for the environment." + + default = null +} + variable "workload_analytics" { type = bool @@ -183,19 +223,6 @@ variable "endpoint_access_scheme" { } -variable "datalake_custom_instance_groups" { - type = list( - object({ - name = string, - instance_type = optional(string) - }) - ) - - description = "A set of custom instance groups for the datalake. Only applicable for CDP deployment on AWS." - - default = null -} - variable "datalake_image" { type = object({ id = optional(string) @@ -228,6 +255,61 @@ variable "datalake_recipes" { default = null } +# ------- CDP Environment Deployment - AWS specific ------- +variable "encryption_key_arn" { + type = string + + description = "ARN of the AWS KMS CMK to use for the server-side encryption of AWS storage resources. Only applicable for CDP deployment on AWS." + + default = null +} + +variable "datalake_custom_instance_groups" { + type = list( + object({ + name = string, + instance_type = optional(string) + }) + ) + + description = "A set of custom instance groups for the datalake. Only applicable for CDP deployment on AWS." + + default = null +} + +variable "s3_guard_table_name" { + type = string + + description = "Name for the DynamoDB table backing S3Guard. Only applicable for CDP deployment on AWS." + + default = null +} + +# ------- CDP Environment Deployment - Azure specific ------- +variable "enable_outbound_load_balancer" { + type = bool + + description = "Create outbound load balancers for Azure environments. Only applicable for CDP deployment on Azure." + + default = null +} + +variable "encryption_key_resource_group_name" { + type = string + + description = "Name of the existing Azure resource group hosting the Azure Key Vault containing customer managed key which will be used to encrypt the Azure Managed Disk. Only applicable for CDP deployment on Azure." + + default = null +} + +variable "encryption_key_url" { + type = string + + description = "URL of the key which will be used to encrypt the Azure Managed Disks. Only applicable for CDP deployment on Azure." + + default = null +} + # ------- Cloud Service Provider Settings - General ------- variable "region" { type = string @@ -235,6 +317,22 @@ variable "region" { } +variable "keypair_name" { + type = string + + description = "SSH Keypair name in Cloud Service Provider. For CDP deployment on AWS, either 'keypair_name' or 'public_key_text' needs to be set." + + default = null +} + +variable "public_key_text" { + type = string + + description = "SSH Public key string for the nodes of the CDP environment. Required for CDP deployment on Azure. For CDP deployment on AWS, either 'keypair_name' or 'public_key_text' needs to be set." + + default = null +} + variable "data_storage_location" { type = string description = "Data storage location. The location has to be in uri format for the cloud provider - i.e. s3a:// for AWS, abfs:// for Azure, gs://" @@ -289,14 +387,6 @@ variable "aws_security_group_knox_id" { default = null } -variable "keypair_name" { - type = string - - description = "SSH Keypair name in Cloud Service Provider. Required for CDP deployment on AWS." - - default = null -} - variable "aws_datalake_admin_role_arn" { type = string @@ -373,6 +463,22 @@ variable "azure_vnet_name" { } +variable "azure_aks_private_dns_zone_id" { + type = string + description = "The ID of an existing private DNS zone used for the AKS." + + default = null + +} + +variable "azure_database_private_dns_zone_id" { + type = string + description = "The ID of an existing private DNS zone used for the database." + + default = null + +} + variable "azure_cdp_subnet_names" { type = list(any) description = "List of Azure Subnet Names for CDP Resources. Required for CDP deployment on Azure." @@ -405,14 +511,6 @@ variable "azure_security_group_knox_uri" { } -variable "public_key_text" { - type = string - - description = "SSH Public key string for the nodes of the CDP environment. Required for CDP deployment on Azure." - - default = null -} - variable "use_public_ips" { type = bool