Merge pull request #59 from along528/multi_master

SUP-5610 Support multiple master nodes using instance groups
Datatamer · Oct 22, 2021 · 6ed18fa · 6ed18fa
2 parents 2b075db + 1097fa5
commit 6ed18fa
Show file tree

Hide file tree

Showing 7 changed files with 90 additions and 45 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,8 @@
 # TAMR AWS EMR module
 
+## v7.1.0 - October 21st, 2021
+* Switch to using instance groups instead of instance fleets when multiple master nodes specified and no spot instances
+
 ## v7.0.0 - September 13th, 2021
 * Adds/renames variables to `aws_emr_ports` module for differentiating between ports required for pre-6x and 6x EMR
 * Adds variables:

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-7.0.0
+7.1.0
diff --git a/modules/aws-emr-cluster/main.tf b/modules/aws-emr-cluster/main.tf
@@ -1,5 +1,11 @@
 locals {
   applications = [for app in var.applications : lower(app)]
+
+  # Instance groups are required if deploying multi-master but instance fleets are required
+  # if using both on-demand and spot instances.
+  # Use instance group only if multi-master and no spot instances.
+  # Otherwise deploy as fleet which will result in AWS error
+  use_instance_groups = (var.master_instance_on_demand_count > 1 && var.core_instance_spot_count == 0) ? true : false
 }
 
 data "aws_s3_bucket_object" "json_config" {
@@ -25,57 +31,93 @@ resource "aws_emr_cluster" "emr-cluster" {
     key_name                          = var.key_pair_name
   }
 
-  master_instance_fleet {
-    name                      = var.master_instance_fleet_name
-    target_on_demand_capacity = var.master_instance_on_demand_count
-    target_spot_capacity      = var.master_instance_spot_count
-    instance_type_configs {
-      bid_price                                  = var.master_bid_price
-      bid_price_as_percentage_of_on_demand_price = var.master_bid_price_as_percentage_of_on_demand_price
-      instance_type                              = var.master_instance_type
-      weighted_capacity                          = 1
+  dynamic "master_instance_group" {
+    for_each = local.use_instance_groups ? [1] : []
+    content {
+      name           = var.master_instance_fleet_name
+      instance_count = var.master_instance_on_demand_count
+      instance_type  = var.master_instance_type
       ebs_config {
         size                 = var.master_ebs_size
         type                 = var.master_ebs_type
         volumes_per_instance = var.master_ebs_volumes_count
       }
     }
-    dynamic "launch_specifications" {
-      for_each = var.master_instance_spot_count > 0 ? [1] : []
-      content {
-        spot_specification {
-          allocation_strategy      = "capacity-optimized"
-          block_duration_minutes   = var.master_block_duration_minutes
-          timeout_action           = var.master_timeout_action
-          timeout_duration_minutes = var.master_timeout_duration_minutes
-        }
-      }
-    }
   }
 
-  core_instance_fleet {
-    name                      = var.core_instance_fleet_name
-    target_on_demand_capacity = var.core_instance_on_demand_count
-    target_spot_capacity      = var.core_instance_spot_count
-    instance_type_configs {
-      bid_price                                  = var.core_bid_price
-      bid_price_as_percentage_of_on_demand_price = var.core_bid_price_as_percentage_of_on_demand_price
-      instance_type                              = var.core_instance_type
-      weighted_capacity                          = 1
+  dynamic "core_instance_group" {
+    for_each = local.use_instance_groups ? [1] : []
+    content {
+      name           = var.core_instance_fleet_name
+      instance_count = var.core_instance_on_demand_count
+      instance_type  = var.core_instance_type
       ebs_config {
         size                 = var.core_ebs_size
         type                 = var.core_ebs_type
         volumes_per_instance = var.core_ebs_volumes_count
       }
     }
-    dynamic "launch_specifications" {
-      for_each = var.core_instance_spot_count > 0 ? [1] : []
-      content {
-        spot_specification {
-          allocation_strategy      = "capacity-optimized"
-          block_duration_minutes   = var.core_block_duration_minutes
-          timeout_action           = var.core_timeout_action
-          timeout_duration_minutes = var.core_timeout_duration_minutes
+  }
+
+  dynamic "master_instance_fleet" {
+    for_each = ! local.use_instance_groups ? [1] : []
+    content {
+      name                      = var.master_instance_fleet_name
+      target_on_demand_capacity = var.master_instance_on_demand_count
+      target_spot_capacity      = var.master_instance_spot_count
+      instance_type_configs {
+        bid_price                                  = var.master_bid_price
+        bid_price_as_percentage_of_on_demand_price = var.master_bid_price_as_percentage_of_on_demand_price
+        instance_type                              = var.master_instance_type
+        weighted_capacity                          = 1
+        ebs_config {
+          size                 = var.master_ebs_size
+          type                 = var.master_ebs_type
+          volumes_per_instance = var.master_ebs_volumes_count
+        }
+      }
+      dynamic "launch_specifications" {
+        for_each = var.master_instance_spot_count > 0 ? [
+        1] : []
+        content {
+          spot_specification {
+            allocation_strategy      = "capacity-optimized"
+            block_duration_minutes   = var.master_block_duration_minutes
+            timeout_action           = var.master_timeout_action
+            timeout_duration_minutes = var.master_timeout_duration_minutes
+          }
+        }
+      }
+    }
+  }
+
+  dynamic "core_instance_fleet" {
+    for_each = ! local.use_instance_groups ? [1] : []
+    content {
+      name                      = var.core_instance_fleet_name
+      target_on_demand_capacity = var.core_instance_on_demand_count
+      target_spot_capacity      = var.core_instance_spot_count
+      instance_type_configs {
+        bid_price                                  = var.core_bid_price
+        bid_price_as_percentage_of_on_demand_price = var.core_bid_price_as_percentage_of_on_demand_price
+        instance_type                              = var.core_instance_type
+        weighted_capacity                          = 1
+        ebs_config {
+          size                 = var.core_ebs_size
+          type                 = var.core_ebs_type
+          volumes_per_instance = var.core_ebs_volumes_count
+        }
+      }
+      dynamic "launch_specifications" {
+        for_each = var.core_instance_spot_count > 0 ? [
+        1] : []
+        content {
+          spot_specification {
+            allocation_strategy      = "capacity-optimized"
+            block_duration_minutes   = var.core_block_duration_minutes
+            timeout_action           = var.core_timeout_action
+            timeout_duration_minutes = var.core_timeout_duration_minutes
+          }
         }
       }
     }

diff --git a/modules/aws-emr-cluster/variables.tf b/modules/aws-emr-cluster/variables.tf
@@ -266,4 +266,4 @@ variable "custom_ami_id" {
   type        = string
   description = "The ID of a custom Amazon EBS-backed Linux AMI"
   default     = null
-}
+}
diff --git a/modules/aws-emr-iam/variables.tf b/modules/aws-emr-iam/variables.tf
@@ -83,4 +83,4 @@ variable "require_abac_for_subnet" {
 variable "vpc_id" {
   type        = string
   description = "VPC ID of the network"
-}
+}
diff --git a/modules/aws-emr-ports/variables.tf b/modules/aws-emr-ports/variables.tf
@@ -20,7 +20,7 @@ variable "master_ports_spark" {
 variable "is_pre_6x" {
   type        = bool
   description = "Is this a pre-6x EMR"
-  default = true
+  default     = true
 }
 
 variable "master_ports_hbase_common" {
@@ -42,15 +42,15 @@ variable "master_ports_hbase_pre_6x" {
   type        = list(number)
   description = "Ports used by AWS EMR Master HBase pre-6.x"
   default = [
-    50070  // HDFS NameNode
+    50070 // HDFS NameNode
   ]
 }
 
 variable "master_ports_hbase_6x" {
   type        = list(number)
   description = "Ports used by AWS EMR Master HBase post-6.x"
   default = [
-    9870  // HDFS NameNode
+    9870 // HDFS NameNode
   ]
 }
 
@@ -75,7 +75,7 @@ variable "core_ports_emr_6x" {
   type        = list(number)
   description = "Ports used by AWS EMR Core 6x"
   default = [
-    9864  // HDFS DataNode
+    9864 // HDFS DataNode
   ]
 }
 

diff --git a/variables.tf b/variables.tf
@@ -370,4 +370,4 @@ variable "require_abac_for_subnet" {
   type        = bool
   description = "If abac_valid_tags is specified, choose whether or not to require ABAC also for actions related to the subnet"
   default     = true
-}
+}