Adds a "standard" pool for jobs (#2154)

* Adds a standard pool * Use the new standard instance * Update ops/tf-modules/warehouse-cluster/main.tf
opensource-observer · Sep 16, 2024 · b49f79b · b49f79b
1 parent b9dab82
commit b49f79b
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 4 deletions.
diff --git a/ops/k8s-apps/base/dagster/dagster.yaml b/ops/k8s-apps/base/dagster/dagster.yaml
@@ -53,17 +53,17 @@ spec:
                   - key: pool_type
                     effect: NoSchedule
                     operator: Equal
-                    value: persistent
+                    value: standard
                 nodeSelector:
-                  pool_type: persistent
+                  pool_type: standard
               containerConfig:
                 resources:
                   limits:
                     cpu: 500m 
-                    memory: 512Mi
+                    memory: 1024Mi
                   requests:
                     cpu: 500m
-                    memory: 512Mi
+                    memory: 768Mi
       dagsterWebserver:
         enableReadOnly: true
         env:

diff --git a/ops/tf-modules/warehouse-cluster/main.tf b/ops/tf-modules/warehouse-cluster/main.tf
@@ -25,6 +25,27 @@ locals {
       preemptible        = false
       initial_node_count = 0
     },
+    # A non-spot non-preemptible set of nodes for job execution. Nothing should run here except temporary jobs
+    {
+      name               = "${var.cluster_name}-standard-node-pool"
+      machine_type       = "e2-standard-4"
+      node_locations     = join(",", var.cluster_zones)
+      min_count          = 0
+      max_count          = 4
+      local_ssd_count    = 0
+      spot               = false
+      disk_size_gb       = 100
+      disk_type          = "pd-standard"
+      image_type         = "COS_CONTAINERD"
+      enable_gcfs        = false
+      enable_gvnic       = false
+      logging_variant    = "DEFAULT"
+      auto_repair        = true
+      auto_upgrade       = true
+      service_account    = local.node_service_account_email
+      preemptible        = false
+      initial_node_count = 0
+    },
     # The spot pool is for workloads that need spot
     {
       name               = "${var.cluster_name}-spot-node-pool"
@@ -74,6 +95,10 @@ locals {
       default_node_pool = true
       pool_type         = "persistent"
     }
+    "${var.cluster_name}-standard-node-pool" = {
+      default_node_pool = false
+      pool_type         = "standard"
+    }
     "${var.cluster_name}-spot-node-pool" = {
       default_node_pool = false
       pool_type         = "spot"
@@ -91,6 +116,13 @@ locals {
   }, var.extra_node_metadata)
 
   node_pool_taints = merge({
+    "${var.cluster_name}-standard-node-pool" = [
+      {
+        key    = "pool_type"
+        value  = "standard"
+        effect = "NO_SCHEDULE"
+      },
+    ]
     "${var.cluster_name}-spot-node-pool" = [
       {
         key    = "pool_type"
@@ -111,6 +143,9 @@ locals {
     "${var.cluster_name}-default-node-pool" = [
       "default-node-pool",
     ]
+    "${var.cluster_name}-standard-node-pool" = [
+      "standard",
+    ]
     "${var.cluster_name}-spot-node-pool" = [
       "spot",
     ]