Fix issues with SQLMesh run (#2651)

* Test setting variable cluster size * fix * Split rolling window work into 1 year at a time * Use websockets for job status and ensure retries * Update mcs worker deployment * more tf updates * fix tests * Move some things around * fix debug
opensource-observer · Dec 17, 2024 · fd86eb6 · fd86eb6
1 parent fd79486
commit fd86eb6
Show file tree

Hide file tree

Showing 15 changed files with 800 additions and 349 deletions.
diff --git a/ops/tf-modules/warehouse-cluster/main.tf b/ops/tf-modules/warehouse-cluster/main.tf
@@ -161,9 +161,9 @@ locals {
       machine_type                      = "n1-highmem-16"
       node_locations                    = join(",", var.cluster_zones)
       min_count                         = 0
-      max_count                         = 10
+      max_count                         = 20
       local_ssd_count                   = 0
-      local_ssd_ephemeral_storage_count = 1
+      local_ssd_ephemeral_storage_count = 2
       spot                              = false
       disk_size_gb                      = 100
       disk_type                         = "pd-standard"
@@ -357,7 +357,7 @@ module "vpc" {
 
 module "gke" {
   source                     = "terraform-google-modules/kubernetes-engine/google"
-  version                    = "~> 33.0"
+  version                    = "~> 35.0.0"
   project_id                 = var.project_id
   name                       = var.cluster_name
   region                     = var.cluster_region

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -76,6 +76,8 @@ aiotrino = "^0.2.3"
 pytest-asyncio = "^0.24.0"
 isort = "^5.13.2"
 uvicorn = { extras = ["standard"], version = "^0.32.1" }
+websockets = "^14.1"
+pydantic-settings = "^2.7.0"
 
 
 [tool.poetry.scripts]