More sqlmesh run tweaks (#2657)

* allow job retries to be configurable

* set flux to this branch

* temp uninstall

* reinstall

* ensure that we use scopes to resolve table deps

* go spot or go home

* more tests

* Improve progress reporting and error logging

* restore main

ops/tf-modules/warehouse-cluster/main.tf

@@ -164,7 +164,7 @@ locals {
       max_count                         = 20
       local_ssd_count                   = 0
       local_ssd_ephemeral_storage_count = 2
-      spot                              = false
+      spot                              = true
       disk_size_gb                      = 100
       disk_type                         = "pd-standard"
       image_type                        = "COS_CONTAINERD"

warehouse/metrics_tools/compute/client.py

@@ -190,6 +190,12 @@ def _handler(response: JobStatusResponse):
 
         if final_status.status == QueryJobStatus.FAILED:
             self.logger.error(f"job[{job_id}] failed with status {final_status.status}")
+            if final_status.exceptions:
+                self.logger.error(f"job[{job_id}] failed with exceptions")
+
+            for exc in final_status.exceptions:
+                self.logger.error(f"job[{job_id}] failed with exceptoin {exc}")
+
             raise Exception(f"job[{job_id}] failed with status {final_status.status}")
 
         self.logger.info(f"job[{job_id}] completed with status {final_status.status}")