ravenac95/debug bigquery failures (#1459)

* Reduce load size to debug bq failures * More debug configurations
opensource-observer · May 20, 2024 · 6fd0733 · 6fd0733
1 parent 1043fc0
commit 6fd0733
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 27 deletions.
diff --git a/ops/k8s-apps/base/dagster/kustomization.yaml b/ops/k8s-apps/base/dagster/kustomization.yaml
@@ -2,9 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: base-dagster
 resources:
-  #- proxy-for-deployments.yaml
-  #- proxy-for-jobs.yaml
-  - dagster.yaml
-# patchesStrategicMerge:
-#   - ./daemon-init-container.yaml
-#   - ./webserver-init-container.yaml
+  - dagster.yaml
diff --git a/test_all.sh b/test_all.sh
diff --git a/warehouse/oso_dagster/goldsky.py b/warehouse/oso_dagster/goldsky.py
@@ -36,8 +36,9 @@ class GoldskyConfig:
     source_name: str
     destination_table_name: str
 
-    # Maximum number of objects we can load into a load job is 10000 so the largest this can be is 10000
-    pointer_size: int = int(os.environ.get("GOLDSKY_CHECKPOINT_SIZE", "10000"))
+    # Maximum number of objects we can load into a load job is 10000 so the
+    # largest this can be is 10000.
+    pointer_size: int = int(os.environ.get("GOLDSKY_CHECKPOINT_SIZE", "5000"))
 
     max_objects_to_load: int = 200_000
 
@@ -1042,6 +1043,6 @@ def load_queues(
                 )
 
         for worker, queue in queues.worker_queues():
-            context.log.debug(f"Worker[{worker}] queue size: {queue.len()}")
+            context.log.info(f"Worker[{worker}] queue size: {queue.len()}")
 
         return (worker_status, queues)