Fix goldsky checkpoint size (#1454)

opensource-observer · May 20, 2024 · 1043fc0 · 1043fc0
1 parent 98d7a60
commit 1043fc0
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/warehouse/oso_dagster/goldsky.py b/warehouse/oso_dagster/goldsky.py
@@ -36,7 +36,8 @@ class GoldskyConfig:
     source_name: str
     destination_table_name: str
 
-    pointer_size: int = int(os.environ.get("GOLDSKY_CHECKPOINT_SIZE", "20000"))
+    # Maximum number of objects we can load into a load job is 10000 so the largest this can be is 10000
+    pointer_size: int = int(os.environ.get("GOLDSKY_CHECKPOINT_SIZE", "10000"))
 
     max_objects_to_load: int = 200_000