From 1043fc04fbf30364eec7f22c8866440168f76c3d Mon Sep 17 00:00:00 2001 From: Reuven Gonzales Date: Mon, 20 May 2024 13:51:36 -0700 Subject: [PATCH] Fix goldsky checkpoint size (#1454) --- warehouse/oso_dagster/goldsky.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/warehouse/oso_dagster/goldsky.py b/warehouse/oso_dagster/goldsky.py index 3e4a27e97..44125c7b1 100644 --- a/warehouse/oso_dagster/goldsky.py +++ b/warehouse/oso_dagster/goldsky.py @@ -36,7 +36,8 @@ class GoldskyConfig: source_name: str destination_table_name: str - pointer_size: int = int(os.environ.get("GOLDSKY_CHECKPOINT_SIZE", "20000")) + # Maximum number of objects we can load into a load job is 10000 so the largest this can be is 10000 + pointer_size: int = int(os.environ.get("GOLDSKY_CHECKPOINT_SIZE", "10000")) max_objects_to_load: int = 200_000