Dagster Fixes: Load job out of order (#1460)

Fix load and pointer update are out of order
opensource-observer · May 21, 2024 · 4d342a4 · 4d342a4
1 parent 9af3b9b
commit 4d342a4
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/warehouse/oso_dagster/goldsky.py b/warehouse/oso_dagster/goldsky.py
@@ -317,9 +317,13 @@ def commit_pointer(
                 job_config=job_config,
                 timeout=self.config.load_table_timeout_seconds,
             )
-            self.update_pointer_table(client, context, checkpoint, pointer_table_mutex)
-            context.log.debug("updated pointer table")
             load_job.result()
+            context.log.info(f"Worker[{self.name}] Data loaded into bigquery")
+
+            self.update_pointer_table(client, context, checkpoint, pointer_table_mutex)
+            context.log.info(
+                f"Worker[{self.name}] Pointer table updated to {checkpoint.worker_checkpoint}"
+            )
 
     def run_load_bigquery_load(
         self,