Fix duplicate rows in first time addresses (#1890)

* add all * update macro from first block timestamp to blockdate --------- Co-authored-by: Carl Cervone <[email protected]>
opensource-observer · Aug 5, 2024 · 554f13a · 554f13a
1 parent c2ce4fd
commit 554f13a
Show file tree

Hide file tree

Showing 7 changed files with 43 additions and 1 deletion.
diff --git a/warehouse/dbt/macros/models/addresses_daily_gas_and_usage.sql b/warehouse/dbt/macros/models/addresses_daily_gas_and_usage.sql
@@ -34,7 +34,7 @@ with daily_gas_fees as (
         ,date_diff(c.date_timestamp, ft.first_active_day, DAY) as days_since_first_active
     from {{ ref('stg_%s__first_time_addresses' % lower_network_name)}} as ft 
     join {{ ref('stg_utility__calendar')}} as c 
-        on c.date_timestamp >= ft.first_block_timestamp
+        on c.date_timestamp >= ft.first_active_day
         {% if is_incremental() %}
             and c.date_timestamp < TIMESTAMP_SUB(_dbt_max_partition, INTERVAL 0 DAY)
         {% else %}

diff --git a/warehouse/dbt/models/staging/base/stg_base__first_time_addresses.sql b/warehouse/dbt/models/staging/base/stg_base__first_time_addresses.sql
@@ -11,4 +11,11 @@
     incremental_strategy="insert_overwrite"
   ) 
 }}
+
+{% set unique_key_col = config.get('unique_id') %}
+
 {{ first_time_addresses("base") }}
+{% if is_incremental() %}
+  where
+    {{ unique_key_col }} not in (select {{ unique_key_col }} from {{ this }})
+{% endif %}
diff --git a/warehouse/dbt/models/staging/frax/stg_frax__first_time_addresses.sql b/warehouse/dbt/models/staging/frax/stg_frax__first_time_addresses.sql
@@ -11,4 +11,11 @@
     incremental_strategy="insert_overwrite"
   ) 
 }}
+
+{% set unique_key_col = config.get('unique_id') %}
+
 {{ first_time_addresses("frax") }}
+{% if is_incremental() %}
+  where
+    {{ unique_key_col }} not in (select {{ unique_key_col }} from {{ this }})
+{% endif %}
diff --git a/warehouse/dbt/models/staging/metal/stg_metal__first_time_addresses.sql b/warehouse/dbt/models/staging/metal/stg_metal__first_time_addresses.sql
@@ -11,4 +11,11 @@
     incremental_strategy="insert_overwrite"
   ) 
 }}
+
+{% set unique_key_col = config.get('unique_id') %}
+
 {{ first_time_addresses("metal") }}
+{% if is_incremental() %}
+  where
+    {{ unique_key_col }} not in (select {{ unique_key_col }} from {{ this }})
+{% endif %}
diff --git a/warehouse/dbt/models/staging/mode/stg_mode__first_time_addresses.sql b/warehouse/dbt/models/staging/mode/stg_mode__first_time_addresses.sql
@@ -11,4 +11,11 @@
     incremental_strategy="insert_overwrite"
   ) 
 }}
+
+{% set unique_key_col = config.get('unique_id') %}
+
 {{ first_time_addresses("mode") }}
+{% if is_incremental() %}
+  where
+    {{ unique_key_col }} not in (select {{ unique_key_col }} from {{ this }})
+{% endif %}
diff --git a/warehouse/dbt/models/staging/pgn/stg_pgn__first_time_addresses.sql b/warehouse/dbt/models/staging/pgn/stg_pgn__first_time_addresses.sql
@@ -11,4 +11,11 @@
     incremental_strategy="insert_overwrite"
   ) 
 }}
+
+{% set unique_key_col = config.get('unique_id') %}
+
 {{ first_time_addresses("pgn") }}
+{% if is_incremental() %}
+  where
+    {{ unique_key_col }} not in (select {{ unique_key_col }} from {{ this }})
+{% endif %}
diff --git a/warehouse/dbt/models/staging/zora/stg_zora__first_time_addresses.sql b/warehouse/dbt/models/staging/zora/stg_zora__first_time_addresses.sql
@@ -11,4 +11,11 @@
     incremental_strategy="insert_overwrite"
   ) 
 }}
+
+{% set unique_key_col = config.get('unique_id') %}
+
 {{ first_time_addresses("zora") }}
+{% if is_incremental() %}
+  where
+    {{ unique_key_col }} not in (select {{ unique_key_col }} from {{ this }})
+{% endif %}