From 554f13a954097f9632189d7f77ba8f901c9704be Mon Sep 17 00:00:00 2001
From: Chuxin Huang <chuxin@oplabs.co>
Date: Tue, 6 Aug 2024 03:15:51 +1000
Subject: [PATCH] Fix duplicate rows in first time addresses (#1890)

* add all

* update macro from first block timestamp to blockdate

---------

Co-authored-by: Carl Cervone <42869436+ccerv1@users.noreply.github.com>
---
 .../dbt/macros/models/addresses_daily_gas_and_usage.sql    | 2 +-
 .../models/staging/base/stg_base__first_time_addresses.sql | 7 +++++++
 .../models/staging/frax/stg_frax__first_time_addresses.sql | 7 +++++++
 .../staging/metal/stg_metal__first_time_addresses.sql      | 7 +++++++
 .../models/staging/mode/stg_mode__first_time_addresses.sql | 7 +++++++
 .../models/staging/pgn/stg_pgn__first_time_addresses.sql   | 7 +++++++
 .../models/staging/zora/stg_zora__first_time_addresses.sql | 7 +++++++
 7 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/warehouse/dbt/macros/models/addresses_daily_gas_and_usage.sql b/warehouse/dbt/macros/models/addresses_daily_gas_and_usage.sql
index f064dae10..bebe24d1b 100644
--- a/warehouse/dbt/macros/models/addresses_daily_gas_and_usage.sql
+++ b/warehouse/dbt/macros/models/addresses_daily_gas_and_usage.sql
@@ -34,7 +34,7 @@ with daily_gas_fees as (
         ,date_diff(c.date_timestamp, ft.first_active_day, DAY) as days_since_first_active
     from {{ ref('stg_%s__first_time_addresses' % lower_network_name)}} as ft 
     join {{ ref('stg_utility__calendar')}} as c 
-        on c.date_timestamp >= ft.first_block_timestamp
+        on c.date_timestamp >= ft.first_active_day
         {% if is_incremental() %}
             and c.date_timestamp < TIMESTAMP_SUB(_dbt_max_partition, INTERVAL 0 DAY)
         {% else %}
diff --git a/warehouse/dbt/models/staging/base/stg_base__first_time_addresses.sql b/warehouse/dbt/models/staging/base/stg_base__first_time_addresses.sql
index eb49f4645..e2b494c52 100644
--- a/warehouse/dbt/models/staging/base/stg_base__first_time_addresses.sql
+++ b/warehouse/dbt/models/staging/base/stg_base__first_time_addresses.sql
@@ -11,4 +11,11 @@
     incremental_strategy="insert_overwrite"
   ) 
 }}
+
+{% set unique_key_col = config.get('unique_id') %}
+
 {{ first_time_addresses("base") }}
+{% if is_incremental() %}
+  where
+    {{ unique_key_col }} not in (select {{ unique_key_col }} from {{ this }})
+{% endif %}
diff --git a/warehouse/dbt/models/staging/frax/stg_frax__first_time_addresses.sql b/warehouse/dbt/models/staging/frax/stg_frax__first_time_addresses.sql
index 1b36c77c0..3fc833727 100644
--- a/warehouse/dbt/models/staging/frax/stg_frax__first_time_addresses.sql
+++ b/warehouse/dbt/models/staging/frax/stg_frax__first_time_addresses.sql
@@ -11,4 +11,11 @@
     incremental_strategy="insert_overwrite"
   ) 
 }}
+
+{% set unique_key_col = config.get('unique_id') %}
+
 {{ first_time_addresses("frax") }}
+{% if is_incremental() %}
+  where
+    {{ unique_key_col }} not in (select {{ unique_key_col }} from {{ this }})
+{% endif %}
diff --git a/warehouse/dbt/models/staging/metal/stg_metal__first_time_addresses.sql b/warehouse/dbt/models/staging/metal/stg_metal__first_time_addresses.sql
index 8b35f33a4..62ac617ae 100644
--- a/warehouse/dbt/models/staging/metal/stg_metal__first_time_addresses.sql
+++ b/warehouse/dbt/models/staging/metal/stg_metal__first_time_addresses.sql
@@ -11,4 +11,11 @@
     incremental_strategy="insert_overwrite"
   ) 
 }}
+
+{% set unique_key_col = config.get('unique_id') %}
+
 {{ first_time_addresses("metal") }}
+{% if is_incremental() %}
+  where
+    {{ unique_key_col }} not in (select {{ unique_key_col }} from {{ this }})
+{% endif %}
diff --git a/warehouse/dbt/models/staging/mode/stg_mode__first_time_addresses.sql b/warehouse/dbt/models/staging/mode/stg_mode__first_time_addresses.sql
index d8e4a7bd8..3e4a1e8e3 100644
--- a/warehouse/dbt/models/staging/mode/stg_mode__first_time_addresses.sql
+++ b/warehouse/dbt/models/staging/mode/stg_mode__first_time_addresses.sql
@@ -11,4 +11,11 @@
     incremental_strategy="insert_overwrite"
   ) 
 }}
+
+{% set unique_key_col = config.get('unique_id') %}
+
 {{ first_time_addresses("mode") }}
+{% if is_incremental() %}
+  where
+    {{ unique_key_col }} not in (select {{ unique_key_col }} from {{ this }})
+{% endif %}
diff --git a/warehouse/dbt/models/staging/pgn/stg_pgn__first_time_addresses.sql b/warehouse/dbt/models/staging/pgn/stg_pgn__first_time_addresses.sql
index 19e667561..2472fe28b 100644
--- a/warehouse/dbt/models/staging/pgn/stg_pgn__first_time_addresses.sql
+++ b/warehouse/dbt/models/staging/pgn/stg_pgn__first_time_addresses.sql
@@ -11,4 +11,11 @@
     incremental_strategy="insert_overwrite"
   ) 
 }}
+
+{% set unique_key_col = config.get('unique_id') %}
+
 {{ first_time_addresses("pgn") }}
+{% if is_incremental() %}
+  where
+    {{ unique_key_col }} not in (select {{ unique_key_col }} from {{ this }})
+{% endif %}
diff --git a/warehouse/dbt/models/staging/zora/stg_zora__first_time_addresses.sql b/warehouse/dbt/models/staging/zora/stg_zora__first_time_addresses.sql
index f2f286a57..af74dd2bb 100644
--- a/warehouse/dbt/models/staging/zora/stg_zora__first_time_addresses.sql
+++ b/warehouse/dbt/models/staging/zora/stg_zora__first_time_addresses.sql
@@ -11,4 +11,11 @@
     incremental_strategy="insert_overwrite"
   ) 
 }}
+
+{% set unique_key_col = config.get('unique_id') %}
+
 {{ first_time_addresses("zora") }}
+{% if is_incremental() %}
+  where
+    {{ unique_key_col }} not in (select {{ unique_key_col }} from {{ this }})
+{% endif %}