Update Gitcoin schemas (#2556)

* fix: include timestamp in matching data

* chore: add round data as static source

* feat(dbt): gitcoin funding rounds

* fix: application title

* fix: linting error

* fix: add round name to staging

* temp: save progress

* refactor: use round names from donations table

* refactor: finalize funding event data

* fix: final refactor of all models

* fix: linting error

* feat: gitcoin mart model

warehouse/dbt/models/intermediate/funding/int_gitcoin_funding_events.sql

@@ -1,65 +1,69 @@
-{#
-  This model combines the donations and matching grants data to create a single table of funding events.
-  The `funding_type` column is used to differentiate between donations and matching grants.
-  The `event_time` data has some issues. It is missing for matching grants, so we use the most recent donation timestamp as a placeholder. To the cGrants data, we use a placeholder date of 2023-07-01.
-#}
-
-
-with last_donation_by_round as (
-  select
-    round_id,
-    round_number,
-    gitcoin_project_id,
-    max(donation_timestamp) as assumed_event_time
-  from {{ ref('stg_gitcoin__donations') }}
-  group by
-    round_id,
-    round_number,
-    gitcoin_project_id
+with unioned_funding_events as (
+  select * from {{ ref('stg_gitcoin__donations') }}
+  union all
+  select * from {{ ref('stg_gitcoin__matching') }}
 ),
 
-unioned_events as (
+labeled_funding_events as (
   select
-    transaction_hash,
-    donation_timestamp as event_time,
-    round_id,
-    round_number,
-    chain_id,
-    gitcoin_project_id,
-    donor_address,
-    amount_in_usd,
-    'DONATIONS' as funding_type
-  from {{ ref('stg_gitcoin__donations') }}
-
-  union all
+    *,
+    case
+      when (
+        gitcoin_data_source in ('MatchFunding', 'CGrants')
+        and round_number between 1 and 15
+      ) then concat('GG-', lpad(cast(round_number as string), 2, '0'))
+      when (
+        gitcoin_data_source in ('MatchFunding', 'Alpha')
+        and round_number = 16
+      ) then 'GG-16'
+      when (
+        gitcoin_data_source in ('MatchFunding', 'GrantsStack')
+        and round_number is not null
+      ) then concat('GG-', lpad(cast(round_number as string), 2, '0'))
+    end as main_round_label,
+    case
+      when (
+        gitcoin_data_source = 'CGrants'
+        and round_number is null
+      ) then 'DirectDonations'
+      when (
+        gitcoin_data_source in ('MatchFunding', 'GrantsStack')
+        and round_number is null
+      ) then 'PartnerRound'
+      else 'MainRound'
+    end as round_type
+  from unioned_funding_events
+),
 
+joined_events as (
   select
-    null as transaction_hash,
-    last_donation_by_round.assumed_event_time as event_time,
-    matching.round_id,
-    matching.round_number,
-    matching.chain_id,
-    matching.gitcoin_project_id,
-    null as donor_address,
-    matching.amount_in_usd,
-    'MATCHING' as funding_type
-  from {{ ref('stg_gitcoin__matching') }} as matching
-  left join last_donation_by_round
-    on
-      matching.round_id = last_donation_by_round.round_id
-      and matching.round_number = last_donation_by_round.round_number
-      and matching.gitcoin_project_id = last_donation_by_round.gitcoin_project_id
+    labeled_funding_events.*,
+    directory.oso_project_id,
+    projects.project_name as oso_project_name,
+    projects.display_name as oso_display_name
+  from labeled_funding_events
+  left join {{ ref('int_gitcoin_project_directory') }} as directory
+    on labeled_funding_events.gitcoin_project_id = directory.gitcoin_project_id
+  left join {{ ref('projects_v1') }} as projects
+    on directory.oso_project_id = projects.project_id
+  where amount_in_usd > 0
 )
 
 select
-  transaction_hash,
-  round_id,
+  event_time,
+  gitcoin_data_source,
+  gitcoin_round_id,
   round_number,
+  round_type,
+  main_round_label,
+  round_name,
   chain_id,
   gitcoin_project_id,
+  project_application_title,
+  oso_project_id,
+  oso_project_name,
+  oso_display_name,
   donor_address,
   amount_in_usd,
-  funding_type,
-  coalesce(event_time, timestamp('2023-07-01')) as event_time
-from unioned_events
-where amount_in_usd > 0
+  transaction_hash
+from joined_events

warehouse/dbt/models/intermediate/funding/int_gitcoin_funding_rounds.sql

@@ -0,0 +1,42 @@
+with partner_rounds as (
+  select *
+  from {{ ref('int_gitcoin_funding_events') }}
+  where
+    round_type = 'PartnerRound'
+    and round_name is not null
+),
+
+main_rounds as (
+  select *
+  from {{ ref('int_gitcoin_funding_events') }}
+  where round_type = 'MainRound'
+),
+
+unioned as (
+  select * from partner_rounds
+  union all
+  select * from main_rounds
+)
+
+select
+  gitcoin_data_source,
+  gitcoin_round_id,
+  round_number,
+  round_name,
+  round_type,
+  main_round_label,
+  array_agg(distinct chain_id) as chain_ids,
+  sum(amount_in_usd) as total_amount_in_usd,
+  count(distinct gitcoin_project_id) as count_projects
+from unioned
+group by
+  gitcoin_data_source,
+  gitcoin_round_id,
+  round_number,
+  round_name,
+  round_type,
+  main_round_label
+order by
+  round_type asc,
+  main_round_label desc,
+  gitcoin_round_id asc

warehouse/dbt/models/intermediate/funding/int_gitcoin_project_directory.sql

@@ -25,15 +25,51 @@ oso_projects as (
     wallets.artifact_type = 'WALLET'
     and repos.artifact_source = 'GITHUB'
     and wallets.project_id = repos.project_id
-)
+),
+
+wallet_matches as (
+  select distinct
+    gitcoin_projects.gitcoin_project_id,
+    gitcoin_projects.latest_project_github,
+    gitcoin_projects.latest_project_recipient_address,
+    oso_projects.oso_project_id as oso_wallet_match
+  from gitcoin_projects
+  left join oso_projects
+    on gitcoin_projects.latest_project_recipient_address = oso_projects.address
+),
 
-select distinct
-  oso_projects.oso_project_id,
-  gitcoin_projects.gitcoin_project_id,
-  gitcoin_projects.latest_project_github,
-  gitcoin_projects.latest_project_recipient_address
-from gitcoin_projects
-inner join oso_projects on (
-  gitcoin_projects.latest_project_github = oso_projects.repo_owner
-  and gitcoin_projects.latest_project_recipient_address = oso_projects.address
+repo_matches as (
+  select distinct
+    wm.*,
+    oso_projects.oso_project_id as oso_repo_match
+  from wallet_matches as wm
+  left join oso_projects
+    on wm.latest_project_github = oso_projects.repo_owner
+),
+
+final_matches as (
+  select distinct
+    gitcoin_project_id,
+    latest_project_github,
+    latest_project_recipient_address,
+    oso_wallet_match,
+    oso_repo_match,
+    case
+      when oso_wallet_match is not null then oso_wallet_match
+      when oso_repo_match is not null then oso_repo_match
+    end as oso_project_id
+  from repo_matches
 )
+
+select
+  final_matches.gitcoin_project_id,
+  final_matches.latest_project_github,
+  final_matches.latest_project_recipient_address,
+  final_matches.oso_wallet_match,
+  final_matches.oso_repo_match,
+  final_matches.oso_project_id,
+  projects.project_name as oso_project_name,
+  projects.display_name as oso_display_name
+from final_matches
+left join {{ ref('projects_v1') }} as projects
+  on final_matches.oso_project_id = projects.project_id

warehouse/dbt/models/intermediate/funding/int_oss_funding_grants_to_project.sql

@@ -6,13 +6,13 @@
 
 with oss_funding_data as (
   select -- noqa: ST06
-    CAST(funding_date as timestamp) as `time`,
+    cast(funding_date as timestamp) as `time`,
     'GRANT_RECEIVED_USD' as event_type,
     '{{ oss_funding_url }}' as event_source_id,
     'OSS_FUNDING' as event_source,
-    LOWER(to_project_name) as to_project_name,
-    LOWER(from_funder_name) as from_project_name,
-    COALESCE(amount, 0) as amount
+    lower(to_project_name) as to_project_name,
+    lower(from_funder_name) as from_project_name,
+    coalesce(amount, 0) as amount
   from {{ source('static_data_sources', 'oss_funding_v1') }}
   where
     to_project_name is not null
@@ -22,21 +22,17 @@ with oss_funding_data as (
 
 gitcoin_data as (
   select -- noqa: ST06
-    events.event_time as `time`,
+    event_time as `time`,
     'GRANT_RECEIVED_USD' as event_type,
-    COALESCE(
-      events.transaction_hash,
-      CAST(events.gitcoin_project_id as string)
+    coalesce(
+      transaction_hash,
+      cast(gitcoin_project_id as string)
     ) as event_source_id,
-    CONCAT('GITCOIN', '_', funding_type) as event_source,
-    projects.project_name as to_project_name,
+    concat('GITCOIN', '_', upper(gitcoin_data_source)) as event_source,
+    oso_project_name as to_project_name,
     'gitcoin' as from_project_name,
-    events.amount_in_usd as amount
-  from {{ ref('int_gitcoin_funding_events') }} as events
-  inner join {{ ref('int_gitcoin_project_directory') }} as project_directory
-    on events.gitcoin_project_id = project_directory.gitcoin_project_id
-  inner join {{ ref('projects_v1') }} as projects
-    on project_directory.oso_project_id = projects.project_id
+    amount_in_usd as amount
+  from {{ ref('int_gitcoin_funding_events') }}
 ),
 
 grants as (

warehouse/dbt/models/marts/gitcoin/gitcoin_funding_events_by_project_v0.sql

@@ -0,0 +1,31 @@
+select
+  event_time,
+  gitcoin_data_source,
+  gitcoin_round_id,
+  round_number,
+  round_type,
+  main_round_label,
+  round_name,
+  gitcoin_project_id,
+  project_application_title,
+  oso_project_id,
+  oso_project_name,
+  oso_display_name,
+  donor_address,
+  sum(amount_in_usd) as amount_in_usd
+from {{ ref('int_gitcoin_funding_events') }}
+where oso_project_id is not null
+group by
+  event_time,
+  gitcoin_data_source,
+  gitcoin_round_id,
+  round_number,
+  round_type,
+  main_round_label,
+  round_name,
+  gitcoin_project_id,
+  project_application_title,
+  oso_project_id,
+  oso_project_name,
+  oso_display_name,
+  donor_address

warehouse/dbt/models/staging/gitcoin/stg_gitcoin__donations.sql

@@ -1,14 +1,22 @@
-select distinct
+with max_dlt as (
+  select max(_dlt_load_id) as max_dlt_load_id
+  from {{ source("gitcoin", "all_donations") }}
+)
+
+select distinct  -- noqa: ST06
   `source` as gitcoin_data_source,
-  `timestamp` as donation_timestamp,
-  round_id,
+  `timestamp` as event_time,
+  round_id as gitcoin_round_id,
   round_num as round_number,
+  round_name,
   chain_id,
   project_id as gitcoin_project_id,
-  amount_in_usd,
-  LOWER(recipient_address) as project_recipient_address,
-  LOWER(donor_address) as donor_address,
-  LOWER(transaction_hash) as transaction_hash,
-  TRIM(project_name) as project_application_title
+  trim(project_name) as project_application_title,
+  lower(recipient_address) as project_recipient_address,
+  lower(donor_address) as donor_address,
+  lower(transaction_hash) as transaction_hash,
+  amount_in_usd
 from {{ source("gitcoin", "all_donations") }}
-where amount_in_usd > 0
+where
+  amount_in_usd > 0
+  and _dlt_load_id = (select max_dlt.max_dlt_load_id from max_dlt)

warehouse/dbt/models/staging/gitcoin/stg_gitcoin__matching.sql

@@ -3,14 +3,19 @@ with max_dlt as (
   from {{ source("gitcoin", "all_matching") }}
 )
 
-select distinct
-  round_id as round_id,
+select distinct  -- noqa: ST06
+  'MatchFunding' as gitcoin_data_source,
+  `timestamp` as event_time,
+  round_id as gitcoin_round_id,
   round_num as round_number,
+  cast(null as string) as round_name,
   chain_id,
   project_id as gitcoin_project_id,
-  match_amount_in_usd as amount_in_usd,
-  trim(title) as round_title,
-  lower(recipient_address) as project_recipient_address
+  trim(title) as project_application_title,
+  lower(recipient_address) as project_recipient_address,
+  cast(null as string) as donor_address,
+  cast(null as string) as transaction_hash,
+  match_amount_in_usd as amount_in_usd
 from {{ source("gitcoin", "all_matching") }}
 where
   match_amount_in_usd > 0

warehouse/dbt/models/static_data_sources.yml

@@ -3,6 +3,8 @@ sources:
     database: opensource-observer
     schema: static_data_sources
     tables:
+      - name: grants_stack_rounds
+        identifier: grants_stack_rounds
       - name: oss_funding_v1
         identifier: oss_funding_v1
       - name: op_rf4_trusted_addresses