Merge branch 'main' into jabolol/defillama-tvl

apps/docs/docs/tutorials/gitcoin-social-networks.md

@@ -0,0 +1,42 @@
+---
+title: Funding in a Social Network
+sidebar_position: 5
+---
+
+Analyze Gitcoin grants funding in a social network. New to OSO? Check out our [Getting Started guide](../get-started/index.md) to set up your BigQuery or API access.
+
+This tutorial combines Farcaster and Gitcoin data to to identify popular projects within a social network.
+
+## BigQuery
+
+If you haven't already, then the first step is to subscribe to OSO public datasets in BigQuery. You can do this by clicking the "Subscribe" button on our [Datasets page](../integrate/datasets/#oso-production-data-pipeline). For this tutorial, you'll need to subscribe to the Gitcoin and Karma3/OpenRank datasets. (You can also use the Farcaster dataset in place of OpenRank.)
+
+The following queries should work if you copy-paste them into your [BigQuery console](https://console.cloud.google.com/bigquery).
+
+### Identify popular projects within your social network
+
+```sql
+select distinct
+  donations.donor_address,
+  users.user_source_id as fid,
+  users.user_name as username,
+  donations.project_name,
+  amount_in_usd,
+  timestamp
+from `gitcoin.all_donations` as donations
+join `oso_production.artifacts_by_user_v1` as users
+  on lower(donations.donor_address) = users.artifact_name
+where
+  user_source = 'FARCASTER'
+  and users.user_source_id in (
+    with max_date as (
+      select max(date) as last_date
+      from `karma3.localtrust`
+    )
+    select cast(j as string) as fid
+    from `karma3.localtrust`
+    where i = 5650
+    order by v desc
+    limit 150
+  )
+```

ops/k8s-apps/base/trino/trino.yaml

@@ -112,7 +112,7 @@ spec:
         hive.metastore-cache-ttl=0s
         hive.metastore-refresh-interval=5s
         hive.metastore.thrift.client.connect-timeout=10s
-        hive.metastore.thrift.client.read-timeout=10s
+        hive.metastore.thrift.client.read-timeout=30s
         iceberg.use-file-size-from-metadata=false
         fs.native-gcs.enabled=true
         fs.cache.enabled=true

ops/k8s-apps/production/metrics-calculation-service/custom-helm-values.yaml

@@ -32,8 +32,8 @@ spec:
         worker:
           threads: "16"
           memory:
-            limit: "96Gi"
-            request: "90Gi"
+            limit: "390Gi"
+            request: "375Gi"
           poolType: "mcs-worker"
           duckdb_path: "/scratch/mcs-local.db"
       trino:

ops/tf-modules/warehouse-cluster/main.tf

@@ -158,12 +158,12 @@ locals {
     # MCS Workers
     {
       name                              = "${var.cluster_name}-mcs-worker-node-pool"
-      machine_type                      = "n1-highmem-16"
+      machine_type                      = "n1-highmem-64"
       node_locations                    = join(",", var.cluster_zones)
       min_count                         = 0
-      max_count                         = 20
+      max_count                         = 50
       local_ssd_count                   = 0
-      local_ssd_ephemeral_storage_count = 2
+      local_ssd_ephemeral_storage_count = 3
       spot                              = true
       disk_size_gb                      = 100
       disk_type                         = "pd-standard"

warehouse/dbt/models/intermediate/directory/int_open_collective_projects.sql

@@ -0,0 +1,59 @@
+with url_registry as (
+  select
+    LOWER(JSON_EXTRACT_SCALAR(to_account, '$.slug')) as project_slug,
+    LOWER(JSON_EXTRACT_SCALAR(link, '$.url')) as github_url,
+    REGEXP_EXTRACT(LOWER(JSON_EXTRACT_SCALAR(link, '$.url')), r'github\.com/([a-z0-9-]+)')
+      as artifact_namespace,
+    REGEXP_EXTRACT(
+      LOWER(JSON_EXTRACT_SCALAR(link, '$.url')), r'github\.com/[a-z0-9-]+/([a-z0-9-._]+)'
+    ) as artifact_name
+  from
+    {{ ref('stg_open_collective__deposits') }},
+    UNNEST(JSON_EXTRACT_ARRAY(to_account, '$.socialLinks')) as link
+  where
+    JSON_EXTRACT_SCALAR(link, '$.url') like '%github.com%'
+),
+
+oso_projects as (
+  select
+    project_id,
+    artifact_namespace,
+    artifact_name
+  from {{ ref('repositories_v0') }}
+  where artifact_source = 'GITHUB'
+),
+
+namespace_counts as (
+  select
+    artifact_namespace,
+    COUNT(distinct project_id) as project_count,
+    MIN(project_id) as project_id
+  from oso_projects
+  group by artifact_namespace
+),
+
+matched_projects as (
+  select
+    ur.*,
+    case
+      when op.project_id is not null then op.project_id
+      when nc.project_count = 1 then nc.project_id
+    end as project_id
+  from url_registry as ur
+  left join oso_projects as op
+    on
+      ur.artifact_namespace = op.artifact_namespace
+      and ur.artifact_name = op.artifact_name
+  left join namespace_counts as nc
+    on ur.artifact_namespace = nc.artifact_namespace
+)
+
+select distinct
+  project_id as project_id,
+  {{ oso_id("'OPEN_COLLECTIVE'", 'project_slug') }} as artifact_id,
+  project_slug as artifact_source_id,
+  'OPEN_COLLECTIVE' as artifact_source,
+  '' as artifact_namespace,
+  project_slug as artifact_name,
+  'https://opencollective.com/' || project_slug as artifact_url
+from matched_projects

warehouse/dbt/models/intermediate/funding/int_open_collective_deposits.sql

@@ -0,0 +1,24 @@
+with open_collective_deposits as (
+  select
+    id as event_source_id,
+    created_at as `time`,
+    JSON_EXTRACT_SCALAR(to_account, '$.name') as project_name,
+    LOWER(JSON_EXTRACT_SCALAR(to_account, '$.slug')) as project_slug,
+    UPPER(JSON_EXTRACT_SCALAR(to_account, '$.type')) as project_type,
+    UPPER(JSON_EXTRACT_SCALAR(amount, '$.currency')) as currency,
+    CAST(JSON_EXTRACT_SCALAR(amount, '$.value') as NUMERIC) as amount
+  from {{ ref('stg_open_collective__deposits') }}
+)
+
+select
+  open_collective_deposits.event_source_id,
+  open_collective_deposits.`time`,
+  projects.project_id,
+  open_collective_deposits.project_name,
+  open_collective_deposits.project_slug,
+  open_collective_deposits.project_type,
+  open_collective_deposits.currency,
+  open_collective_deposits.amount
+from open_collective_deposits
+left join {{ ref('int_open_collective_projects') }} as projects
+  on open_collective_deposits.project_slug = projects.artifact_source_id

warehouse/dbt/models/intermediate/funding/int_oss_funding_grants_to_project.sql

@@ -17,8 +17,7 @@ with oss_funding_data as (
     parse_json(metadata_json) as metadata_json
   from {{ source('static_data_sources', 'oss_funding_v1') }}
   where
-    to_project_name is not null
-    and from_funder_name is not null
+    from_funder_name is not null
     and amount is not null
 ),
 
@@ -52,6 +51,55 @@ gitcoin_data as (
   from {{ ref('int_gitcoin_funding_events') }}
 ),
 
+open_collective_data as (
+  select -- noqa: ST06
+    ocd.`time`,
+    'GRANT_RECEIVED_USD' as event_type,
+    ocd.event_source_id,
+    'OPEN_COLLECTIVE' as event_source,
+    projects.project_name as to_project_name,
+    'opencollective' as from_project_name,
+    ocd.amount,
+    'contributions' as grant_pool_name,
+    to_json(struct(
+      ocd.project_name as open_collective_project_name,
+      ocd.project_slug as open_collective_project_slug,
+      ocd.project_type as open_collective_project_type,
+      ocd.currency as open_collective_currency,
+      ocd.amount as open_collective_amount
+    )) as metadata_json
+  from {{ ref('int_open_collective_deposits') }} as ocd
+  left join {{ ref('projects_v1') }} as projects
+    on ocd.project_id = projects.project_id
+  where ocd.currency = 'USD'
+),
+
+oso_indexed_data as (
+  select
+    gitcoin_data.time,
+    gitcoin_data.event_type,
+    gitcoin_data.event_source_id,
+    gitcoin_data.event_source,
+    gitcoin_data.to_project_name,
+    gitcoin_data.from_project_name,
+    gitcoin_data.amount,
+    gitcoin_data.grant_pool_name,
+    gitcoin_data.metadata_json
+  from gitcoin_data
+  union all
+  select
+    open_collective_data.time,
+    open_collective_data.event_type,
+    open_collective_data.event_source_id,
+    open_collective_data.event_source,
+    open_collective_data.to_project_name,
+    open_collective_data.from_project_name,
+    open_collective_data.amount,
+    open_collective_data.grant_pool_name,
+    open_collective_data.metadata_json
+  from open_collective_data
+),
+
 grants as (
   select
     oss_funding_data.time,
@@ -68,16 +116,16 @@ grants as (
   union all
 
   select
-    gitcoin_data.time,
-    gitcoin_data.event_type,
-    gitcoin_data.event_source_id,
-    gitcoin_data.event_source,
-    gitcoin_data.to_project_name,
-    gitcoin_data.from_project_name,
-    gitcoin_data.amount,
-    gitcoin_data.grant_pool_name,
-    gitcoin_data.metadata_json
-  from gitcoin_data
+    oso_indexed_data.time,
+    oso_indexed_data.event_type,
+    oso_indexed_data.event_source_id,
+    oso_indexed_data.event_source,
+    oso_indexed_data.to_project_name,
+    oso_indexed_data.from_project_name,
+    oso_indexed_data.amount,
+    oso_indexed_data.grant_pool_name,
+    oso_indexed_data.metadata_json
+  from oso_indexed_data
 )
 
 select
@@ -87,7 +135,7 @@ select
   grants.event_source,
   grants.to_project_name,
   to_projects.project_id as to_project_id,
-  'WALLET' as to_type,
+  'FISCAL_HOST' as to_type,
   grants.from_project_name,
   from_projects.project_id as from_project_id,
   'WALLET' as from_type,

warehouse/metrics_mesh/models/metric_names_from_artifact.sql

@@ -0,0 +1,8 @@
+MODEL (
+  name metrics.metric_names_from_artifact,
+  kind FULL
+);
+
+SELECT DISTINCT
+  metric
+FROM metrics.timeseries_metrics_to_artifact

warehouse/metrics_mesh/models/metric_names_from_collection.sql

@@ -0,0 +1,8 @@
+MODEL (
+  name metrics.metric_names_from_collection,
+  kind FULL
+);
+
+SELECT DISTINCT
+  metric
+FROM metrics.timeseries_metrics_to_collection

warehouse/metrics_mesh/models/metric_names_from_project.sql

@@ -0,0 +1,8 @@
+MODEL (
+  name metrics.metric_names_from_project,
+  kind FULL
+);
+
+SELECT DISTINCT
+  metric
+FROM metrics.timeseries_metrics_to_project