feat: implement latest RF4 OSS checks (#1659)

* add: static data for repos containing contracts

* feat: implement latest OSS checks logic

warehouse/dbt/models/intermediate/metrics/int_repo_metrics_by_project.sql

@@ -45,11 +45,11 @@ repo_stats as (
 
 
 select
-  int_all_artifacts.project_id,
-  int_all_artifacts.artifact_id,
-  int_all_artifacts.artifact_namespace,
-  int_all_artifacts.artifact_name,
-  int_all_artifacts.artifact_source,
+  int_artifacts_in_ossd_by_project.project_id,
+  int_artifacts_in_ossd_by_project.artifact_id,
+  int_artifacts_in_ossd_by_project.artifact_namespace,
+  int_artifacts_in_ossd_by_project.artifact_name,
+  int_artifacts_in_ossd_by_project.artifact_source,
   repo_snapshot.is_fork,
   repo_snapshot.fork_count,
   repo_snapshot.star_count,
@@ -61,11 +61,11 @@ select
   repo_stats.days_with_commits_count,
   repo_stats.contributors_to_repo_count,
   repo_stats.commit_count
-from {{ ref('int_all_artifacts') }}
+from {{ ref('int_artifacts_in_ossd_by_project') }}
 left join repo_snapshot
-  on int_all_artifacts.artifact_id = repo_snapshot.artifact_id
+  on int_artifacts_in_ossd_by_project.artifact_id = repo_snapshot.artifact_id
 left join repo_stats
-  on int_all_artifacts.artifact_id = repo_stats.artifact_id
+  on int_artifacts_in_ossd_by_project.artifact_id = repo_stats.artifact_id
 where
-  int_all_artifacts.artifact_source = 'GITHUB'
-  and UPPER(int_all_artifacts.artifact_type) = 'REPOSITORY'
+  int_artifacts_in_ossd_by_project.artifact_source = 'GITHUB'
+  and int_artifacts_in_ossd_by_project.artifact_type = 'REPOSITORY'

warehouse/dbt/models/marts/superchain/metrics/rf4_check_oss_requirements.sql

@@ -0,0 +1,18 @@
+with repo_stats as (
+  select
+    project_id,
+    count(distinct url) as total_repos,
+    sum(case
+      when license_check = 'Permissive' and repo_activity_check = 'OK' then 1
+      else 0
+    end) as eligible_repos
+  from {{ ref('rf4_repo_stats_by_project') }}
+  group by project_id
+)
+
+select
+  project_id,
+  'check_oss_requirements' as metric,
+  cast((eligible_repos = total_repos) as int64) as amount
+from repo_stats
+where total_repos > 0

warehouse/dbt/models/marts/superchain/rf4_impact_metrics_by_project.sql

@@ -3,6 +3,8 @@
 #}
 
 with metrics as (
+  select * from {{ ref('rf4_check_oss_requirements') }}
+  union all
   select * from {{ ref('rf4_gas_fees') }}
   union all
   select * from {{ ref('rf4_transactions') }}
@@ -33,6 +35,9 @@ with metrics as (
 pivot_metrics as (
   select
     project_id,
+    MAX(
+      case when metric = 'check_oss_requirements' then amount else 0 end
+    ) as check_oss_requirements,
     MAX(
       case when metric = 'gas_fees' then amount else 0 end
     ) as gas_fees,
@@ -79,7 +84,7 @@ pivot_metrics as (
 select
   pivot_metrics.project_id,
   projects_v1.project_name,
-  rf4_project_verification.check_oss_requirements,
+  pivot_metrics.check_oss_requirements,
   pivot_metrics.gas_fees,
   pivot_metrics.transaction_count,
   pivot_metrics.trusted_transaction_count,
@@ -96,5 +101,3 @@ select
 from pivot_metrics
 left join {{ ref('projects_v1') }}
   on pivot_metrics.project_id = projects_v1.project_id
-left join {{ ref('rf4_project_verification') }}
-  on pivot_metrics.project_id = rf4_project_verification.project_id

warehouse/dbt/models/marts/superchain/rf4_repo_stats_by_project.sql

@@ -1,5 +1,4 @@
 {# TODO: Review licenses https://spdx.org/licenses/ for OSI Approved #}
-{# TODO: update with actual collection for RF4 #}
 with repo_snapshot as (
   select
     project_id,
@@ -29,39 +28,48 @@ with repo_snapshot as (
       when license_spdx_id = 'NOASSERTION'
         then 'Custom'
       else 'Unspecified'
-    end as license_type
+    end as license_check,
+    case
+      when (
+        commit_count >= 10
+        and days_with_commits_count >= 3
+        and first_commit_time < '2024-05-01'
+        and star_count >= 10
+        and language in ('Solidity', 'JavaScript', 'TypeScript')
+      ) then 'OK'
+      else 'Review'
+    end as repo_activity_check,
+    concat(artifact_namespace, '/', artifact_name) as url
   from {{ ref('int_repo_metrics_by_project') }}
+),
+
+rf4_repos as (
+  select lower(replace(artifact, 'https://github.com/', '')) as url
+  from {{ source("static_data_sources", "agora_rf4_repos_with_contracts") }}
+),
+
+filtered_repos as (
+  select * from repo_snapshot
+  where url in (select url from rf4_repos)
 )
 
 select
-  repo_snapshot.project_id,
+  filtered_repos.project_id,
   projects_v1.project_name,
-  repo_snapshot.artifact_namespace,
-  repo_snapshot.artifact_name,
-  repo_snapshot.is_fork,
-  repo_snapshot.fork_count,
-  repo_snapshot.star_count,
-  repo_snapshot.first_commit_time,
-  repo_snapshot.last_commit_time,
-  repo_snapshot.days_with_commits_count,
-  repo_snapshot.commit_count,
-  repo_snapshot.language,
-  repo_snapshot.license_spdx_id,
-  repo_snapshot.license_type,
-  case
-    when (
-      repo_snapshot.commit_count >= 10
-      and repo_snapshot.days_with_commits_count >= 3
-      and repo_snapshot.first_commit_time < '2024-05-01'
-      and repo_snapshot.star_count >= 10
-      and repo_snapshot.language in ('Solidity', 'JavaScript', 'TypeScript')
-    ) then 'approved'
-    else 'review'
-  end as approval_status
-from repo_snapshot
+  filtered_repos.artifact_namespace,
+  filtered_repos.artifact_name,
+  filtered_repos.url,
+  filtered_repos.is_fork,
+  filtered_repos.fork_count,
+  filtered_repos.star_count,
+  filtered_repos.first_commit_time,
+  filtered_repos.last_commit_time,
+  filtered_repos.days_with_commits_count,
+  filtered_repos.commit_count,
+  filtered_repos.language,
+  filtered_repos.license_spdx_id,
+  filtered_repos.license_check,
+  filtered_repos.repo_activity_check
+from filtered_repos
 left join {{ ref('projects_v1') }}
-  on repo_snapshot.project_id = projects_v1.project_id
-left join {{ ref('projects_by_collection_v1') }}
-  on repo_snapshot.project_id = projects_by_collection_v1.project_id
-where
-  repo_snapshot.license_type != 'Unspecified'
+  on filtered_repos.project_id = projects_v1.project_id

warehouse/dbt/models/static_data_sources.yml

@@ -10,4 +10,6 @@ sources:
       - name: agora_rf4_artifacts_by_app
         identifier: agora_rf4_artifacts_by_app
       - name: agora_rf4_to_ossd
-        identifier: agora_rf4_to_ossd
+        identifier: agora_rf4_to_ossd
+      - name: agora_rf4_repos_with_contracts
+        identifier: agora_rf4_repos_with_contracts