diff --git a/warehouse/dbt/models/intermediate/metrics/int_repo_metrics_by_project.sql b/warehouse/dbt/models/intermediate/metrics/int_repo_metrics_by_project.sql index 0be828205..e6a56f35f 100644 --- a/warehouse/dbt/models/intermediate/metrics/int_repo_metrics_by_project.sql +++ b/warehouse/dbt/models/intermediate/metrics/int_repo_metrics_by_project.sql @@ -45,11 +45,11 @@ repo_stats as ( select - int_all_artifacts.project_id, - int_all_artifacts.artifact_id, - int_all_artifacts.artifact_namespace, - int_all_artifacts.artifact_name, - int_all_artifacts.artifact_source, + int_artifacts_in_ossd_by_project.project_id, + int_artifacts_in_ossd_by_project.artifact_id, + int_artifacts_in_ossd_by_project.artifact_namespace, + int_artifacts_in_ossd_by_project.artifact_name, + int_artifacts_in_ossd_by_project.artifact_source, repo_snapshot.is_fork, repo_snapshot.fork_count, repo_snapshot.star_count, @@ -61,11 +61,11 @@ select repo_stats.days_with_commits_count, repo_stats.contributors_to_repo_count, repo_stats.commit_count -from {{ ref('int_all_artifacts') }} +from {{ ref('int_artifacts_in_ossd_by_project') }} left join repo_snapshot - on int_all_artifacts.artifact_id = repo_snapshot.artifact_id + on int_artifacts_in_ossd_by_project.artifact_id = repo_snapshot.artifact_id left join repo_stats - on int_all_artifacts.artifact_id = repo_stats.artifact_id + on int_artifacts_in_ossd_by_project.artifact_id = repo_stats.artifact_id where - int_all_artifacts.artifact_source = 'GITHUB' - and UPPER(int_all_artifacts.artifact_type) = 'REPOSITORY' + int_artifacts_in_ossd_by_project.artifact_source = 'GITHUB' + and int_artifacts_in_ossd_by_project.artifact_type = 'REPOSITORY' diff --git a/warehouse/dbt/models/marts/superchain/metrics/rf4_check_oss_requirements.sql b/warehouse/dbt/models/marts/superchain/metrics/rf4_check_oss_requirements.sql new file mode 100644 index 000000000..0a9be4816 --- /dev/null +++ b/warehouse/dbt/models/marts/superchain/metrics/rf4_check_oss_requirements.sql @@ -0,0 +1,18 @@ +with repo_stats as ( + select + project_id, + count(distinct url) as total_repos, + sum(case + when license_check = 'Permissive' and repo_activity_check = 'OK' then 1 + else 0 + end) as eligible_repos + from {{ ref('rf4_repo_stats_by_project') }} + group by project_id +) + +select + project_id, + 'check_oss_requirements' as metric, + cast((eligible_repos = total_repos) as int64) as amount +from repo_stats +where total_repos > 0 diff --git a/warehouse/dbt/models/marts/superchain/rf4_impact_metrics_by_project.sql b/warehouse/dbt/models/marts/superchain/rf4_impact_metrics_by_project.sql index 58bfd222d..3d9857f55 100644 --- a/warehouse/dbt/models/marts/superchain/rf4_impact_metrics_by_project.sql +++ b/warehouse/dbt/models/marts/superchain/rf4_impact_metrics_by_project.sql @@ -3,6 +3,8 @@ #} with metrics as ( + select * from {{ ref('rf4_check_oss_requirements') }} + union all select * from {{ ref('rf4_gas_fees') }} union all select * from {{ ref('rf4_transactions') }} @@ -33,6 +35,9 @@ with metrics as ( pivot_metrics as ( select project_id, + MAX( + case when metric = 'check_oss_requirements' then amount else 0 end + ) as check_oss_requirements, MAX( case when metric = 'gas_fees' then amount else 0 end ) as gas_fees, @@ -79,7 +84,7 @@ pivot_metrics as ( select pivot_metrics.project_id, projects_v1.project_name, - rf4_project_verification.check_oss_requirements, + pivot_metrics.check_oss_requirements, pivot_metrics.gas_fees, pivot_metrics.transaction_count, pivot_metrics.trusted_transaction_count, @@ -96,5 +101,3 @@ select from pivot_metrics left join {{ ref('projects_v1') }} on pivot_metrics.project_id = projects_v1.project_id -left join {{ ref('rf4_project_verification') }} - on pivot_metrics.project_id = rf4_project_verification.project_id diff --git a/warehouse/dbt/models/marts/superchain/rf4_project_verification.sql b/warehouse/dbt/models/marts/superchain/rf4_project_verification.sql deleted file mode 100644 index c18d384d8..000000000 --- a/warehouse/dbt/models/marts/superchain/rf4_project_verification.sql +++ /dev/null @@ -1,89 +0,0 @@ -{# - Preliminary project verification query. - - TODO: - - Review thresholds for unique_addresses, date_first_transaction, and - days_with_onchain_activity_in_range. - - Filter on contracts that are linked to Retro Funding applications (from Agora data) -#} - -with repo_stats as ( - select - project_id, - ARRAY_AGG(repo_name) as eligible_repos - from ( - select - project_id, - CONCAT(artifact_namespace, '/', artifact_name) as repo_name - from {{ ref('rf4_repo_stats_by_project') }} - where approval_status = 'approved' - ) - group by project_id -), - -onchain_stats as ( - select - project_id, - project_name, - COUNT(distinct from_artifact_name) as unique_addresses, - MIN(bucket_day) as date_first_transaction, - COUNT( - distinct - case - when bucket_day between '2024-02-01' and '2024-04-01' then bucket_day - end - ) as days_with_onchain_activity_in_range - from {{ ref('rf4_events_daily_to_project') }} - where - event_type = 'CONTRACT_INVOCATION_SUCCESS_DAILY_COUNT' - and bucket_day >= '2023-10-01' - group by - project_id, - project_name -), - -project_stats as ( - select - onchain_stats.*, - COALESCE(repo_stats.eligible_repos, array[]) as eligible_repos - from onchain_stats - left join repo_stats - on onchain_stats.project_id = repo_stats.project_id - left join {{ ref('projects_by_collection_v1') }} - on onchain_stats.project_id = projects_by_collection_v1.project_id - where - projects_by_collection_v1.collection_name = 'op-onchain' -), - -checks as ( - select - project_id, - project_name, - eligible_repos, - unique_addresses, - date_first_transaction, - days_with_onchain_activity_in_range, - COALESCE(ARRAY_LENGTH(eligible_repos), 0) >= 1 as check_oss_requirements, - unique_addresses >= 420 as check_unique_addresses, - date_first_transaction < '2024-03-01' as check_date_first_transaction, - days_with_onchain_activity_in_range >= 10 - as check_days_with_onchain_activity_in_range - from project_stats -) - -select - project_id, - project_name, - unique_addresses, - date_first_transaction, - days_with_onchain_activity_in_range, - check_oss_requirements, - check_unique_addresses, - check_date_first_transaction, - check_days_with_onchain_activity_in_range, - ( - check_unique_addresses - and check_date_first_transaction - and check_days_with_onchain_activity_in_range - ) as meets_all_requirements -from checks diff --git a/warehouse/dbt/models/marts/superchain/rf4_repo_stats_by_project.sql b/warehouse/dbt/models/marts/superchain/rf4_repo_stats_by_project.sql index 48441fe15..ebfb21e52 100644 --- a/warehouse/dbt/models/marts/superchain/rf4_repo_stats_by_project.sql +++ b/warehouse/dbt/models/marts/superchain/rf4_repo_stats_by_project.sql @@ -1,5 +1,4 @@ {# TODO: Review licenses https://spdx.org/licenses/ for OSI Approved #} -{# TODO: update with actual collection for RF4 #} with repo_snapshot as ( select project_id, @@ -29,39 +28,48 @@ with repo_snapshot as ( when license_spdx_id = 'NOASSERTION' then 'Custom' else 'Unspecified' - end as license_type + end as license_check, + case + when ( + commit_count >= 10 + and days_with_commits_count >= 3 + and first_commit_time < '2024-05-01' + and star_count >= 10 + and language in ('Solidity', 'JavaScript', 'TypeScript') + ) then 'OK' + else 'Review' + end as repo_activity_check, + concat(artifact_namespace, '/', artifact_name) as url from {{ ref('int_repo_metrics_by_project') }} +), + +rf4_repos as ( + select lower(replace(artifact, 'https://github.com/', '')) as url + from {{ source("static_data_sources", "agora_rf4_repos_with_contracts") }} +), + +filtered_repos as ( + select * from repo_snapshot + where url in (select url from rf4_repos) ) select - repo_snapshot.project_id, + filtered_repos.project_id, projects_v1.project_name, - repo_snapshot.artifact_namespace, - repo_snapshot.artifact_name, - repo_snapshot.is_fork, - repo_snapshot.fork_count, - repo_snapshot.star_count, - repo_snapshot.first_commit_time, - repo_snapshot.last_commit_time, - repo_snapshot.days_with_commits_count, - repo_snapshot.commit_count, - repo_snapshot.language, - repo_snapshot.license_spdx_id, - repo_snapshot.license_type, - case - when ( - repo_snapshot.commit_count >= 10 - and repo_snapshot.days_with_commits_count >= 3 - and repo_snapshot.first_commit_time < '2024-05-01' - and repo_snapshot.star_count >= 10 - and repo_snapshot.language in ('Solidity', 'JavaScript', 'TypeScript') - ) then 'approved' - else 'review' - end as approval_status -from repo_snapshot + filtered_repos.artifact_namespace, + filtered_repos.artifact_name, + filtered_repos.url, + filtered_repos.is_fork, + filtered_repos.fork_count, + filtered_repos.star_count, + filtered_repos.first_commit_time, + filtered_repos.last_commit_time, + filtered_repos.days_with_commits_count, + filtered_repos.commit_count, + filtered_repos.language, + filtered_repos.license_spdx_id, + filtered_repos.license_check, + filtered_repos.repo_activity_check +from filtered_repos left join {{ ref('projects_v1') }} - on repo_snapshot.project_id = projects_v1.project_id -left join {{ ref('projects_by_collection_v1') }} - on repo_snapshot.project_id = projects_by_collection_v1.project_id -where - repo_snapshot.license_type != 'Unspecified' + on filtered_repos.project_id = projects_v1.project_id diff --git a/warehouse/dbt/models/static_data_sources.yml b/warehouse/dbt/models/static_data_sources.yml index e8e3198ed..8a12a525f 100644 --- a/warehouse/dbt/models/static_data_sources.yml +++ b/warehouse/dbt/models/static_data_sources.yml @@ -10,4 +10,6 @@ sources: - name: agora_rf4_artifacts_by_app identifier: agora_rf4_artifacts_by_app - name: agora_rf4_to_ossd - identifier: agora_rf4_to_ossd \ No newline at end of file + identifier: agora_rf4_to_ossd + - name: agora_rf4_repos_with_contracts + identifier: agora_rf4_repos_with_contracts \ No newline at end of file