Skip to content

Commit

Permalink
rebase (#1615)
Browse files Browse the repository at this point in the history
* Attempts to set the correct readonly values (#1607)

* more readonly fixes (#1608)

* Proper read only fix (#1610)

* Working fix for read only

* reset to main

* feat: rf4 contract verification models (#1611)

* feat: rf4 contract verification models

* fix: linting issue

* fix: rename source

* refactor: contract discovery for rf4 (#1612)

* feat: rf4 contract discovery part 2 (#1613)

* refactor: contract discovery for rf4

* feat: discovery 3rd level contracts

* fix: remove web of trust signal from TU model (#1614)

---------

Co-authored-by: Reuven Gonzales <[email protected]>
  • Loading branch information
ccerv1 and ravenac95 authored Jun 10, 2024
1 parent c384239 commit e6728cf
Show file tree
Hide file tree
Showing 11 changed files with 397 additions and 32 deletions.
2 changes: 1 addition & 1 deletion ops/helm-charts/oso-dagster/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ name: oso-dagster
description: Extension of the dagster template

type: application
version: 0.1.10
version: 0.1.17
appVersion: "1.0.0"
dependencies:
- name: dagster
Expand Down
26 changes: 24 additions & 2 deletions ops/helm-charts/oso-dagster/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,32 @@ postgres_db:
{{- end }}
{{- end }}

{{/*
This is copied due to some kind of error with helm and flux when overriding
portions of this
*/}}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "dagster.override-fullname" -}}
{{- if .Values.global.fullnameOverride -}}
{{- .Values.global.fullnameOverride | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- $name := "dagster" -}}
{{- if contains $name .Release.Name -}}
{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
{{- else -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
{{- end -}}

# Fix issues with the full name
{{- define "dagster.webserver.fullname" -}}
{{- $name := default "webserver" .Values.dagsterWebserver.nameOverride -}}
{{- $fullname := include "dagster.fullname" . -}}
{{- $fullname := include "dagster.override-fullname" . -}}
{{- printf "%s-%s" $fullname $name | trunc 63 | trimSuffix "-" -}}
{{- if .webserverReadOnly -}} --read-only {{- end -}}
{{- if .webserverReadOnly -}} -read-only {{- end -}}
{{- end -}}
6 changes: 5 additions & 1 deletion ops/helm-charts/oso-dagster/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,11 @@ dagster:
postgresql:
enabled: false
postgresqlPassword: "gcp:secretmanager:dagster-db-password/versions/1"
enableReadOnly: true
dagsterWebserver:
enableReadOnly: true

global:
dagsterInternalName: "dagster"

configMap:
name: "dagster-extra-env-config-map"
Expand Down
1 change: 1 addition & 0 deletions ops/k8s-apps/base/dagster/dagster.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ spec:
nodeSelector:
pool_type: spot
dagsterWebserver:
enableReadOnly: true
env:
- name: DAGSTER_DBT_GENERATE_AND_AUTH_GCP
value: "1"
Expand Down
27 changes: 0 additions & 27 deletions warehouse/dbt/models/marts/superchain/rf4_trusted_users.sql
Original file line number Diff line number Diff line change
Expand Up @@ -22,27 +22,6 @@ eigentrust_top_users as (
limit 50000
),

web_of_trust as (
{# draft model for testing #}
select
1 as vitalik_verification,
CAST(fof_id as string) as farcaster_id
from (
select
l2.peer_farcaster_id as fof_id,
COUNT(distinct l1.peer_farcaster_id) as edge_count
from {{ ref('stg_karma3__localtrust') }} as l1
left join {{ ref('stg_karma3__localtrust') }} as l2
on l1.peer_farcaster_id = l2.farcaster_id
where
l1.farcaster_id = 5650
and l1.strategy_id = 1
and l2.strategy_id = 1
group by l2.peer_farcaster_id
)
where edge_count > 1
),

optimist_nft_holders as (
select
optimist_address as address,
Expand Down Expand Up @@ -80,8 +59,6 @@ trusted_user_model as (
as farcaster_prepermissionless,
COALESCE(eigentrust_top_users.eigentrust_verification, 0)
as eigentrust_verification,
COALESCE(web_of_trust.vitalik_verification, 0)
as vitalik_verification,
COALESCE(passport_scores.passport_user, 0)
as passport_user,
COALESCE(passport_scores.passport_verification, 0)
Expand All @@ -93,8 +70,6 @@ trusted_user_model as (
on all_addresses.address = farcaster_users.address
left join eigentrust_top_users
on farcaster_users.farcaster_id = eigentrust_top_users.farcaster_id
left join web_of_trust
on farcaster_users.farcaster_id = web_of_trust.farcaster_id
left join passport_scores
on all_addresses.address = passport_scores.address
left join optimist_nft_holders
Expand All @@ -106,15 +81,13 @@ select
farcaster_user,
farcaster_prepermissionless,
eigentrust_verification,
vitalik_verification,
passport_user,
passport_verification,
optimist_nft_verification,
(
farcaster_user
+ farcaster_prepermissionless
+ eigentrust_verification
+ vitalik_verification
+ passport_verification
+ optimist_nft_verification
) > 1 as is_trusted_user
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
with agora as (
select
application_id,
project_name,
artifact as address,
artifact_source as network,
artifact_type,
'agora_verification' as discovery_method
from {{ source('static_data_sources', 'agora_rf4_artifacts_by_app') }}
where artifact_type in ('CONTRACT', 'DEPLOYER')
),

discovered_contracts as (
select
agora.application_id,
agora.project_name,
derived_contracts.contract_address as address,
derived_contracts.network,
--derived_contracts.factory_deployer_address,
'CONTRACT' as artifact_type,
'discovered_contract_from_agora_verified_deployer' as discovery_method
from agora
inner join {{ ref('int_derived_contracts') }} as derived_contracts
on
agora.address = derived_contracts.deployer_address
and agora.network = derived_contracts.network
where
agora.address != derived_contracts.contract_address
)

select
application_id,
project_name,
address,
network,
artifact_type,
discovery_method
from discovered_contracts
union all
select
application_id,
project_name,
address,
network,
artifact_type,
discovery_method
from agora
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora"] %}

{% set union_factory_queries = [] %}

{% for network in networks %}

{% set network_upper = network.upper() %}

{% set factory_table = "stg_" ~ network ~ "__factories" %}
{% set query %}
select
factory_address,
contract_address,
'{{ network_upper }}' as network
from {{ ref(factory_table) }}
{% endset %}
{% do union_factory_queries.append(query) %}

{% endfor %}

{% set all_factories = union_factory_queries | join(' union all ') %}

with factories as (
{{ all_factories }}
),

app_contracts as (
select
application_id,
project_name,
address,
network,
artifact_type,
discovery_method
from {{ ref('rf4_oso_contract_discovery') }}
where artifact_type = 'CONTRACT'
union all
select
application_id,
project_name,
address,
network,
artifact_type,
discovery_method
from {{ ref('rf4_agora_contract_discovery') }}
where artifact_type = 'CONTRACT'
),

discovered_contracts as (
select
app_contracts.application_id,
app_contracts.project_name,
factories.contract_address as address,
factories.network,
'CONTRACT' as artifact_type,
'discovered_contract_from_verified_factory' as discovery_method
from factories
left join app_contracts
on
factories.factory_address = app_contracts.address
and factories.network = app_contracts.network
),

contracts as (
select
application_id,
address,
network,
discovery_method
from discovered_contracts
union all
select
application_id,
address,
network,
discovery_method
from app_contracts
)

select distinct
application_id,
address as contract_address,
network,
discovery_method
from contracts
where application_id is not null
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
with projects as (
select
apps.application_id,
apps.project_name,
current_projects.blockchain
from {{ source('static_data_sources', 'agora_rf4_applications') }} as apps
left join
{{ ref('stg_ossd__current_projects') }} as current_projects
on apps.oso_project_name = current_projects.project_name
),

oso_blockchain_artifacts as (
select
projects.application_id,
projects.project_name,
'oso_verification' as discovery_method,
UPPER(tag) as artifact_type,
UPPER(network) as network,
LOWER(JSON_VALUE(blockchains.address)) as address
from projects
cross join
UNNEST(JSON_QUERY_ARRAY(projects.blockchain)) as blockchains
cross join
UNNEST(JSON_VALUE_ARRAY(blockchains.networks)) as network
cross join
UNNEST(JSON_VALUE_ARRAY(blockchains.tags)) as tag
where tag in ('contract', 'deployer')
),

networks as (
select *
from UNNEST([
struct('OPTIMISM' as network),
struct('BASE' as network),
struct('FRAX' as network),
struct('METAL' as network),
struct('MODE' as network),
struct('ZORA' as network)
])
),

oso_any_evm_deployers as (
select
oso_blockchain_artifacts.application_id,
oso_blockchain_artifacts.project_name,
oso_blockchain_artifacts.artifact_type,
networks.network,
oso_blockchain_artifacts.address,
oso_blockchain_artifacts.discovery_method
from oso_blockchain_artifacts
cross join networks
where
oso_blockchain_artifacts.artifact_type = 'DEPLOYER'
and oso_blockchain_artifacts.network = 'ANY_EVM'
),

oso_other_addresses as (
select
application_id,
project_name,
artifact_type,
network,
address,
discovery_method
from oso_blockchain_artifacts
where
network in (select network from networks)
),

oso_addresses as (
select *
from oso_any_evm_deployers
union all
select *
from oso_other_addresses
),

discovered_contracts as (
select
oso_addresses.application_id,
oso_addresses.project_name,
derived_contracts.contract_address as address,
derived_contracts.network,
--derived_contracts.factory_deployer_address,
'CONTRACT' as artifact_type,
'discovered_contract_from_oso_verified_deployer' as discovery_method
from oso_addresses
inner join {{ ref('int_derived_contracts') }} as derived_contracts
on
oso_addresses.address = derived_contracts.deployer_address
and oso_addresses.network = derived_contracts.network
where
oso_addresses.address != derived_contracts.contract_address
)

select
application_id,
project_name,
address,
network,
artifact_type,
discovery_method
from discovered_contracts
union all
select
application_id,
project_name,
address,
network,
artifact_type,
discovery_method
from oso_addresses
Loading

0 comments on commit e6728cf

Please sign in to comment.