From 7d0e7524955d7153f82bd702f262b551098aba07 Mon Sep 17 00:00:00 2001 From: Carl Cervone <42869436+ccerv1@users.noreply.github.com> Date: Wed, 25 Sep 2024 00:16:54 -0400 Subject: [PATCH] feat(dbt): adds source and staging models for arb1 events (#2197) * feat(dbt): adds source and staging models for arb1 events * fix: update int_arbitrum_one_traces * Update int_arbitrum_one_transactions.sql * Adds gas_price conversion * make sure the playgrounds work * Reduce the need for 80 char line length --------- Co-authored-by: Reuven V. Gonzales --- .sqlfluff | 2 +- warehouse/dbt/models/arbitrum_one_sources.yml | 13 +++ warehouse/dbt/models/arbitrum_sources.yml | 15 ---- .../dbt/models/base_playground_sources.yml | 9 ++ .../blockchain_artifacts/int_deployers.sql | 2 +- .../blockchain_artifacts/int_factories.sql | 2 +- .../blockchain_artifacts/int_proxies.sql | 2 +- .../blockchain_artifacts/int_safes.sql | 2 +- ...rbitrum_one_contract_invocation_events.sql | 19 +++++ .../int_arbitrum_one_traces.sql | 14 ++++ .../int_arbitrum_one_transactions.sql | 82 +++++++++++++++++++ .../models/intermediate/events/int_events.sql | 2 + warehouse/dbt/models/playground_sources.yml | 9 ++ .../stg_arbitrum_one__deployers.sql | 14 ++++ .../stg_arbitrum_one__factories.sql} | 4 +- .../stg_arbitrum_one__proxies.sql | 19 +++++ 16 files changed, 188 insertions(+), 22 deletions(-) create mode 100644 warehouse/dbt/models/arbitrum_one_sources.yml delete mode 100644 warehouse/dbt/models/arbitrum_sources.yml create mode 100644 warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_contract_invocation_events.sql create mode 100644 warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_traces.sql create mode 100644 warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_transactions.sql create mode 100644 warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__deployers.sql rename warehouse/dbt/models/staging/{arbitrum/stg_arbitrum__deployers.sql => arbitrum_one/stg_arbitrum_one__factories.sql} (85%) create mode 100644 warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__proxies.sql diff --git a/.sqlfluff b/.sqlfluff index 9b166bd25..5f2ede1bd 100644 --- a/.sqlfluff +++ b/.sqlfluff @@ -2,7 +2,7 @@ dialect = bigquery templater = dbt runaway_limit = 10 -max_line_length = 80 +max_line_length = 100 indent_unit = space exclude_rules = AL09, RF02 diff --git a/warehouse/dbt/models/arbitrum_one_sources.yml b/warehouse/dbt/models/arbitrum_one_sources.yml new file mode 100644 index 000000000..3aa2bbb19 --- /dev/null +++ b/warehouse/dbt/models/arbitrum_one_sources.yml @@ -0,0 +1,13 @@ +sources: + - name: arbitrum_one + database: opensource-observer + schema: arbitrum_one + tables: + - name: transactions + identifier: arbitrum_transactions + + - name: blocks + identifier: arbitrum_blocks + + - name: traces + identifier: arbitrum_traces \ No newline at end of file diff --git a/warehouse/dbt/models/arbitrum_sources.yml b/warehouse/dbt/models/arbitrum_sources.yml deleted file mode 100644 index 0c8f390ca..000000000 --- a/warehouse/dbt/models/arbitrum_sources.yml +++ /dev/null @@ -1,15 +0,0 @@ -sources: - - name: arbitrum - - database: bigquery-public-data - - schema: goog_blockchain_arbitrum_one_us - tables: - - name: transactions - identifier: transactions - - - name: logs - identifier: logs - - - name: receipts - identifier: receipts \ No newline at end of file diff --git a/warehouse/dbt/models/base_playground_sources.yml b/warehouse/dbt/models/base_playground_sources.yml index a5fe6c224..449db9d06 100644 --- a/warehouse/dbt/models/base_playground_sources.yml +++ b/warehouse/dbt/models/base_playground_sources.yml @@ -8,6 +8,15 @@ sources: - name: arbitrum_deployers identifier: stg_arbitrum__deployers + - name: arbitrum_one_traces + identifier: int_arbitrum_one_traces + + - name: arbitrum_one_transactions + identifier: int_arbitrum_one_transactions + + - name: arbitrum_one_deployers + identifier: stg_arbitrum_one__deployers + - name: base_traces identifier: int_base_traces diff --git a/warehouse/dbt/models/intermediate/blockchain_artifacts/int_deployers.sql b/warehouse/dbt/models/intermediate/blockchain_artifacts/int_deployers.sql index 8644f8910..5f3992569 100644 --- a/warehouse/dbt/models/intermediate/blockchain_artifacts/int_deployers.sql +++ b/warehouse/dbt/models/intermediate/blockchain_artifacts/int_deployers.sql @@ -4,7 +4,7 @@ ) }} -{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora"] %} +{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora", "arbitrum_one"] %} {% set union_queries = [] %} diff --git a/warehouse/dbt/models/intermediate/blockchain_artifacts/int_factories.sql b/warehouse/dbt/models/intermediate/blockchain_artifacts/int_factories.sql index 1e4271fca..897a3bfb3 100644 --- a/warehouse/dbt/models/intermediate/blockchain_artifacts/int_factories.sql +++ b/warehouse/dbt/models/intermediate/blockchain_artifacts/int_factories.sql @@ -4,7 +4,7 @@ ) }} -{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora"] %} +{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora", "arbitrum_one"] %} {% set union_queries = [] %} diff --git a/warehouse/dbt/models/intermediate/blockchain_artifacts/int_proxies.sql b/warehouse/dbt/models/intermediate/blockchain_artifacts/int_proxies.sql index abb17c5f9..8fbbf2b0c 100644 --- a/warehouse/dbt/models/intermediate/blockchain_artifacts/int_proxies.sql +++ b/warehouse/dbt/models/intermediate/blockchain_artifacts/int_proxies.sql @@ -1,6 +1,6 @@ {# any from address coming out of a proxy #} -{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora"] %} +{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora", "arbitrum_one"] %} {% set union_queries = [] %} diff --git a/warehouse/dbt/models/intermediate/blockchain_artifacts/int_safes.sql b/warehouse/dbt/models/intermediate/blockchain_artifacts/int_safes.sql index 88701ad68..30816b0d1 100644 --- a/warehouse/dbt/models/intermediate/blockchain_artifacts/int_safes.sql +++ b/warehouse/dbt/models/intermediate/blockchain_artifacts/int_safes.sql @@ -4,7 +4,7 @@ ) }} -{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora"] %} +{% set networks = ["optimism", "base", "frax", "metal", "mode", "zora", "arbitrum_one"] %} {% set union_queries = [] %} diff --git a/warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_contract_invocation_events.sql b/warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_contract_invocation_events.sql new file mode 100644 index 000000000..e34749322 --- /dev/null +++ b/warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_contract_invocation_events.sql @@ -0,0 +1,19 @@ +{{ + config( + materialized='incremental', + partition_by={ + "field": "time", + "data_type": "timestamp", + "granularity": "day", + }, + unique_id="id", + on_schema_change="append_new_columns", + incremental_strategy="insert_overwrite" + ) +}} +{% if is_incremental() %} + {% set start = "TIMESTAMP_SUB(_dbt_max_partition, INTERVAL 1 DAY)" %} +{% else %} + {% set start = "'1970-01-01'" %} +{% endif %} +{{ contract_invocation_events_with_l1("arbitrum_one", start) }} diff --git a/warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_traces.sql b/warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_traces.sql new file mode 100644 index 000000000..ff66e0cf9 --- /dev/null +++ b/warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_traces.sql @@ -0,0 +1,14 @@ +{{ + config( + materialized='incremental', + partition_by={ + "field": "block_timestamp", + "data_type": "timestamp", + "granularity": "day", + }, + unique_id="id", + on_schema_change="append_new_columns", + incremental_strategy="insert_overwrite" + ) +}} +{{ filtered_blockchain_events("ARBITRUM_ONE", "arbitrum_one", "traces") }} diff --git a/warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_transactions.sql b/warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_transactions.sql new file mode 100644 index 000000000..525485c29 --- /dev/null +++ b/warehouse/dbt/models/intermediate/blockchain_events/int_arbitrum_one_transactions.sql @@ -0,0 +1,82 @@ +{{ + config( + materialized='incremental', + partition_by={ + "field": "block_timestamp", + "data_type": "timestamp", + "granularity": "day", + }, + unique_id="id", + on_schema_change="append_new_columns", + incremental_strategy="insert_overwrite", + sql_header=''' +CREATE TEMP FUNCTION from_string_to_double(input STRING) +RETURNS FLOAT64 +LANGUAGE js AS r""" + yourNumber = BigInt(input, 10); + return parseFloat(yourNumber); + """; + ''' + ) +}} + +with filtered_transactions as ( + {{ + filtered_blockchain_events( + "ARBITRUM_ONE", + "arbitrum_one", + "transactions" + ) + }} +) + +select + id, + `hash`, + nonce, + block_hash, + block_number, + transaction_index, + from_address, + to_address, + `value`, + gas, + {% if target.name == 'production' %} + {# + We need to convert the arbitrum gas_price from bytes to a double. We + intentionally lose some precision by doing this. The gas_price is + actually a utf-8 string. So we convert this to a string which is a + string of the integer. This transformation should only happen on + production and not on the playgrounds as the playgrounds source their + data originally from production + #} + from_string_to_double(CAST(gas_price AS STRING FORMAT 'UTF-8')) as gas_price, + {% else %} + gas_price, + {% endif %} + input, + max_fee_per_gas, + max_priority_fee_per_gas, + transaction_type, + receipt_cumulative_gas_used, + receipt_gas_used, + receipt_contract_address, + receipt_status, + receipt_effective_gas_price, + receipt_root_hash, + receipt_l1_fee, + receipt_l1_gas_used, + receipt_l1_gas_price, + receipt_l1_fee_scalar, + receipt_l1_blob_base_fee, + receipt_l1_blob_base_fee_scalar, + blob_versioned_hashes, + max_fee_per_blob_gas, + receipt_l1_block_number, + receipt_l1_base_fee_scalar, + gateway_fee, + fee_currency, + gateway_fee_recipient, + ingestion_time, + block_timestamp +from filtered_transactions diff --git a/warehouse/dbt/models/intermediate/events/int_events.sql b/warehouse/dbt/models/intermediate/events/int_events.sql index 66e5ae25b..860cda3c8 100644 --- a/warehouse/dbt/models/intermediate/events/int_events.sql +++ b/warehouse/dbt/models/intermediate/events/int_events.sql @@ -212,6 +212,8 @@ all_events as ( select * from {{ ref('int_pgn_contract_invocation_events') }} union all select * from {{ ref('int_zora_contract_invocation_events') }} + union all + select * from {{ ref('int_arbitrum_one_contract_invocation_events') }} ) union all select diff --git a/warehouse/dbt/models/playground_sources.yml b/warehouse/dbt/models/playground_sources.yml index 88257f462..3f803cf98 100644 --- a/warehouse/dbt/models/playground_sources.yml +++ b/warehouse/dbt/models/playground_sources.yml @@ -12,6 +12,15 @@ sources: - name: arbitrum_deployers identifier: stg_arbitrum__deployers + - name: arbitrum_one_traces + identifier: int_arbitrum_one_traces + + - name: arbitrum_one_transactions + identifier: int_arbitrum_one_transactions + + - name: arbitrum_one_deployers + identifier: stg_arbitrum_one__deployers + - name: base_traces identifier: int_base_traces diff --git a/warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__deployers.sql b/warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__deployers.sql new file mode 100644 index 000000000..7f96eec1e --- /dev/null +++ b/warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__deployers.sql @@ -0,0 +1,14 @@ +{{ + config( + materialized='incremental', + partition_by={ + "field": "block_timestamp", + "data_type": "timestamp", + "granularity": "day", + }, + unique_id="transaction_hash", + on_schema_change="append_new_columns", + incremental_strategy="insert_overwrite" + ) +}} +{{ transactions_with_receipts_deployers("arbitrum_one") }} diff --git a/warehouse/dbt/models/staging/arbitrum/stg_arbitrum__deployers.sql b/warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__factories.sql similarity index 85% rename from warehouse/dbt/models/staging/arbitrum/stg_arbitrum__deployers.sql rename to warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__factories.sql index c86f107ee..faa00964a 100644 --- a/warehouse/dbt/models/staging/arbitrum/stg_arbitrum__deployers.sql +++ b/warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__factories.sql @@ -9,6 +9,6 @@ unique_id="transaction_hash", on_schema_change="append_new_columns", incremental_strategy="insert_overwrite" - ) + ) }} -{{ goog_blockchain_deployers("arbitrum") }} +{{ factory_deployments("arbitrum_one") }} diff --git a/warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__proxies.sql b/warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__proxies.sql new file mode 100644 index 000000000..6040afd77 --- /dev/null +++ b/warehouse/dbt/models/staging/arbitrum_one/stg_arbitrum_one__proxies.sql @@ -0,0 +1,19 @@ +{{ + config( + materialized='incremental', + partition_by={ + "field": "block_timestamp", + "data_type": "timestamp", + "granularity": "day", + }, + unique_key="id", + on_schema_change="append_new_columns", + incremental_strategy="insert_overwrite" + ) +}} +{% if is_incremental() %} + {% set start = "TIMESTAMP_SUB(_dbt_max_partition, INTERVAL 1 DAY)" %} +{% else %} + {% set start = "'1970-01-01'" %} +{% endif %} +{{ known_proxies("arbitrum_one", start) }}