Improve metrics macros (#2133)

* Improves metrics macros

* set concurrent tasks

warehouse/metrics_mesh/config.py

@@ -1,11 +1,11 @@
 import os
-import dotenv
 
+import dotenv
 from sqlmesh.core.config import (
     Config,
-    ModelDefaultsConfig,
-    GatewayConfig,
     DuckDBConnectionConfig,
+    GatewayConfig,
+    ModelDefaultsConfig,
 )
 from sqlmesh.core.config.connection import (
     ClickhouseConnectionConfig,
@@ -31,6 +31,9 @@
                 username=os.environ.get("SQLMESH_CLICKHOUSE_USERNAME", ""),
                 password=os.environ.get("SQLMESH_CLICKHOUSE_PASSWORD", ""),
                 port=int(os.environ.get("SQLMESH_CLICKHOUSE_PORT", "443")),
+                concurrent_tasks=int(
+                    os.environ.get("SQLMESH_CLICKHOUSE_CONCURRENT_TASKS", "8")
+                ),
             ),
             state_connection=GCPPostgresConnectionConfig(
                 instance_connection_string=os.environ.get(

warehouse/metrics_mesh/oso_metrics/active_addresses.sql

@@ -1,12 +1,12 @@
-select STR_TO_DATE(@end_ds, '%Y-%m-%d') as metrics_sample_date,
+select @metrics_sample_date(events.bucket_day) as metrics_sample_date,
   events.event_source,
   events.to_artifact_id,
   '' as from_artifact_id,
   @metric_name() as metric,
   COUNT(distinct events.from_artifact_id) as amount
 from metrics.events_daily_to_artifact as events
 where event_type in ('CONTRACT_INVOCATION_SUCCESS_DAILY_COUNT')
-  and events.bucket_day BETWEEN STR_TO_DATE(@start_ds, '%Y-%m-%d') AND STR_TO_DATE(@end_ds, '%Y-%m-%d')
+  and events.bucket_day BETWEEN @metrics_start(DATE) AND @metrics_end(DATE)
 group by 1,
   metric,
   from_artifact_id,

warehouse/metrics_mesh/oso_metrics/active_days.sql

@@ -1,15 +1,12 @@
-select STR_TO_DATE(@end_ds, '%Y-%m-%d') as metrics_sample_date,
+select @metrics_sample_date(events.bucket_day) as metrics_sample_date,
   events.event_source,
   events.to_artifact_id,
   '' as from_artifact_id,
   @metric_name() as metric,
   COUNT(DISTINCT events.bucket_day) amount
 from metrics.events_daily_to_artifact as events
 where event_type in @activity_event_types
-  and events.bucket_day BETWEEN (
-    STR_TO_DATE(@end_ds, '%Y-%m-%d') - INTERVAL @rolling_window DAY
-  )
-  AND STR_TO_DATE(@end_ds, '%Y-%m-%d')
+  and events.bucket_day BETWEEN @metrics_start(DATE) AND @metrics_end(DATE)
 group by 1,
   metric,
   from_artifact_id,

warehouse/metrics_mesh/oso_metrics/commits.sql

@@ -1,12 +1,12 @@
-select STR_TO_DATE(@end_ds, '%Y-%m-%d') as metrics_sample_date,
+select @metrics_sample_date(events.bucket_day) as metrics_sample_date,
   events.event_source,
   events.to_artifact_id,
   '' as from_artifact_id,
   @metric_name() as metric,
   SUM(events.amount) as amount
 from metrics.events_daily_to_artifact as events
 where event_type in ('COMMIT_CODE')
-  and events.bucket_day BETWEEN STR_TO_DATE(@start_ds, '%Y-%m-%d') AND STR_TO_DATE(@end_ds, '%Y-%m-%d')
+  and events.bucket_day BETWEEN @metrics_start(DATE) AND @metrics_end(DATE)
 group by 1,
   metric,
   from_artifact_id,

warehouse/metrics_mesh/oso_metrics/contributor_activity_classification.sql

@@ -1,6 +1,6 @@
 select active.metrics_sample_date,
   active.event_source,
-  @entity_type_col('to_%s_id', table_alias := active),
+  @metrics_entity_type_col('to_%s_id', table_alias := active),
   '' as from_artifact_id,
   @metric_name('full_time_contributors') as metric,
   COUNT(DISTINCT active.from_artifact_id) as amount
@@ -12,13 +12,13 @@ from metrics_peer_ref(
 where active.amount / @rolling_window >= @full_time_ratio
 group by metric,
   from_artifact_id,
-  @entity_type_col('to_%s_id', table_alias := active),
+  @metrics_entity_type_col('to_%s_id', table_alias := active),
   event_source,
   metrics_sample_date
 union all
 select active.metrics_sample_date,
   active.event_source,
-  @entity_type_col('to_%s_id', table_alias := active),
+  @metrics_entity_type_col('to_%s_id', table_alias := active),
   '' as from_artifact_id,
   @metric_name('part_time_contributors') as metric,
   COUNT(DISTINCT active.from_artifact_id) as amount
@@ -30,13 +30,13 @@ from metrics_peer_ref(
 where active.amount / @rolling_window < @full_time_ratio
 group by metric,
   from_artifact_id,
-  @entity_type_col('to_%s_id', table_alias := active),
+  @metrics_entity_type_col('to_%s_id', table_alias := active),
   event_source,
   metrics_sample_date
 union all
 select active.metrics_sample_date,
   active.event_source,
-  @entity_type_col('to_%s_id', table_alias := active),
+  @metrics_entity_type_col('to_%s_id', table_alias := active),
   '' as from_artifact_id,
   @metric_name('active_contributors') as metric,
   COUNT(DISTINCT active.from_artifact_id) as amount
@@ -47,6 +47,6 @@ from metrics_peer_ref(
   ) as active
 group by metric,
   from_artifact_id,
-  @entity_type_col('to_%s_id', table_alias := active),
+  @metrics_entity_type_col('to_%s_id', table_alias := active),
   event_source,
   metrics_sample_date

warehouse/metrics_mesh/oso_metrics/developer_activity_classification.sql

@@ -1,6 +1,6 @@
 select active.metrics_sample_date,
   active.event_source,
-  @entity_type_col('to_%s_id', table_alias := active),
+  @metrics_entity_type_col('to_%s_id', table_alias := active),
   '' as from_artifact_id,
   @metric_name('full_time_developers') as metric,
   COUNT(DISTINCT active.from_artifact_id) as amount
@@ -12,13 +12,13 @@ from metrics_peer_ref(
 where active.amount / @rolling_window >= @full_time_ratio
 group by metric,
   from_artifact_id,
-  @entity_type_col('to_%s_id', table_alias := active),
+  @metrics_entity_type_col('to_%s_id', table_alias := active),
   event_source,
   metrics_sample_date
 union all
 select active.metrics_sample_date,
   active.event_source,
-  @entity_type_col('to_%s_id', table_alias := active),
+  @metrics_entity_type_col('to_%s_id', table_alias := active),
   '' as from_artifact_id,
   @metric_name('part_time_developers') as metric,
   COUNT(DISTINCT active.from_artifact_id) as amount
@@ -30,13 +30,13 @@ from metrics_peer_ref(
 where active.amount / @rolling_window < @full_time_ratio
 group by metric,
   from_artifact_id,
-  @entity_type_col('to_%s_id', table_alias := active),
+  @metrics_entity_type_col('to_%s_id', table_alias := active),
   event_source,
   metrics_sample_date
 union all
 select active.metrics_sample_date,
   active.event_source,
-  @entity_type_col('to_%s_id', table_alias := active),
+  @metrics_entity_type_col('to_%s_id', table_alias := active),
   '' as from_artifact_id,
   @metric_name('active_developers') as metric,
   COUNT(DISTINCT active.from_artifact_id) as amount
@@ -47,6 +47,6 @@ from metrics_peer_ref(
   ) as active
 group by metric,
   from_artifact_id,
-  @entity_type_col('to_%s_id', table_alias := active),
+  @metrics_entity_type_col('to_%s_id', table_alias := active),
   event_source,
   metrics_sample_date

warehouse/metrics_mesh/oso_metrics/forks.sql

@@ -1,12 +1,12 @@
-select STR_TO_DATE(@end_ds, '%Y-%m-%d') as metrics_sample_date,
+select @metrics_sample_date(events.bucket_day) as metrics_sample_date,
   events.event_source,
   events.to_artifact_id,
   '' as from_artifact_id,
   @metric_name() as metric,
   SUM(events.amount) as amount,
   from metrics.events_daily_to_artifact as events
 where event_type in ('FORKED')
-  and events.bucket_day BETWEEN STR_TO_DATE(@start_ds, '%Y-%m-%d') AND STR_TO_DATE(@end_ds, '%Y-%m-%d')
+  and events.bucket_day BETWEEN @metrics_start(DATE) AND @metrics_end(DATE)
 group by 1,
   metric,
   from_artifact_id,

warehouse/metrics_mesh/oso_metrics/gas_fees.sql

@@ -1,12 +1,12 @@
-select STR_TO_DATE(@end_ds, '%Y-%m-%d') as metrics_sample_date,
+select @metrics_sample_date(events.bucket_day) as metrics_sample_date,
   events.event_source,
   events.to_artifact_id,
   '' as from_artifact_id,
   @metric_name() as metric,
   SUM(events.amount) as amount
 from metrics.events_daily_to_artifact as events
 where event_type in ('CONTRACT_INVOCATION_DAILY_L2_GAS_USED')
-  and events.bucket_day BETWEEN STR_TO_DATE(@start_ds, '%Y-%m-%d') AND STR_TO_DATE(@end_ds, '%Y-%m-%d')
+  and events.bucket_day BETWEEN @metrics_start(DATE) AND @metrics_end(DATE)
 group by 1,
   metric,
   from_artifact_id,

warehouse/metrics_mesh/oso_metrics/issues_closed.sql

@@ -1,12 +1,12 @@
-select STR_TO_DATE(@end_ds, '%Y-%m-%d') as metrics_sample_date,
+select @metrics_sample_date(events.bucket_day) as metrics_sample_date,
   events.event_source,
   events.to_artifact_id,
   '' as from_artifact_id,
   @metric_name() as metric,
   SUM(events.amount) as amount
 from metrics.events_daily_to_artifact as events
 where event_type in ('ISSUE_CLOSED')
-  and events.bucket_day BETWEEN STR_TO_DATE(@start_ds, '%Y-%m-%d') AND STR_TO_DATE(@end_ds, '%Y-%m-%d')
+  and events.bucket_day BETWEEN @metrics_start(DATE) AND @metrics_end(DATE)
 group by 1,
   metric,
   from_artifact_id,

warehouse/metrics_mesh/oso_metrics/issues_opened.sql

@@ -1,12 +1,12 @@
-select STR_TO_DATE(@end_ds, '%Y-%m-%d') as metrics_sample_date,
+select @metrics_sample_date(events.bucket_day) as metrics_sample_date,
   events.event_source,
   events.to_artifact_id,
   '' as from_artifact_id,
   @metric_name() as metric,
   SUM(events.amount) as amount
 from metrics.events_daily_to_artifact as events
 where event_type in ('ISSUE_OPENED')
-  and events.bucket_day BETWEEN STR_TO_DATE(@start_ds, '%Y-%m-%d') AND STR_TO_DATE(@end_ds, '%Y-%m-%d')
+  and events.bucket_day BETWEEN @metrics_start(DATE) AND @metrics_end(DATE)
 group by 1,
   metric,
   from_artifact_id,

warehouse/metrics_mesh/oso_metrics/prs_merged.sql

@@ -1,12 +1,12 @@
-select STR_TO_DATE(@end_ds, '%Y-%m-%d') as metrics_sample_date,
+select @metrics_sample_date(events.bucket_day) as metrics_sample_date,
   events.event_source,
   events.to_artifact_id,
   '' as from_artifact_id,
   @metric_name() as metric,
   SUM(events.amount) as amount
 from metrics.events_daily_to_artifact as events
 where event_type in ('PULL_REQUEST_MERGED')
-  and events.bucket_day BETWEEN STR_TO_DATE(@start_ds, '%Y-%m-%d') AND STR_TO_DATE(@end_ds, '%Y-%m-%d')
+  and events.bucket_day BETWEEN @metrics_start(DATE) AND @metrics_end(DATE)
 group by 1,
   metric,
   from_artifact_id,

warehouse/metrics_mesh/oso_metrics/prs_opened.sql

@@ -1,12 +1,12 @@
-select STR_TO_DATE(@end_ds, '%Y-%m-%d') as metrics_sample_date,
+select @metrics_sample_date(events.bucket_day) as metrics_sample_date,
   events.event_source,
   events.to_artifact_id,
   '' as from_artifact_id,
   @metric_name() as metric,
   SUM(events.amount) as amount
 from metrics.events_daily_to_artifact as events
 where event_type in ('PULL_REQUEST_OPENED')
-  and events.bucket_day BETWEEN STR_TO_DATE(@start_ds, '%Y-%m-%d') AND STR_TO_DATE(@end_ds, '%Y-%m-%d')
+  and events.bucket_day BETWEEN @metrics_start(DATE) AND @metrics_end(DATE)
 group by 1,
   metric,
   from_artifact_id,

warehouse/metrics_mesh/oso_metrics/stars.sql

@@ -1,12 +1,12 @@
-select @TIME_AGGREGATION_BUCKET(@end_ds, @time_aggregation) as metrics_sample_date,
+select @metrics_sample_date(events.bucket_day) as metrics_sample_date,
   events.event_source,
   events.to_artifact_id,
   '' as from_artifact_id,
   @metric_name() as metric,
   SUM(events.amount) as amount
 from metrics.events_daily_to_artifact as events
 where event_type in ('STARRED')
-  and events.bucket_day BETWEEN STR_TO_DATE(@start_ds, '%Y-%m-%d') AND STR_TO_DATE(@end_ds, '%Y-%m-%d')
+  and events.bucket_day BETWEEN @metrics_start(DATE) AND @metrics_end(DATE)
 group by 1,
   metric,
   from_artifact_id,

warehouse/metrics_mesh/oso_metrics/transactions.sql

@@ -1,12 +1,12 @@
-select STR_TO_DATE(@end_ds, '%Y-%m-%d') as metrics_sample_date,
+select @metrics_sample_date(events.bucket_day) as metrics_sample_date,
   events.event_source,
   events.to_artifact_id,
   '' as from_artifact_id,
   @metric_name() as metric,
   SUM(events.amount) as amount
 from metrics.events_daily_to_artifact as events
 where event_type in ('CONTRACT_INVOCATION_SUCCESS_DAILY_COUNT')
-  and events.bucket_day BETWEEN STR_TO_DATE(@start_ds, '%Y-%m-%d') AND STR_TO_DATE(@end_ds, '%Y-%m-%d')
+  and events.bucket_day BETWEEN @metrics_start(DATE) AND @metrics_end(DATE)
 group by 1,
   metric,
   from_artifact_id,