Add SQLMesh (#1967)

* Update deps * Adds initial sqlmesh skeleton
opensource-observer · Aug 20, 2024 · 5fb2c96 · 5fb2c96
1 parent 8bbc1ae
commit 5fb2c96
Show file tree

Hide file tree

Showing 11 changed files with 603 additions and 497 deletions.
diff --git a/.gitignore b/.gitignore
@@ -71,3 +71,6 @@ supabase/.temp/
 base_playground/
 playground/
 production/
+
+# sqlmesh
+.cache/
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -54,7 +54,7 @@ sqlalchemy = "^2.0.25"
 textual = "^0.52.1"
 redis = "^5.0.7"
 githubkit = "^0.11.6"
-sqlmesh = "^0.113.0"
+sqlmesh = "^0.117.0"
 dagster-duckdb = "^0.24.0"
 dagster-duckdb-polars = "^0.24.0"
 

diff --git a/warehouse/clickhouse-mesh/audits/.gitkeep b/warehouse/clickhouse-mesh/audits/.gitkeep
diff --git a/warehouse/clickhouse-mesh/config.yaml b/warehouse/clickhouse-mesh/config.yaml
@@ -0,0 +1,11 @@
+gateways:
+  local:
+    connection:
+      type: duckdb
+      database: /data/int_events/test.db
+
+default_gateway: local
+
+model_defaults:
+  dialect: duckdb
+  start: 2024-08-19
diff --git a/warehouse/clickhouse-mesh/macros/.gitkeep b/warehouse/clickhouse-mesh/macros/.gitkeep
diff --git a/warehouse/clickhouse-mesh/macros/__init__.py b/warehouse/clickhouse-mesh/macros/__init__.py
diff --git a/warehouse/clickhouse-mesh/models/.gitkeep b/warehouse/clickhouse-mesh/models/.gitkeep
diff --git a/warehouse/clickhouse-mesh/models/incremental_model.sql b/warehouse/clickhouse-mesh/models/incremental_model.sql
@@ -0,0 +1,20 @@
+MODEL (
+  name metrics.incremental_model,
+  kind INCREMENTAL_BY_TIME_RANGE (
+    time_column bucket_day,
+    batch_size 1
+  ),
+  start '2018-01-01',
+  cron '@daily',
+  grain (bucket_day, event_type)
+);
+
+SELECT
+  time_bucket(INTERVAL 1 DAY, @start_date) as bucket_day,
+  event_type,
+  count(event_type) as events
+FROM
+  sources.int_events
+WHERE
+  time BETWEEN (@start_date - INTERVAL 30 DAY) AND @end_date
+GROUP BY bucket_day, event_type
diff --git a/warehouse/clickhouse-mesh/seeds/.gitkeep b/warehouse/clickhouse-mesh/seeds/.gitkeep
diff --git a/warehouse/clickhouse-mesh/tests/.gitkeep b/warehouse/clickhouse-mesh/tests/.gitkeep