Upgrade DLT and Duckdb (#2441)

* Update dlt and duckdb * Make schedules stopped by default
opensource-observer · Oct 31, 2024 · 038806c · 038806c
1 parent d197275
commit 038806c
Show file tree

Hide file tree

Showing 5 changed files with 99 additions and 70 deletions.
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -30,8 +30,8 @@ dask-kubernetes = "^2024.4.2"
 dbt-bigquery = "^1.7.0"
 dbt-core = "^1.8.2"
 discord-webhook = "^1.3.1"
-dlt = { extras = ["duckdb"], version = "^0.5.2" }
-duckdb = "^0.10.2"
+dlt = "^1.3.0"
+duckdb = "^1.1.0"
 gcsfs = "^2024.6.1"
 google-api-python-client = "^2.116.0"
 google-cloud-bigquery = "^3.17.1"

diff --git a/warehouse/oso_dagster/definitions.py b/warehouse/oso_dagster/definitions.py
@@ -7,6 +7,7 @@
 from dagster_k8s import k8s_job_executor
 from dagster_embedded_elt.dlt import DagsterDltResource
 from dotenv import load_dotenv
+from oso_dagster.utils.dbt import support_home_dir_profiles
 from . import constants
 from .schedules import schedules, get_partitioned_schedules
 from .cbt import CBTResource
@@ -114,7 +115,9 @@ def load_definitions():
     }
     for target in constants.main_dbt_manifests:
         resources[f"{target}_dbt"] = DbtCliResource(
-            project_dir=os.fspath(constants.main_dbt_project_dir), target=target
+            project_dir=os.fspath(constants.main_dbt_project_dir),
+            target=target,
+            profiles_dir=support_home_dir_profiles(),
         )
 
     extra_kwargs = {}