From ef004bd5c93f81ec7c1511b20e4144bc88ce978d Mon Sep 17 00:00:00 2001 From: Javier Date: Sat, 21 Dec 2024 21:41:05 +0000 Subject: [PATCH] fix: support `prefixed` dlt assets --- warehouse/oso_dagster/factories/dlt.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/warehouse/oso_dagster/factories/dlt.py b/warehouse/oso_dagster/factories/dlt.py index 2f71e5baf..8b377b954 100644 --- a/warehouse/oso_dagster/factories/dlt.py +++ b/warehouse/oso_dagster/factories/dlt.py @@ -191,12 +191,24 @@ def _dlt_asset( context.log.debug("dlt pipeline setup with bigquery and jsonl") dlt_run_options["loader_file_format"] = "jsonl" + dlt_prefix_keys = ( + [key_prefix] + if isinstance(key_prefix, str) + else (key_prefix or []) + ) + dlt_source_name = ( + dlt_prefix_keys[-1] if dlt_prefix_keys else key_prefix_str + ) + dlt_key_prefix = dlt_prefix_keys[:-1] + results = dlt.run( context=context, dlt_source=dlt_source, dlt_pipeline=pipeline, dagster_dlt_translator=PrefixedDltTranslator( - source_name=key_prefix_str, tags=dict(tags) + prefix=dlt_key_prefix, + source_name=dlt_source_name, + tags=dict(tags), ), **dlt_run_options, )