Skip to content

Commit

Permalink
Fix one more issue with running dlt locally (#1939)
Browse files Browse the repository at this point in the history
  • Loading branch information
ravenac95 authored Aug 12, 2024
1 parent 885a248 commit c1fd898
Showing 1 changed file with 12 additions and 1 deletion.
13 changes: 12 additions & 1 deletion warehouse/oso_dagster/factories/dlt.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
)
from ..utils import SecretResolver, resolve_secrets_for_func
from .sql import PrefixedDltTranslator
from .. import constants


class DltAssetConfig(Config):
Expand Down Expand Up @@ -118,12 +119,22 @@ def _dlt_asset(
config: config_type,
**extra_source_args,
) -> Iterable[R]:
# Hack for now. Staging cannot be used if running locally.
# We need to change this interface. Instead of being reliant
# on the constant defining bigquery we should use some kind
# of generic function to wire this pipeline together.
pipeline = dltlib.pipeline(
f"{key_prefix_str}_{name}",
destination=dlt_warehouse_destination,
staging=dlt_staging_destination,
dataset_name=dataset_name,
)
if constants.enable_bigquery:
pipeline = dltlib.pipeline(
f"{key_prefix_str}_{name}",
destination=dlt_warehouse_destination,
staging=dlt_staging_destination,
dataset_name=dataset_name,
)

dlt = cast(DagsterDltResource, getattr(context.resources, "dlt"))

Expand Down

0 comments on commit c1fd898

Please sign in to comment.