Merge pull request #1653 from dlt-hub/devel

master merge for `0.5.2` release

.github/weaviate-compose.yml

@@ -11,8 +11,6 @@ services:
     image: semitechnologies/weaviate:1.21.1
     ports:
     - 8080:8080
-    volumes:
-    - weaviate_data
     restart: on-failure:0
     environment:
       QUERY_DEFAULTS_LIMIT: 25

.github/workflows/test_destination_clickhouse.yml

@@ -1,4 +1,3 @@
-
 name: test | clickhouse
 
 on:
@@ -8,7 +7,7 @@ on:
       - devel
   workflow_dispatch:
   schedule:
-    - cron:  '0 2 * * *'
+    - cron: '0 2 * * *'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -20,7 +19,7 @@ env:
   DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
 
   ACTIVE_DESTINATIONS: "[\"clickhouse\"]"
-  ALL_FILESYSTEM_DRIVERS: "[\"memory\"]"
+  ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\"]"
 
 jobs:
   get_docs_changes:
@@ -67,12 +66,51 @@ jobs:
       - name: create secrets.toml
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
+      # OSS ClickHouse
+      - run: |
+          docker compose -f "tests/load/clickhouse/clickhouse-compose.yml" up -d
+          echo "Waiting for ClickHouse to be healthy..."
+          timeout 30s bash -c 'until docker compose -f "tests/load/clickhouse/clickhouse-compose.yml" ps | grep -q "healthy"; do sleep 1; done'
+          echo "ClickHouse is up and running"
+        name: Start ClickHouse OSS
+
+
+      - run: poetry run pytest tests/load -m "essential"
+        name: Run essential tests Linux (ClickHouse OSS)
+        if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}}
+        env:
+          DESTINATION__CLICKHOUSE__CREDENTIALS__HOST: localhost
+          DESTINATION__CLICKHOUSE__CREDENTIALS__DATABASE: dlt_data
+          DESTINATION__CLICKHOUSE__CREDENTIALS__USERNAME: loader
+          DESTINATION__CLICKHOUSE__CREDENTIALS__PASSWORD: loader
+          DESTINATION__CLICKHOUSE__CREDENTIALS__PORT: 9000
+          DESTINATION__CLICKHOUSE__CREDENTIALS__HTTP_PORT: 8123
+          DESTINATION__CLICKHOUSE__CREDENTIALS__SECURE: 0
+
+      - run: poetry run pytest tests/load
+        name: Run all tests Linux (ClickHouse OSS)
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}}
+        env:
+          DESTINATION__CLICKHOUSE__CREDENTIALS__HOST: localhost
+          DESTINATION__CLICKHOUSE__CREDENTIALS__DATABASE: dlt_data
+          DESTINATION__CLICKHOUSE__CREDENTIALS__USERNAME: loader
+          DESTINATION__CLICKHOUSE__CREDENTIALS__PASSWORD: loader
+          DESTINATION__CLICKHOUSE__CREDENTIALS__PORT: 9000
+          DESTINATION__CLICKHOUSE__CREDENTIALS__HTTP_PORT: 8123
+          DESTINATION__CLICKHOUSE__CREDENTIALS__SECURE: 0
+
+      - name: Stop ClickHouse OSS
+        if: always()
+        run: docker compose -f "tests/load/clickhouse/clickhouse-compose.yml" down -v
+
+      # ClickHouse Cloud
       - run: |
           poetry run pytest tests/load -m "essential"
-        name: Run essential tests Linux
+        name: Run essential tests Linux (ClickHouse Cloud)
         if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}}
 
       - run: |
           poetry run pytest tests/load
-        name: Run all tests Linux
+        name: Run all tests Linux (ClickHouse Cloud)
         if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}}
+

.github/workflows/test_destination_dremio.yml

@@ -43,7 +43,7 @@ jobs:
         uses: actions/checkout@master
 
       - name: Start dremio
-        run: docker-compose -f "tests/load/dremio/docker-compose.yml" up -d
+        run: docker compose -f "tests/load/dremio/docker-compose.yml" up -d
 
       - name: Setup Python
         uses: actions/setup-python@v4
@@ -87,4 +87,4 @@ jobs:
 
       - name: Stop dremio
         if: always()
-        run: docker-compose -f "tests/load/dremio/docker-compose.yml" down -v
+        run: docker compose -f "tests/load/dremio/docker-compose.yml" down -v

.github/workflows/test_doc_snippets.yml

@@ -60,7 +60,7 @@ jobs:
         uses: actions/checkout@master
 
       - name: Start weaviate
-        run: docker-compose -f ".github/weaviate-compose.yml" up -d
+        run: docker compose -f ".github/weaviate-compose.yml" up -d
 
       - name: Setup Python
         uses: actions/setup-python@v4

.github/workflows/test_local_destinations.yml

@@ -73,7 +73,7 @@ jobs:
         uses: actions/checkout@master
 
       - name: Start weaviate
-        run: docker-compose -f ".github/weaviate-compose.yml" up -d
+        run: docker compose -f ".github/weaviate-compose.yml" up -d
 
       - name: Setup Python
         uses: actions/setup-python@v4
@@ -109,4 +109,4 @@ jobs:
 
       - name: Stop weaviate
         if: always()
-        run: docker-compose -f ".github/weaviate-compose.yml" down -v
+        run: docker compose -f ".github/weaviate-compose.yml" down -v

.github/workflows/test_pyarrow17.yml

@@ -0,0 +1,77 @@
+
+name: tests marked as needspyarrow17
+
+on:
+  pull_request:
+    branches:
+      - master
+      - devel
+  workflow_dispatch:
+  schedule:
+    - cron:  '0 2 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+
+  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
+
+  RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752
+  RUNTIME__LOG_LEVEL: ERROR
+  RUNTIME__DLTHUB_TELEMETRY_ENDPOINT: ${{ secrets.RUNTIME__DLTHUB_TELEMETRY_ENDPOINT }}
+
+  ACTIVE_DESTINATIONS: "[\"filesystem\"]"
+
+jobs:
+  get_docs_changes:
+    name: docs changes
+    uses: ./.github/workflows/get_docs_changes.yml
+    if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
+
+  run_pyarrow17:
+    name: needspyarrow17 tests
+    needs: get_docs_changes
+    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
+    defaults:
+      run:
+        shell: bash
+    runs-on: "ubuntu-latest"
+
+    steps:
+
+      - name: Check out
+        uses: actions/checkout@master
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10.x"
+
+      - name: Install Poetry
+        uses: snok/[email protected]
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-pyarrow17
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --with sentry-sdk --with pipeline -E deltalake -E gs -E s3 -E az
+
+      - name: Upgrade pyarrow
+        run: poetry run pip install pyarrow==17.0.0
+
+      - name: create secrets.toml
+        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
+
+      - run: |
+          poetry run pytest tests/libs tests/load -m needspyarrow17
+        name: Run needspyarrow17 tests Linux

deploy/dlt/Dockerfile

@@ -31,7 +31,7 @@ RUN apk update &&\
 # add build labels and envs
 ARG COMMIT_SHA=""
 ARG IMAGE_VERSION=""
-LABEL commit_sha = ${COMMIT_SHA}
+LABEL commit_sha=${COMMIT_SHA}
 LABEL version=${IMAGE_VERSION}
 ENV COMMIT_SHA=${COMMIT_SHA}
 ENV IMAGE_VERSION=${IMAGE_VERSION}

deploy/dlt/Dockerfile.airflow

@@ -14,7 +14,7 @@ WORKDIR /tmp/pydlt
 # add build labels and envs
 ARG COMMIT_SHA=""
 ARG IMAGE_VERSION=""
-LABEL commit_sha = ${COMMIT_SHA}
+LABEL commit_sha=${COMMIT_SHA}
 LABEL version=${IMAGE_VERSION}
 ENV COMMIT_SHA=${COMMIT_SHA}
 ENV IMAGE_VERSION=${IMAGE_VERSION}

dlt/cli/init_command.py

@@ -1,6 +1,7 @@
 import os
 import ast
 import shutil
+import tomlkit
 from types import ModuleType
 from typing import Dict, List, Sequence, Tuple
 from importlib.metadata import version as pkg_version
@@ -488,16 +489,19 @@ def init_command(
 
     # generate tomls with comments
     secrets_prov = SecretsTomlProvider()
-    # print(secrets_prov._toml)
-    write_values(secrets_prov._toml, required_secrets.values(), overwrite_existing=False)
+    secrets_toml = tomlkit.document()
+    write_values(secrets_toml, required_secrets.values(), overwrite_existing=False)
+    secrets_prov._config_doc = secrets_toml
+
     config_prov = ConfigTomlProvider()
-    write_values(config_prov._toml, required_config.values(), overwrite_existing=False)
+    config_toml = tomlkit.document()
+    write_values(config_toml, required_config.values(), overwrite_existing=False)
+    config_prov._config_doc = config_toml
+
     # write toml files
     secrets_prov.write_toml()
     config_prov.write_toml()
 
-    # telemetry_status_command()
-
     # if there's no dependency system write the requirements file
     if dependency_system is None:
         requirements_txt = "\n".join(source_files.requirements.compiled())

dlt/cli/telemetry_command.py

@@ -1,6 +1,6 @@
 import os
+import tomlkit
 
-from dlt.common.configuration import resolve_configuration
 from dlt.common.configuration.container import Container
 from dlt.common.configuration.providers.toml import ConfigTomlProvider
 from dlt.common.configuration.specs import RunConfiguration
@@ -29,15 +29,21 @@ def change_telemetry_status_command(enabled: bool) -> None:
     ]
     # write local config
     config = ConfigTomlProvider(add_global_config=False)
+    config_toml = tomlkit.document()
     if not config.is_empty:
-        write_values(config._toml, telemetry_value, overwrite_existing=True)
+        write_values(config_toml, telemetry_value, overwrite_existing=True)
+        config._config_doc = config_toml
         config.write_toml()
+
     # write global config
     global_path = ConfigTomlProvider.global_config_path()
     os.makedirs(global_path, exist_ok=True)
     config = ConfigTomlProvider(project_dir=global_path, add_global_config=False)
-    write_values(config._toml, telemetry_value, overwrite_existing=True)
+    config_toml = tomlkit.document()
+    write_values(config_toml, telemetry_value, overwrite_existing=True)
+    config._config_doc = config_toml
     config.write_toml()
+
     if enabled:
         fmt.echo("Telemetry switched %s" % fmt.bold("ON"))
     else:

dlt/common/configuration/accessors.py

@@ -80,6 +80,13 @@ def _get_value(self, field: str, type_hint: Type[Any] = None) -> Tuple[Any, List
                 break
         return value, traces
 
+    @staticmethod
+    def register_provider(provider: ConfigProvider) -> None:
+        """Registers `provider` to participate in the configuration resolution. `provider`
+        is added after all existing providers and will be used if all others do not resolve.
+        """
+        Container()[ConfigProvidersContext].add_provider(provider)
+
 
 class _ConfigAccessor(_Accessor):
     """Provides direct access to configured values that are not secrets."""

dlt/common/configuration/providers/__init__.py

@@ -4,12 +4,13 @@
 from .toml import (
     SecretsTomlProvider,
     ConfigTomlProvider,
-    TomlFileProvider,
+    ProjectDocProvider,
     CONFIG_TOML,
     SECRETS_TOML,
     StringTomlProvider,
-    SECRETS_TOML_KEY,
+    CustomLoaderDocProvider,
 )
+from .vault import SECRETS_TOML_KEY
 from .google_secrets import GoogleSecretsProvider
 from .context import ContextProvider
 
@@ -19,11 +20,12 @@
     "DictionaryProvider",
     "SecretsTomlProvider",
     "ConfigTomlProvider",
-    "TomlFileProvider",
+    "ProjectDocProvider",
     "CONFIG_TOML",
     "SECRETS_TOML",
     "StringTomlProvider",
     "SECRETS_TOML_KEY",
     "GoogleSecretsProvider",
     "ContextProvider",
+    "CustomLoaderDocProvider",
 ]

dlt/common/configuration/providers/airflow.py

@@ -1,10 +1,10 @@
 import io
 import contextlib
 
-from .toml import VaultTomlProvider
+from .vault import VaultDocProvider
 
 
-class AirflowSecretsTomlProvider(VaultTomlProvider):
+class AirflowSecretsTomlProvider(VaultDocProvider):
     def __init__(self, only_secrets: bool = False, only_toml_fragments: bool = False) -> None:
         super().__init__(only_secrets, only_toml_fragments)