feat: Import contents of aspects-dbt repo

Moves the contents of aspects-dbt to this repository. There are a couple
of current issues with this:

- Old requirements for our clickhouse dbt stack conflict with the rest
of our constraints
- dbt docs generation / hosting won't work until we change the github
pages for it

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/profiles.yml

@@ -0,0 +1,16 @@
+# This is a dbt profile for CI use.
+# https://docs.getdbt.com/docs/core/connect-data-platform/connection-profiles
+aspects: # this needs to match the profile in your dbt_project.yml file
+  target: dev
+  outputs:
+    dev:
+      type: clickhouse
+      schema: xapi
+      host: localhost
+      port: 8123
+      user: ch_admin
+      password: "ch_password"
+      secure: False
+
+      custom_settings:
+        check_table_dependencies: 0

.github/workflows/ci-dbt-coverage.yml

@@ -0,0 +1,58 @@
+# Check documentation coverage
+
+name: dbt Tests & Coverage
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+env:
+  DBT_PROFILES_DIR: ./.github/
+  CLICKHOUSE_DB: "xapi"
+  CLICKHOUSE_USER: "ch_admin"
+  CLICKHOUSE_PASSWORD: "ch_password"
+  TUTOR_ROOT: ./dbt/.ci/
+
+jobs:
+  build:
+    name: Check coverage & run tests
+    runs-on: ubuntu-latest
+    permissions:
+      contents: "read"
+      id-token: "write"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11.x"
+      - name: Install dependencies
+        run: |
+          pip install tutor-contrib-aspects
+          pip install -r requirements/dbt.txt
+          cd dbt
+          dbt deps
+      - name: Initialize Open edX
+        continue-on-error: false
+        run: |
+          tutor plugins enable aspects
+          tutor config save
+          tutor local start ralph -d
+          tutor local do init-clickhouse
+          tutor local do alembic -c "upgrade head"
+          tutor local do dbt
+          tutor local do load-xapi-test-data
+      - name: Check dbt tests
+        run: |
+          mv unit-test-seeds ci-seeds
+          dbt seed --full-refresh --selector all_tests
+          dbt run --full-refresh --selector all_tests
+          dbt test --selector all_tests
+          mv ci-seeds unit-test-seeds
+      - name: Check docs coverage
+        run: |
+          dbt docs generate
+          dbt-coverage compute doc --cov-fail-under 1.0 --model-path-filter models/

.github/workflows/ci-dbt-documentation.yml

@@ -0,0 +1,58 @@
+# Build documentation for dbt-aspects on every PR to main.
+# Deploy documentation to gh-pages branch on every push to main.
+
+name: Build documentation
+
+on:
+  push:
+    branches:
+      - main
+
+env:
+  DBT_PROFILES_DIR: ./.github/
+  CLICKHOUSE_DB: "xapi"
+  CLICKHOUSE_USER: "ch_admin"
+  CLICKHOUSE_PASSWORD: "ch_password"
+  TUTOR_ROOT: ./.ci/
+
+jobs:
+  build:
+    name: Deploy dbt docs to github pages
+    runs-on: ubuntu-latest
+    permissions:
+      contents: "read"
+      id-token: "write"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.SEMANTIC_RELEASE_GITHUB_TOKEN }}
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11.x"
+      - name: Install dependencies
+        run: |
+          pip install tutor-contrib-aspects
+          pip install -r requirements/dbt.txt
+          cd dbt
+          dbt deps
+      - name: Initialize Open edX
+        continue-on-error: true
+        run: |
+          tutor plugins enable aspects
+          tutor config save
+          tutor local start -d
+          tutor local do init -l aspects
+          tutor local do load-xapi-test-data
+      - name: Build docs
+        run: |
+          dbt run
+          dbt docs generate
+      - name: Deploy
+        if: github.event_name == 'push'
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.SEMANTIC_RELEASE_GITHUB_TOKEN }}
+          publish_dir: ./target
+          commit_message: "docs: update docs for "

.github/workflows/ci-dbt-format.yml

@@ -0,0 +1,23 @@
+name: Validate SQL format
+
+on:
+  - pull_request
+
+jobs:
+  format:
+    name: Validate SQL format
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11.x"
+      - name: Install dependencies
+        run: |
+          pip install -r requirements/dbt.txt
+      - name: Format SQL
+        run: |
+          cd dbt
+          sqlfmt models macros --check

.gitignore

@@ -1,21 +1,14 @@
-build
-docs/_build
+__pycache__/
 .DS_Store
+.coverage
 .idea/
 .python-version
-.coverage
+.tox/
+.venv
 *.pyc
 *.egg-info/
-tests/__init__.py
-.tox/
+**/tests/__init__.py
+build
 coverage.xml
 dist/
-.venv
-.dir-locals.el
-*/private_configs/
-*/logs/
-__pycache__/
-dbt/target/
-dbt/dbt_packages/
-dbt/logs/
-coverage.json
+docs/_build

Makefile

@@ -31,14 +31,19 @@ upgrade: ## update the requirements/*.txt files with the latest packages satisfy
 	$(PIP_COMPILE) -o requirements/quality.txt requirements/quality.in
 	$(PIP_COMPILE) -o requirements/ci.txt requirements/ci.in
 	$(PIP_COMPILE) -o requirements/dev.txt requirements/dev.in
+	$(PIP_COMPILE) -o requirements/dbt.txt requirements/dbt.in
 
 requirements:
 	pip install -r requirements/pip-tools.txt
 	pip-sync requirements/base.txt
 
 dev-requirements:
 	pip install -r requirements/pip-tools.txt
-	pip-sync requirements/base.txt requirements/dev.txt
+	pip-sync requirements/base.txt requirements/dev.txt requirements/dbt.txt
+
+dbt-requirements:
+	pip install -r requirements/pip-tools.txt
+	pip-sync requirements/dbt.txt
 
 quality: ## check coding style with pycodestyle and pylint
 	pylint aspects/ *.py
@@ -69,11 +74,17 @@ clean: ## remove generated byte code, coverage reports, and build artifacts
 test: clean ## run tests in the current virtualenv
 	pytest
 
-test-all: quality clean## run tests on every supported Python/Django combination
+test-all: quality clean ## run tests on every supported Python/Django combination
 	tox
 
 validate: quality test ## run tests and quality checks
 
+dbt-format:
+	sqlfmt dbt/models dbt/macros
+
+dbt-coverage: ## Must be run after a dbt compile or run as it needs manifest.json
+	dbt-coverage compute doc --cov-fail-under 1.0
+
 selfcheck: ## check that the Makefile is well-formed
 	@echo "The Makefile is well-formed."
 

pyproject.toml

@@ -0,0 +1,2 @@
+[tool.sqlfmt]
+dialect="clickhouse"

requirements/base.in

@@ -2,7 +2,7 @@
 -c constraints.txt
 
 click
-clickhouse-connect>=0.5,<0.7
+clickhouse-connect
 pyyaml
 requests
 smart_open[s3]