forked from duneanalytics/spellbook
-
Notifications
You must be signed in to change notification settings - Fork 0
89 lines (74 loc) · 2.62 KB
/
dbt_slim_CI.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
# .github/workflows/app.yaml
name: dbt slim ci
on:
push:
paths:
- 'spellbook/**'
jobs:
dbt-test:
runs-on: ubuntu-latest
steps:
- name: Set DBT schema
run: echo "PR_SCHEMA=$(echo sha_${{ github.sha}} | tr - _)" >> $GITHUB_ENV
- name: Check out repository code
uses: actions/checkout@v2
- name: Set up Python 3.9
uses: actions/setup-python@v1
with:
python-version: 3.9
cache: 'pipenv'
- name: Install pipenv
run: pip install pipenv
- name: Cache pipenv virtualenv
id: cache-pipenv
uses: actions/cache@v1
with:
path: ~/.local/share/virtualenvs
key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
- name: Install dependencies
if: steps.cache-pipenv.outputs.cache-hit != 'true'
run: |
pipenv install
- name: Get latest manifest
working-directory: ./spellbook
run: "pipenv run python fetch_latest_manifest.py --dbt_api_token ${{ secrets.DBT_API_TOKEN }}"
- name: DBT configure
working-directory: ./spellbook
continue-on-error: true
env:
DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
DATABRICKS_HTTPS_PATH: ${{ secrets.DATABRICKS_HTTPS_PATH }}
DATABRICKS_ACCESS_TOKEN: ${{ secrets.DATABRICKS_ACCESS_TOKEN }}
run: |
echo "2
$DATABRICKS_HOST
$DATABRICKS_HTTPS_PATH
$DATABRICKS_ACCESS_TOKEN
$PR_SCHEMA
1" | pipenv run dbt init
- name: DBT configure take 2 (sometimes dbt init switches the order of db options)
continue-on-error: true
working-directory: ./spellbook
env:
DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }}
DATABRICKS_HTTPS_PATH: ${{ secrets.DATABRICKS_HTTPS_PATH }}
DATABRICKS_ACCESS_TOKEN: ${{ secrets.DATABRICKS_ACCESS_TOKEN }}
run: |
echo "1
$DATABRICKS_HOST
$DATABRICKS_HTTPS_PATH
$DATABRICKS_ACCESS_TOKEN
$PR_SCHEMA
1" | pipenv run dbt init
- name: DBT compile to create manifest to compare to
working-directory: ./spellbook
run: "pipenv run dbt compile"
- name: DBT seed
working-directory: ./spellbook
run: "pipenv run dbt seed --select +state:modified --state ."
- name: DBT run
working-directory: ./spellbook
run: "pipenv run dbt run --select +state:modified --state ."
- name: DBT test
working-directory: ./spellbook
run: "pipenv run dbt test --select +state:new +state:modified --state ."