Skip to content

Commit

Permalink
Use poetry as dependenct tool
Browse files Browse the repository at this point in the history
  • Loading branch information
aalexfvk committed Aug 9, 2023
1 parent 20c47d8 commit ea4e542
Show file tree
Hide file tree
Showing 129 changed files with 2,401 additions and 302 deletions.
35 changes: 35 additions & 0 deletions .github/actions/setup_dependencies/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
name: Setup Python dependencies
description: Create Python environment with all dependencies installed
inputs:
python-version:
required: true
poetry-version:
required: true
runs:
using: "composite"
steps:
- name: Setup Python ${{ inputs.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ inputs.python-version }}

- name: Get full Python version
id: full-python-version
shell: bash
run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT

- name: Cache virtual environments
uses: actions/cache@v3
with:
path: |
~/.cache/pip
.venv
key: ${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-poetry-${{ hashFiles('poetry.lock') }}

- name: Install Python dependencies
shell: bash
env:
POETRY_VERSION: ${{ inputs.poetry-version }}
run: |
sudo make install-poetry
make install-deps
162 changes: 82 additions & 80 deletions .github/workflows/workflow.yml
Original file line number Diff line number Diff line change
@@ -1,19 +1,29 @@
name: main
on:
push: { branches: [main] }
pull_request: { branches: [main] }
push:
branches: [main]
tags: "v*"
pull_request:
branches: [main]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
BUILD_DEB_OUTPUT_DIR: out
BUILD_PYTHON_OUTPUT_DIR: dist
# The latest version supporting Python 3.6
POETRY_VERSION: 1.1.15


jobs:
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v4
with: { python-version: "3.11", cache: pip }
- run: pip install .[test]
- uses: actions/checkout@v3
- uses: ./.github/actions/setup_dependencies
with:
python-version: "3.10.12"
poetry-version: ${{ env.POETRY_VERSION }}
- name: "lint: isort"
run: make isort
- name: "lint: black"
Expand All @@ -24,6 +34,42 @@ jobs:
run: make pylint
- name: "lint: mypy"
run: make mypy

tests:
needs: lint
runs-on: ubuntu-${{ matrix.target.ubuntu }}
env:
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
strategy:
fail-fast: false
matrix:
target:
- { python: "3.6.15", ubuntu: "20.04" }
- { python: "3.10.12", ubuntu: "latest" }
clickhouse:
- "21.8.15.7"
- "22.3.20.29"
- "22.8.19.10"
- "23.3.4.17"
- "23.4.4.16"
- "23.5.3.24"
- "latest"
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup_dependencies
with:
python-version: ${{ matrix.target.python }}
poetry-version: ${{ env.POETRY_VERSION }}

- run: make unit-tests
- run: make integration-tests

- name: publish test report
uses: mikepenz/action-junit-report@v3
if: always()
with:
report_paths: 'tests/reports/*.xml'


push_to_dockerhub:
runs-on: ubuntu-latest
Expand All @@ -43,129 +89,85 @@ jobs:
files: tests/bake.hcl
push: true


build:
needs: lint
needs: tests
strategy:
fail-fast: false
matrix:
target:
- {python: "3.6.15", ubuntu: "20.04"}
- {python: "3.10.12", ubuntu: "latest"}
# a copy-paste of the above as github CI can't use env context in matrices
clickhouse:
- "21.8.15.7"
- "22.3.20.29"
- "22.8.19.10"
- "23.3.4.17"
- "23.4.4.16"
- "23.5.3.24"
- "latest"

runs-on: ubuntu-${{ matrix.target.ubuntu }}
steps:
- uses: actions/checkout@v3
- name: set up python ${{ matrix.target.python }}
uses: actions/setup-python@v4

- uses: ./.github/actions/setup_dependencies
with:
python-version: ${{ matrix.target.python }}
cache: pip

- name: install dependencies
# language=sh
run: |
pip install '.[test]'
poetry-version: ${{ env.POETRY_VERSION }}

- name: build project
# language=sh
run: |
make prepare-version
flit build --no-use-vcs
run: make build-python-packages

- name: upload wheel
uses: actions/upload-artifact@v3
if: ${{ matrix.clickhouse == 'latest' }}
with:
name: ch_tools_py${{ matrix.target.python }}.whl
path: dist/*.whl
path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.whl
if-no-files-found: error

- name: upload sdist
uses: actions/upload-artifact@v3
if: ${{ matrix.clickhouse == 'latest' }}
with:
name: ch_tools_py${{ matrix.target.python }}.tar.gz
path: dist/*.tar.gz
path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.tar.gz
if-no-files-found: error

- {name: run unit tests, run: pytest}

- name: prepare integration tests
# language=sh
run: |
cd tests
CLICKHOUSE_VERSION=${{ matrix.clickhouse }} python3 -m env_control create
cd ..
- name: run integration tests
# language=sh
run: |
cd tests
behave --show-timings --junit -D skip_setup
cd ..
- name: publish test report
uses: mikepenz/action-junit-report@v3
if: always()
with:
report_paths: 'tests/reports/*.xml'

- name: prepare build deb
if: ${{ matrix.clickhouse == 'latest' }}
run: sudo apt install python3-venv debhelper devscripts
- name: install prerequisites for DEB packaging tools
run: sudo make prepare-build-deb

- name: build deb
if: ${{ matrix.clickhouse == 'latest' }}
# language=sh
run: |
# deb building implicitly cleans dist/
cp dist/*.tar.gz ch_tools.tar.gz
cp dist/*.whl ch_tools.whl
- name: build DEB package
run: |
# Speedup Debian package building
echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io
sudo make build-deb-package
- name: test deb
if: ${{ matrix.clickhouse == 'latest' }}
# language=sh
- name: test DEB package
run: |
sudo make uninstall
sudo apt-get install -q -y ./out/ch-tools*.deb
sudo apt-get install -q -y ./${{ env.BUILD_DEB_OUTPUT_DIR }}/ch-tools*.deb
sudo chadmin --help
sudo ch-monitoring --no-user-check --help
sudo keeper-monitoring --help
sudo ch-s3-credentials --help
- name: upload deb package artifact
- name: upload DEB package artifact
uses: actions/upload-artifact@v3
if: ${{ matrix.clickhouse == 'latest' }}
with:
name: ch-tools_py-${{ matrix.target.python }}_ubuntu-${{ matrix.target.ubuntu }}.deb
path: out/ch-tools*.deb
path: ${{ env.BUILD_DEB_OUTPUT_DIR }}/ch-tools*.deb
if-no-files-found: error

- name: create a release
uses: softprops/action-gh-release@v1
if: ${{ matrix.clickhouse == 'latest' && matrix.target.ubuntu == 'latest' && startsWith(github.ref, 'refs/tags/') }}
if: ${{ matrix.target.ubuntu == 'latest' && startsWith(github.ref, 'refs/tags/') }}
with:
generate_release_notes: true
fail_on_unmatched_files: true
files: |
*.whl
*.tar.gz
out/ch-tools*.deb
- name: publish to pypi
if: ${{ matrix.clickhouse == 'latest' && matrix.target.ubuntu == 'latest' && startsWith(github.ref, 'refs/tags/') }}
run: flit publish --no-use-vcs
${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.whl
${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.tar.gz
${{ env.BUILD_DEB_OUTPUT_DIR }}/ch-tools*.deb
- name: publish to PYPI
if: ${{ matrix.target.ubuntu == 'latest' && startsWith(github.ref, 'refs/tags/') }}
continue-on-error: true
run: make publish
env:
FLIT_USERNAME: "__token__"
FLIT_PASSWORD: "${{ secrets.PYPI_TOKEN }}"
POETRY_HTTP_BASIC_PYPI_USERNAME: "__token__"
POETRY_HTTP_BASIC_PYPI_PASSWORD: "${{ secrets.PYPI_TOKEN }}"
# Disable using keyring for poetry to get password from env variables
PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring
16 changes: 9 additions & 7 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
.DS_Store
.idea
build/
dist/
.python-version
__pycache__
venv/
.mypy_cache/
.python-version
.pytype/
.session_conf.sav
__pycache__
build/
cython_debug/
tests/staging/
dist/
out/
tests/reports/
.session_conf.sav
tests/staging/
venv/
version.txt
.install-deps
3 changes: 2 additions & 1 deletion .pylintrc
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,8 @@ ignore=
# ignore-list. The regex matches against paths and can be in Posix or Windows
# format. Because '\\' represents the directory delimiter on Windows systems,
# it can't be used as an escape character.
ignore-paths=
ignore-paths='tests/staging/.*',
'tests/images/.*'

# Files or directories matching the regular expression patterns are skipped.
# The regex matches against base names, not paths. The default value ignores
Expand Down
Loading

0 comments on commit ea4e542

Please sign in to comment.