Use Poetry as a dependency tool (#33)

* Use poetry as dependenct tool

.github/actions/setup_dependencies/action.yml

@@ -0,0 +1,31 @@
+name: Setup Python dependencies
+description: Create Python environment with all dependencies installed
+inputs:
+  python-version:
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Setup Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Get full Python version
+      id: full-python-version
+      shell: bash
+      run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT
+
+    - name: Cache virtual environments
+      uses: actions/cache@v3
+      with:
+        path: |        
+          ~/.cache/pip
+          .venv        
+        key: ${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-poetry-${{ hashFiles('poetry.lock') }}
+
+    - name: Install Python dependencies
+      shell: bash
+      run: |        
+        sudo make install-poetry
+        make install-deps

.github/workflows/workflow.yml

@@ -1,19 +1,27 @@
 name: main
 on:
-  push: { branches: [main] }
-  pull_request: { branches: [main] }
+  push: 
+    branches: [main]
+    tags: "v*"
+  pull_request:
+    branches: [main]
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
+env:
+  BUILD_DEB_OUTPUT_DIR: out
+  BUILD_PYTHON_OUTPUT_DIR: dist
+  PROJECT_NAME: clickhouse-tools
+  PROJECT_NAME_UNDESCORE: clickhouse_tools
 
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      with: { python-version: "3.11", cache: pip }
-    - run: pip install .[test]
+    - uses: actions/checkout@v3                
+    - uses: ./.github/actions/setup_dependencies
+      with:
+        python-version: "3.10"
     - name: "lint: isort"
       run: make isort
     - name: "lint: black"
@@ -24,6 +32,41 @@ jobs:
       run: make pylint
     - name: "lint: mypy"
       run: make mypy
+
+  tests:
+    needs: lint
+    runs-on: ubuntu-${{ matrix.target.ubuntu }}
+    env:
+      CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}        
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+          - { python: "3.6", ubuntu: "20.04" }
+          - { python: "3.10", ubuntu: "latest" }
+        clickhouse:
+          - "21.8.15.7"
+          - "22.3.20.29"
+          - "22.8.19.10"
+          - "23.3.4.17"
+          - "23.4.4.16"
+          - "23.5.3.24"
+          - "latest"    
+    steps:
+      - uses: actions/checkout@v3                
+      - uses: ./.github/actions/setup_dependencies
+        with:
+          python-version: ${{ matrix.target.python }}
+
+      - run: make test-unit
+      - run: make test-integration
+
+      - name: publish test report
+        uses: mikepenz/action-junit-report@v3
+        if: always()
+        with:
+          report_paths: 'tests/reports/*.xml'
+
 
   push_to_dockerhub:
     runs-on: ubuntu-latest
@@ -43,129 +86,83 @@ jobs:
         files: tests/bake.hcl
         push: true
 
-  build:
-    needs: lint
+
+  release:
+    needs: tests
     strategy:
       fail-fast: false
       matrix:
         target:
-          - {python: "3.6.15", ubuntu: "20.04"}
-          - {python: "3.10.12", ubuntu: "latest"}
-        # a copy-paste of the above as github CI can't use env context in matrices
-        clickhouse:
-          - "21.8.15.7"
-          - "22.3.20.29"
-          - "22.8.19.10"
-          - "23.3.4.17"
-          - "23.4.4.16"
-          - "23.5.3.24"
-          - "latest"
+          - {python: "3.6", ubuntu: "20.04"}
+          - {python: "3.10", ubuntu: "latest"}
 
     runs-on: ubuntu-${{ matrix.target.ubuntu }}
     steps:
     - uses: actions/checkout@v3
-    - name: set up python ${{ matrix.target.python }}
-      uses: actions/setup-python@v4
+
+    - uses: ./.github/actions/setup_dependencies
       with:
         python-version: ${{ matrix.target.python }}
-        cache: pip
-
-    - name: install dependencies
-      # language=sh
-      run: |
-        pip install '.[test]'
 
     - name: build project
-      # language=sh
-      run: |
-        make prepare-version
-        flit build --no-use-vcs
+      run: make build-python-packages
 
     - name: upload wheel
       uses: actions/upload-artifact@v3
-      if: ${{ matrix.clickhouse == 'latest' }}
       with:
-        name: ch_tools_py${{ matrix.target.python }}.whl
-        path: dist/*.whl
+        name: ${{ env.PROJECT_NAME_UNDESCORE }}_py${{ matrix.target.python }}.whl
+        path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.whl
         if-no-files-found: error
 
     - name: upload sdist
       uses: actions/upload-artifact@v3
-      if: ${{ matrix.clickhouse == 'latest' }}
       with:
-        name: ch_tools_py${{ matrix.target.python }}.tar.gz
-        path: dist/*.tar.gz
+        name: ${{ env.PROJECT_NAME_UNDESCORE }}_py${{ matrix.target.python }}.tar.gz
+        path: ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.tar.gz
         if-no-files-found: error
 
-    - {name: run unit tests, run: pytest}
-
-    - name: prepare integration tests
-      # language=sh
-      run: |
-        cd tests
-        CLICKHOUSE_VERSION=${{ matrix.clickhouse }} python3 -m env_control create
-        cd ..
-
-    - name: run integration tests
-      # language=sh
-      run: |
-        cd tests
-        behave --show-timings --junit -D skip_setup
-        cd ..
-
-    - name: publish test report
-      uses: mikepenz/action-junit-report@v3
-      if: always()
-      with:
-        report_paths: 'tests/reports/*.xml'
-
-    - name: prepare build deb
-      if: ${{ matrix.clickhouse == 'latest' }}
-      run: sudo apt install python3-venv debhelper devscripts
+    - name: install prerequisites for DEB packaging tools
+      run: sudo make prepare-build-deb
 
-    - name: build deb
-      if: ${{ matrix.clickhouse == 'latest' }}
-      # language=sh
-      run: |
-        # deb building implicitly cleans dist/
-        cp dist/*.tar.gz ch_tools.tar.gz
-        cp dist/*.whl ch_tools.whl
+    - name: build DEB package
+      run: |      
+        # Speedup Debian package building
         echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io
-        sudo make build-deb-package
+        make build-deb-package
 
-    - name: test deb
-      if: ${{ matrix.clickhouse == 'latest' }}
-      # language=sh
+    - name: test DEB package
       run: |
         sudo make uninstall
-        sudo apt-get install -q -y ./out/ch-tools*.deb
+        sudo apt-get install -q -y ./${{ env.BUILD_DEB_OUTPUT_DIR }}/*.deb
         sudo chadmin --help
         sudo ch-monitoring --no-user-check --help
         sudo keeper-monitoring --help
         sudo ch-s3-credentials --help
 
-    - name: upload deb package artifact
+    - name: upload DEB package artifact
       uses: actions/upload-artifact@v3
-      if: ${{ matrix.clickhouse == 'latest' }}
       with:
-        name: ch-tools_py-${{ matrix.target.python }}_ubuntu-${{ matrix.target.ubuntu }}.deb
-        path: out/ch-tools*.deb
+        name: ${{ env.PROJECT_NAME }}_py-${{ matrix.target.python }}_ubuntu-${{ matrix.target.ubuntu }}.deb
+        path: ${{ env.BUILD_DEB_OUTPUT_DIR }}/*.deb
         if-no-files-found: error
 
     - name: create a release
       uses: softprops/action-gh-release@v1
-      if: ${{ matrix.clickhouse == 'latest' && matrix.target.ubuntu == 'latest' && startsWith(github.ref, 'refs/tags/') }}
+      if: ${{ matrix.target.ubuntu == 'latest' && startsWith(github.ref, 'refs/tags/') }}
       with:
         generate_release_notes: true
         fail_on_unmatched_files: true
         files: |
-          *.whl
-          *.tar.gz
-          out/ch-tools*.deb
-
-    - name: publish to pypi
-      if: ${{ matrix.clickhouse == 'latest' && matrix.target.ubuntu == 'latest' && startsWith(github.ref, 'refs/tags/') }}
-      run: flit publish --no-use-vcs
+          ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.whl
+          ${{ env.BUILD_PYTHON_OUTPUT_DIR }}/*.tar.gz
+          ${{ env.BUILD_DEB_OUTPUT_DIR }}/*.deb
+
+    - name: publish to PYPI
+      if: ${{ matrix.target.ubuntu == 'latest' && startsWith(github.ref, 'refs/tags/') }}
+      continue-on-error: true
+      run: make publish
       env:
-        FLIT_USERNAME: "__token__"
-        FLIT_PASSWORD: "${{ secrets.PYPI_TOKEN }}"
+        POETRY_HTTP_BASIC_PYPI_USERNAME: "__token__"
+        POETRY_HTTP_BASIC_PYPI_PASSWORD: "${{ secrets.PYPI_TOKEN }}"
+        # Disable using keyring for poetry to get password from env variable
+        PYTHON_KEYRING_BACKEND: keyring.backends.null.Keyring

.gitignore

@@ -1,14 +1,16 @@
 .DS_Store
 .idea
-build/
-dist/
-.python-version
-__pycache__
-venv/
 .mypy_cache/
+.python-version
 .pytype/
+.session_conf.sav
+__pycache__
+build/
 cython_debug/
-tests/staging/
+dist/
+out/
 tests/reports/
-.session_conf.sav
+tests/staging/
+venv/
 version.txt
+.install-deps

.pylintrc

@@ -52,7 +52,8 @@ ignore=
 # ignore-list. The regex matches against paths and can be in Posix or Windows
 # format. Because '\\' represents the directory delimiter on Windows systems,
 # it can't be used as an escape character.
-ignore-paths=
+ignore-paths='tests/staging/.*',
+             'tests/images/.*'
 
 # Files or directories matching the regular expression patterns are skipped.
 # The regex matches against base names, not paths. The default value ignores

AUTHORS

@@ -1,4 +1,4 @@
-The following authors have created the source code of "ch-tools"
+The following authors have created the source code of "clickhouse-tools"
 published and distributed by YANDEX LLC as the owner:
 
 Alexander Burmak <[email protected]>