diff --git a/.github/workflows/build-deb-package.yml b/.github/workflows/build-deb-package.yml new file mode 100644 index 00000000..ff7058ba --- /dev/null +++ b/.github/workflows/build-deb-package.yml @@ -0,0 +1,69 @@ +name: Build Debian package +on: + push: + branches: + - main +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: false + +jobs: + build-deb-package: + strategy: + matrix: + target: + - { python: "3.6.15", ubuntu: 20.04 } + - { python: "3.10.12", ubuntu: 22.04 } + runs-on: ubuntu-${{ matrix.target.ubuntu }} + steps: + - uses: actions/checkout@v3 + - name: set up python ${{ matrix.target.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.target.python }} + cache: pip + + - name: build amd64 package + # language=sh + run: | + sed -i -E -e "s/amd64|arm64/amd64/g" debian/control debian/rules + make build-deb-package + + - name: build arm64 package + uses: uraimo/run-on-arch-action@v2 + with: + arch: aarch64 + distro: ubuntu${{ matrix.target.ubuntu }} + githubToken: ${{ github.token }} + # language=sh + setup: | + mkdir -p "${PWD}/artifacts" + # language=sh + dockerRunArgs: | + --volume "${PWD}/artifacts:/artifacts" + # language=sh + shell: /bin/sh + # language=sh + install: | + apt-get update -q -y + apt-get install -q -y git python3=${{ matrix.target.python }} python3-venv python3-pip debhelper devscripts + # language=sh + run: | + sed -i -E -e "s/amd64|arm64/arm64/g" debian/control debian/rules + make build-deb-package + mv ../mdb-ch-tools*.deb /artifacts/mdb-ch-tools_py${{ matrix.target.python }}_arm64.deb + echo "Produced artifact: `ls /artifacts/`" + + - name: upload amd64 artifacts + uses: actions/upload-artifact@v3 + with: + name: mdb-ch-tools_py${{ matrix.target.python }}_amd64.deb + path: ../mdb-ch-tools*.deb + if-no-files-found: error + + - name: upload arm64 artifacts + uses: actions/upload-artifact@v3 + with: + name: mdb-ch-tools_py${{ matrix.target.python }}_arm64.deb + path: artifacts/mdb-ch-tools*.deb + if-no-files-found: error diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 59c16550..b38631e0 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -66,7 +66,7 @@ jobs: python-version: ${{ matrix.target.python }} cache: pip - - {name: install dependencies, run: "pip install .[test]" } + - {name: install dependencies, run: "pip install '.[test]'" } - {name: build project, run: flit build --no-use-vcs } - name: upload wheel @@ -88,12 +88,14 @@ jobs: - {name: run unit tests, run: pytest} - name: prepare integration tests + # language=sh run: | cd tests CLICKHOUSE_VERSION=${{ matrix.clickhouse }} python3 -m env_control create cd .. - name: run integration tests + # language=sh run: | cd tests behave --show-timings --junit -D skip_setup diff --git a/.gitignore b/.gitignore index 29ca99e0..685d90b4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ cython_debug/ tests/staging/ tests/reports/ .session_conf.sav +version.txt diff --git a/Makefile b/Makefile index bf2847d0..0e42a1b4 100644 --- a/Makefile +++ b/Makefile @@ -4,11 +4,47 @@ PREFIX=/opt/yandex/mdb-ch-tools INSTALL_DIR=$(DESTDIR)$(PREFIX) .PHONY: install -install: install-symlinks install-bash-completions install-logrotate ; +install: install-python-package install-symlinks install-bash-completions configure-logs ; .PHONY: uninstall uninstall: uninstall-python-package uninstall-symlinks uninstall-bash-completions uninstall-logrotate ; +.PHONY: install-python-package +install-python-package: build-python-package + @echo 'Installing mdb-ch-tools' + + python3 -m venv $(INSTALL_DIR) + rm -f $(INSTALL_DIR)/bin/activate* + $(INSTALL_DIR)/bin/pip install -U pip + + $(INSTALL_DIR)/bin/pip install --no-compile chtools.tar.gz + + find $(INSTALL_DIR) -name __pycache__ -type d -exec rm -rf {} + + test -n '$(DESTDIR)' \ + && grep -l -r -F '#!$(INSTALL_DIR)' $(INSTALL_DIR) \ + | xargs sed -i -e 's|$(INSTALL_DIR)|$(PREFIX)|' \ + || true + +.PHONY: build-python-package +build-python-package: chtools.tar.gz ; + @echo 'Cleaning up residuals from building of Python package' + rm -rf venv dist + +chtools.tar.gz: venv-build prepare-version + @echo 'Building Python package' + + venv/bin/flit build --no-use-vcs + mv dist/*.tar.gz $@ + +.PHONY: venv-build +venv-build: venv/bin/python3 ; + +venv/bin/python3: + @echo 'Setting up build venv' + + python3 -m venv venv + venv/bin/pip install -U pip flit + .PHONY: uninstall-python-package uninstall-python-package: @echo 'Uninstalling mdb-ch-tools' @@ -48,14 +84,14 @@ uninstall-bash-completions: rm -f $(DESTDIR)/etc/bash_completion.d/$(bin) ; \ ) -.PHONY: install-logrotate -install-logrotate: - @echo 'Creating log rotation rules' +.PHONY: configure-logs +configure-logs: + @echo 'Configuring logging' mkdir -p $(DESTDIR)/etc/logrotate.d/ $(foreach bin, chadmin clickhouse-monitoring keeper-monitoring, \ - mkdir -p $(DESTDIR)/var/log/$(folder) ; \ - chmod 775 $(DESTDIR)/var/log/$(folder) ; \ + mkdir -p $(DESTDIR)/var/log/$(bin) ; \ + chmod 775 $(DESTDIR)/var/log/$(bin) ; \ cp resources/logrotate/$(bin).logrotate $(DESTDIR)/etc/logrotate.d/$(bin) ; \ ) @@ -67,9 +103,9 @@ uninstall-logrotate: rm -f $(DESTDIR)/etc/logrotate.d/$(bin) ; \ ) - .PHONY: prepare-changelog prepare-changelog: prepare-version + @echo 'Bumping version into Debian package changelog' dch --force-bad-version --distribution stable -v `cat version.txt` Autobuild .PHONY: prepare-version @@ -78,16 +114,29 @@ prepare-version: version.txt version.txt: @echo "2.$$(git rev-list HEAD --count).$$(git rev-parse --short HEAD | perl -ne 'print hex $$_')" > $@ + @# Replace version in pyproject.toml + sed -ie "s/__version__ = \"[0-9\.]\+\"/__version__ = \"`cat $@`\"/" src/chtools/__init__.py +.PHONY: build-deb-package build-deb-package: prepare-changelog cd debian && debuild --check-dirname-level 0 --preserve-env --no-lintian --no-tgz-check -uc -us +.PHONY: clean +clean: + @echo 'Cleaning up' + + rm -rf build + rm -rf debian/files debian/.debhelper + rm -rf debian/mdb-ch-tools* + .PHONY: help help: @echo "Base targets:" @echo " prepare-changelog Add an autobuild version entity to changelog" @echo " prepare-version Update version based on latest commit" + @echo " build-python-package Build 'ch-tools' Python package" @echo " build-deb-package Build 'mdb-ch-tools' debian package" + @echo " clean Clean up after building debian package" @echo "" @echo "--------------------------------------------------------------------------------" @echo "" @@ -95,12 +144,13 @@ help: @echo " install Install 'mdb-ch-tools' debian package" @echo " uninstall Uninstall 'mdb-ch-tools' debian package" @echo "" + @echo " install-python-package Install 'ch-tools' python package" @echo " uninstall-python-package Uninstall 'ch-tools' python package" @echo " install-symlinks Install symlinks to /usr/bin/" @echo " uninstall-symlinks Uninstall symlinks from /usr/bin/" @echo " install-bash-completions Install to /etc/bash_completion.d/" @echo " uninstall-bash-completions Uninstall from /etc/bash_completion.d/" - @echo " install-logrotate Install log rotation rules to /etc/logrotate.d/" + @echo " configure-logs Install log rotation rules to /etc/logrotate.d/ and create log dirs" @echo " uninstall-logrotate Uninstall log rotation rules from /etc/logrotate.d/" @echo "" @echo "--------------------------------------------------------------------------------" diff --git a/debian/control b/debian/control index 901bcb21..a3407f44 100644 --- a/debian/control +++ b/debian/control @@ -11,7 +11,7 @@ Uploaders: Alexander Burmak , Aleksei Filatov , Evgenii Kopanev , Mikhail Kot -Build-Depends: debhelper (>= 8.0.0), python3, python3-venv, python3-pip, python3-setuptools +Build-Depends: debhelper (>= 9.0.0), python3, python3-venv, python3-pip, python3-setuptools Standards-Version: 4.1.4 Homepage: https://github.com/yandex/ch-tools Vcs-Browser: https://github.com/yandex/ch-tools.git diff --git a/pyproject.toml b/pyproject.toml index 8b1db502..9456881d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,9 +4,8 @@ build-backend = "flit_core.buildapi" [project] name = "chtools" -version = "1.0.0" -description = "A set of tools for administration and diagnostics of ClickHouse DBMS." license = { file = "LICENSE" } +dynamic = ["version", "description"] authors = [ {name = "Alexander Burmak", email = "alex-burmak@yandex-team.ru"}, diff --git a/src/chtools/__init__.py b/src/chtools/__init__.py index e69de29b..8526f52a 100644 --- a/src/chtools/__init__.py +++ b/src/chtools/__init__.py @@ -0,0 +1,3 @@ +"""A set of tools for administration and diagnostics of ClickHouse DBMS.""" + +__version__ = "1.0.0"