Bring the deb-package building back

.github/workflows/build-deb-package.yml

@@ -0,0 +1,69 @@
+name: Build Debian package
+on:
+  push:
+    branches:
+      - main
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  build-deb-package:
+    strategy:
+      matrix:
+        target:
+          - { python: "3.6.15", ubuntu: 20.04 }
+          - { python: "3.10.12", ubuntu: 22.04 }
+    runs-on: ubuntu-${{ matrix.target.ubuntu }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: set up python ${{ matrix.target.python }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.target.python }}
+          cache: pip
+
+      - name: build amd64 package
+        # language=sh
+        run: |
+          sed -i -E -e "s/amd64|arm64/amd64/g" debian/control debian/rules
+          make build-deb-package
+
+      - name: build arm64 package
+        uses: uraimo/run-on-arch-action@v2
+        with:
+          arch: aarch64
+          distro: ubuntu${{ matrix.target.ubuntu }}
+          githubToken: ${{ github.token }}
+          # language=sh
+          setup: |
+            mkdir -p "${PWD}/artifacts"
+          # language=sh
+          dockerRunArgs: |
+            --volume "${PWD}/artifacts:/artifacts"
+          # language=sh
+          shell: /bin/sh
+          # language=sh
+          install: |
+            apt-get update -q -y
+            apt-get install -q -y git python3=${{ matrix.target.python }} python3-venv python3-pip debhelper devscripts
+          # language=sh
+          run: |
+            sed -i -E -e "s/amd64|arm64/arm64/g" debian/control debian/rules
+            make build-deb-package
+            mv ../mdb-ch-tools*.deb /artifacts/mdb-ch-tools_py${{ matrix.target.python }}_arm64.deb
+            echo "Produced artifact: `ls /artifacts/`"
+
+      - name: upload amd64 artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: mdb-ch-tools_py${{ matrix.target.python }}_amd64.deb
+          path: ../mdb-ch-tools*.deb
+          if-no-files-found: error
+
+      - name: upload arm64 artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: mdb-ch-tools_py${{ matrix.target.python }}_arm64.deb
+          path: artifacts/mdb-ch-tools*.deb
+          if-no-files-found: error

.github/workflows/workflow.yml

@@ -66,7 +66,7 @@ jobs:
         python-version: ${{ matrix.target.python }}
         cache: pip
 
-    - {name: install dependencies, run: "pip install .[test]" }
+    - {name: install dependencies, run: "pip install '.[test]'" }
     - {name: build project, run: flit build --no-use-vcs }
 
     - name: upload wheel
@@ -88,12 +88,14 @@ jobs:
     - {name: run unit tests, run: pytest}
 
     - name: prepare integration tests
+      # language=sh
       run: |
         cd tests
         CLICKHOUSE_VERSION=${{ matrix.clickhouse }} python3 -m env_control create
         cd ..
 
     - name: run integration tests
+      # language=sh
       run: |
         cd tests
         behave --show-timings --junit -D skip_setup

.gitignore

@@ -11,3 +11,4 @@ cython_debug/
 tests/staging/
 tests/reports/
 .session_conf.sav
+version.txt

Makefile

@@ -4,11 +4,47 @@ PREFIX=/opt/yandex/mdb-ch-tools
 INSTALL_DIR=$(DESTDIR)$(PREFIX)
 
 .PHONY: install
-install: install-symlinks install-bash-completions install-logrotate ;
+install: install-python-package install-symlinks install-bash-completions configure-logs ;
 
 .PHONY: uninstall
 uninstall: uninstall-python-package uninstall-symlinks uninstall-bash-completions uninstall-logrotate ;
 
+.PHONY: install-python-package
+install-python-package: build-python-package
+	@echo 'Installing mdb-ch-tools'
+
+	python3 -m venv $(INSTALL_DIR)
+	rm -f $(INSTALL_DIR)/bin/activate*
+	$(INSTALL_DIR)/bin/pip install -U pip
+
+	$(INSTALL_DIR)/bin/pip install --no-compile chtools.tar.gz
+
+	find $(INSTALL_DIR) -name __pycache__ -type d -exec rm -rf {} +
+	test -n '$(DESTDIR)' \
+		&& grep -l -r -F '#!$(INSTALL_DIR)' $(INSTALL_DIR) \
+			| xargs sed -i -e 's|$(INSTALL_DIR)|$(PREFIX)|' \
+		|| true
+
+.PHONY: build-python-package
+build-python-package: chtools.tar.gz ;
+	@echo 'Cleaning up residuals from building of Python package'
+	rm -rf venv dist
+
+chtools.tar.gz: venv-build prepare-version
+	@echo 'Building Python package'
+
+	venv/bin/flit build --no-use-vcs
+	mv dist/*.tar.gz $@
+
+.PHONY: venv-build
+venv-build: venv/bin/python3 ;
+
+venv/bin/python3:
+	@echo 'Setting up build venv'
+
+	python3 -m venv venv
+	venv/bin/pip install -U pip flit
+
 .PHONY: uninstall-python-package
 uninstall-python-package:
 	@echo 'Uninstalling mdb-ch-tools'
@@ -48,14 +84,14 @@ uninstall-bash-completions:
 		rm -f $(DESTDIR)/etc/bash_completion.d/$(bin) ; \
 	)
 
-.PHONY: install-logrotate
-install-logrotate:
-	@echo 'Creating log rotation rules'
+.PHONY: configure-logs
+configure-logs:
+	@echo 'Configuring logging'
 
 	mkdir -p $(DESTDIR)/etc/logrotate.d/
 	$(foreach bin, chadmin clickhouse-monitoring keeper-monitoring, \
-		mkdir -p $(DESTDIR)/var/log/$(folder) ; \
-		chmod 775 $(DESTDIR)/var/log/$(folder) ; \
+		mkdir -p $(DESTDIR)/var/log/$(bin) ; \
+		chmod 775 $(DESTDIR)/var/log/$(bin) ; \
 		cp resources/logrotate/$(bin).logrotate $(DESTDIR)/etc/logrotate.d/$(bin) ; \
 	)
 
@@ -67,9 +103,9 @@ uninstall-logrotate:
 		rm -f $(DESTDIR)/etc/logrotate.d/$(bin) ; \
 	)
 
-
 .PHONY: prepare-changelog
 prepare-changelog: prepare-version
+	@echo 'Bumping version into Debian package changelog'
 	dch --force-bad-version --distribution stable -v `cat version.txt` Autobuild
 
 .PHONY: prepare-version
@@ -78,29 +114,43 @@ prepare-version: version.txt
 
 version.txt:
 	@echo "2.$$(git rev-list HEAD --count).$$(git rev-parse --short HEAD | perl -ne 'print hex $$_')" > $@
+	@# Replace version in pyproject.toml
+	sed -ie "s/__version__ = \"[0-9\.]\+\"/__version__ = \"`cat $@`\"/" src/chtools/__init__.py
 
+.PHONY: build-deb-package
 build-deb-package: prepare-changelog
 	cd debian && debuild --check-dirname-level 0 --preserve-env --no-lintian --no-tgz-check -uc -us
 
+.PHONY: clean
+clean:
+	@echo 'Cleaning up'
+
+	rm -rf build
+	rm -rf debian/files debian/.debhelper
+	rm -rf debian/mdb-ch-tools*
+
 .PHONY: help
 help:
 	@echo "Base targets:"
 	@echo "  prepare-changelog          Add an autobuild version entity to changelog"
 	@echo "  prepare-version            Update version based on latest commit"
+	@echo "  build-python-package       Build 'ch-tools' Python package"
 	@echo "  build-deb-package          Build 'mdb-ch-tools' debian package"
+	@echo "  clean                      Clean up after building debian package"
 	@echo ""
 	@echo "--------------------------------------------------------------------------------"
 	@echo ""
 	@echo "Debian package build targets:"
 	@echo "  install                    Install 'mdb-ch-tools' debian package"
 	@echo "  uninstall                  Uninstall 'mdb-ch-tools' debian package"
 	@echo ""
+	@echo "  install-python-package     Install 'ch-tools' python package"
 	@echo "  uninstall-python-package   Uninstall 'ch-tools' python package"
 	@echo "  install-symlinks           Install symlinks to /usr/bin/"
 	@echo "  uninstall-symlinks         Uninstall symlinks from /usr/bin/"
 	@echo "  install-bash-completions   Install to /etc/bash_completion.d/"
 	@echo "  uninstall-bash-completions Uninstall from /etc/bash_completion.d/"
-	@echo "  install-logrotate          Install log rotation rules to /etc/logrotate.d/"
+	@echo "  configure-logs             Install log rotation rules to /etc/logrotate.d/ and create log dirs"
 	@echo "  uninstall-logrotate        Uninstall log rotation rules from /etc/logrotate.d/"
 	@echo ""
 	@echo "--------------------------------------------------------------------------------"

debian/control

@@ -11,7 +11,7 @@ Uploaders: Alexander Burmak <[email protected]>,
  Aleksei Filatov <[email protected]>,
  Evgenii Kopanev <[email protected]>,
  Mikhail Kot <[email protected]>
-Build-Depends: debhelper (>= 8.0.0), python3, python3-venv, python3-pip, python3-setuptools
+Build-Depends: debhelper (>= 9.0.0), python3, python3-venv, python3-pip, python3-setuptools
 Standards-Version: 4.1.4
 Homepage: https://github.com/yandex/ch-tools
 Vcs-Browser: https://github.com/yandex/ch-tools.git

pyproject.toml

@@ -4,9 +4,8 @@ build-backend = "flit_core.buildapi"
 
 [project]
 name = "chtools"
-version = "1.0.0"
-description = "A set of tools for administration and diagnostics of ClickHouse DBMS."
 license = { file = "LICENSE" }
+dynamic = ["version", "description"]
 
 authors = [
     {name = "Alexander Burmak", email = "[email protected]"},

src/chtools/__init__.py

@@ -0,0 +1,3 @@
+"""A set of tools for administration and diagnostics of ClickHouse DBMS."""
+
+__version__ = "1.0.0"