use dockerhub for loading base images (#16)

* use dockerhub to speed up builds * Little cleanup Let's also add `.log` into gitignore * Remove redundant makefile targets from help and deps --------- Co-authored-by: Dmitry Starov <[email protected]>
yandex · Jul 10, 2023 · 5c8e8b2 · 5c8e8b2
1 parent ae9368a
commit 5c8e8b2
Show file tree

Hide file tree

Showing 18 changed files with 191 additions and 180 deletions.
diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml
@@ -1,43 +1,62 @@
-name: on push
+name: main
 on:
   push: { branches: [main] }
   pull_request: { branches: [main] }
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
-  lint: # fail-fast on linting
+  lint:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
-    - name: set up python ${{ matrix.target.python }}
-      uses: actions/setup-python@v4
-      with:
-        python-version: "3.11"
-        cache: pip
-
-    - {name: "lint: prepare", run: make install-deps }
-    - {name: "lint: black", run: make black }
-    - {name: "lint: isort", run: make isort }
-
+    - uses: actions/setup-python@v4
+      with: { python-version: "3.11", cache: pip }
+    - run: pip install .[test]
+    - {name: "lint: black", run: black --check --diff . }
+    - {name: "lint: isort", run: isort --check --diff . }
     # Temporarily disabled due to bugs in typing in click library
     # https://github.com/pallets/click/pull/2559
     # https://github.com/python/mypy/issues/13250
     # https://github.com/python/mypy/issues/13449
     #- {name: "lint: mypy", run: make mypy }
 
+  push_to_dockerhub:
+    runs-on: ubuntu-latest
+    env:
+      CLICKHOUSE_VERSIONS: "21.8.15.7, 22.3.20.29, 22.8.19.10, 23.3.4.17, 23.4.4.16, 23.5.3.24, latest"
+    if: ${{ github.event_name == 'push' }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: login to dockerhub
+      uses: docker/login-action@v2
+      with:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_TOKEN }}
+    - name: build and push base images
+      uses: docker/bake-action@v3
+      with:
+        files: tests/bake.hcl
+        push: true
+
   build:
+    needs: lint
     strategy:
       matrix:
         target:
           - {python: "3.6.15", ubuntu: "20.04"}
           - {python: "3.10.12", ubuntu: "latest"}
+        # a copy-paste of the above as github CI can't use env context in matrices
         clickhouse:
-        - "21.8.15.7"
-        - "22.3.20.29"
-        - "22.8.19.10"
-        - "23.3.4.17"
-        - "23.4.4.16"
-        - "23.5.3.24"
-        - "latest"
+          - "21.8.15.7"
+          - "22.3.20.29"
+          - "22.8.19.10"
+          - "23.3.4.17"
+          - "23.4.4.16"
+          - "23.5.3.24"
+          - "latest"
+
     runs-on: ubuntu-${{ matrix.target.ubuntu }}
     steps:
     - uses: actions/checkout@v3
@@ -47,29 +66,30 @@ jobs:
         python-version: ${{ matrix.target.python }}
         cache: pip
 
-    - run: make install-deps
-    - run: make build-python-package
+    - {name: install dependencies, run: "pip install .[test]" }
+    - {name: build project, run: flit build --no-use-vcs }
 
     - name: upload wheel
       uses: actions/upload-artifact@v3
+      if: ${{ matrix.clickhouse == 'latest' }}
       with:
         name: chtools-py${{ matrix.target.python }}.whl
         path: dist/*.whl
         if-no-files-found: error
 
     - name: upload sdist
       uses: actions/upload-artifact@v3
+      if: ${{ matrix.clickhouse == 'latest' }}
       with:
         name: chtools-py${{ matrix.target.python }}.tar.gz
         path: dist/*.tar.gz
         if-no-files-found: error
 
-    - name: run unit tests
-      run: make test-unit
+    - {name: run unit tests, run: pytest}
 
     - name: prepare docker images for integration tests
       run: CLICKHOUSE_VERSION=${{ matrix.clickhouse }} make test-integration-prepare
-
+        
     - name: run integration tests
       run: make test-integration
 
@@ -78,3 +98,4 @@ jobs:
       if: always()
       with:
         report_paths: 'tests/reports/*.xml'
+
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,5 @@
+.DS_Store
+.idea
 build/
 dist/
 .python-version

diff --git a/Makefile b/Makefile
@@ -4,36 +4,16 @@ PREFIX=/opt/yandex/mdb-ch-tools
 INSTALL_DIR=$(DESTDIR)$(PREFIX)
 
 .PHONY: install
-install: install-python-package install-symlinks install-bash-completions install-logrotate ;
+install: install-symlinks install-bash-completions install-logrotate ;
 
 .PHONY: uninstall
 uninstall: uninstall-python-package uninstall-symlinks uninstall-bash-completions uninstall-logrotate ;
 
-
-.PHONY: install-python-package
-install-python-package:
-	@echo 'Installing mdb-ch-tools'
-
-	python3 -m venv $(INSTALL_DIR)
-	rm -rf $(INSTALL_DIR)/bin/activate*
-
-	$(INSTALL_DIR)/bin/pip install -U pip
-	$(INSTALL_DIR)/bin/pip install -U -r requirements.txt
-	$(INSTALL_DIR)/bin/pip install -U --no-compile .
-
-	find $(INSTALL_DIR) -name __pycache__ -type d -exec rm -rf {} +
-	test -n '$(DESTDIR)' \
-	    && grep -l -r -F '#!$(INSTALL_DIR)' $(INSTALL_DIR) \
-	        | xargs sed -i -e 's|$(INSTALL_DIR)|$(PREFIX)|' \
-	    || true
-
 .PHONY: uninstall-python-package
 uninstall-python-package:
 	@echo 'Uninstalling mdb-ch-tools'
-
 	rm -rf $(INSTALL_DIR)
 
-
 .PHONY: install-symlinks
 install-symlinks:
 	@echo 'Creating symlinks to /usr/bin/'
@@ -68,7 +48,6 @@ uninstall-bash-completions:
 		rm -f $(DESTDIR)/etc/bash_completion.d/$(bin) ; \
 	)
 
-
 .PHONY: install-logrotate
 install-logrotate:
 	@echo 'Creating log rotation rules'
@@ -93,77 +72,38 @@ uninstall-logrotate:
 prepare-changelog: prepare-version
 	dch --force-bad-version --distribution stable -v `cat version.txt` Autobuild
 
-
 .PHONY: prepare-version
 prepare-version: version.txt
 	@echo "Version: `cat version.txt`"
 
-
 version.txt:
 	@echo "2.$$(git rev-list HEAD --count).$$(git rev-parse --short HEAD | perl -ne 'print hex $$_')" > $@
 
-install-deps:
-	pip install flit
-	pip install ".[test]"
-
-lint: black isort mypy #pylint bandit
-
-black:
-	black --check src/ tests/
-isort:
-	isort src/ tests/
-mypy:
-	mypy src/ tests/ --exclude tests/staging
-
-build-python-package: install-deps $(dist/*)
-	flit build --no-use-vcs
-
-build-deb-package: install-deps prepare-changelog
+build-deb-package: prepare-changelog
 	cd debian && debuild --check-dirname-level 0 --preserve-env --no-lintian --no-tgz-check -uc -us
 
-test-unit: install-deps
-	pytest tests/unit
-
 export CLICKHOUSE_VERSION?=latest
 
-test-integration-prepare: install-deps build-python-package
-	cp dist/*.whl tests/ && \
-	cd tests && \
-		CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION} python3 -m env_control create
-
-test-integration: test-integration-prepare
-	cd tests && \
-		behave --show-timings -D skip_setup --junit
-
-.PHONY: clean
-clean:
-	@echo 'Cleaning up'
-
-	rm -f version.txt
-	rm -rf build
-	rm -rf dist
-	rm -rf *.egg-info
-	rm -f debian/files
-	rm -rf debian/mdb-ch-tools*
-	rm -f ../mdb-ch-tools_*
-	find . -name __pycache__ -type d -exec rm -rf {} +
+test-integration-prepare:
+	cp dist/*.whl tests/
+	cd tests && CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION} python3 -m env_control create
 
+test-integration:
+	cd tests && behave --show-timings -D skip_setup --junit
 
 .PHONY: help
 help:
 	@echo "Base targets:"
 	@echo "  prepare-changelog          Add an autobuild version entity to changelog"
 	@echo "  prepare-version            Update version based on latest commit"
 	@echo "  build-deb-package          Build 'mdb-ch-tools' debian package"
-	@echo "  clean                      Clean up after building debian package"
 	@echo ""
 	@echo "--------------------------------------------------------------------------------"
 	@echo ""
 	@echo "Debian package build targets:"
 	@echo "  install                    Install 'mdb-ch-tools' debian package"
 	@echo "  uninstall                  Uninstall 'mdb-ch-tools' debian package"
 	@echo ""
-	@echo "  install-python-package     Install 'ch-tools' python package"
 	@echo "  uninstall-python-package   Uninstall 'ch-tools' python package"
 	@echo "  install-symlinks           Install symlinks to /usr/bin/"
 	@echo "  uninstall-symlinks         Uninstall symlinks from /usr/bin/"

diff --git a/README.md b/README.md
@@ -4,33 +4,30 @@
 
 ## Tools
 
-**ch-tools** consists of following components:
+**ch-tools** consist of following components:
 - [chadmin](./src/chtools/chadmin/README.md) - ClickHouse administration tool
 - [ch-monitoring](./src/chtools/monrun_checks/README.md) - ClickHouse monitoring tool
 - [keeper-monitoring](./src/chtools/monrun_checks_keeper/README.md) - ClickHouse Keeper / ZooKeeper monitoring tool
 - [ch-s3-credentials](./src/chtools/s3_credentials/README.md) - ClickHouse S3 credentials management tool
 
 All of these tools must be run on the same host as ClickHouse server is running.
 
-## Installation
-
-TBD
-
-## Testing
+## Local development (using venv)
 
 ```bash
-cd tests
-make venv
+python3 -m venv venv
 source venv/bin/activate
-make test
-```
+pip install .[test]
+flit install
+flit build --no-use-vcs
+
+# lint
+black .
+isort .
 
-If you don't have access to dbaas infra build cache: replace `prebuild_cmd` in `configuration.py`
+# run tests
+pytest
 
-```python
-'prebuild_cmd': [
-    'mkdir -p images/minio/bin',
-    'wget -N https://dl.min.io/server/minio/release/linux-amd64/minio -O bin/minio',
-    'wget -N https://dl.min.io/client/mc/release/linux-amd64/mc -O bin/mc',
-],
+CLICKHOUSE_VERSION="1.2.3.4" make test-integration-prepare
+make test-integration
 ```
diff --git a/debian/.gitignore b/debian/.gitignore
@@ -0,0 +1 @@
+*.log
diff --git a/pyproject.toml b/pyproject.toml
@@ -50,10 +50,20 @@ classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Intended Audience :: Developers",
     "Intended Audience :: System Administrators",
+    "License :: OSI Approved :: MIT License",
     "Operating System :: MacOS",
     "Operating System :: POSIX :: BSD",
     "Operating System :: POSIX :: Linux",
     "Operating System :: Unix",
+    "Environment :: Console",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.6",
+    "Programming Language :: Python :: 3.7",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: Database",
     "Typing :: Typed",
 ]
@@ -85,11 +95,15 @@ dependencies = [
     "typing-extensions"
 ]
 
+[tool.flit.sdist]
+include = ["resources/"]
+
 [project.urls]
 Source = "https://github.com/yandex/ch-tools"
 
 [project.optional-dependencies]
 test = [
+    "flit",
     "black",
     "isort",
     "mypy",

diff --git a/src/chtools/monrun_checks/ch_backup.py b/src/chtools/monrun_checks/ch_backup.py
@@ -3,7 +3,7 @@
 """
 
 import json
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, timedelta, timezone
 from os.path import exists
 from typing import Dict, List, Optional
 

diff --git a/tests/all.featureset b/tests/all.featureset
diff --git a/tests/bake.hcl b/tests/bake.hcl
@@ -0,0 +1,34 @@
+group "default" {
+  targets = ["http-mock", "minio", "zookeeper", "clickhouse"]
+}
+
+target "http-mock" {
+  dockerfile = "images/http_mock/http-mock.Dockerfile"
+  tags = ["chtools/test-http-mock"]
+  context = "tests"
+}
+
+target "minio" {
+  dockerfile = "images/minio/minio.Dockerfile"
+  tags = ["chtools/test-minio"]
+  context = "tests"
+}
+
+target "zookeeper" {
+  dockerfile = "images/zookeeper/zookeeper.Dockerfile"
+  tags = ["chtools/test-zookeeper"]
+  context = "tests"
+}
+
+variable "CLICKHOUSE_VERSIONS" {
+  default = ""
+}
+
+target "clickhouse" {
+  name = "clickhouse-${replace(ch_version, ".", "_")}"
+  context = "tests"
+  dockerfile = "images/clickhouse/clickhouse.Dockerfile"
+  tags = ["chtools/test-clickhouse:${ch_version}"]
+  matrix = { ch_version = split(",", replace("${CLICKHOUSE_VERSIONS}", " " , "")) }
+  args = { CLICKHOUSE_VERSION = "${ch_version}" }
+}