diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 4c4a95b7..4f890c1c 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -1,43 +1,62 @@ -name: on push +name: main on: push: { branches: [main] } pull_request: { branches: [main] } +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true jobs: - lint: # fail-fast on linting + lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: set up python ${{ matrix.target.python }} - uses: actions/setup-python@v4 - with: - python-version: "3.11" - cache: pip - - - {name: "lint: prepare", run: make install-deps } - - {name: "lint: black", run: make black } - - {name: "lint: isort", run: make isort } - + - uses: actions/setup-python@v4 + with: { python-version: "3.11", cache: pip } + - run: pip install .[test] + - {name: "lint: black", run: black --check --diff . } + - {name: "lint: isort", run: isort --check --diff . } # Temporarily disabled due to bugs in typing in click library # https://github.com/pallets/click/pull/2559 # https://github.com/python/mypy/issues/13250 # https://github.com/python/mypy/issues/13449 #- {name: "lint: mypy", run: make mypy } + push_to_dockerhub: + runs-on: ubuntu-latest + env: + CLICKHOUSE_VERSIONS: "21.8.15.7, 22.3.20.29, 22.8.19.10, 23.3.4.17, 23.4.4.16, 23.5.3.24, latest" + if: ${{ github.event_name == 'push' }} + steps: + - uses: actions/checkout@v3 + - name: login to dockerhub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: build and push base images + uses: docker/bake-action@v3 + with: + files: tests/bake.hcl + push: true + build: + needs: lint strategy: matrix: target: - {python: "3.6.15", ubuntu: "20.04"} - {python: "3.10.12", ubuntu: "latest"} + # a copy-paste of the above as github CI can't use env context in matrices clickhouse: - - "21.8.15.7" - - "22.3.20.29" - - "22.8.19.10" - - "23.3.4.17" - - "23.4.4.16" - - "23.5.3.24" - - "latest" + - "21.8.15.7" + - "22.3.20.29" + - "22.8.19.10" + - "23.3.4.17" + - "23.4.4.16" + - "23.5.3.24" + - "latest" + runs-on: ubuntu-${{ matrix.target.ubuntu }} steps: - uses: actions/checkout@v3 @@ -47,11 +66,12 @@ jobs: python-version: ${{ matrix.target.python }} cache: pip - - run: make install-deps - - run: make build-python-package + - {name: install dependencies, run: "pip install .[test]" } + - {name: build project, run: flit build --no-use-vcs } - name: upload wheel uses: actions/upload-artifact@v3 + if: ${{ matrix.clickhouse == 'latest' }} with: name: chtools-py${{ matrix.target.python }}.whl path: dist/*.whl @@ -59,17 +79,17 @@ jobs: - name: upload sdist uses: actions/upload-artifact@v3 + if: ${{ matrix.clickhouse == 'latest' }} with: name: chtools-py${{ matrix.target.python }}.tar.gz path: dist/*.tar.gz if-no-files-found: error - - name: run unit tests - run: make test-unit + - {name: run unit tests, run: pytest} - name: prepare docker images for integration tests run: CLICKHOUSE_VERSION=${{ matrix.clickhouse }} make test-integration-prepare - + - name: run integration tests run: make test-integration @@ -78,3 +98,4 @@ jobs: if: always() with: report_paths: 'tests/reports/*.xml' + diff --git a/.gitignore b/.gitignore index 3abf5b58..29ca99e0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +.DS_Store +.idea build/ dist/ .python-version diff --git a/Makefile b/Makefile index 74443336..90c0a613 100644 --- a/Makefile +++ b/Makefile @@ -4,36 +4,16 @@ PREFIX=/opt/yandex/mdb-ch-tools INSTALL_DIR=$(DESTDIR)$(PREFIX) .PHONY: install -install: install-python-package install-symlinks install-bash-completions install-logrotate ; +install: install-symlinks install-bash-completions install-logrotate ; .PHONY: uninstall uninstall: uninstall-python-package uninstall-symlinks uninstall-bash-completions uninstall-logrotate ; - -.PHONY: install-python-package -install-python-package: - @echo 'Installing mdb-ch-tools' - - python3 -m venv $(INSTALL_DIR) - rm -rf $(INSTALL_DIR)/bin/activate* - - $(INSTALL_DIR)/bin/pip install -U pip - $(INSTALL_DIR)/bin/pip install -U -r requirements.txt - $(INSTALL_DIR)/bin/pip install -U --no-compile . - - find $(INSTALL_DIR) -name __pycache__ -type d -exec rm -rf {} + - test -n '$(DESTDIR)' \ - && grep -l -r -F '#!$(INSTALL_DIR)' $(INSTALL_DIR) \ - | xargs sed -i -e 's|$(INSTALL_DIR)|$(PREFIX)|' \ - || true - .PHONY: uninstall-python-package uninstall-python-package: @echo 'Uninstalling mdb-ch-tools' - rm -rf $(INSTALL_DIR) - .PHONY: install-symlinks install-symlinks: @echo 'Creating symlinks to /usr/bin/' @@ -68,7 +48,6 @@ uninstall-bash-completions: rm -f $(DESTDIR)/etc/bash_completion.d/$(bin) ; \ ) - .PHONY: install-logrotate install-logrotate: @echo 'Creating log rotation rules' @@ -93,61 +72,24 @@ uninstall-logrotate: prepare-changelog: prepare-version dch --force-bad-version --distribution stable -v `cat version.txt` Autobuild - .PHONY: prepare-version prepare-version: version.txt @echo "Version: `cat version.txt`" - version.txt: @echo "2.$$(git rev-list HEAD --count).$$(git rev-parse --short HEAD | perl -ne 'print hex $$_')" > $@ -install-deps: - pip install flit - pip install ".[test]" - -lint: black isort mypy #pylint bandit - -black: - black --check src/ tests/ -isort: - isort src/ tests/ -mypy: - mypy src/ tests/ --exclude tests/staging - -build-python-package: install-deps $(dist/*) - flit build --no-use-vcs - -build-deb-package: install-deps prepare-changelog +build-deb-package: prepare-changelog cd debian && debuild --check-dirname-level 0 --preserve-env --no-lintian --no-tgz-check -uc -us -test-unit: install-deps - pytest tests/unit - export CLICKHOUSE_VERSION?=latest -test-integration-prepare: install-deps build-python-package - cp dist/*.whl tests/ && \ - cd tests && \ - CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION} python3 -m env_control create - -test-integration: test-integration-prepare - cd tests && \ - behave --show-timings -D skip_setup --junit - -.PHONY: clean -clean: - @echo 'Cleaning up' - - rm -f version.txt - rm -rf build - rm -rf dist - rm -rf *.egg-info - rm -f debian/files - rm -rf debian/mdb-ch-tools* - rm -f ../mdb-ch-tools_* - find . -name __pycache__ -type d -exec rm -rf {} + +test-integration-prepare: + cp dist/*.whl tests/ + cd tests && CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION} python3 -m env_control create +test-integration: + cd tests && behave --show-timings -D skip_setup --junit .PHONY: help help: @@ -155,7 +97,6 @@ help: @echo " prepare-changelog Add an autobuild version entity to changelog" @echo " prepare-version Update version based on latest commit" @echo " build-deb-package Build 'mdb-ch-tools' debian package" - @echo " clean Clean up after building debian package" @echo "" @echo "--------------------------------------------------------------------------------" @echo "" @@ -163,7 +104,6 @@ help: @echo " install Install 'mdb-ch-tools' debian package" @echo " uninstall Uninstall 'mdb-ch-tools' debian package" @echo "" - @echo " install-python-package Install 'ch-tools' python package" @echo " uninstall-python-package Uninstall 'ch-tools' python package" @echo " install-symlinks Install symlinks to /usr/bin/" @echo " uninstall-symlinks Uninstall symlinks from /usr/bin/" diff --git a/README.md b/README.md index f8b173c2..9fb1f559 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ## Tools -**ch-tools** consists of following components: +**ch-tools** consist of following components: - [chadmin](./src/chtools/chadmin/README.md) - ClickHouse administration tool - [ch-monitoring](./src/chtools/monrun_checks/README.md) - ClickHouse monitoring tool - [keeper-monitoring](./src/chtools/monrun_checks_keeper/README.md) - ClickHouse Keeper / ZooKeeper monitoring tool @@ -12,25 +12,22 @@ All of these tools must be run on the same host as ClickHouse server is running. -## Installation - -TBD - -## Testing +## Local development (using venv) ```bash -cd tests -make venv +python3 -m venv venv source venv/bin/activate -make test -``` +pip install .[test] +flit install +flit build --no-use-vcs + +# lint +black . +isort . -If you don't have access to dbaas infra build cache: replace `prebuild_cmd` in `configuration.py` +# run tests +pytest -```python -'prebuild_cmd': [ - 'mkdir -p images/minio/bin', - 'wget -N https://dl.min.io/server/minio/release/linux-amd64/minio -O bin/minio', - 'wget -N https://dl.min.io/client/mc/release/linux-amd64/mc -O bin/mc', -], +CLICKHOUSE_VERSION="1.2.3.4" make test-integration-prepare +make test-integration ``` diff --git a/debian/.gitignore b/debian/.gitignore new file mode 100644 index 00000000..397b4a76 --- /dev/null +++ b/debian/.gitignore @@ -0,0 +1 @@ +*.log diff --git a/pyproject.toml b/pyproject.toml index 0d29092c..8b1db502 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,10 +50,20 @@ classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: System Administrators", + "License :: OSI Approved :: MIT License", "Operating System :: MacOS", "Operating System :: POSIX :: BSD", "Operating System :: POSIX :: Linux", "Operating System :: Unix", + "Environment :: Console", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Database", "Typing :: Typed", ] @@ -85,11 +95,15 @@ dependencies = [ "typing-extensions" ] +[tool.flit.sdist] +include = ["resources/"] + [project.urls] Source = "https://github.com/yandex/ch-tools" [project.optional-dependencies] test = [ + "flit", "black", "isort", "mypy", diff --git a/src/chtools/monrun_checks/ch_backup.py b/src/chtools/monrun_checks/ch_backup.py index 65c9c053..315eb08b 100644 --- a/src/chtools/monrun_checks/ch_backup.py +++ b/src/chtools/monrun_checks/ch_backup.py @@ -3,7 +3,7 @@ """ import json -from datetime import datetime, timezone, timedelta +from datetime import datetime, timedelta, timezone from os.path import exists from typing import Dict, List, Optional diff --git a/tests/all.featureset b/tests/all.featureset deleted file mode 100644 index 4f637374..00000000 --- a/tests/all.featureset +++ /dev/null @@ -1,3 +0,0 @@ -features/s3_credentials.feature -features/monrun.feature -features/monrun_keeper.feature diff --git a/tests/bake.hcl b/tests/bake.hcl new file mode 100644 index 00000000..7b774f07 --- /dev/null +++ b/tests/bake.hcl @@ -0,0 +1,34 @@ +group "default" { + targets = ["http-mock", "minio", "zookeeper", "clickhouse"] +} + +target "http-mock" { + dockerfile = "images/http_mock/http-mock.Dockerfile" + tags = ["chtools/test-http-mock"] + context = "tests" +} + +target "minio" { + dockerfile = "images/minio/minio.Dockerfile" + tags = ["chtools/test-minio"] + context = "tests" +} + +target "zookeeper" { + dockerfile = "images/zookeeper/zookeeper.Dockerfile" + tags = ["chtools/test-zookeeper"] + context = "tests" +} + +variable "CLICKHOUSE_VERSIONS" { + default = "" +} + +target "clickhouse" { + name = "clickhouse-${replace(ch_version, ".", "_")}" + context = "tests" + dockerfile = "images/clickhouse/clickhouse.Dockerfile" + tags = ["chtools/test-clickhouse:${ch_version}"] + matrix = { ch_version = split(",", replace("${CLICKHOUSE_VERSIONS}", " " , "")) } + args = { CLICKHOUSE_VERSION = "${ch_version}" } +} diff --git a/tests/configuration.py b/tests/configuration.py index c1b90adf..19447648 100644 --- a/tests/configuration.py +++ b/tests/configuration.py @@ -19,7 +19,6 @@ def create(): "depends_on": ["zookeeper"], "args": { "CLICKHOUSE_VERSION": "$CLICKHOUSE_VERSION", - "NETWORK_NAME": network_name, }, "db": { "user": "reader", diff --git a/tests/images/clickhouse/Dockerfile b/tests/images/clickhouse/Dockerfile index 587bb9f7..e8024662 100644 --- a/tests/images/clickhouse/Dockerfile +++ b/tests/images/clickhouse/Dockerfile @@ -1,50 +1,5 @@ ARG CLICKHOUSE_VERSION -FROM clickhouse/clickhouse-server:$CLICKHOUSE_VERSION -ARG NETWORK_NAME - -# Clickhouse changed deb package url so old builds' paths (e.g 21.8.15.7) yield an error on update -RUN rm /etc/apt/sources.list.d/clickhouse.list || true -RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ - apt-get install -y supervisor python3-pip && \ - rm -rf /var/lib/apt/lists/* /var/cache/debconf && \ - apt-get clean - -COPY images/clickhouse/config/monitor-ch-backup /etc/sudoers.d/monitor-ch-backup -COPY images/clickhouse/config/regions_names_ru.txt /opt/geo/regions_names_ru.txt -COPY images/clickhouse/config/regions_hierarchy.txt /opt/geo/regions_hierarchy.txt - -RUN rm -rf /etc/supervisor && \ - ln -s /config/supervisor /etc/supervisor && \ - mkdir /var/log/monrun && \ - ln -s /config/dbaas.conf /etc/dbaas.conf && \ - mkdir -p /etc/yandex/ch-backup && \ - ln -s /config/ch-backup.conf /etc/yandex/ch-backup/ch-backup.conf && \ - ln -s /config/config.xml /etc/clickhouse-server/config.d/ && \ - rm /etc/clickhouse-server/users.xml && \ - ln -s /config/users.xml /etc/clickhouse-server/ && \ - \ - mkdir -p /etc/clickhouse-server/ssl && \ - openssl rand -writerand /root/.rnd && \ - openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 \ - -keyout /etc/clickhouse-server/ssl/server.key \ - -addext "subjectAltName=DNS:clickhouse01,DNS:clickhouse02,DNS:clickhouse01.$NETWORK_NAME,DNS:clickhouse02.$NETWORK_NAME" \ - -out /etc/clickhouse-server/ssl/server.crt && \ - ln -s /etc/clickhouse-server/ssl/server.crt /etc/clickhouse-server/ssl/allCAs.pem && \ - \ - groupadd monitor && useradd -g monitor -G monitor,clickhouse monitor && \ - mkdir /var/log/clickhouse-monitoring && \ - chown monitor:monitor -R /var/log/clickhouse-monitoring && \ - mkdir /var/log/keeper-monitoring && \ - chown monitor:monitor -R /var/log/keeper-monitoring && \ - chmod 755 /var/lib/clickhouse && \ - chmod 777 /etc && \ - chmod 755 /etc/clickhouse-server && \ - chmod 750 /etc/clickhouse-server/ssl && \ - chmod 640 /etc/clickhouse-server/ssl/server.crt && \ - chmod 640 /etc/clickhouse-server/ssl/allCAs.pem && \ - chown -R clickhouse:clickhouse /etc/clickhouse-server/ /usr/bin/clickhouse +FROM chtools/test-clickhouse:$CLICKHOUSE_VERSION COPY *.whl / RUN pip3 install *.whl - -ENTRYPOINT ["supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/tests/images/clickhouse/clickhouse.Dockerfile b/tests/images/clickhouse/clickhouse.Dockerfile new file mode 100644 index 00000000..4ef7fa71 --- /dev/null +++ b/tests/images/clickhouse/clickhouse.Dockerfile @@ -0,0 +1,48 @@ +ARG CLICKHOUSE_VERSION +FROM clickhouse/clickhouse-server:${CLICKHOUSE_VERSION} + +# Clickhouse changed deb package url so old builds' paths (e.g 21.8.15.7) yield an error on update +# TODO this should be removed once 21.8 is not supported +RUN rm /etc/apt/sources.list.d/clickhouse.list || true + +RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ + apt-get install -y supervisor python3-pip && \ + rm -rf /var/lib/apt/lists/* /var/cache/debconf && \ + apt-get clean + +COPY images/clickhouse/config/monitor-ch-backup /etc/sudoers.d/monitor-ch-backup +COPY images/clickhouse/config/regions_names_ru.txt /opt/geo/regions_names_ru.txt +COPY images/clickhouse/config/regions_hierarchy.txt /opt/geo/regions_hierarchy.txt + +RUN rm -rf /etc/supervisor && \ + ln -s /config/supervisor /etc/supervisor && \ + mkdir /var/log/monrun && \ + ln -s /config/dbaas.conf /etc/dbaas.conf && \ + mkdir -p /etc/yandex/ch-backup && \ + ln -s /config/ch-backup.conf /etc/yandex/ch-backup/ch-backup.conf && \ + ln -s /config/config.xml /etc/clickhouse-server/config.d/ && \ + rm /etc/clickhouse-server/users.xml && \ + ln -s /config/users.xml /etc/clickhouse-server/ && \ + \ + mkdir -p /etc/clickhouse-server/ssl && \ + openssl rand -writerand /root/.rnd && \ + openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 \ + -keyout /etc/clickhouse-server/ssl/server.key \ + -addext "subjectAltName=DNS:clickhouse01,DNS:clickhouse02,DNS:clickhouse01.ch_tools_test,DNS:clickhouse02.ch_tools_test" \ + -out /etc/clickhouse-server/ssl/server.crt && \ + ln -s /etc/clickhouse-server/ssl/server.crt /etc/clickhouse-server/ssl/allCAs.pem && \ + \ + groupadd monitor && useradd -g monitor -G monitor,clickhouse monitor && \ + mkdir /var/log/clickhouse-monitoring && \ + chown monitor:monitor -R /var/log/clickhouse-monitoring && \ + mkdir /var/log/keeper-monitoring && \ + chown monitor:monitor -R /var/log/keeper-monitoring && \ + chmod 755 /var/lib/clickhouse && \ + chmod 777 /etc && \ + chmod 755 /etc/clickhouse-server && \ + chmod 750 /etc/clickhouse-server/ssl && \ + chmod 640 /etc/clickhouse-server/ssl/server.crt && \ + chmod 640 /etc/clickhouse-server/ssl/allCAs.pem && \ + chown -R clickhouse:clickhouse /etc/clickhouse-server/ /usr/bin/clickhouse + +ENTRYPOINT ["supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/tests/images/http_mock/Dockerfile b/tests/images/http_mock/Dockerfile index 978dd9a1..f48206d8 100644 --- a/tests/images/http_mock/Dockerfile +++ b/tests/images/http_mock/Dockerfile @@ -1,5 +1 @@ -FROM python:3-alpine -RUN pip3 install flask --no-cache -ENV FLASK_APP /service.py -COPY images/http_mock/service.py / -CMD ["python3", "-m", "flask", "run", "--host=0.0.0.0", "--port=8080"] +FROM chtools/test-http-mock diff --git a/tests/images/http_mock/http-mock.Dockerfile b/tests/images/http_mock/http-mock.Dockerfile new file mode 100644 index 00000000..978dd9a1 --- /dev/null +++ b/tests/images/http_mock/http-mock.Dockerfile @@ -0,0 +1,5 @@ +FROM python:3-alpine +RUN pip3 install flask --no-cache +ENV FLASK_APP /service.py +COPY images/http_mock/service.py / +CMD ["python3", "-m", "flask", "run", "--host=0.0.0.0", "--port=8080"] diff --git a/tests/images/minio/Dockerfile b/tests/images/minio/Dockerfile index a39d53c8..331c4a11 100644 --- a/tests/images/minio/Dockerfile +++ b/tests/images/minio/Dockerfile @@ -1,7 +1,3 @@ -FROM alpine -RUN apk add --no-cache minio minio-client -COPY images/minio/config/mc.json /root/.mcli/config.json +FROM chtools/test-minio ENV MINIO_ACCESS_KEY {{ conf.s3.access_key_id }} ENV MINIO_SECRET_KEY {{ conf.s3.access_secret_key }} -VOLUME ["/export"] -CMD ["minio", "server", "/export"] diff --git a/tests/images/minio/minio.Dockerfile b/tests/images/minio/minio.Dockerfile new file mode 100644 index 00000000..ba5da80c --- /dev/null +++ b/tests/images/minio/minio.Dockerfile @@ -0,0 +1,5 @@ +FROM alpine +RUN apk add --no-cache minio minio-client +COPY images/minio/config/mc.json /root/.mcli/config.json +VOLUME ["/export"] +CMD ["minio", "server", "/export"] diff --git a/tests/images/zookeeper/Dockerfile b/tests/images/zookeeper/Dockerfile index 277e53f6..168d64ec 100644 --- a/tests/images/zookeeper/Dockerfile +++ b/tests/images/zookeeper/Dockerfile @@ -1,14 +1,4 @@ -FROM ubuntu/zookeeper - -RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y supervisor python3-pip && \ - rm -rf /var/lib/apt/lists/* /var/cache/debconf && \ - apt-get clean - -COPY images/zookeeper/config/zookeeper.conf /etc/supervisor/supervisord.conf -COPY images/zookeeper/config/zoo.cfg /etc/zookeeper/conf/zoo.cfg -COPY images/zookeeper/config/log4j.properties /etc/zookeeper/conf/log4j.properties +FROM chtools/test-zookeeper COPY *.whl / RUN python3 -m pip install *.whl - -ENTRYPOINT ["supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/tests/images/zookeeper/zookeeper.Dockerfile b/tests/images/zookeeper/zookeeper.Dockerfile new file mode 100644 index 00000000..7e9a577e --- /dev/null +++ b/tests/images/zookeeper/zookeeper.Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu/zookeeper + +RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y supervisor python3-pip && \ + rm -rf /var/lib/apt/lists/* /var/cache/debconf && \ + apt-get clean + +COPY images/zookeeper/config/zookeeper.conf /etc/supervisor/supervisord.conf +COPY images/zookeeper/config/zoo.cfg /etc/zookeeper/conf/zoo.cfg +COPY images/zookeeper/config/log4j.properties /etc/zookeeper/conf/log4j.properties + +ENTRYPOINT ["supervisord", "-c", "/etc/supervisor/supervisord.conf"]