diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml new file mode 100644 index 00000000..fa34b25f --- /dev/null +++ b/.github/workflows/workflow.yml @@ -0,0 +1,73 @@ +name: on push +on: [push] + +jobs: + lint: # fail-fast on linting + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: set up python ${{ matrix.target.python }} + uses: actions/setup-python@v4 + with: + python-version: "3.11" + cache: pip + + - run: make install-deps + + - name: "lint: black" + run: make black + - name: "lint: isort" + run: make isort + + build: + strategy: + matrix: + target: + - {python: "3.6.15", ubuntu: "20.04"} + - {python: "3.10.12", ubuntu: "latest"} + clickhouse: + - "21.8.15.7" + - "22.3.20.29" + - "22.8.19.10" + - "23.3.4.17" + - "23.4.4.16" + - "23.5.3.24" + - "latest" + runs-on: ubuntu-${{ matrix.target.ubuntu }} + steps: + - uses: actions/checkout@v3 + - name: set up python ${{ matrix.target.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.target.python }} + cache: pip + + - run: make install-deps + - run: make build-python-package + + - name: upload wheel + uses: actions/upload-artifact@v3 + with: + path: dist/*.whl + if-no-files-found: error + + - name: upload sdist + uses: actions/upload-artifact@v3 + with: + path: dist/*.tar.gz + if-no-files-found: error + + - name: run unit tests + run: make test-unit + + - name: prepare docker images for integration tests + run: CLICKHOUSE_VERSION=${{ matrix.clickhouse }} make test-integration-prepare + + - name: run integration tests + run: make test-integration + + - name: publish test report + uses: mikepenz/action-junit-report@v3 + if: always() + with: + report_paths: 'tests/reports/*.xml' diff --git a/.gitignore b/.gitignore index 2ceaaabc..3abf5b58 100644 --- a/.gitignore +++ b/.gitignore @@ -1,81 +1,11 @@ -# MacOS -.DS_Store - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python build/ -develop-eggs/ dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# pyenv .python-version - -# poetry -poetry.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Environments -.env -.venv -env/ +__pycache__ venv/ -ENV/ -env.bak/ -venv.bak/ - -# mypy .mypy_cache/ -.dmypy.json -dmypy.json - -# pytype static type analyzer .pytype/ - -# Cython debug symbols cython_debug/ - -# JetBrains IDEs -.idea +tests/staging/ +tests/reports/ +.session_conf.sav diff --git a/tests/.pylintrc b/.pylintrc similarity index 100% rename from tests/.pylintrc rename to .pylintrc diff --git a/tests/.style.yapf b/.style.yapf similarity index 100% rename from tests/.style.yapf rename to .style.yapf diff --git a/Makefile b/Makefile index a1ee7332..ece76d5e 100644 --- a/Makefile +++ b/Makefile @@ -102,22 +102,42 @@ prepare-version: version.txt version.txt: @echo "2.$$(git rev-list HEAD --count).$$(git rev-parse --short HEAD | perl -ne 'print hex $$_')" > $@ +install-deps: + pip install flit + pip install ".[test]" -.PHONY: build-deb-package -build-deb-package: prepare-changelog - @echo 'Building debian package' +lint: black isort #pylint mypy bandit - rm -f pyproject.toml # for forcing usage of setup.py - cd debian && \ - debuild --check-dirname-level 0 --preserve-env --no-lintian --no-tgz-check -uc -us +black: install-deps + black --check src/ tests/ +isort: install-deps + isort src/ tests/ +build-python-package: install-deps $(dist/*) + flit build --no-use-vcs + +build-deb-package: install-deps prepare-changelog + cd debian && debuild --check-dirname-level 0 --preserve-env --no-lintian --no-tgz-check -uc -us + +test-unit: install-deps + pytest tests/unit + +export CLICKHOUSE_VERSION?=latest + +test-integration-prepare: install-deps build-python-package + cp dist/*.whl tests/ && \ + cd tests && \ + CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION} python3 -m env_control create + +test-integration: test-integration-prepare + cd tests && \ + behave --show-timings -D skip_setup --junit .PHONY: clean clean: @echo 'Cleaning up' rm -f version.txt - rm -rf venv rm -rf build rm -rf dist rm -rf *.egg-info diff --git a/tests/mypy.ini b/mypy.ini similarity index 100% rename from tests/mypy.ini rename to mypy.ini diff --git a/pyproject.toml b/pyproject.toml index 09ccd8f4..22e0d664 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,48 +1,43 @@ [build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +requires = ["flit_core >=3.2,<4"] +build-backend = "flit_core.buildapi" -[tool.poetry] -name = "ch-tools" +[project] +name = "chtools" version = "1.0.0" description = "A set of tools for administration and diagnostics of ClickHouse DBMS." -license = "MIT" +license = { file = "LICENSE" } authors = [ - "Alexander Burmak ", - "Dmitry Starov ", - "Anton Ivashkin ", - "Grigory Pervakov ", - "Petr Nuzhnov ", - "Egor Medvedev ", - "Aleksei Filatov ", - "Evgeny Dyukov ", - "Evgeny Strizhnev ", - "Vadim Volodin ", - "Anton Chaporgin ", - "Evgenii Kopanev ", - "Mikhail Kot ", + {name = "Alexander Burmak", email = "alex-burmak@yandex-team.ru"}, + #{name = "Dmitry Starov", + #{name = "Anton Ivashkin ", + #{name = "Grigory Pervakov ", + #{name = "Petr Nuzhnov ", + #{name = "Egor Medvedev ", + #{name = "Aleksei Filatov ", + #{name = "Evgeny Dyukov ", + #{name = "Evgeny Strizhnev ", + #{name = "Vadim Volodin ", + #{name = "Anton Chaporgin ", + #{name = "Evgenii Kopanev ", + #{name = "Mikhail Kot ", ] + maintainers = [ - "Alexander Burmak ", - "Dmitry Starov ", - "Anton Ivashkin ", - "Grigory Pervakov ", - "Petr Nuzhnov ", - "Egor Medvedev ", - "Aleksei Filatov ", - "Evgenii Kopanev ", - "Mikhail Kot ", -] -readme = [ - "README.md", - "src/chtools/chadmin/README.md", - "src/chtools/monrun_checks/README.md", - "src/chtools/monrun_checks_keeper/README.md", - "src/chtools/s3_credentials/README.md", + {name = "Alexander Burmak", email = "alex-burmak@yandex-team.ru"}, + #"Alexander Burmak ", + #"Dmitry Starov ", + #"Anton Ivashkin ", + #"Grigory Pervakov ", + #"Petr Nuzhnov ", + #"Egor Medvedev ", + #"Aleksei Filatov ", + #"Evgenii Kopanev ", + #"Mikhail Kot ", ] -#homepage="" -repository="https://github.com/yandex/ch-tools" -#documentation="" + +readme = "README.md" + keywords=[ "clickhouse", "database", @@ -50,6 +45,7 @@ keywords=[ "diagnostics", "administration", ] + classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", @@ -61,62 +57,53 @@ classifiers = [ "Topic :: Database", "Typing :: Typed", ] -packages=[ - { include="chtools", from="src" }, -] -include=[ - "LICENSE", - "resources" + +requires-python = ">=3.6.2" + +dependencies = [ + "boto3", + "click", + "cloup", + "deepdiff", + "dnspython", + "file_read_backwards", + "humanfriendly", + "jinja2", + "kazoo", + "lxml", + "psutil", + "pygments", + "pyopenssl", + "python-dateutil", + "pyyaml", + "requests", + "tabulate", + "tenacity", + "termcolor", + "tqdm", + "xmltodict" ] -#exclude=[ -# "tests", -# "debian", -#] -[tool.poetry.dependencies] -python = "^3.6.2" -boto3 = "^1.23" -click = "^8" -cloup = "^0.15" -deepdiff = "^5" -dnspython = "^2" -file_read_backwards = "^3" -humanfriendly = "^10" -jinja2 = "^3" -kazoo = "^2.9" -lxml = "^4.9" -psutil = "^5.9" -pygments = "^2" -pyopenssl = "^23.2" -python-dateutil = "^2" -pyyaml = "^5" -requests = "^2.27" -tabulate = "^0.8" -tenacity = "^8" -termcolor = "^1" -tqdm = "^4" -xmltodict = "^0.13" +[project.urls] +Source = "https://github.com/yandex/ch-tools" -[tool.poetry.group.dev.dependencies] -behave = "^1" -black = "*" -docker = "^5" -docker-compose = "^1.29" -flake8 = "^5" -flake8-commas = "*" -flake8-print = "*" -flake8-quotes = "*" -flake8-string-format = "*" -isort = "^5" -mypy = "^0.971" -pep8 = "^1.7" -pyhamcrest = "^2" -pylint = "^2.13" -pytest = "^7.0" -types-pyyaml = "*" -types-requests = "*" +[project.optional-dependencies] +test = [ + "black", + "isort", +#pylint = "^2.13" +#mypy = "^0.971" +#types-pyyaml = "*" +#types-requests = "*" +#bandit = "*" + "behave", + "docker", + "docker-compose", + "pyhamcrest", + "pytest" +] -[tool.poetry.scripts] +[project.scripts] chadmin = "chtools.chadmin.chadmin_cli:main" ch-monitoring = "chtools.monrun_checks.main:main" keeper-monitoring = "chtools.monrun_checks_keeper.main:main" diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index bad0f54f..00000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,15 +0,0 @@ -behave >= 1.2.0 -black >= 22.8.0 -docker >= 5.0.0 -docker-compose >= 1.29.0 -flake8 >= 5.0.0 -flake8-commas >= 2.1.0 -flake8-print >= 4.0.0 -flake8-quotes >= 3.3.0 -flake8-string-format >= 0.3.0 -isort >= 5.10.0 -mypy >= 0.971 -pep8 >= 1.7.0 -PyHamcrest >= 2.0.0 -pylint >= 2.13.0 -pytest >= 7.0.0 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index df37517a..00000000 --- a/requirements.txt +++ /dev/null @@ -1,22 +0,0 @@ -Jinja2 >= 3.0.0 -PyYAML >= 5.4.0, < 6 -Pygments >= 2.14.0 -boto3 >= 1.23.0 -click >= 8.0.0 -cloup >= 0.15.0 -deepdiff >= 5.7.0 -dnspython >= 2.2.0 -humanfriendly >= 10.0 -kazoo >= 2.9 -lxml >= 4.9.0 -psutil >= 5.9.0 -pyOpenSSL >= 23.2.0 -python-dateutil >= 2.8.0 -requests >= 2.27.0 -setuptools >= 59.6.0 -tabulate >= 0.8.0 -tenacity >= 8.2.0 -termcolor >= 1.1.0 -tqdm >= 4.64.0 -xmltodict >= 0.13.0 -file_read_backwards>=3.0.0 diff --git a/setup.py b/setup.py deleted file mode 100644 index 1ca39f5b..00000000 --- a/setup.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -# encoding: utf-8 -""" -Package configuration for ch-tools. -""" - -from setuptools import find_packages, setup - -with open('requirements.txt') as f: - REQUIREMENTS = list(map(lambda l: l.strip('\n'), f.readlines())) - -with open('version.txt') as f: - VERSION = f.read().strip() - -setup( - name='ch-tools', - version=VERSION, - description='A set of tools for administration and diagnostics of ClickHouse DBMS.', - license='MIT', - url='https://github.com/yandex/ch-tools', - author='Yandex LLC', - author_email='opensource@yandex-team.ru', - zip_safe=False, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Intended Audience :: System Administrators", - "License :: OSI Approved :: MIT License", - "Operating System :: MacOS", - "Operating System :: POSIX :: BSD", - "Operating System :: POSIX :: Linux", - "Operating System :: Unix", - "Environment :: Console", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Database", - "Typing :: Typed", - ], - platforms=['Linux', 'BSD', 'MacOS'], - packages=find_packages(where='src'), - package_dir={'': 'src'}, - package_data={'': ['version.txt']}, - entry_points={ - 'console_scripts': [ - 'chadmin = chtools.chadmin.chadmin_cli:main', - 'ch-monitoring = chtools.monrun_checks.main:main', - 'keeper-monitoring = chtools.monrun_checks_keeper.main:main', - 'ch-s3-credentials = chtools.s3_credentials.main:main', - ] - }, - install_requires=REQUIREMENTS, -) diff --git a/src/chtools/chadmin/chadmin_cli.py b/src/chtools/chadmin/chadmin_cli.py index da80254e..bb422735 100755 --- a/src/chtools/chadmin/chadmin_cli.py +++ b/src/chtools/chadmin/chadmin_cli.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import logging +import os import warnings import os diff --git a/src/chtools/chadmin/cli/disk_group.py b/src/chtools/chadmin/cli/disk_group.py index 10f2f68d..1ea013c7 100644 --- a/src/chtools/chadmin/cli/disk_group.py +++ b/src/chtools/chadmin/cli/disk_group.py @@ -25,7 +25,7 @@ def check_s3_metadata_command(path, cleanup): def check_dir(path, cleanup): corrupted_dirs = [] - for (dirpath, _, filenames) in os.walk(path): + for dirpath, _, filenames in os.walk(path): for filename in filenames: if not check_file(f"{dirpath}/{filename}"): print(f"{dirpath}/{filename}") diff --git a/src/chtools/chadmin/cli/part_log_group.py b/src/chtools/chadmin/cli/part_log_group.py index 24dd131a..01c14c37 100644 --- a/src/chtools/chadmin/cli/part_log_group.py +++ b/src/chtools/chadmin/cli/part_log_group.py @@ -74,7 +74,7 @@ def _table_formatter(record): max_date=max_date, min_time=min_time, max_time=max_time, - **kwargs + **kwargs, ) print_response( diff --git a/src/chtools/chadmin/cli/query_log_group.py b/src/chtools/chadmin/cli/query_log_group.py index 7cf43863..6fd40b46 100644 --- a/src/chtools/chadmin/cli/query_log_group.py +++ b/src/chtools/chadmin/cli/query_log_group.py @@ -101,7 +101,7 @@ def list_queries_command( max_date=max_date, min_time=min_time, max_time=max_time, - **kwargs + **kwargs, ) ) diff --git a/src/chtools/monrun_checks_keeper/main.py b/src/chtools/monrun_checks_keeper/main.py index 12e0e5aa..7e52118d 100644 --- a/src/chtools/monrun_checks_keeper/main.py +++ b/src/chtools/monrun_checks_keeper/main.py @@ -1,4 +1,5 @@ import logging +import os from functools import wraps import os diff --git a/tests/.flake8 b/tests/.flake8 deleted file mode 100644 index 1f88a276..00000000 --- a/tests/.flake8 +++ /dev/null @@ -1,9 +0,0 @@ -[flake8] -; ignoring the checks: -; E122 (continuation line missing indentation or outdented), -; E126 (continuation line over-indented for hanging indent), -; E131 (continuation line unaligned for hanging indent) as indentation is responsibility of yapf -; F401 (module imported but unused) in favor of corresponding pylint check -select = E,F,W,C,S -ignore = E122,E126,E131,F401 -max-line-length = 120 diff --git a/tests/Makefile b/tests/Makefile deleted file mode 100644 index 481b63b0..00000000 --- a/tests/Makefile +++ /dev/null @@ -1,104 +0,0 @@ -VERSION=1.0 -SESSION_FILE=.session_conf.sav - -export PYTHONIOENCODING?=utf8 -export CLICKHOUSE_VERSION?=latest - -ENV=env -i \ - PATH=venv/bin:$$PATH \ - PYTHONIOENCODING=${PYTHONIOENCODING} \ - CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION} \ - DOCKER_HOST=$$DOCKER_HOST \ - DOCKER_TLS_VERIFY=$$DOCKER_TLS_VERIFY \ - DOCKER_CERT_PATH=$$DOCKER_CERT_PATH - -.PHONY: build -build: staging/tools/ - -staging/tools/: build-deb-package - mkdir -p staging/tools/ - cp ../../mdb-ch-tools_*.deb staging/tools/mdb-ch-tools.deb - -.PHONY: build-deb-package -build-deb-package: - cd ../ && make build-deb-package - -.PHONY: all -all: build lint test - -.PHONY: lint -lint: isort flake8 pylint mypy - -.PHONY: isort -isort: venv - venv/bin/isort --check-only --ignore-whitespace --project modules --diff modules steps *.py - -.PHONY: flake8 -flake8: venv - venv/bin/flake8 modules steps *.py - -.PHONY: pylint -pylint: venv - venv/bin/pylint --reports=no --score=no modules steps *.py - -.PHONY: mypy -mypy: venv - venv/bin/mypy modules steps *.py - -.PHONY: format -format: venv - venv/bin/isort --project modules modules steps *.py - -.PHONY: test -test: create_env - ${ENV} behave --show-timings --stop -D skip_setup @all.featureset - -.PHONY: clean -clean: clean_env clean_pycache - rm -rf venv .cache *.egg-info htmlcov .coverage* - -.PHONY: clean_pycache -clean_pycache: - find . -name __pycache__ -type d -exec rm -rf {} + - -.PHONY: create_env -create_env: venv build ${SESSION_FILE} - -${SESSION_FILE}: - ${ENV} python3 -m env_control create - -.PHONY: start_env -start_env: create_env - ${ENV} python3 -m env_control start - -.PHONY: stop_env -stop_env: - test -d venv/bin && test -f ${SESSION_FILE} && ${ENV} python3 -m env_control stop || true - -.PHONY: clean_env -clean_env: stop_env - rm -rf staging ${SESSION_FILE} - cd ../ && make clean uninstall - -venv: requirements.txt - python3 -m venv venv - venv/bin/pip install --no-cache-dir --disable-pip-version-check -r requirements.txt - -.PHONY: help -help: - @echo "Targets:" - @echo " build (default) Build mdb-ch-tools package." - @echo " all Alias for \"build lint test\"." - @echo " lint Run linter tools." - @echo " test Run tests." - @echo " clean Clean up build and test artifacts." - @echo " create_env Create test environment." - @echo " start_env Start test environment runtime." - @echo " stop_env Stop test environment runtime." - @echo " clean_env Clean up test environment." - @echo " format Re-format source code to conform style settings enforced by" - @echo " isort tool." - @echo " help Show this help message." - @echo - @echo "Environment Variables:" - @echo " CLICKHOUSE_VERSION ClickHouse version to use in test target (default: \"$(CLICKHOUSE_VERSION)\")." diff --git a/tests/images/clickhouse/Dockerfile b/tests/images/clickhouse/Dockerfile index 9c92a9f1..587bb9f7 100644 --- a/tests/images/clickhouse/Dockerfile +++ b/tests/images/clickhouse/Dockerfile @@ -5,7 +5,7 @@ ARG NETWORK_NAME # Clickhouse changed deb package url so old builds' paths (e.g 21.8.15.7) yield an error on update RUN rm /etc/apt/sources.list.d/clickhouse.list || true RUN DEBIAN_FRONTEND=noninteractive apt-get update && \ - apt-get install -y supervisor && \ + apt-get install -y supervisor python3-pip && \ rm -rf /var/lib/apt/lists/* /var/cache/debconf && \ apt-get clean @@ -32,6 +32,10 @@ RUN rm -rf /etc/supervisor && \ ln -s /etc/clickhouse-server/ssl/server.crt /etc/clickhouse-server/ssl/allCAs.pem && \ \ groupadd monitor && useradd -g monitor -G monitor,clickhouse monitor && \ + mkdir /var/log/clickhouse-monitoring && \ + chown monitor:monitor -R /var/log/clickhouse-monitoring && \ + mkdir /var/log/keeper-monitoring && \ + chown monitor:monitor -R /var/log/keeper-monitoring && \ chmod 755 /var/lib/clickhouse && \ chmod 777 /etc && \ chmod 755 /etc/clickhouse-server && \ @@ -40,8 +44,7 @@ RUN rm -rf /etc/supervisor && \ chmod 640 /etc/clickhouse-server/ssl/allCAs.pem && \ chown -R clickhouse:clickhouse /etc/clickhouse-server/ /usr/bin/clickhouse -COPY staging/tools/ /tmp/tools/ -RUN dpkg -i /tmp/tools/mdb-ch-tools.deb && \ - chown monitor:monitor -R /var/log/clickhouse-monitoring +COPY *.whl / +RUN pip3 install *.whl ENTRYPOINT ["supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/tests/images/clickhouse/config/users.xml b/tests/images/clickhouse/config/users.xml index a8bb1c24..b0e53ba9 100644 --- a/tests/images/clickhouse/config/users.xml +++ b/tests/images/clickhouse/config/users.xml @@ -2,7 +2,7 @@ - 5 + 20 diff --git a/tests/images/zookeeper/Dockerfile b/tests/images/zookeeper/Dockerfile index 7b613473..277e53f6 100644 --- a/tests/images/zookeeper/Dockerfile +++ b/tests/images/zookeeper/Dockerfile @@ -1,14 +1,14 @@ FROM ubuntu/zookeeper -COPY staging/tools/ /tmp/tools/ -RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y supervisor && \ +RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y supervisor python3-pip && \ rm -rf /var/lib/apt/lists/* /var/cache/debconf && \ apt-get clean -RUN dpkg -i /tmp/tools/mdb-ch-tools.deb - COPY images/zookeeper/config/zookeeper.conf /etc/supervisor/supervisord.conf COPY images/zookeeper/config/zoo.cfg /etc/zookeeper/conf/zoo.cfg COPY images/zookeeper/config/log4j.properties /etc/zookeeper/conf/log4j.properties +COPY *.whl / +RUN python3 -m pip install *.whl + ENTRYPOINT ["supervisord", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index 626aad11..00000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,22 +0,0 @@ -requests -PyYAML < 6 -tenacity -isort -flake8 -flake8-string-format -flake8-commas -flake8-quotes -flake8-print -pep8 -pylint -mypy -pytest -pyhamcrest -behave -docker -docker-compose -Jinja2 -humanfriendly -kazoo >= 2.9 -types-requests -types-PyYAML