From f7dcb33871c131e32f32f12cba3888defdd0ca5f Mon Sep 17 00:00:00 2001 From: Mike Kot Date: Thu, 6 Jul 2023 17:15:08 +0300 Subject: [PATCH] add mypy for ci (#11) --- .github/workflows/workflow.yml | 12 ++--- Makefile | 8 +-- pyproject.toml | 52 ++++++++++--------- src/chtools/chadmin/chadmin_cli.py | 19 ++++--- src/chtools/chadmin/cli/__init__.py | 4 +- src/chtools/chadmin/cli/chs3_backup_group.py | 5 +- src/chtools/chadmin/cli/data_store_group.py | 11 ++-- .../chadmin/cli/diagnostics_command.py | 6 ++- .../chadmin/cli/object_storage_group.py | 12 ++--- .../chadmin/internal/diagnostics/data.py | 14 +++-- .../chadmin/internal/diagnostics/diagnose.py | 4 +- src/chtools/chadmin/internal/system.py | 5 +- src/chtools/common/cli/formatting.py | 2 +- .../clickhouse/client/clickhouse_client.py | 19 +++---- src/chtools/common/clickhouse/client/retry.py | 4 +- src/chtools/monrun_checks/ch_backup.py | 13 +++-- src/chtools/monrun_checks/ch_keeper.py | 2 +- src/chtools/monrun_checks/ch_log_errors.py | 4 +- .../monrun_checks/ch_replication_lag.py | 3 +- src/chtools/monrun_checks/ch_tls.py | 17 +++--- .../monrun_checks/clickhouse_client.py | 9 ++-- .../monrun_checks_keeper/keeper_commands.py | 9 ++-- src/chtools/monrun_checks_keeper/main.py | 1 - src/chtools/s3_credentials/main.py | 2 +- tests/modules/clickhouse.py | 10 ++-- tests/modules/templates.py | 3 +- tests/unit/common/type/test_typed_enum.py | 4 +- 27 files changed, 141 insertions(+), 113 deletions(-) diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index fa34b25f..6a9296fc 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -12,12 +12,10 @@ jobs: python-version: "3.11" cache: pip - - run: make install-deps - - - name: "lint: black" - run: make black - - name: "lint: isort" - run: make isort + - {name: "lint: prepare", run: make install-deps } + - {name: "lint: black", run: make black } + - {name: "lint: isort", run: make isort } + - {name: "lint: mypy", run: make mypy } build: strategy: @@ -48,12 +46,14 @@ jobs: - name: upload wheel uses: actions/upload-artifact@v3 with: + name: chtools-py${{ matrix.target.python }}.whl path: dist/*.whl if-no-files-found: error - name: upload sdist uses: actions/upload-artifact@v3 with: + name: chtools-py${{ matrix.target.python }}.tar.gz path: dist/*.tar.gz if-no-files-found: error diff --git a/Makefile b/Makefile index ece76d5e..74443336 100644 --- a/Makefile +++ b/Makefile @@ -106,12 +106,14 @@ install-deps: pip install flit pip install ".[test]" -lint: black isort #pylint mypy bandit +lint: black isort mypy #pylint bandit -black: install-deps +black: black --check src/ tests/ -isort: install-deps +isort: isort src/ tests/ +mypy: + mypy src/ tests/ --exclude tests/staging build-python-package: install-deps $(dist/*) flit build --no-use-vcs diff --git a/pyproject.toml b/pyproject.toml index 22e0d664..86aef0f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,33 +7,33 @@ name = "chtools" version = "1.0.0" description = "A set of tools for administration and diagnostics of ClickHouse DBMS." license = { file = "LICENSE" } + authors = [ {name = "Alexander Burmak", email = "alex-burmak@yandex-team.ru"}, - #{name = "Dmitry Starov", - #{name = "Anton Ivashkin ", - #{name = "Grigory Pervakov ", - #{name = "Petr Nuzhnov ", - #{name = "Egor Medvedev ", - #{name = "Aleksei Filatov ", - #{name = "Evgeny Dyukov ", - #{name = "Evgeny Strizhnev ", - #{name = "Vadim Volodin ", - #{name = "Anton Chaporgin ", - #{name = "Evgenii Kopanev ", - #{name = "Mikhail Kot ", + {name = "Dmitry Starov", email="dstaroff@yandex-team.ru"}, + {name = "Anton Ivashkin", email="iantonspb@yandex-team.ru"}, + {name = "Grigory Pervakov", email="pervakovg@yandex-team.ru"}, + {name = "Petr Nuzhnov", email="petrnuzhnov@yandex-team.ru"}, + {name = "Egor Medvedev", email="egor-medvedev@yandex-team.ru"}, + {name = "Aleksei Filatov", email="alexfvk@yandex-team.ru"}, + {name = "Evgeny Dyukov", email="secwall@yandex-team.ru"}, + {name = "Evgeny Strizhnev", email="estrizhnev@yandex-team.ru"}, + {name = "Vadim Volodin", email="vadim-volodin@yandex-team.ru"}, + {name = "Anton Chaporgin", email="chapson@yandex-team.ru"}, + {name = "Evgenii Kopanev", email="ekopanev@yandex-team.ru"}, + {name = "Mikhail Kot", email="myrrc@yandex-team.ru"}, ] maintainers = [ {name = "Alexander Burmak", email = "alex-burmak@yandex-team.ru"}, - #"Alexander Burmak ", - #"Dmitry Starov ", - #"Anton Ivashkin ", - #"Grigory Pervakov ", - #"Petr Nuzhnov ", - #"Egor Medvedev ", - #"Aleksei Filatov ", - #"Evgenii Kopanev ", - #"Mikhail Kot ", + {name = "Dmitry Starov", email="dstaroff@yandex-team.ru"}, + {name = "Anton Ivashkin", email="iantonspb@yandex-team.ru"}, + {name = "Grigory Pervakov", email="pervakovg@yandex-team.ru"}, + {name = "Petr Nuzhnov", email="petrnuzhnov@yandex-team.ru"}, + {name = "Egor Medvedev", email="egor-medvedev@yandex-team.ru"}, + {name = "Aleksei Filatov", email="alexfvk@yandex-team.ru"}, + {name = "Evgenii Kopanev", email="ekopanev@yandex-team.ru"}, + {name = "Mikhail Kot", email="myrrc@yandex-team.ru"}, ] readme = "README.md" @@ -91,10 +91,14 @@ Source = "https://github.com/yandex/ch-tools" test = [ "black", "isort", + "mypy", + "types-pyyaml", + "types-requests", + "types-python-dateutil", + "types-tabulate", + "types-pyOpenSSL", + "types-setuptools", #pylint = "^2.13" -#mypy = "^0.971" -#types-pyyaml = "*" -#types-requests = "*" #bandit = "*" "behave", "docker", diff --git a/src/chtools/chadmin/chadmin_cli.py b/src/chtools/chadmin/chadmin_cli.py index bb422735..6c94da44 100755 --- a/src/chtools/chadmin/chadmin_cli.py +++ b/src/chtools/chadmin/chadmin_cli.py @@ -2,7 +2,7 @@ import logging import os import warnings -import os +from typing import Any, List warnings.filterwarnings(action="ignore", message="Python 3.6 is no longer supported") @@ -86,16 +86,15 @@ def cli(ctx, format_, settings, timeout, port, debug): """ClickHouse administration tool.""" os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True) - handlers = [logging.FileHandler(LOG_FILE)] + handlers: List[logging.Handler] = [logging.FileHandler(LOG_FILE)] if debug: handlers.append(logging.StreamHandler()) - log_config = { - "level": logging.DEBUG if debug else logging.INFO, - "format": "%(asctime)s [%(levelname)s]:%(message)s", - "handlers": handlers, - } - logging.basicConfig(**log_config) + logging.basicConfig( + level=logging.DEBUG if debug else logging.INFO, + format="%(asctime)s [%(levelname)s]:%(message)s", + handlers=handlers, + ) timeout_seconds = timeout.total_seconds() if timeout else None settings = {item[0]: item[1] for item in settings} @@ -105,7 +104,7 @@ def cli(ctx, format_, settings, timeout, port, debug): ctx.obj = dict(chcli_conf=ch_cli_conf, format=format_, debug=debug) -commands = [ +commands: List[Any] = [ config_command, diagnostics_command, list_async_metrics_command, @@ -119,7 +118,7 @@ def cli(ctx, format_, settings, timeout, port, debug): wait_started_command, ] -groups = [ +groups: List[Any] = [ chs3_backup_group, crash_log_group, data_store_group, diff --git a/src/chtools/chadmin/cli/__init__.py b/src/chtools/chadmin/cli/__init__.py index 93922951..22b41423 100644 --- a/src/chtools/chadmin/cli/__init__.py +++ b/src/chtools/chadmin/cli/__init__.py @@ -1,7 +1,9 @@ +from click import Context + from chtools.common.clickhouse.config import ClickhouseConfig -def get_config(ctx, try_preprocessed=True) -> ClickhouseConfig: +def get_config(ctx: Context, try_preprocessed: bool = True) -> ClickhouseConfig: if "clickhouse_config" not in ctx.obj: ctx.obj["clickhouse_config"] = ClickhouseConfig.load(try_preprocessed) diff --git a/src/chtools/chadmin/cli/chs3_backup_group.py b/src/chtools/chadmin/cli/chs3_backup_group.py index bde3fc6b..82273f19 100644 --- a/src/chtools/chadmin/cli/chs3_backup_group.py +++ b/src/chtools/chadmin/cli/chs3_backup_group.py @@ -1,7 +1,8 @@ import os +from typing import List import requests -from click import ClickException, argument, group, option, pass_context +from click import ClickException, Context, argument, group, option, pass_context from chtools.chadmin.internal.backup import unfreeze_backup, unfreeze_table from chtools.chadmin.internal.system import match_ch_version @@ -66,7 +67,7 @@ def cleanup_backups(ctx, dry_run, keep_going): ) -def delete_chs3_backups(ctx, chs3_backups: [str], *, keep_going=False, dry_run=False): +def delete_chs3_backups(ctx, chs3_backups, *, keep_going=False, dry_run=False): """ Delete CHS3 backups. """ diff --git a/src/chtools/chadmin/cli/data_store_group.py b/src/chtools/chadmin/cli/data_store_group.py index d57ce11c..7ec429f7 100644 --- a/src/chtools/chadmin/cli/data_store_group.py +++ b/src/chtools/chadmin/cli/data_store_group.py @@ -1,6 +1,7 @@ import os import shutil import subprocess +from typing import Optional from click import group, option @@ -36,7 +37,7 @@ def clean_orphaned_tables_command(column, remove): process_path(path, prefix, column, remove) -def process_path(path: str, prefix: str, column: str, remove: bool): +def process_path(path: str, prefix: str, column: str, remove: bool) -> None: print(f"Processing path {path} with prefix {prefix}:") file = prefix_exists_in_metadata(prefix) @@ -65,7 +66,7 @@ def process_path(path: str, prefix: str, column: str, remove: bool): ) -def prefix_exists_in_metadata(prefix: str): +def prefix_exists_in_metadata(prefix: str) -> Optional[str]: for w in os.walk(CLICKHOUSE_PATH): dirName = w[0] filenames = w[2] @@ -81,7 +82,7 @@ def prefix_exists_in_metadata(prefix: str): return None -def additional_check_successed(column: str, path: str): +def additional_check_successed(column: str, path: str) -> bool: if not column: return False @@ -95,11 +96,11 @@ def additional_check_successed(column: str, path: str): return False -def du(path: str): +def du(path: str) -> str: return subprocess.check_output(["du", "-sh", path]).split()[0].decode("utf-8") -def remove_data(path: str): +def remove_data(path: str) -> None: def onerror(*args): errors = [x for x in args] diff --git a/src/chtools/chadmin/cli/diagnostics_command.py b/src/chtools/chadmin/cli/diagnostics_command.py index 4305791c..2398946f 100644 --- a/src/chtools/chadmin/cli/diagnostics_command.py +++ b/src/chtools/chadmin/cli/diagnostics_command.py @@ -1,5 +1,5 @@ import cloup -from click import pass_context +from click import Context, pass_context from chtools.chadmin.internal.diagnostics.diagnose import diagnose from chtools.common.cli.parameters import env_var_help @@ -28,7 +28,9 @@ + env_var_help("CHADMIN_DIAGNOSTICS_NORMALIZE_QUERIES"), ) @pass_context -def diagnostics_command(ctx, output_format: str, normalize_queries: bool): +def diagnostics_command( + ctx: Context, output_format: str, normalize_queries: bool +) -> None: """ Collect diagnostics data. """ diff --git a/src/chtools/chadmin/cli/object_storage_group.py b/src/chtools/chadmin/cli/object_storage_group.py index 6a0d2026..299f5ea3 100644 --- a/src/chtools/chadmin/cli/object_storage_group.py +++ b/src/chtools/chadmin/cli/object_storage_group.py @@ -4,7 +4,7 @@ import sys from datetime import datetime, timedelta, timezone from gzip import GzipFile -from io import BufferedIOBase, TextIOWrapper +from io import IOBase, TextIOWrapper from pathlib import Path from typing import Dict, Iterator, List, Optional, Union @@ -173,7 +173,7 @@ def list_objects( help="Input stream is compressed using GZIP format", ) @pass_context -def clean_object_storage(ctx: Context, file: BufferedIOBase, compressed: bool) -> None: +def clean_object_storage(ctx, file, compressed): disk_conf: S3DiskConfiguration = ctx.obj["disk_configuration"] if compressed: @@ -189,14 +189,10 @@ def clean_object_storage(ctx: Context, file: BufferedIOBase, compressed: bool) - @contextlib.contextmanager -def dump_writer( - compressed: bool, file_path: Optional[Path] = None -) -> Iterator[Union[BufferedIOBase, GzipFile]]: +def dump_writer(compressed, file_path=None): writer = open(file_path, "wb") if file_path is not None else sys.stdout.buffer - if compressed: - writer = GzipFile(mode="wb", fileobj=writer) try: - yield writer + yield GzipFile(mode="wb", fileobj=writer) if compressed else writer finally: if file_path is not None or compressed: writer.close() diff --git a/src/chtools/chadmin/internal/diagnostics/data.py b/src/chtools/chadmin/internal/diagnostics/data.py index 19fe1950..7ded66f7 100644 --- a/src/chtools/chadmin/internal/diagnostics/data.py +++ b/src/chtools/chadmin/internal/diagnostics/data.py @@ -3,11 +3,12 @@ import json import subprocess import sys +from typing import Any, Dict, List import yaml from requests.exceptions import RequestException -from chtools.common.clickhouse.client import OutputFormat +from chtools.common.clickhouse.client import ClickhouseClient, OutputFormat from .utils import delayed @@ -16,7 +17,7 @@ class DiagnosticsData: def __init__(self, host: str, normalize_queries: bool): self.host = host self.normalize_queries = normalize_queries - self._sections = [{"section": None, "data": {}}] + self._sections: List[Dict[str, Any]] = [{"section": None, "data": {}}] @delayed def add_string(self, name, value, section=None): @@ -193,7 +194,7 @@ def _write_result(buffer_, result, format_=None): @delayed -def add_query(diagnostics, name, client, query, format_: OutputFormat, section=None): +def add_query(diagnostics, name, client, query, format_, section=None): query_args = { "normalize_queries": diagnostics.normalize_queries, } @@ -207,8 +208,11 @@ def add_query(diagnostics, name, client, query, format_: OutputFormat, section=N def execute_query( - client, query, render_query=True, format_: OutputFormat = OutputFormat.Default -): + client: ClickhouseClient, + query: str, + render_query: bool = True, + format_: OutputFormat = OutputFormat.Default, +) -> Any: if render_query: query = client.render_query(query) diff --git a/src/chtools/chadmin/internal/diagnostics/diagnose.py b/src/chtools/chadmin/internal/diagnostics/diagnose.py index 4e7a052b..884e902a 100644 --- a/src/chtools/chadmin/internal/diagnostics/diagnose.py +++ b/src/chtools/chadmin/internal/diagnostics/diagnose.py @@ -1,5 +1,7 @@ from datetime import datetime +from click import Context + import chtools.chadmin.internal.diagnostics.formatter as formatter import chtools.chadmin.internal.diagnostics.query as query from chtools.common.cli.formatting import format_duration @@ -17,7 +19,7 @@ from .data import DiagnosticsData, add_command, add_query, execute_query -def diagnose(ctx, output_format: str, normalize_queries: bool): +def diagnose(ctx: Context, output_format: str, normalize_queries: bool) -> None: timestamp = datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S") client = clickhouse_client(ctx) dbaas_config = DbaasConfig.load() diff --git a/src/chtools/chadmin/internal/system.py b/src/chtools/chadmin/internal/system.py index b5ee212b..7ff21bac 100644 --- a/src/chtools/chadmin/internal/system.py +++ b/src/chtools/chadmin/internal/system.py @@ -1,16 +1,17 @@ +from click import Context from pkg_resources import parse_version from chtools.chadmin.internal.utils import clickhouse_client -def get_version(ctx): +def get_version(ctx: Context) -> str: """ Get ClickHouse version. """ return clickhouse_client(ctx).get_clickhouse_version() -def match_ch_version(ctx, min_version: str) -> bool: +def match_ch_version(ctx: Context, min_version: str) -> bool: """ Returns True if ClickHouse version >= min_version. """ diff --git a/src/chtools/common/cli/formatting.py b/src/chtools/common/cli/formatting.py index f63faf44..75e2300e 100644 --- a/src/chtools/common/cli/formatting.py +++ b/src/chtools/common/cli/formatting.py @@ -41,7 +41,7 @@ def print_header(header): def print_response( - ctx: Context, + ctx, value, format_=None, default_format=None, diff --git a/src/chtools/common/clickhouse/client/clickhouse_client.py b/src/chtools/common/clickhouse/client/clickhouse_client.py index 59429cfc..d6a43053 100644 --- a/src/chtools/common/clickhouse/client/clickhouse_client.py +++ b/src/chtools/common/clickhouse/client/clickhouse_client.py @@ -1,10 +1,11 @@ import logging import socket from datetime import timedelta -from typing import Any, Dict +from typing import Any, Dict, Optional import requests from jinja2 import Environment +from typing_extensions import Self from chtools.common.utils import version_ge @@ -59,15 +60,15 @@ def get_uptime(self): @retry(requests.exceptions.ConnectionError) def query( - self, + self: Self, query: str, - query_args: Dict[str, Any] = None, - format_=None, - post_data=None, - timeout=None, - echo=False, - dry_run=False, - ): + query_args: Optional[Dict[str, Any]] = None, + format_: Optional[str] = None, + post_data: Any = None, + timeout: Optional[int] = None, + echo: bool = False, + dry_run: bool = False, + ) -> Any: """ Execute query. """ diff --git a/src/chtools/common/clickhouse/client/retry.py b/src/chtools/common/clickhouse/client/retry.py index 4877eb39..5aeee551 100644 --- a/src/chtools/common/clickhouse/client/retry.py +++ b/src/chtools/common/clickhouse/client/retry.py @@ -1,4 +1,4 @@ -from typing import Tuple, Type, Union +from typing import Any, Tuple, Type, Union import tenacity @@ -7,7 +7,7 @@ def retry( exception_types: Union[Type[BaseException], Tuple[Type[BaseException]]], max_attempts: int = 5, max_interval: int = 5, -): +) -> Any: """ Function decorator that retries wrapped function on failures. """ diff --git a/src/chtools/monrun_checks/ch_backup.py b/src/chtools/monrun_checks/ch_backup.py index bc67823d..a8b32f2d 100644 --- a/src/chtools/monrun_checks/ch_backup.py +++ b/src/chtools/monrun_checks/ch_backup.py @@ -5,7 +5,7 @@ import json from datetime import datetime, timezone from os.path import exists -from typing import Dict, List +from typing import Dict, List, Optional import click @@ -100,13 +100,16 @@ def check_backup_age(ch_client, backups, age_threshold=1, crit=0): if uptime < age_threshold: return - checking_backup = None + checking_backup: Dict[str, str] = {} for i, backup in enumerate(backups): state = backup["state"] if state == "created" or (state == "creating" and i == 0): checking_backup = backup break + if len(checking_backup) == 0: + die(2, "Didn't find a backup to check") + backup_age = get_backup_age(checking_backup) if backup_age.days < age_threshold: return @@ -168,7 +171,7 @@ def get_backup_age(backup): return datetime.now(timezone.utc) - backup_time -def parse_str_datetime(input: str) -> datetime: +def parse_str_datetime(input: str) -> Optional[datetime]: """ Parse input string to datetime. """ @@ -181,7 +184,9 @@ def parse_str_datetime(input: str) -> datetime: return None -def check_now_pass_threshold(date_time: datetime, hours_threshold: int = 25) -> bool: +def check_now_pass_threshold( + date_time: Optional[datetime], hours_threshold: int = 25 +) -> bool: """ Check that hours threshold is passed since input date """ diff --git a/src/chtools/monrun_checks/ch_keeper.py b/src/chtools/monrun_checks/ch_keeper.py index 655a1a45..ed304e8a 100644 --- a/src/chtools/monrun_checks/ch_keeper.py +++ b/src/chtools/monrun_checks/ch_keeper.py @@ -31,7 +31,7 @@ default=False, help="Allow unverified SSL certificates, e.g. self-signed ones", ) -def keeper_command(retries, timeout, no_verify_ssl_certs) -> Result: +def keeper_command(retries: int, timeout: int, no_verify_ssl_certs: bool) -> Result: """ Check ClickHouse Keeper is alive. """ diff --git a/src/chtools/monrun_checks/ch_log_errors.py b/src/chtools/monrun_checks/ch_log_errors.py index 5cd0cd96..7ef2a747 100644 --- a/src/chtools/monrun_checks/ch_log_errors.py +++ b/src/chtools/monrun_checks/ch_log_errors.py @@ -1,5 +1,5 @@ -import datetime import re +from datetime import datetime, timedelta import click from file_read_backwards import FileReadBackwards @@ -51,7 +51,7 @@ def log_errors_command(crit, warn, watch_seconds, exclude, logfile): """ Check errors in ClickHouse server logs. """ - datetime_start = datetime.now() - datetime.timedelta(seconds=watch_seconds) + datetime_start = datetime.now() - timedelta(seconds=watch_seconds) errors = 0 with FileReadBackwards(logfile, encoding="utf-8") as f: diff --git a/src/chtools/monrun_checks/ch_replication_lag.py b/src/chtools/monrun_checks/ch_replication_lag.py index 6e18bc32..aff3352f 100644 --- a/src/chtools/monrun_checks/ch_replication_lag.py +++ b/src/chtools/monrun_checks/ch_replication_lag.py @@ -1,4 +1,5 @@ import logging +from typing import Any, Dict import click from tabulate import tabulate @@ -184,7 +185,7 @@ def get_replication_lag(): """ tables = get_tables_with_replication_delay() - chart = {} + chart: Dict[str, Dict[str, Any]] = {} for t in tables: key = "{database}.{table}".format(database=t["database"], table=t["table"]) chart[key] = {} diff --git a/src/chtools/monrun_checks/ch_tls.py b/src/chtools/monrun_checks/ch_tls.py index 639601ca..03d87484 100644 --- a/src/chtools/monrun_checks/ch_tls.py +++ b/src/chtools/monrun_checks/ch_tls.py @@ -36,9 +36,9 @@ def tls_command(crit: int, warn: int, ports: Optional[str]) -> Result: for port in get_ports(ports): try: - certificate, days_to_expire = load_certificate_info( - ssl.get_server_certificate((socket.getfqdn(), port)) - ) + addr: Tuple[str, int] = socket.getfqdn(), int(port) + cert: str = ssl.get_server_certificate(addr) + certificate, days_to_expire = load_certificate_info(str.encode(cert)) except Exception as e: return Result(1, f"Failed to get certificate: {repr(e)}") @@ -67,9 +67,12 @@ def read_cert_file() -> Tuple[str, int]: return load_certificate_info(stdout) -def load_certificate_info(certificate) -> Tuple[str, int]: +def load_certificate_info(certificate: bytes) -> Tuple[str, int]: x509 = load_certificate(FILETYPE_PEM, certificate) - expire_date = datetime.strptime( - x509.get_notAfter().decode("ascii"), "%Y%m%d%H%M%SZ" + x509_not_after: Optional[bytes] = x509.get_notAfter() + assert x509_not_after is not None + expire_date = datetime.strptime(x509_not_after.decode("ascii"), "%Y%m%d%H%M%SZ") + return ( + dump_certificate(FILETYPE_PEM, x509).decode(), + (expire_date - datetime.now()).days, ) - return dump_certificate(FILETYPE_PEM, x509), (expire_date - datetime.now()).days diff --git a/src/chtools/monrun_checks/clickhouse_client.py b/src/chtools/monrun_checks/clickhouse_client.py index 36dde886..f6b247fe 100644 --- a/src/chtools/monrun_checks/clickhouse_client.py +++ b/src/chtools/monrun_checks/clickhouse_client.py @@ -3,6 +3,7 @@ import subprocess import xml.etree.ElementTree as xml from enum import Enum +from typing import Dict import requests @@ -35,7 +36,7 @@ def list(cls): class ClickhouseClient: - port_settings = {} + port_settings: Dict[str, str] = {} cert_path = "/etc/clickhouse-server/ssl/allCAs.pem" def __init__(self): @@ -121,19 +122,19 @@ def get_port(self, port): return 0 def __get_settings(self): - result = {} + result: Dict[str, str] = {} try: root = xml.parse("/var/lib/clickhouse/preprocessed_configs/config.xml") for setting in ClickhousePortHelper.list(): node = root.find(setting) if node is not None: - result[ClickhousePortHelper.get(setting)] = node.text + result[ClickhousePortHelper.get(setting)] = str(node.text) self.port_settings = result if not result: die(2, "Can't find any port in clickhouse-server config") node = root.find("./openSSL/server/caConfig") if node is not None: - self.cert_path = node.text + self.cert_path = str(node.text) except FileNotFoundError as e: die(2, f"clickhouse-server config not found: {e.filename}") diff --git a/src/chtools/monrun_checks_keeper/keeper_commands.py b/src/chtools/monrun_checks_keeper/keeper_commands.py index 665d974b..f58d5722 100644 --- a/src/chtools/monrun_checks_keeper/keeper_commands.py +++ b/src/chtools/monrun_checks_keeper/keeper_commands.py @@ -3,6 +3,7 @@ import socket import ssl import time +from typing import Dict from click import command, pass_context from kazoo.client import KazooClient @@ -168,9 +169,11 @@ def get_snapshot_files(snapshots_dir): ] -def read_zookeeper_config(): - """Read Zookeeper configuration file and return content as dict""" - config = {} +def read_zookeeper_config() -> Dict[str, str]: + """ + Read Zookeeper configuration file and return content as dict + """ + config: Dict[str, str] = {} if not os.path.exists(ZOOKEEPER_CFG_FILE): return config with open(ZOOKEEPER_CFG_FILE) as f: diff --git a/src/chtools/monrun_checks_keeper/main.py b/src/chtools/monrun_checks_keeper/main.py index 7e52118d..46774115 100644 --- a/src/chtools/monrun_checks_keeper/main.py +++ b/src/chtools/monrun_checks_keeper/main.py @@ -2,7 +2,6 @@ import os from functools import wraps -import os import click from click import group, option, pass_context diff --git a/src/chtools/s3_credentials/main.py b/src/chtools/s3_credentials/main.py index 14ef6da7..3484cc3e 100644 --- a/src/chtools/s3_credentials/main.py +++ b/src/chtools/s3_credentials/main.py @@ -100,7 +100,7 @@ def _update_config(args): file.write(doc.toprettyxml(indent=4 * " ", encoding="utf-8")) -def _delta_to_hours(delta: timedelta): +def _delta_to_hours(delta: timedelta) -> str: return f"{(delta.total_seconds() / 3600):.2f}" diff --git a/tests/modules/clickhouse.py b/tests/modules/clickhouse.py index 7a526b9a..78ab4c63 100644 --- a/tests/modules/clickhouse.py +++ b/tests/modules/clickhouse.py @@ -3,7 +3,7 @@ """ import logging -from typing import Any, Sequence, Tuple, Union +from typing import Any, Optional, Sequence, Tuple, Union from urllib.parse import urljoin from requests import HTTPError, Session @@ -127,10 +127,10 @@ def _get_all_user_tables(self) -> dict: def _query( self, method: str, - query: str = None, - url: str = None, - params: dict = None, - data: Union[bytes, str] = None, + query: Optional[str] = None, + url: Optional[str] = None, + params: Optional[dict] = None, + data: Union[None, bytes, str] = None, ) -> Any: if url: url = urljoin(self._url, url) diff --git a/tests/modules/templates.py b/tests/modules/templates.py index 813aaa29..85b0adf3 100644 --- a/tests/modules/templates.py +++ b/tests/modules/templates.py @@ -3,6 +3,7 @@ """ import json import os +from typing import Optional from jinja2 import BaseLoader, Environment, FileSystemLoader, StrictUndefined @@ -56,7 +57,7 @@ def _render_file(context: ContextT, directory: str, basename: str) -> None: os.rename(temp_file_path, path) -def _environment(context: ContextT, loader: BaseLoader = None) -> Environment: +def _environment(context: ContextT, loader: Optional[BaseLoader] = None) -> Environment: """ Create Environment object. """ diff --git a/tests/unit/common/type/test_typed_enum.py b/tests/unit/common/type/test_typed_enum.py index 9fb35513..4efddf22 100644 --- a/tests/unit/common/type/test_typed_enum.py +++ b/tests/unit/common/type/test_typed_enum.py @@ -40,9 +40,9 @@ class IEnum(IntEnum): ) def test_typed_enum( inputs: Sequence[TypedEnum], stringified_expected: Sequence[str], summed_expected: T -): +) -> None: stringified: Sequence[str] = [str(i) for i in inputs] assert_that(stringified, equal_to(stringified_expected)) - summed = reduce(lambda a, b: a + b, inputs) + summed = reduce(lambda a, b: a + b, inputs) # type: ignore assert_that(summed, equal_to(summed_expected))