add mypy for ci (#11)

.github/workflows/workflow.yml

@@ -12,12 +12,10 @@ jobs:
         python-version: "3.11"
         cache: pip
 
-    - run: make install-deps
-
-    - name: "lint: black"
-      run: make black
-    - name: "lint: isort"
-      run: make isort
+    - {name: "lint: prepare", run: make install-deps }
+    - {name: "lint: black", run: make black }
+    - {name: "lint: isort", run: make isort }
+    - {name: "lint: mypy", run: make mypy }
 
   build:
     strategy:
@@ -48,12 +46,14 @@ jobs:
     - name: upload wheel
       uses: actions/upload-artifact@v3
       with:
+        name: chtools-py${{ matrix.target.python }}.whl
         path: dist/*.whl
         if-no-files-found: error
 
     - name: upload sdist
       uses: actions/upload-artifact@v3
       with:
+        name: chtools-py${{ matrix.target.python }}.tar.gz
         path: dist/*.tar.gz
         if-no-files-found: error
 

Makefile

@@ -106,12 +106,14 @@ install-deps:
 	pip install flit
 	pip install ".[test]"
 
-lint: black isort #pylint mypy bandit
+lint: black isort mypy #pylint bandit
 
-black: install-deps
+black:
 	black --check src/ tests/
-isort: install-deps
+isort:
 	isort src/ tests/
+mypy:
+	mypy src/ tests/ --exclude tests/staging
 
 build-python-package: install-deps $(dist/*)
 	flit build --no-use-vcs

pyproject.toml

@@ -7,33 +7,33 @@ name = "chtools"
 version = "1.0.0"
 description = "A set of tools for administration and diagnostics of ClickHouse DBMS."
 license = { file = "LICENSE" }
+
 authors = [
     {name = "Alexander Burmak", email = "[email protected]"},
-    #{name = "Dmitry Starov<[email protected]>",
-    #{name = "Anton Ivashkin <[email protected]>",
-    #{name = "Grigory Pervakov <[email protected]>",
-    #{name = "Petr Nuzhnov <[email protected]>",
-    #{name = "Egor Medvedev <[email protected]>",
-    #{name = "Aleksei Filatov <[email protected]>",
-    #{name = "Evgeny Dyukov <[email protected]>",
-    #{name = "Evgeny Strizhnev <[email protected]>",
-    #{name = "Vadim Volodin <[email protected]>",
-    #{name = "Anton Chaporgin <[email protected]>",
-    #{name = "Evgenii Kopanev <[email protected]>",
-    #{name = "Mikhail Kot <[email protected]>",
+    {name = "Dmitry Starov", email="[email protected]"},
+    {name = "Anton Ivashkin", email="[email protected]"},
+    {name = "Grigory Pervakov", email="[email protected]"},
+    {name = "Petr Nuzhnov", email="[email protected]"},
+    {name = "Egor Medvedev", email="[email protected]"},
+    {name = "Aleksei Filatov", email="[email protected]"},
+    {name = "Evgeny Dyukov", email="[email protected]"},
+    {name = "Evgeny Strizhnev", email="[email protected]"},
+    {name = "Vadim Volodin", email="[email protected]"},
+    {name = "Anton Chaporgin", email="[email protected]"},
+    {name = "Evgenii Kopanev", email="[email protected]"},
+    {name = "Mikhail Kot", email="[email protected]"},
 ]
 
 maintainers = [
     {name = "Alexander Burmak", email = "[email protected]"},
-    #"Alexander Burmak <[email protected]>",
-    #"Dmitry Starov <[email protected]>",
-    #"Anton Ivashkin <[email protected]>",
-    #"Grigory Pervakov <[email protected]>",
-    #"Petr Nuzhnov <[email protected]>",
-    #"Egor Medvedev <[email protected]>",
-    #"Aleksei Filatov <[email protected]>",
-    #"Evgenii Kopanev <[email protected]>",
-    #"Mikhail Kot <[email protected]>",
+    {name = "Dmitry Starov", email="[email protected]"},
+    {name = "Anton Ivashkin", email="[email protected]"},
+    {name = "Grigory Pervakov", email="[email protected]"},
+    {name = "Petr Nuzhnov", email="[email protected]"},
+    {name = "Egor Medvedev", email="[email protected]"},
+    {name = "Aleksei Filatov", email="[email protected]"},
+    {name = "Evgenii Kopanev", email="[email protected]"},
+    {name = "Mikhail Kot", email="[email protected]"},
 ]
 
 readme = "README.md"
@@ -91,10 +91,14 @@ Source = "https://github.com/yandex/ch-tools"
 test = [
     "black",
     "isort",
+    "mypy",
+    "types-pyyaml",
+    "types-requests",
+    "types-python-dateutil",
+    "types-tabulate",
+    "types-pyOpenSSL",
+    "types-setuptools",
 #pylint = "^2.13"
-#mypy = "^0.971"
-#types-pyyaml = "*"
-#types-requests = "*"
 #bandit = "*"
     "behave",
     "docker",

src/chtools/chadmin/chadmin_cli.py

@@ -2,7 +2,7 @@
 import logging
 import os
 import warnings
-import os
+from typing import Any, List
 
 warnings.filterwarnings(action="ignore", message="Python 3.6 is no longer supported")
 
@@ -86,16 +86,15 @@ def cli(ctx, format_, settings, timeout, port, debug):
     """ClickHouse administration tool."""
 
     os.makedirs(os.path.dirname(LOG_FILE), exist_ok=True)
-    handlers = [logging.FileHandler(LOG_FILE)]
+    handlers: List[logging.Handler] = [logging.FileHandler(LOG_FILE)]
     if debug:
         handlers.append(logging.StreamHandler())
 
-    log_config = {
-        "level": logging.DEBUG if debug else logging.INFO,
-        "format": "%(asctime)s [%(levelname)s]:%(message)s",
-        "handlers": handlers,
-    }
-    logging.basicConfig(**log_config)
+    logging.basicConfig(
+        level=logging.DEBUG if debug else logging.INFO,
+        format="%(asctime)s [%(levelname)s]:%(message)s",
+        handlers=handlers,
+    )
 
     timeout_seconds = timeout.total_seconds() if timeout else None
     settings = {item[0]: item[1] for item in settings}
@@ -105,7 +104,7 @@ def cli(ctx, format_, settings, timeout, port, debug):
     ctx.obj = dict(chcli_conf=ch_cli_conf, format=format_, debug=debug)
 
 
-commands = [
+commands: List[Any] = [
     config_command,
     diagnostics_command,
     list_async_metrics_command,
@@ -119,7 +118,7 @@ def cli(ctx, format_, settings, timeout, port, debug):
     wait_started_command,
 ]
 
-groups = [
+groups: List[Any] = [
     chs3_backup_group,
     crash_log_group,
     data_store_group,

src/chtools/chadmin/cli/__init__.py

@@ -1,7 +1,9 @@
+from click import Context
+
 from chtools.common.clickhouse.config import ClickhouseConfig
 
 
-def get_config(ctx, try_preprocessed=True) -> ClickhouseConfig:
+def get_config(ctx: Context, try_preprocessed: bool = True) -> ClickhouseConfig:
     if "clickhouse_config" not in ctx.obj:
         ctx.obj["clickhouse_config"] = ClickhouseConfig.load(try_preprocessed)
 

src/chtools/chadmin/cli/chs3_backup_group.py

@@ -1,7 +1,8 @@
 import os
+from typing import List
 
 import requests
-from click import ClickException, argument, group, option, pass_context
+from click import ClickException, Context, argument, group, option, pass_context
 
 from chtools.chadmin.internal.backup import unfreeze_backup, unfreeze_table
 from chtools.chadmin.internal.system import match_ch_version
@@ -66,7 +67,7 @@ def cleanup_backups(ctx, dry_run, keep_going):
     )
 
 
-def delete_chs3_backups(ctx, chs3_backups: [str], *, keep_going=False, dry_run=False):
+def delete_chs3_backups(ctx, chs3_backups, *, keep_going=False, dry_run=False):
     """
     Delete CHS3 backups.
     """

src/chtools/chadmin/cli/data_store_group.py

@@ -1,6 +1,7 @@
 import os
 import shutil
 import subprocess
+from typing import Optional
 
 from click import group, option
 
@@ -36,7 +37,7 @@ def clean_orphaned_tables_command(column, remove):
         process_path(path, prefix, column, remove)
 
 
-def process_path(path: str, prefix: str, column: str, remove: bool):
+def process_path(path: str, prefix: str, column: str, remove: bool) -> None:
     print(f"Processing path {path} with prefix {prefix}:")
 
     file = prefix_exists_in_metadata(prefix)
@@ -65,7 +66,7 @@ def process_path(path: str, prefix: str, column: str, remove: bool):
     )
 
 
-def prefix_exists_in_metadata(prefix: str):
+def prefix_exists_in_metadata(prefix: str) -> Optional[str]:
     for w in os.walk(CLICKHOUSE_PATH):
         dirName = w[0]
         filenames = w[2]
@@ -81,7 +82,7 @@ def prefix_exists_in_metadata(prefix: str):
     return None
 
 
-def additional_check_successed(column: str, path: str):
+def additional_check_successed(column: str, path: str) -> bool:
     if not column:
         return False
 
@@ -95,11 +96,11 @@ def additional_check_successed(column: str, path: str):
     return False
 
 
-def du(path: str):
+def du(path: str) -> str:
     return subprocess.check_output(["du", "-sh", path]).split()[0].decode("utf-8")
 
 
-def remove_data(path: str):
+def remove_data(path: str) -> None:
     def onerror(*args):
         errors = [x for x in args]
 

src/chtools/chadmin/cli/diagnostics_command.py

@@ -1,5 +1,5 @@
 import cloup
-from click import pass_context
+from click import Context, pass_context
 
 from chtools.chadmin.internal.diagnostics.diagnose import diagnose
 from chtools.common.cli.parameters import env_var_help
@@ -28,7 +28,9 @@
     + env_var_help("CHADMIN_DIAGNOSTICS_NORMALIZE_QUERIES"),
 )
 @pass_context
-def diagnostics_command(ctx, output_format: str, normalize_queries: bool):
+def diagnostics_command(
+    ctx: Context, output_format: str, normalize_queries: bool
+) -> None:
     """
     Collect diagnostics data.
     """

src/chtools/chadmin/cli/object_storage_group.py

@@ -4,7 +4,7 @@
 import sys
 from datetime import datetime, timedelta, timezone
 from gzip import GzipFile
-from io import BufferedIOBase, TextIOWrapper
+from io import IOBase, TextIOWrapper
 from pathlib import Path
 from typing import Dict, Iterator, List, Optional, Union
 
@@ -173,7 +173,7 @@ def list_objects(
     help="Input stream is compressed using GZIP format",
 )
 @pass_context
-def clean_object_storage(ctx: Context, file: BufferedIOBase, compressed: bool) -> None:
+def clean_object_storage(ctx, file, compressed):
     disk_conf: S3DiskConfiguration = ctx.obj["disk_configuration"]
 
     if compressed:
@@ -189,14 +189,10 @@ def clean_object_storage(ctx: Context, file: BufferedIOBase, compressed: bool) -
 
 
 @contextlib.contextmanager
-def dump_writer(
-    compressed: bool, file_path: Optional[Path] = None
-) -> Iterator[Union[BufferedIOBase, GzipFile]]:
+def dump_writer(compressed, file_path=None):
     writer = open(file_path, "wb") if file_path is not None else sys.stdout.buffer
-    if compressed:
-        writer = GzipFile(mode="wb", fileobj=writer)
     try:
-        yield writer
+        yield GzipFile(mode="wb", fileobj=writer) if compressed else writer
     finally:
         if file_path is not None or compressed:
             writer.close()

src/chtools/chadmin/internal/diagnostics/data.py

@@ -3,11 +3,12 @@
 import json
 import subprocess
 import sys
+from typing import Any, Dict, List
 
 import yaml
 from requests.exceptions import RequestException
 
-from chtools.common.clickhouse.client import OutputFormat
+from chtools.common.clickhouse.client import ClickhouseClient, OutputFormat
 
 from .utils import delayed
 
@@ -16,7 +17,7 @@ class DiagnosticsData:
     def __init__(self, host: str, normalize_queries: bool):
         self.host = host
         self.normalize_queries = normalize_queries
-        self._sections = [{"section": None, "data": {}}]
+        self._sections: List[Dict[str, Any]] = [{"section": None, "data": {}}]
 
     @delayed
     def add_string(self, name, value, section=None):
@@ -193,7 +194,7 @@ def _write_result(buffer_, result, format_=None):
 
 
 @delayed
-def add_query(diagnostics, name, client, query, format_: OutputFormat, section=None):
+def add_query(diagnostics, name, client, query, format_, section=None):
     query_args = {
         "normalize_queries": diagnostics.normalize_queries,
     }
@@ -207,8 +208,11 @@ def add_query(diagnostics, name, client, query, format_: OutputFormat, section=N
 
 
 def execute_query(
-    client, query, render_query=True, format_: OutputFormat = OutputFormat.Default
-):
+    client: ClickhouseClient,
+    query: str,
+    render_query: bool = True,
+    format_: OutputFormat = OutputFormat.Default,
+) -> Any:
     if render_query:
         query = client.render_query(query)
 

src/chtools/chadmin/internal/diagnostics/diagnose.py

@@ -1,5 +1,7 @@
 from datetime import datetime
 
+from click import Context
+
 import chtools.chadmin.internal.diagnostics.formatter as formatter
 import chtools.chadmin.internal.diagnostics.query as query
 from chtools.common.cli.formatting import format_duration
@@ -17,7 +19,7 @@
 from .data import DiagnosticsData, add_command, add_query, execute_query
 
 
-def diagnose(ctx, output_format: str, normalize_queries: bool):
+def diagnose(ctx: Context, output_format: str, normalize_queries: bool) -> None:
     timestamp = datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S")
     client = clickhouse_client(ctx)
     dbaas_config = DbaasConfig.load()

src/chtools/chadmin/internal/system.py

@@ -1,16 +1,17 @@
+from click import Context
 from pkg_resources import parse_version
 
 from chtools.chadmin.internal.utils import clickhouse_client
 
 
-def get_version(ctx):
+def get_version(ctx: Context) -> str:
     """
     Get ClickHouse version.
     """
     return clickhouse_client(ctx).get_clickhouse_version()
 
 
-def match_ch_version(ctx, min_version: str) -> bool:
+def match_ch_version(ctx: Context, min_version: str) -> bool:
     """
     Returns True if ClickHouse version >= min_version.
     """

src/chtools/common/cli/formatting.py

@@ -41,7 +41,7 @@ def print_header(header):
 
 
 def print_response(
-    ctx: Context,
+    ctx,
     value,
     format_=None,
     default_format=None,