diff --git a/ch_tools/chadmin/chadmin_cli.py b/ch_tools/chadmin/chadmin_cli.py index 8e0348e2..4218aa06 100755 --- a/ch_tools/chadmin/chadmin_cli.py +++ b/ch_tools/chadmin/chadmin_cli.py @@ -4,8 +4,8 @@ import cloup +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.cli.move_group import move_group -from ch_tools.common import logging from ch_tools.common.config import load_config warnings.filterwarnings(action="ignore", message="Python 3.6 is no longer supported") @@ -54,6 +54,7 @@ @cloup.group( "chadmin", + cls=Chadmin, help="ClickHouse administration tool.", context_settings=CONTEXT_SETTINGS, ) @@ -82,8 +83,6 @@ def cli(ctx, format_, settings, timeout, port, debug): """ClickHouse administration tool.""" config = load_config() - logging.configure(config["loguru"], "chadmin") - if port: config["clickhouse"]["port"] = port if timeout: @@ -137,10 +136,12 @@ def cli(ctx, format_, settings, timeout, port, debug): zookeeper_group, ] -for section_title, entries in {"Commands": commands, "Groups": groups}.items(): - section = cloup.Section(section_title) - for entry in entries: - cli.add_command(entry, section=section) +section = cloup.Section("Commands") +for command in commands: + cli.add_command(command, section=section) +section = cloup.Section("Groups") +for group in groups: + cli.add_group(group, section=section) def main(): diff --git a/ch_tools/chadmin/cli/chadmin_group.py b/ch_tools/chadmin/cli/chadmin_group.py new file mode 100644 index 00000000..9d43a628 --- /dev/null +++ b/ch_tools/chadmin/cli/chadmin_group.py @@ -0,0 +1,78 @@ +from functools import wraps +from typing import Optional + +import click +import cloup + +from ch_tools import __version__ +from ch_tools.common import logging + +# pylint: disable=too-many-ancestors + + +class Chadmin(cloup.Group): + def add_command( + self, + cmd: click.Command, + name: Optional[str] = None, + section: Optional[cloup.Section] = None, + fallback_to_default_section: bool = True, + ) -> None: + if cmd.callback is None: + super().add_command( + cmd, + name=name, + section=section, + fallback_to_default_section=fallback_to_default_section, + ) + return + + cmd_callback = cmd.callback + + @wraps(cmd_callback) + @cloup.pass_context + def wrapper(ctx, *a, **kw): + logging.configure( + ctx.obj["config"]["loguru"], "chadmin", {"cmd_name": cmd.name} + ) + + logging.debug( + "Executing command '{}', params: {}, args: {}, version: {}", + cmd.name, + { + **ctx.parent.params, + **ctx.params, + }, + ctx.args, + __version__, + ) + + try: + result = cmd_callback(*a, **kw) + logging.debug("Command '{}' completed", cmd.name) + except Exception: + logging.exception("Command '{}' failed with error:", cmd.name) + + return result + + cmd.callback = wrapper + super().add_command( + cmd, + name=name, + section=section, + fallback_to_default_section=fallback_to_default_section, + ) + + def add_group( + self, + cmd: click.Group, + name: Optional[str] = None, + section: Optional[cloup.Section] = None, + fallback_to_default_section: bool = True, + ) -> None: + super().add_command( + cmd, + name=name, + section=section, + fallback_to_default_section=fallback_to_default_section, + ) diff --git a/ch_tools/chadmin/cli/chs3_backup_group.py b/ch_tools/chadmin/cli/chs3_backup_group.py index 0fb9e294..f7749119 100644 --- a/ch_tools/chadmin/cli/chs3_backup_group.py +++ b/ch_tools/chadmin/cli/chs3_backup_group.py @@ -2,6 +2,7 @@ from click import ClickException, argument, group, option, pass_context +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.backup import unfreeze_backup from ch_tools.common import logging from ch_tools.common.backup import ( @@ -12,7 +13,7 @@ from ch_tools.common.utils import clear_empty_directories_recursively -@group("chs3-backup") +@group("chs3-backup", cls=Chadmin) def chs3_backup_group(): """Commands to manage ClickHouse over S3 backups (backups for data stored in S3).""" pass diff --git a/ch_tools/chadmin/cli/crash_log_group.py b/ch_tools/chadmin/cli/crash_log_group.py index 5b4ab29e..8b8123e8 100644 --- a/ch_tools/chadmin/cli/crash_log_group.py +++ b/ch_tools/chadmin/cli/crash_log_group.py @@ -1,11 +1,12 @@ from click import group, option, pass_context +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.utils import execute_query from ch_tools.common import logging from ch_tools.common.clickhouse.config import get_cluster_name -@group("crash-log") +@group("crash-log", cls=Chadmin) def crash_log_group(): """ Commands for retrieving information from system.crash_log. diff --git a/ch_tools/chadmin/cli/data_store_group.py b/ch_tools/chadmin/cli/data_store_group.py index c1f74da5..2e338a6f 100644 --- a/ch_tools/chadmin/cli/data_store_group.py +++ b/ch_tools/chadmin/cli/data_store_group.py @@ -5,6 +5,7 @@ from click import group, option, pass_context +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.clickhouse_disks import ( CLICKHOUSE_METADATA_PATH, CLICKHOUSE_PATH, @@ -18,7 +19,7 @@ from ch_tools.common.process_pool import WorkerTask, execute_tasks_in_parallel -@group("data-store") +@group("data-store", cls=Chadmin) def data_store_group(): """ Commands for manipulating data stored by ClickHouse. diff --git a/ch_tools/chadmin/cli/database_group.py b/ch_tools/chadmin/cli/database_group.py index e34f1413..246a3f71 100644 --- a/ch_tools/chadmin/cli/database_group.py +++ b/ch_tools/chadmin/cli/database_group.py @@ -1,12 +1,13 @@ from cloup import argument, group, option, option_group, pass_context from cloup.constraints import RequireAtLeast +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.utils import execute_query from ch_tools.common import logging from ch_tools.common.clickhouse.config import get_cluster_name -@group("database") +@group("database", cls=Chadmin) def database_group(): """Commands to manage databases.""" pass diff --git a/ch_tools/chadmin/cli/dictionary_group.py b/ch_tools/chadmin/cli/dictionary_group.py index 90afbe41..6562a8e2 100644 --- a/ch_tools/chadmin/cli/dictionary_group.py +++ b/ch_tools/chadmin/cli/dictionary_group.py @@ -1,11 +1,12 @@ from click import group, option, pass_context +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.dictionary import list_dictionaries, reload_dictionary from ch_tools.common import logging from ch_tools.common.cli.formatting import print_response -@group("dictionary") +@group("dictionary", cls=Chadmin) def dictionary_group(): """Commands to manage external dictionaries.""" pass diff --git a/ch_tools/chadmin/cli/disk_group.py b/ch_tools/chadmin/cli/disk_group.py index b0620e9b..6ba74a72 100644 --- a/ch_tools/chadmin/cli/disk_group.py +++ b/ch_tools/chadmin/cli/disk_group.py @@ -4,10 +4,11 @@ from click import group, option +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.common import logging -@group("disks") +@group("disks", cls=Chadmin) def disks_group(): """Commands to manage disks.""" pass diff --git a/ch_tools/chadmin/cli/merge_group.py b/ch_tools/chadmin/cli/merge_group.py index a2dd5846..fb344581 100644 --- a/ch_tools/chadmin/cli/merge_group.py +++ b/ch_tools/chadmin/cli/merge_group.py @@ -2,6 +2,7 @@ from click import group, option, pass_context +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.process import list_merges from ch_tools.common.cli.formatting import ( format_bytes, @@ -21,7 +22,7 @@ } -@group("merge") +@group("merge", cls=Chadmin) def merge_group(): """Commands to manage merges (retrieve information from system.merges).""" pass diff --git a/ch_tools/chadmin/cli/move_group.py b/ch_tools/chadmin/cli/move_group.py index a61e9a8f..9e462208 100644 --- a/ch_tools/chadmin/cli/move_group.py +++ b/ch_tools/chadmin/cli/move_group.py @@ -2,6 +2,7 @@ from cloup import group, option, pass_context +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.process import list_moves from ch_tools.common.cli.formatting import format_bytes, format_float, print_response from ch_tools.common.clickhouse.config import get_cluster_name @@ -12,7 +13,7 @@ } -@group("move") +@group("move", cls=Chadmin) def move_group(): """Commands to manage moves (retrieve information from system.moves).""" pass diff --git a/ch_tools/chadmin/cli/mutation_group.py b/ch_tools/chadmin/cli/mutation_group.py index 4c1991ec..d1fb6eca 100644 --- a/ch_tools/chadmin/cli/mutation_group.py +++ b/ch_tools/chadmin/cli/mutation_group.py @@ -1,11 +1,12 @@ from click import argument, group, option, pass_context +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.utils import execute_query from ch_tools.common import logging from ch_tools.common.clickhouse.config import get_cluster_name -@group("mutation") +@group("mutation", cls=Chadmin) def mutation_group(): """ Commands to manage mutations. diff --git a/ch_tools/chadmin/cli/object_storage_group.py b/ch_tools/chadmin/cli/object_storage_group.py index 5844eca7..c7e89875 100644 --- a/ch_tools/chadmin/cli/object_storage_group.py +++ b/ch_tools/chadmin/cli/object_storage_group.py @@ -5,6 +5,7 @@ from click import Context, group, option, pass_context from humanfriendly import format_size +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.common.cli.formatting import print_response from ch_tools.common.cli.parameters import TimeSpanParamType from ch_tools.common.clickhouse.config import get_clickhouse_config @@ -17,7 +18,7 @@ ORPHANED_OBJECTS_SIZE_FIELD = "orphaned_objects_size" -@group("object-storage") +@group("object-storage", cls=Chadmin) @option( "-d", "--disk", diff --git a/ch_tools/chadmin/cli/part_group.py b/ch_tools/chadmin/cli/part_group.py index b57bf4bb..ec0ef35e 100644 --- a/ch_tools/chadmin/cli/part_group.py +++ b/ch_tools/chadmin/cli/part_group.py @@ -3,6 +3,7 @@ from cloup import Choice, group, option, option_group, pass_context from cloup.constraints import RequireAtLeast +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.part import ( attach_part, detach_part, @@ -22,7 +23,7 @@ } -@group("part") +@group("part", cls=Chadmin) def part_group(): """ Commands to manage data parts. diff --git a/ch_tools/chadmin/cli/part_log_group.py b/ch_tools/chadmin/cli/part_log_group.py index 75c06aec..238e258b 100644 --- a/ch_tools/chadmin/cli/part_log_group.py +++ b/ch_tools/chadmin/cli/part_log_group.py @@ -2,6 +2,7 @@ from click import Choice, group, option, pass_context +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.part import list_part_log from ch_tools.common.cli.formatting import format_bytes, print_response from ch_tools.common.clickhouse.config import get_cluster_name @@ -13,7 +14,7 @@ } -@group("part-log") +@group("part-log", cls=Chadmin) def part_log_group(): """ Commands for retrieving information from system.part_log. diff --git a/ch_tools/chadmin/cli/partition_group.py b/ch_tools/chadmin/cli/partition_group.py index 4fe855da..d6323e69 100644 --- a/ch_tools/chadmin/cli/partition_group.py +++ b/ch_tools/chadmin/cli/partition_group.py @@ -1,6 +1,7 @@ from cloup import Choice, group, option, option_group, pass_context from cloup.constraints import RequireAtLeast +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.partition import ( attach_partition, detach_partition, @@ -13,7 +14,7 @@ from ch_tools.common.cli.parameters import BytesParamType -@group("partition") +@group("partition", cls=Chadmin) def partition_group(): """ Commands to manage partitions. diff --git a/ch_tools/chadmin/cli/process_group.py b/ch_tools/chadmin/cli/process_group.py index e03d2eee..acecf354 100644 --- a/ch_tools/chadmin/cli/process_group.py +++ b/ch_tools/chadmin/cli/process_group.py @@ -1,6 +1,7 @@ from cloup import Choice, argument, group, option, option_group, pass_context from cloup.constraints import RequireAtLeast +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.process import get_process, kill_process, list_processes from ch_tools.chadmin.internal.utils import format_query from ch_tools.common.cli.formatting import print_response @@ -11,7 +12,7 @@ } -@group("process") +@group("process", cls=Chadmin) def process_group(): """ Commands to manage processes. diff --git a/ch_tools/chadmin/cli/query_log_group.py b/ch_tools/chadmin/cli/query_log_group.py index 12d201bf..0e2c8039 100644 --- a/ch_tools/chadmin/cli/query_log_group.py +++ b/ch_tools/chadmin/cli/query_log_group.py @@ -2,12 +2,13 @@ from click import Choice, argument, group, option, pass_context +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.utils import execute_query from ch_tools.common import logging from ch_tools.common.clickhouse.config import get_cluster_name -@group("query-log") +@group("query-log", cls=Chadmin) def query_log_group(): """ Commands for retrieving information from system.query_log. diff --git a/ch_tools/chadmin/cli/replica_group.py b/ch_tools/chadmin/cli/replica_group.py index 61f75db1..6e7ccf19 100644 --- a/ch_tools/chadmin/cli/replica_group.py +++ b/ch_tools/chadmin/cli/replica_group.py @@ -13,6 +13,7 @@ ) from cloup.constraints import AnySet, If, RequireAtLeast, accept_none, require_all +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.table_replica import ( get_table_replica, list_table_replicas, @@ -26,7 +27,7 @@ from ch_tools.common.process_pool import WorkerTask, execute_tasks_in_parallel -@group("replica") +@group("replica", cls=Chadmin) def replica_group(): """Commands to manage table replicas.""" pass diff --git a/ch_tools/chadmin/cli/replicated_fetch_group.py b/ch_tools/chadmin/cli/replicated_fetch_group.py index 881f58b1..c5154e5d 100644 --- a/ch_tools/chadmin/cli/replicated_fetch_group.py +++ b/ch_tools/chadmin/cli/replicated_fetch_group.py @@ -2,6 +2,7 @@ from click import group, option, pass_context +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.process import list_replicated_fetches from ch_tools.common.cli.formatting import ( format_bytes, @@ -18,7 +19,7 @@ } -@group("replicated-fetch") +@group("replicated-fetch", cls=Chadmin) def replicated_fetch_group(): """Commands to manage fetches (retrieve information from system.replicated_fetches).""" pass diff --git a/ch_tools/chadmin/cli/replication_queue_group.py b/ch_tools/chadmin/cli/replication_queue_group.py index da1e85ec..790cbd08 100644 --- a/ch_tools/chadmin/cli/replication_queue_group.py +++ b/ch_tools/chadmin/cli/replication_queue_group.py @@ -2,6 +2,7 @@ from click import group, option, pass_context +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.utils import execute_query from ch_tools.chadmin.internal.zookeeper import delete_zk_node from ch_tools.common import logging @@ -9,7 +10,7 @@ from ch_tools.common.clickhouse.config import get_cluster_name -@group("replication-queue") +@group("replication-queue", cls=Chadmin) def replication_queue_group(): """ Commands to manage replication queue. diff --git a/ch_tools/chadmin/cli/table_group.py b/ch_tools/chadmin/cli/table_group.py index 518000b8..1901a0a0 100644 --- a/ch_tools/chadmin/cli/table_group.py +++ b/ch_tools/chadmin/cli/table_group.py @@ -12,6 +12,7 @@ require_all, ) +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.table import ( attach_table, delete_detached_table, @@ -33,7 +34,7 @@ } -@group("table") +@group("table", cls=Chadmin) def table_group(): """Commands to manage tables.""" pass diff --git a/ch_tools/chadmin/cli/thread_log_group.py b/ch_tools/chadmin/cli/thread_log_group.py index 9f9b3f84..065cb2bb 100644 --- a/ch_tools/chadmin/cli/thread_log_group.py +++ b/ch_tools/chadmin/cli/thread_log_group.py @@ -1,10 +1,11 @@ from click import argument, group, option, pass_context +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.utils import execute_query from ch_tools.common import logging -@group("thread-log") +@group("thread-log", cls=Chadmin) def thread_log_group(): """ Commands for retrieving information from system.query_thread_log. diff --git a/ch_tools/chadmin/cli/wait_group.py b/ch_tools/chadmin/cli/wait_group.py index 7236afad..5691446d 100644 --- a/ch_tools/chadmin/cli/wait_group.py +++ b/ch_tools/chadmin/cli/wait_group.py @@ -5,6 +5,7 @@ from click import FloatRange, group, option, pass_context from requests.exceptions import ReadTimeout +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.clickhouse_disks import S3_METADATA_STORE_PATH from ch_tools.chadmin.internal.table_replica import list_table_replicas from ch_tools.chadmin.internal.utils import execute_query @@ -19,7 +20,7 @@ S3_PART_LOAD_SPEED = 0.5 # in data parts per second -@group("wait") +@group("wait", cls=Chadmin) def wait_group(): """Commands to wait until Clickhouse is in a certain state.""" pass diff --git a/ch_tools/chadmin/cli/zookeeper_group.py b/ch_tools/chadmin/cli/zookeeper_group.py index 3dff0e16..7ae92dba 100644 --- a/ch_tools/chadmin/cli/zookeeper_group.py +++ b/ch_tools/chadmin/cli/zookeeper_group.py @@ -4,6 +4,7 @@ from click import argument, group, option, pass_context from kazoo.security import make_digest_acl +from ch_tools.chadmin.cli.chadmin_group import Chadmin from ch_tools.chadmin.internal.table_replica import get_table_replica from ch_tools.chadmin.internal.zookeeper import ( check_zk_node, @@ -22,7 +23,7 @@ from ch_tools.common.config import load_config -@group("zookeeper") +@group("zookeeper", cls=Chadmin) @option("--port", help="ZooKeeper port.", type=int, default=2181) @option("--host", help="ZooKeeper host.", type=str) @option("--secure", help="Use secure connection.", default=False, is_flag=True) diff --git a/ch_tools/common/config.py b/ch_tools/common/config.py index 76f87fcc..30cae446 100644 --- a/ch_tools/common/config.py +++ b/ch_tools/common/config.py @@ -14,7 +14,7 @@ S3_LOG_CONFIG = { "sink": f"{CHADMIN_LOG_FILE}", "level": "WARNING", - "format": "chadmin", + "format": "default", } DEFAULT_CONFIG = { @@ -133,15 +133,14 @@ }, "loguru": { "formatters": { - "chadmin": "{time:YYYY-MM-DD HH:mm:ss,SSS} {process.name:11} {process.id:5} [{level:8}] {extra[logger_name]}: {message}", - "monrun": "{time:YYYY-MM-DD HH:mm:ss,SSS} {process.name:11} {process.id:5} [{level:8}] {extra[logger_name]}: {extra[cmd_name]}: {message}", + "default": "{time:YYYY-MM-DD HH:mm:ss,SSS} {process.name:11} {process.id:5} [{level:8}] {extra[logger_name]}: {extra[cmd_name]}: {message}", }, "handlers": { "chadmin": { "chadmin": { "sink": CHADMIN_LOG_FILE, "level": "DEBUG", - "format": "chadmin", + "format": "default", }, "boto3": S3_LOG_CONFIG, "botocore": S3_LOG_CONFIG, @@ -154,19 +153,19 @@ "ch-monitoring": { "sink": CH_MONITORING_LOG_FILE, "level": "DEBUG", - "format": "monrun", + "format": "default", }, "urllib3.connectionpool": { "sink": CH_MONITORING_LOG_FILE, "level": "CRITICAL", - "format": "monrun", + "format": "default", }, }, "keeper-monitoring": { "keeper-monitoring": { "sink": KEEPER_MONITORING_LOG_FILE, "level": "DEBUG", - "format": "monrun", + "format": "default", }, }, }, diff --git a/ch_tools/monrun_checks/main.py b/ch_tools/monrun_checks/main.py index 10f2a8a2..d0ff6966 100644 --- a/ch_tools/monrun_checks/main.py +++ b/ch_tools/monrun_checks/main.py @@ -69,7 +69,16 @@ def callback_wrapper(ctx, *args, **kwargs): ctx.obj["config"]["loguru"], "ch-monitoring", {"cmd_name": cmd.name} ) - logging.debug("Start executing") + logging.debug( + "Executing command '{}', params: {}, args: {}, version: {}", + cmd.name, + { + **ctx.parent.params, + **ctx.params, + }, + ctx.args, + __version__, + ) status = Status() try: diff --git a/ch_tools/monrun_checks_keeper/main.py b/ch_tools/monrun_checks_keeper/main.py index 4c3c61aa..847020e3 100644 --- a/ch_tools/monrun_checks_keeper/main.py +++ b/ch_tools/monrun_checks_keeper/main.py @@ -58,7 +58,16 @@ def wrapper(ctx, *a, **kw): ctx.obj["config"]["loguru"], "keeper-monitoring", {"cmd_name": cmd.name} ) - logging.debug("Start executing") + logging.debug( + "Executing command '{}', params: {}, args: {}, version: {}", + cmd.name, + { + **ctx.parent.params, + **ctx.params, + }, + ctx.args, + __version__, + ) status = Status() try: