Added --store-state-local option. Returned storing statistics in a file

ch_tools/chadmin/cli/object_storage_group.py

@@ -20,6 +20,7 @@
 STREAM_TIMEOUT = 10 * 60
 
 ORPHANED_OBJECTS_SIZE_FIELD = "orphaned_objects_size"
+STORE_STATE_LOCAL_PATH = "/tmp/object_storage_cleanup_state.json"
 
 
 @group("object-storage", cls=Chadmin)
@@ -105,8 +106,14 @@ def object_storage_group(ctx: Context, disk_name: str) -> None:
     help=("Use saved object list without traversing object storage again."),
 )
 @option(
-    "--store-state-path",
-    "store_state_path",
+    "--store-state-local",
+    "store_state_local",
+    is_flag=True,
+    help=("Write total size of orphaned objects to log file."),
+)
+@option(
+    "--store-state-zk-path",
+    "store_state_zk_path",
     default="",
     help=("Zookeeper node path for storage total size of orphaned objects."),
 )
@@ -121,7 +128,8 @@ def clean_command(
     dry_run: bool,
     keep_paths: bool,
     use_saved_list: bool,
-    store_state_path: str,
+    store_state_local: bool,
+    store_state_zk_path: str,
 ) -> None:
     """
     Clean orphaned S3 objects.
@@ -138,17 +146,25 @@ def clean_command(
         use_saved_list,
     )
 
-    if store_state_path:
-        _store_state_save(ctx, store_state_path, total_size)
+    if store_state_zk_path:
+        _store_state_zk_save(ctx, store_state_zk_path, total_size)
+
+    if store_state_local:
+        _store_state_local_save(ctx, total_size)
 
     _print_response(ctx, dry_run, deleted, total_size)
 
 
-def _store_state_save(ctx: Context, store_state_path: str, total_size: int) -> None:
-    if not check_zk_node(ctx, store_state_path):
-        create_zk_nodes(ctx, [store_state_path], make_parents=True)
+def _store_state_zk_save(ctx: Context, path: str, total_size: int) -> None:
+    if not check_zk_node(ctx, path):
+        create_zk_nodes(ctx, [path], make_parents=True)
     state_data = json.dumps({ORPHANED_OBJECTS_SIZE_FIELD: total_size}, indent=4)
-    update_zk_nodes(ctx, [store_state_path], state_data.encode("utf-8"))
+    update_zk_nodes(ctx, [path], state_data.encode("utf-8"))
+
+
+def _store_state_local_save(_: Context, total_size: int) -> None:
+    with open(STORE_STATE_LOCAL_PATH, "w", encoding="utf-8") as file:
+        json.dump({ORPHANED_OBJECTS_SIZE_FIELD: total_size}, file, indent=4)
 
 
 def _print_response(ctx: Context, dry_run: bool, deleted: int, total_size: int) -> None:

ch_tools/monrun_checks/ch_orphaned_objects.py

@@ -2,17 +2,24 @@
 
 import click
 
-from ch_tools.chadmin.cli.object_storage_group import ORPHANED_OBJECTS_SIZE_FIELD
+from ch_tools.chadmin.cli.object_storage_group import (
+    ORPHANED_OBJECTS_SIZE_FIELD,
+    STORE_STATE_LOCAL_PATH,
+)
 from ch_tools.chadmin.internal.zookeeper import check_zk_node, get_zk_node
-from ch_tools.common import logging
 from ch_tools.common.result import CRIT, OK, WARNING, Result
 
 
 @click.command("orphaned-objects")
 @click.option(
-    "--store-state-path",
-    "store_state_path",
-    required=True,
+    "--store-state-local",
+    "store_state_local",
+    is_flag=True,
+    help="Get total size of orphaned objects from local file.",
+)
+@click.option(
+    "--store-state-zk-path",
+    "store_state_zk_path",
     help="Zookeeper node path from which the total size of orphaned objects will be taken.",
 )
 @click.option(
@@ -34,23 +41,54 @@
 @click.pass_context
 def orphaned_objects_command(
     ctx: click.Context,
-    store_state_path: str,
+    store_state_local: bool,
+    store_state_zk_path: str,
     crit: int,
     warn: int,
 ) -> Result:
-    if not check_zk_node(ctx, store_state_path):
-        result = Result(
-            CRIT, f'Zookeeper node with path "{store_state_path}" not exists'
-        )
-        logging.exception(result.message)
-        return result
+    _check_mutually_exclusive(store_state_local, store_state_zk_path)
+
+    total_size = 0
+    if store_state_zk_path:
+        total_size = _zk_get_total_size(ctx, store_state_zk_path)
+
+    if store_state_local:
+        total_size = _local_get_total_size()
 
-    total_size = json.loads(get_zk_node(ctx, store_state_path)).get(
-        ORPHANED_OBJECTS_SIZE_FIELD
-    )
     msg = f"Total size: {total_size}"
     if total_size >= crit:
         return Result(CRIT, msg)
     if total_size >= warn:
         return Result(WARNING, msg)
     return Result(OK, msg)
+
+
+def _check_mutually_exclusive(store_state_local, store_state_zk_path):
+    if not store_state_local and not store_state_zk_path:
+        raise click.UsageError(
+            "One of these options must be provided: --store_state_local, --store_state_zk_path"
+        )
+
+    if store_state_local and store_state_zk_path:
+        raise click.UsageError(
+            "Options --store-state-local and --store-state-zk-path are mutually exclusive."
+        )
+
+
+def _local_get_total_size() -> int:
+    try:
+        with open(STORE_STATE_LOCAL_PATH, mode="r", encoding="utf-8") as file:
+            total_size = json.load(file).get(ORPHANED_OBJECTS_SIZE_FIELD)
+    except FileNotFoundError:
+        total_size = 0
+
+    return total_size
+
+
+def _zk_get_total_size(ctx: click.Context, store_state_zk_path: str) -> int:
+    total_size = 0
+    if check_zk_node(ctx, store_state_zk_path):
+        total_size = json.loads(get_zk_node(ctx, store_state_zk_path)).get(
+            ORPHANED_OBJECTS_SIZE_FIELD
+        )
+    return total_size

tests/features/monrun.feature

@@ -389,14 +389,14 @@ Feature: ch-monitoring tool
     2;KazooTimeoutError('Connection time-out')
     """
 
-  Scenario: Check clickhouse orphaned objects state
+  Scenario: Check clickhouse orphaned objects with store-state-zk-path option
     When we execute command on clickhouse01
     """
-    chadmin object-storage clean --dry-run --to-time 0h --on-cluster --keep-paths --store-state-path /tmp/shard_1
+    chadmin object-storage clean --dry-run --to-time 0h --on-cluster --keep-paths --store-state-zk-path /tmp/shard_1
     """
     When we execute command on clickhouse01
     """
-    ch-monitoring orphaned-objects --store-state-path /tmp/shard_1
+    ch-monitoring orphaned-objects --store-state-zk-path /tmp/shard_1
     """
     Then we get response
     """
@@ -410,29 +410,95 @@ Feature: ch-monitoring tool
     """
     When we execute command on clickhouse01
     """
-    chadmin object-storage clean --dry-run --to-time 0h --on-cluster --keep-paths --store-state-path /tmp/shard_1
+    chadmin object-storage clean --dry-run --to-time 0h --on-cluster --keep-paths --store-state-zk-path /tmp/shard_1
     """
     When we execute command on clickhouse01
     """
-    ch-monitoring orphaned-objects --store-state-path /tmp/shard_1
+    ch-monitoring orphaned-objects --store-state-zk-path /tmp/shard_1
     """
     Then we get response contains
     """
     0;Total size: 10
     """
     When we execute command on clickhouse01
     """
-    ch-monitoring orphaned-objects -w 9 -c 19 --store-state-path /tmp/shard_1
+    ch-monitoring orphaned-objects -w 9 -c 19 --store-state-zk-path /tmp/shard_1
     """
     Then we get response contains
     """
     1;Total size: 10
     """
     When we execute command on clickhouse01
     """
-    ch-monitoring orphaned-objects -w 4 -c 9 --store-state-path /tmp/shard_1
+    ch-monitoring orphaned-objects -w 4 -c 9 --store-state-zk-path /tmp/shard_1
     """
     Then we get response contains
     """
     2;Total size: 10
-    """
+    """
+
+  Scenario: Check clickhouse orphaned objects with store-state-local option
+    When we execute command on clickhouse01
+    """
+    chadmin object-storage clean --dry-run --to-time 0h --on-cluster --keep-paths --store-state-local
+    """
+    When we execute command on clickhouse01
+    """
+    ch-monitoring orphaned-objects --store-state-local
+    """
+    Then we get response
+    """
+    0;Total size: 0
+    """
+    When we put object in S3
+    """
+      bucket: cloud-storage-test
+      path: /data/orpaned_object.tsv
+      data: '1234567890'
+    """
+    When we execute command on clickhouse01
+    """
+    chadmin object-storage clean --dry-run --to-time 0h --on-cluster --keep-paths --store-state-local
+    """
+    When we execute command on clickhouse01
+    """
+    ch-monitoring orphaned-objects --store-state-local
+    """
+    Then we get response contains
+    """
+    0;Total size: 10
+    """
+    When we execute command on clickhouse01
+    """
+    ch-monitoring orphaned-objects -w 9 -c 19 --store-state-local
+    """
+    Then we get response contains
+    """
+    1;Total size: 10
+    """
+    When we execute command on clickhouse01
+    """
+    ch-monitoring orphaned-objects -w 4 -c 9 --store-state-local
+    """
+    Then we get response contains
+    """
+    2;Total size: 10
+    """
+
+  Scenario: Check clickhouse orphaned objects --store-state-local and --store-state-zk-path are mutually exclusive
+    When we execute command on clickhouse01
+    """
+    ch-monitoring orphaned-objects -w 9 -c 19 --store-state-local --store-state-zk-path /tmp/shard_1
+    """
+    Then we get response contains
+    """
+    1;Unknown error: Options --store-state-local and --store-state-zk-path are mutually exclusive.
+    """
+    When we execute command on clickhouse01
+    """
+    ch-monitoring orphaned-objects -w 9 -c 19
+    """
+    Then we get response contains
+    """
+    1;Unknown error: Options --store-state-local and --store-state-zk-path are mutually exclusive.
+    """

tests/features/object_storage.feature

@@ -170,10 +170,10 @@ Feature: chadmin object-storage commands
       TotalSize: 3
     """
 
-  Scenario: Clean with store-state-path works
+  Scenario: Clean with store-state-zk-path works
     When we execute command on clickhouse01
     """
-    chadmin --format yaml object-storage clean --dry-run --to-time 0h --on-cluster --keep-paths --store-state-path /tmp/shard_1
+    chadmin --format yaml object-storage clean --dry-run --to-time 0h --on-cluster --keep-paths --store-state-zk-path /tmp/shard_1
     """
     Then we get zookeeper node with "/tmp/shard_1" path
     """
@@ -189,11 +189,39 @@ Feature: chadmin object-storage commands
     """
     When we execute command on clickhouse01
     """
-    chadmin --format yaml object-storage clean --dry-run --to-time 0h --on-cluster --keep-paths --store-state-path /tmp/shard_1
+    chadmin --format yaml object-storage clean --dry-run --to-time 0h --on-cluster --keep-paths --store-state-zk-path /tmp/shard_1
     """
     Then we get zookeeper node with "/tmp/shard_1" path
     """
     {
         "orphaned_objects_size": 10
     }
     """
+
+  Scenario: Clean with store-state-local works
+    When we execute command on clickhouse01
+    """
+    chadmin --format yaml object-storage clean --dry-run --to-time 0h --on-cluster --keep-paths --store-state-local
+    """
+    Then we get file /tmp/object_storage_cleanup_state.json
+    """
+    {
+        "orphaned_objects_size": 0
+    }
+    """
+    When we put object in S3
+    """
+      bucket: cloud-storage-test
+      path: /data/orpaned_object.tsv
+      data: '1234567890'
+    """
+    When we execute command on clickhouse01
+    """
+    chadmin --format yaml object-storage clean --dry-run --to-time 0h --on-cluster --keep-paths --store-state-local
+    """
+    Then we get file /tmp/object_storage_cleanup_state.json
+    """
+    {
+        "orphaned_objects_size": 10
+    }
+    """