replace clickhouse-keeper to zookeeper, until ClickHouse/ClickHouse#7…

…2057 not resolved Signed-off-by: Slach <[email protected]>
Altinity · Nov 19, 2024 · 94b2083 · 94b2083
1 parent 477da72
commit 94b2083
Show file tree

Hide file tree

Showing 3 changed files with 68 additions and 7 deletions.
diff --git a/test/integration/clickhouse-config.xml b/test/integration/clickhouse-config.xml
@@ -26,4 +26,36 @@
         <level>trace</level>
         <console>1</console>
     </logger>
+
+    <!-- low memory tuning -->
+    <mysql_port remove="1"/>
+    <postgresql_port remove="1"/>
+    <trace_log remove="1"/>
+    <query_thread_log remove="1"/>
+    <opentelemetry_span_log remove="1"/>
+    <processors_profile_log remove="1"/>
+
+    <max_thread_pool_size>2000</max_thread_pool_size>
+    <max_connections>64</max_connections>
+    <max_concurrent_queries>64</max_concurrent_queries>
+    <max_server_memory_usage_to_ram_ratio>0.99</max_server_memory_usage_to_ram_ratio>
+
+    <mark_cache_size>268435456</mark_cache_size>
+    <index_mark_cache_size>67108864</index_mark_cache_size>
+    <uncompressed_cache_size>16777216</uncompressed_cache_size>
+    <mlock_executable>false</mlock_executable>
+
+    <background_buffer_flush_schedule_pool_size>1</background_buffer_flush_schedule_pool_size>
+    <background_pool_size>2</background_pool_size>
+    <background_merges_mutations_concurrency_ratio>2</background_merges_mutations_concurrency_ratio>
+    <background_merges_mutations_scheduling_policy>round_robin</background_merges_mutations_scheduling_policy>
+    <background_move_pool_size>1</background_move_pool_size>
+    <background_fetches_pool_size>1</background_fetches_pool_size>
+    <background_common_pool_size>2</background_common_pool_size>
+    <background_schedule_pool_size>8</background_schedule_pool_size>
+    <background_message_broker_schedule_pool_size>1</background_message_broker_schedule_pool_size>
+    <background_distributed_schedule_pool_size>1</background_distributed_schedule_pool_size>
+    <tables_loader_foreground_pool_size>0</tables_loader_foreground_pool_size>
+    <tables_loader_background_pool_size>0</tables_loader_background_pool_size>
+
 </yandex>
diff --git a/test/integration/docker-compose_advanced.yml b/test/integration/docker-compose_advanced.yml
@@ -131,16 +131,27 @@ services:
     volumes:
       - pgsql:/var/lib/postgresql
 
+  # @todo return keeper back when resolve https://github.com/ClickHouse/ClickHouse/issues/72057
+  #  zookeeper:
+  #    image: docker.io/clickhouse/clickhouse-keeper:${CLICKHOUSE_KEEPER_VERSION:-latest-alpine}
+  #    hostname: zookeeper
+  #    volumes:
+  #      - ./clickhouse-keeper.xml:/etc/clickhouse-keeper/conf.d/clickhouse-keeper.xml
+  #      - /var/lib/clickhouse
+  #      - /var/lib/clickhouse-keeper
+  #    environment:
+  #      - CLICKHOUSE_RUN_AS_ROOT=1
+  #    healthcheck:
+  #      test: bash -c 'if [[ "$$(echo 'ruok' | nc 127.0.0.1 2181)" == "imok" ]]; then exit 0; else exit 1; fi'
+  #      interval: 1s
+  #      timeout: 2s
+  #      retries: 10
+  #      start_period: 1s
   zookeeper:
-    image: docker.io/clickhouse/clickhouse-keeper:${CLICKHOUSE_KEEPER_VERSION:-latest-alpine}
+    image: ${ZOOKEEPER_IMAGE:-docker.io/zookeeper}:${ZOOKEEPER_VERSION:-latest}
     hostname: zookeeper
-    volumes:
-      - ./clickhouse-keeper.xml:/etc/clickhouse-keeper/conf.d/clickhouse-keeper.xml
-      - /var/lib/clickhouse
-      - /var/lib/clickhouse-keeper
     environment:
-      - CLICKHOUSE_UID=0
-      - CLICKHOUSE_GID=0
+      ZOO_4LW_COMMANDS_WHITELIST: "*"
     healthcheck:
       test: bash -c 'if [[ "$$(echo 'ruok' | nc 127.0.0.1 2181)" == "imok" ]]; then exit 0; else exit 1; fi'
       interval: 1s

diff --git a/test/integration/dynamic_settings.sh b/test/integration/dynamic_settings.sh
@@ -490,4 +490,22 @@ cat <<EOT > /etc/clickhouse-server/config.d/blob_storage_log.xml
 </yandex>
 EOT
 
+fi
+
+if [[ "$CLICKHOUSE_VERSION" == "head" || "${CLICKHOUSE_VERSION}" =~ ^19\.1[1-9] || "${CLICKHOUSE_VERSION}" =~ ^2[4-9\.[1-9] ]]; then
+
+cat <<EOT > /etc/clickhouse-server/confid.d/merge_tree_low_memory.xml
+<yandex>
+    <merge_tree>
+        <merge_max_block_size>1024</merge_max_block_size>
+        <max_bytes_to_merge_at_max_space_in_pool>1073741824</max_bytes_to_merge_at_max_space_in_pool>
+        <number_of_free_entries_in_pool_to_lower_max_size_of_merge>0
+        </number_of_free_entries_in_pool_to_lower_max_size_of_merge>
+        <number_of_free_entries_in_pool_to_execute_mutation>1</number_of_free_entries_in_pool_to_execute_mutation>
+        <number_of_free_entries_in_pool_to_execute_optimize_entire_partition>1
+        </number_of_free_entries_in_pool_to_execute_optimize_entire_partition>
+    </merge_tree>
+</yandex>
+EOT
+
 fi