From cedddf6fa48c8f84088cb3f3c98ec89fe9d7a849 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 26 Aug 2024 19:10:49 +0200 Subject: [PATCH 01/13] PoC --- .../integration/helper_container/Dockerfile | 4 +- tests/integration/helpers/network.py | 135 ++++++++++++++++-- 2 files changed, 129 insertions(+), 10 deletions(-) diff --git a/docker/test/integration/helper_container/Dockerfile b/docker/test/integration/helper_container/Dockerfile index 49a3d3cd84b8..1084d087e53b 100644 --- a/docker/test/integration/helper_container/Dockerfile +++ b/docker/test/integration/helper_container/Dockerfile @@ -3,6 +3,8 @@ FROM alpine:3.18 RUN apk add --no-cache -U iproute2 \ - && for bin in iptables iptables-restore iptables-save; \ + && for bin in \ + iptables iptables-restore iptables-save \ + ip6tables ip6tables-restore ip6tables-save; \ do ln -sf xtables-nft-multi "/sbin/$bin"; \ done diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index e6e79dc79478..5219ac22f715 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -3,6 +3,7 @@ import time import logging import docker +import ipaddress class PartitionManager: @@ -26,25 +27,76 @@ def drop_instance_zk_connections(self, instance, action="DROP"): self._check_instance(instance) self._add_rule( - {"source": instance.ip_address, "destination_port": 2181, "action": action} + { + "source": instance.ipv4_address, + "destination_port": 2181, + "action": action, + } ) self._add_rule( - {"destination": instance.ip_address, "source_port": 2181, "action": action} + { + "destination": instance.ipv4_address, + "source_port": 2181, + "action": action, + } ) + if instance.ipv6_address: + self._add_rule( + { + "source": instance.ipv6_address, + "destination_port": 2181, + "action": action, + } + ) + self._add_rule( + { + "destination": instance.ipv6_address, + "source_port": 2181, + "action": action, + } + ) + def dump_rules(self): - return _NetworkManager.get().dump_rules() + v4 = _NetworkManager.get().dump_rules() + v6 = _NetworkManager.get().dump_v6_rules() + + return v4 + v6 def restore_instance_zk_connections(self, instance, action="DROP"): self._check_instance(instance) self._delete_rule( - {"source": instance.ip_address, "destination_port": 2181, "action": action} + { + "source": instance.ipv4_address, + "destination_port": 2181, + "action": action, + } ) self._delete_rule( - {"destination": instance.ip_address, "source_port": 2181, "action": action} + { + "destination": instance.ipv4_address, + "source_port": 2181, + "action": action, + } ) + if instance.ipv6_address: + self._delete_rule( + { + "source": instance.ipv6_address, + "destination_port": 2181, + "action": action, + } + ) + self._delete_rule( + { + "destination": instance.ipv6_address, + "source_port": 2181, + "action": action, + } + ) + def partition_instances(self, left, right, port=None, action="DROP"): self._check_instance(left) self._check_instance(right) @@ -59,16 +111,36 @@ def create_rule(src, dst): rule["destination_port"] = port return rule + def create_rule_v6(src, dst): + rule = { + "source": src.ipv6_address, + "destination": dst.ipv6_address, + "action": action, + } + if port is not None: + rule["destination_port"] = port + return rule + self._add_rule(create_rule(left, right)) self._add_rule(create_rule(right, left)) + if left.ipv6_address and right.ipv6_address: + self._add_rule(create_rule_v6(left, right)) + self._add_rule(create_rule_v6(right, left)) + def add_network_delay(self, instance, delay_ms): self._add_tc_netem_delay(instance, delay_ms) def heal_all(self): while self._iptables_rules: rule = self._iptables_rules.pop() - _NetworkManager.get().delete_iptables_rule(**rule) + + if self._is_ipv6_rule(rule): + _NetworkManager.get().delete_ip6tables_rule(**rule) + else: + _NetworkManager.get().delete_iptables_rule(**rule) + # _NetworkManager.get().delete_iptables_rule(**rule) + # _NetworkManager.get().delete_ip6tables_rule(**rule) while self._netem_delayed_instances: instance = self._netem_delayed_instances.pop() @@ -90,12 +162,29 @@ def _check_instance(instance): if instance.ip_address is None: raise Exception("Instance + " + instance.name + " is not launched!") + @staticmethod + def _is_ipv6_rule(rule): + is_ipv6 = False + + if "source" in rule: + is_ipv6 = ipaddress.ip_address(rule["source"]).version == 6 + if "destination" in rule: + is_ipv6 = ipaddress.ip_address(rule["source"]).version == 6 + + return is_ipv6 + def _add_rule(self, rule): - _NetworkManager.get().add_iptables_rule(**rule) + if self._is_ipv6_rule(rule): + _NetworkManager.get().add_ip6tables_rule(**rule) + else: + _NetworkManager.get().add_iptables_rule(**rule) self._iptables_rules.append(rule) def _delete_rule(self, rule): - _NetworkManager.get().delete_iptables_rule(**rule) + if self._is_ipv6_rule(rule): + _NetworkManager.get().delete_ip6tables_rule(**rule) + else: + _NetworkManager.get().delete_iptables_rule(**rule) self._iptables_rules.remove(rule) def _add_tc_netem_delay(self, instance, delay_ms): @@ -155,15 +244,29 @@ def add_iptables_rule(self, **kwargs): cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) + def add_ip6tables_rule(self, **kwargs): + cmd = ["ip6tables-legacy", "--wait", "-I", "DOCKER-USER", "1"] + cmd.extend(self._iptables_cmd_suffix(**kwargs)) + self._exec_run(cmd, privileged=True) + def delete_iptables_rule(self, **kwargs): cmd = ["iptables", "--wait", "-D", "DOCKER-USER"] cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) + def delete_ip6tables_rule(self, **kwargs): + cmd = ["ip6tables-legacy", "--wait", "-D", "DOCKER-USER"] + cmd.extend(self._iptables_cmd_suffix(**kwargs)) + self._exec_run(cmd, privileged=True) + def dump_rules(self): cmd = ["iptables", "-L", "DOCKER-USER"] return self._exec_run(cmd, privileged=True) + def dump_v6_rules(self): + cmd = ["ip6tables-legacy", "-L", "DOCKER-USER"] + return self._exec_run(cmd, privileged=True) + @staticmethod def clean_all_user_iptables_rules(): for i in range(1000): @@ -178,6 +281,20 @@ def clean_all_user_iptables_rules(): + " iterations, last error: " + str(res.stderr) ) + break + + for i in range(1000): + iptables_iter = i + # when rules will be empty, it will return error + res = subprocess.run("ip6tables-legacy --wait -D DOCKER-USER 1", shell=True) + + if res.returncode != 0: + logging.info( + "All ip6tables rules cleared, " + + str(iptables_iter) + + " iterations, last error: " + + str(res.stderr) + ) return @staticmethod @@ -244,7 +361,7 @@ def __init__( def _ensure_container(self): if self._container is None or self._container_expire_time <= time.time(): image_name = "clickhouse/integration-helper:" + os.getenv( - "DOCKER_HELPER_TAG", "latest" + "DOCKER_HELPER_TAG", "" ) for i in range(5): if self._container is not None: From fe42299928dddd9d2e705305524566ded756f6ec Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 26 Aug 2024 19:11:12 +0200 Subject: [PATCH 02/13] Typo --- tests/integration/helpers/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 5219ac22f715..a92843a313b3 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -361,7 +361,7 @@ def __init__( def _ensure_container(self): if self._container is None or self._container_expire_time <= time.time(): image_name = "clickhouse/integration-helper:" + os.getenv( - "DOCKER_HELPER_TAG", "" + "DOCKER_HELPER_TAG", "latest" ) for i in range(5): if self._container is not None: From d49d413c8d0ae7995bf4188ff8ce87ac85587c25 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 26 Aug 2024 19:11:48 +0200 Subject: [PATCH 03/13] No legacy --- tests/integration/helpers/network.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index a92843a313b3..e62034a51044 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -245,7 +245,7 @@ def add_iptables_rule(self, **kwargs): self._exec_run(cmd, privileged=True) def add_ip6tables_rule(self, **kwargs): - cmd = ["ip6tables-legacy", "--wait", "-I", "DOCKER-USER", "1"] + cmd = ["ip6tables", "--wait", "-I", "DOCKER-USER", "1"] cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) @@ -255,7 +255,7 @@ def delete_iptables_rule(self, **kwargs): self._exec_run(cmd, privileged=True) def delete_ip6tables_rule(self, **kwargs): - cmd = ["ip6tables-legacy", "--wait", "-D", "DOCKER-USER"] + cmd = ["ip6tables", "--wait", "-D", "DOCKER-USER"] cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) @@ -264,7 +264,7 @@ def dump_rules(self): return self._exec_run(cmd, privileged=True) def dump_v6_rules(self): - cmd = ["ip6tables-legacy", "-L", "DOCKER-USER"] + cmd = ["ip6tables", "-L", "DOCKER-USER"] return self._exec_run(cmd, privileged=True) @staticmethod @@ -286,7 +286,7 @@ def clean_all_user_iptables_rules(): for i in range(1000): iptables_iter = i # when rules will be empty, it will return error - res = subprocess.run("ip6tables-legacy --wait -D DOCKER-USER 1", shell=True) + res = subprocess.run("ip6tables --wait -D DOCKER-USER 1", shell=True) if res.returncode != 0: logging.info( From 5ee5c8224ea715b9a78a9ee79f79419ae36db6f2 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 26 Aug 2024 22:16:05 +0200 Subject: [PATCH 04/13] Fix --- tests/integration/helpers/network.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index e62034a51044..c35ab65eef57 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -164,14 +164,10 @@ def _check_instance(instance): @staticmethod def _is_ipv6_rule(rule): - is_ipv6 = False - if "source" in rule: - is_ipv6 = ipaddress.ip_address(rule["source"]).version == 6 + return ipaddress.ip_address(rule["source"]).version == 6 if "destination" in rule: - is_ipv6 = ipaddress.ip_address(rule["source"]).version == 6 - - return is_ipv6 + return ipaddress.ip_address(rule["destination"]).version == 6 def _add_rule(self, rule): if self._is_ipv6_rule(rule): From 9be79614a3fd3d97ed1be76dddcd4182e75d3f8b Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Wed, 28 Aug 2024 15:20:56 +0200 Subject: [PATCH 05/13] Fix --- tests/integration/helpers/network.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index c35ab65eef57..d3de4660acba 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -164,11 +164,13 @@ def _check_instance(instance): @staticmethod def _is_ipv6_rule(rule): - if "source" in rule: + if rule.get("source"): return ipaddress.ip_address(rule["source"]).version == 6 - if "destination" in rule: + if rule.get("destination"): return ipaddress.ip_address(rule["destination"]).version == 6 + return False + def _add_rule(self, rule): if self._is_ipv6_rule(rule): _NetworkManager.get().add_ip6tables_rule(**rule) From 335ca75174329ecabb9d8de59c1cfd4f799842ba Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Sep 2024 00:16:40 +0200 Subject: [PATCH 06/13] Lint --- tests/integration/helpers/network.py | 43 +++++++++------------------- 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index d3de4660acba..4b02f99876f1 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -139,8 +139,6 @@ def heal_all(self): _NetworkManager.get().delete_ip6tables_rule(**rule) else: _NetworkManager.get().delete_iptables_rule(**rule) - # _NetworkManager.get().delete_iptables_rule(**rule) - # _NetworkManager.get().delete_ip6tables_rule(**rule) while self._netem_delayed_instances: instance = self._netem_delayed_instances.pop() @@ -267,33 +265,20 @@ def dump_v6_rules(self): @staticmethod def clean_all_user_iptables_rules(): - for i in range(1000): - iptables_iter = i - # when rules will be empty, it will return error - res = subprocess.run("iptables --wait -D DOCKER-USER 1", shell=True) - - if res.returncode != 0: - logging.info( - "All iptables rules cleared, " - + str(iptables_iter) - + " iterations, last error: " - + str(res.stderr) - ) - break - - for i in range(1000): - iptables_iter = i - # when rules will be empty, it will return error - res = subprocess.run("ip6tables --wait -D DOCKER-USER 1", shell=True) - - if res.returncode != 0: - logging.info( - "All ip6tables rules cleared, " - + str(iptables_iter) - + " iterations, last error: " - + str(res.stderr) - ) - return + for iptables in ("iptables", "ip6tables"): + for i in range(1000): + iptables_iter = i + # when rules will be empty, it will return error + res = subprocess.run(f"{iptables}--wait -D DOCKER-USER 1", shell=True) + + if res.returncode != 0: + logging.info( + f"All {iptables} rules cleared, " + + str(iptables_iter) + + " iterations, last error: " + + str(res.stderr) + ) + break @staticmethod def _iptables_cmd_suffix( From d46065360b13c9dfbab11f642d54702610cd550d Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Sep 2024 05:24:03 +0200 Subject: [PATCH 07/13] Fix --- tests/integration/helpers/network.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 4b02f99876f1..8d96add9501d 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -28,14 +28,14 @@ def drop_instance_zk_connections(self, instance, action="DROP"): self._add_rule( { - "source": instance.ipv4_address, + "source": instance.ip_address, "destination_port": 2181, "action": action, } ) self._add_rule( { - "destination": instance.ipv4_address, + "destination": instance.ip_address, "source_port": 2181, "action": action, } @@ -68,14 +68,14 @@ def restore_instance_zk_connections(self, instance, action="DROP"): self._delete_rule( { - "source": instance.ipv4_address, + "source": instance.ip_address, "destination_port": 2181, "action": action, } ) self._delete_rule( { - "destination": instance.ipv4_address, + "destination": instance.ip_address, "source_port": 2181, "action": action, } From 58b84e9e3d75846428b6839aa03207a4ce8f5cbd Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Sep 2024 17:29:44 +0200 Subject: [PATCH 08/13] Update network.py --- tests/integration/helpers/network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 8d96add9501d..c9e2df0b2a04 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -269,7 +269,7 @@ def clean_all_user_iptables_rules(): for i in range(1000): iptables_iter = i # when rules will be empty, it will return error - res = subprocess.run(f"{iptables}--wait -D DOCKER-USER 1", shell=True) + res = subprocess.run(f"{iptables} --wait -D DOCKER-USER 1", shell=True) if res.returncode != 0: logging.info( From 1f5eb2cd2c1de8639d017f769d04975d21cc590a Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Sep 2024 18:36:32 +0200 Subject: [PATCH 09/13] Set IPv6 instance to the node object --- tests/integration/helpers/cluster.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 53f4f1e1f26e..821bb887435c 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -2112,6 +2112,7 @@ def restart_instance_with_ip_change(self, node, new_ip): self.base_cmd + ["up", "--force-recreate", "--no-deps", "-d", node.name] ) node.ip_address = self.get_instance_ip(node.name) + node.ipv6_address = self.get_instance_global_ipv6(node.name) node.client = Client(node.ip_address, command=self.client_bin_path) logging.info("Restart node with ip change") @@ -3182,6 +3183,7 @@ def logging_azurite_initialization(exception, retry_number, sleep_time): for instance in self.instances.values(): instance.docker_client = self.docker_client instance.ip_address = self.get_instance_ip(instance.name) + instance.ipv6_address = self.get_instance_global_ipv6(instance.name) logging.debug( f"Waiting for ClickHouse start in {instance.name}, ip: {instance.ip_address}..." From 26add45c70cef4dd2c0fc18891c3db9bf90bb06e Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Tue, 3 Sep 2024 18:49:04 +0200 Subject: [PATCH 10/13] Poke CI From da3a7d069173b7508584a63190a5e24a9452e5bd Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Thu, 5 Sep 2024 06:23:20 +0200 Subject: [PATCH 11/13] Setup missing ip6tables chain --- tests/integration/helpers/network.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index c9e2df0b2a04..39f413cdf3b0 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -235,12 +235,29 @@ def get(cls, **kwargs): cls._instance = cls(**kwargs) return cls._instance + def setup_ip6tables_docker_user_chain(self): + _rules = subprocess.check_output( + f"ip6tables-save", shell=True + ) + if "DOCKER-USER" in _rules.decode("utf-8"): + return + + setup_cmds = [ + ["ip6tables", "--wait", "-N", "DOCKER-USER"], + ["ip6tables", "--wait", "-I", "FORWARD", "-j", "DOCKER-USER"], + ["ip6tables", "--wait", "-A", "DOCKER-USER", "-j", "RETURN"], + ] + for cmd in setup_cmds: + self._exec_run(cmd, privileged=True) + def add_iptables_rule(self, **kwargs): cmd = ["iptables", "--wait", "-I", "DOCKER-USER", "1"] cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) def add_ip6tables_rule(self, **kwargs): + self.setup_ip6tables_docker_user_chain() + cmd = ["ip6tables", "--wait", "-I", "DOCKER-USER", "1"] cmd.extend(self._iptables_cmd_suffix(**kwargs)) self._exec_run(cmd, privileged=True) From 798f4b4c3b4667b6cd09760449758b6b7e503e56 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 5 Sep 2024 04:35:04 +0000 Subject: [PATCH 12/13] Automatic style fix --- tests/integration/helpers/network.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/helpers/network.py b/tests/integration/helpers/network.py index 39f413cdf3b0..065836396f35 100644 --- a/tests/integration/helpers/network.py +++ b/tests/integration/helpers/network.py @@ -236,9 +236,7 @@ def get(cls, **kwargs): return cls._instance def setup_ip6tables_docker_user_chain(self): - _rules = subprocess.check_output( - f"ip6tables-save", shell=True - ) + _rules = subprocess.check_output(f"ip6tables-save", shell=True) if "DOCKER-USER" in _rules.decode("utf-8"): return From a463b2b44c9162a9ee7cb3d830a3ad6a3b5a7ff6 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Thu, 5 Sep 2024 15:59:21 +0200 Subject: [PATCH 13/13] Poke CI