From e888ffc7acba712bea9c9bfd7c69bfdff2774eac Mon Sep 17 00:00:00 2001 From: Petr Balogh Date: Mon, 16 Sep 2024 12:54:26 +0200 Subject: [PATCH] Add multi-zone deployment for ROKS IBM Cloud Signed-off-by: Petr Balogh --- ...m_cloud_managed_3az_0m_3w_vpc_cluster.yaml | 30 +++++++++++++++++++ .../ibmcloud/ibm_cloud_vpc_cluster.yaml | 2 ++ docs/supported_platforms.md | 7 +++++ ocs_ci/ocs/node.py | 1 - ocs_ci/utility/ibmcloud.py | 26 ++++++++++++++-- 5 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 conf/deployment/ibmcloud/ibm_cloud_managed_3az_0m_3w_vpc_cluster.yaml diff --git a/conf/deployment/ibmcloud/ibm_cloud_managed_3az_0m_3w_vpc_cluster.yaml b/conf/deployment/ibmcloud/ibm_cloud_managed_3az_0m_3w_vpc_cluster.yaml new file mode 100644 index 00000000000..c96c095e5d9 --- /dev/null +++ b/conf/deployment/ibmcloud/ibm_cloud_managed_3az_0m_3w_vpc_cluster.yaml @@ -0,0 +1,30 @@ +# This is the config for IBM cloud ROKS deployment +# for 3 AZ setup. +--- +ENV_DATA: + platform: 'ibm_cloud' + deployment_type: 'managed' + region: 'eu-de' + zone: 'eu-de-3' + provider: "vpc-gen2" + worker_instance_type: "bx2.16x64" + master_replicas: 0 + worker_replicas: 3 + worker_availability_zones: + - 'eu-de-1' + - 'eu-de-2' + - 'eu-de-3' + rhel_workers: true + # Following values needs to be set in separate config and passed to ocs-ci in + # order to deploy OCP/OCS cluster on IBM Cloud + # vpc_id: VPC ID PLACEHOLDER + # subnet_id: SUBNET ID PLACEHOLDER + # cos_instance: COS INSTANCE PLACEHOLDER +#AUTH: +# ibmcloud: +# api_key: IBM CLOUD API KEY PLACEHOLDER +# account_id: ACCOUNT ID PLACEHOLDER +# ibm_cos_access_key_id: KEY PLACEHOLDER +# ibm_cos_secret_access_key: SECRET PLACEHOLDER +#DEPLOYMENT: +# ocs_secret_dockerconfigjson: BASE64 OF QUAY SECRET PLACEHOLDER diff --git a/conf/deployment/ibmcloud/ibm_cloud_vpc_cluster.yaml b/conf/deployment/ibmcloud/ibm_cloud_vpc_cluster.yaml index c4c7f2b29e7..8f3b4044b08 100644 --- a/conf/deployment/ibmcloud/ibm_cloud_vpc_cluster.yaml +++ b/conf/deployment/ibmcloud/ibm_cloud_vpc_cluster.yaml @@ -10,6 +10,8 @@ ENV_DATA: worker_instance_type: "bx2.16x64" master_replicas: 0 worker_replicas: 3 + worker_availability_zones: + - 'eu-de-3' rhel_workers: true # Following values needs to be set in separate config and passed to ocs-ci in # order to deploy OCP/OCS cluster on IBM Cloud diff --git a/docs/supported_platforms.md b/docs/supported_platforms.md index 37667b5c280..d879b0f9257 100644 --- a/docs/supported_platforms.md +++ b/docs/supported_platforms.md @@ -86,6 +86,13 @@ But an example is pasted also here: ENV_DATA: vpc_id: VPC ID PLACEHOLDER subnet_id: SUBNET ID PLACEHOLDER + # Subnet id from dict below will be used if user defines worker_availability_zones + # This is for multi zone deployment support. So If you have worker_availability_zones and + # subnet_ids_per_zone defined in credential conf it will be used instead of subnet_id. + subnet_ids_per_zone: + "eu-de-1": "PLACEHOLDER_SUBNET_ID" + "eu-de-2": "PLACEHOLDER_SUBNET_ID" + "eu-de-3": "PLACEHOLDER_SUBNET_ID" cos_instance: COS INSTANCE PLACEHOLDER AUTH: ibmcloud: diff --git a/ocs_ci/ocs/node.py b/ocs_ci/ocs/node.py index 5514b469239..590798392af 100644 --- a/ocs_ci/ocs/node.py +++ b/ocs_ci/ocs/node.py @@ -1697,7 +1697,6 @@ def verify_all_nodes_created(): expected_num_nodes += 3 else: expected_num_nodes += config.ENV_DATA.get("infra_replicas", 0) - existing_num_nodes = len(get_all_nodes()) # Some nodes will take time to create due to the issue https://issues.redhat.com/browse/SDA-6346 diff --git a/ocs_ci/utility/ibmcloud.py b/ocs_ci/utility/ibmcloud.py index 8f7ed4b5a0b..0f4a683a31d 100644 --- a/ocs_ci/utility/ibmcloud.py +++ b/ocs_ci/utility/ibmcloud.py @@ -189,16 +189,22 @@ def create_cluster(cluster_name): """ provider = config.ENV_DATA["provider"] + worker_availability_zones = config.ENV_DATA.get("worker_availability_zones", []) + worker_zones_number = len(worker_availability_zones) zone = config.ENV_DATA["zone"] flavor = config.ENV_DATA["worker_instance_type"] worker_replicas = config.ENV_DATA["worker_replicas"] + if worker_zones_number > 1: + worker_replicas = 2 ocp_version = get_ibmcloud_ocp_version() - cmd = ( f"ibmcloud ks cluster create {provider} --name {cluster_name}" f" --flavor {flavor} --workers {worker_replicas}" f" --kube-version {ocp_version}" ) + # Reloading correct number of worker replica for later usage. + if worker_zones_number > 1: + worker_replicas = int(config.ENV_DATA["worker_replicas"] / worker_zones_number) if provider == "vpc-gen2": semantic_ocp_version = util_version.get_semantic_ocp_version_from_config() if semantic_ocp_version >= util_version.VERSION_4_15: @@ -215,11 +221,27 @@ def create_cluster(cluster_name): cluster_info = get_cluster_details(cluster_name) # Create metadata file to store the cluster name cluster_info["clusterName"] = cluster_name - cluster_info["clusterID"] = cluster_info["id"] + cluster_id = cluster_info["id"] + cluster_info["clusterID"] = cluster_id cluster_path = config.ENV_DATA["cluster_path"] metadata_file = os.path.join(cluster_path, "metadata.json") with open(metadata_file, "w+") as f: json.dump(cluster_info, f) + for worker_zone in worker_availability_zones: + if worker_zone == zone: + continue + subnet = config.ENV_DATA["subnet_ids_per_zone"][worker_zone] + cmd = ( + f"ibmcloud oc zone add {provider} --subnet-id {subnet} " + f"--cluster {cluster_id} --zone {worker_zone} --worker-pool default" + ) + run_ibmcloud_cmd(cmd) + if worker_zones_number > 1: + cmd = ( + f"ibmcloud ks worker-pool resize --cluster {cluster_name} --worker-pool " + f"default --size-per-zone {worker_replicas}" + ) + run_ibmcloud_cmd(cmd) # Temporary increased timeout to 10 hours cause of issue with deployment on # IBM cloud timeout = 36000