From 63ef705385046ab7e77b27eeadeb96b2731e3292 Mon Sep 17 00:00:00 2001 From: Tibor Djurica Potpara Date: Mon, 31 May 2021 16:26:01 +0100 Subject: [PATCH] feat: Make primary IP family configurable (#13) * feat: Make primary IP family configurable * fixup * fixup --- .github/workflows/test.yml | 6 ++++-- README.md | 12 ++++++++---- setup.tf => addons.tf | 12 ++++++------ examples/cloud_init.tf | 2 -- master.tf | 8 +++++--- scripts/{cluster-setup.sh => install-addons.sh} | 0 templates/kubeadm.yaml.tpl | 6 +++++- variables.tf | 15 ++++++++++----- 8 files changed, 38 insertions(+), 23 deletions(-) rename setup.tf => addons.tf (71%) rename scripts/{cluster-setup.sh => install-addons.sh} (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be95b94..0bb289f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -36,7 +36,9 @@ jobs: run: terraform apply -auto-approve working-directory: test + # Disable locking of statefile in case previous step was canceled, we + # still want to clean up any resources created. - name: Terraform Destroy - if: ${{ always() }} - run: terraform destroy -auto-approve + if: ${{ always() }} + run: terraform destroy -auto-approve -lock=false working-directory: test diff --git a/README.md b/README.md index 52e9a4b..a3075bf 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ Creates a Kubernetes cluster on the [Hetzner cloud](https://registry.terraform.i - Single or multiple control plane nodes (in [HA configuration with stacked `etcd`](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/)) - containerd for container runtime -- [Wigglenet](https://github.com/tibordp/wigglenet) as a network plugin - - the primary address family for the cluster is IPv6, which is used for control plane communication +- [Wigglenet](https://github.com/tibordp/wigglenet) for the network plugin + - the primary address family for the cluster is configurable, but defaults to IPv6, which is used for control plane communication - pods are allocated a private IPv4 address and a public IPv6 from the /64 subnet that Hetzner gives to every node. No masquerading needed for outbound IPv6 traffic! 🎉 (stateful firewall rules are still in place, so direct ingress traffic to pods is blocked by default, prefer to expose workloads through Service) - Dual-stack and IPv6-only `Service`s get a private (ULA) IPv6 address - A full-mesh dynamic overlay network using Wireguard, so pod-to-pod traffic is encrypted (Hetzner private networks [are not encrypted](https://docs.hetzner.com/cloud/networks/faq#is-traffic-inside-hetzner-cloud-networks-encrypted), just segregated) @@ -31,6 +31,7 @@ Create a simple Kubernetes cluster: ```hcl module "k8s" { source = "tibordp/dualstack-k8s/hcloud" + version = "0.6.0" name = "k8s" hcloud_ssh_key = hcloud_ssh_key.key.id @@ -92,7 +93,8 @@ First master node is special in that it is used by the provisioning process (e.g ```hcl module "k8s" { source = "tibordp/dualstack-k8s/hcloud" - + version = "0.6.0" + ... kubeadm_host = "" @@ -143,11 +145,13 @@ Read these notes carefully before using this module in production. - Node replacement (see notes above for control plane nodes) - Vertical scaling of node (changing the server type) - Horizontal scaling (changing node count). + - Changing cluster addons settings (Wigglenet firewall settings, Hetzner API token for the Hetzner CCM and CSI). - As kube-proxy is configured to use IPVS mode, `load-balancer.hetzner.cloud/hostname: ` must be set on all `LoadBalancer` services, otherwise healthchecks will fail and the service will not be accessible from outsie the cluster (see [this issue](https://github.com/kubernetes/kubernetes/issues/79783) for more details) In addition some caveats for dual-stack clusters in general: -- `Services` are single-stack by default. Since IPv6 is the primary IP family of the clusters created with this modules, this means the `ClusterIP` will be IPv6 only, leading to issues for workloads that only bind on IPv4. Pass `ipFamilyPolicy: PreferDualStack` when creating services to assign both IPv4 and IPv6 ClusterIPs. +- `Services` are single-stack by default. Since IPv6 is the primary IP family of the clusters created with this modules, this means the `ClusterIP` will be IPv6 only, leading to issues for workloads that only bind on IPv4. Pass `ipFamilyPolicy: PreferDualStack` when creating services to assign both IPv4 and IPv6 ClusterIPs. You can use the [prefer-dual-stack-webhook](https://github.com/tibordp/prefer-dual-stack-webhook) admission controller to change the default to `PreferDualStack` for all newly creted services that don't specify IP family policy. +- the apiserver Service (`kubernetes.default.svc.cluster.local`) has to be single-stack, as `--apiserver-advertise-address` does not support dual-stack yet. The default address family for the cluster can be selected with `primary_ip_family` variable (defaults to `ipv6`). ## Acknowledgements diff --git a/setup.tf b/addons.tf similarity index 71% rename from setup.tf rename to addons.tf index 60f435f..7af51f0 100644 --- a/setup.tf +++ b/addons.tf @@ -1,4 +1,4 @@ -resource "null_resource" "setup_cluster" { +resource "null_resource" "install_addons" { depends_on = [ null_resource.cluster_bootstrap ] @@ -24,14 +24,14 @@ resource "null_resource" "setup_cluster" { } provisioner "file" { - source = "${path.module}/scripts/cluster-setup.sh" - destination = "/root/cluster-setup.sh" + source = "${path.module}/scripts/install-addons.sh" + destination = "/root/install-addons.sh" } provisioner "remote-exec" { inline = [ - "chmod +x /root/cluster-setup.sh", - "HCLOUD_TOKEN='${var.hcloud_token}' /root/cluster-setup.sh", + "chmod +x /root/install-addons.sh", + "HCLOUD_TOKEN='${var.hcloud_token}' /root/install-addons.sh", ] } -} \ No newline at end of file +} diff --git a/examples/cloud_init.tf b/examples/cloud_init.tf index 7c946e1..3166013 100644 --- a/examples/cloud_init.tf +++ b/examples/cloud_init.tf @@ -29,8 +29,6 @@ module "k8s" { location = "hel1" master_server_type = "cx31" worker_server_type = "cx31" - - generate_join_configuration = true } // After control plane is set up, additional workers can be joined diff --git a/master.tf b/master.tf index 76de835..b0f6c1f 100644 --- a/master.tf +++ b/master.tf @@ -5,6 +5,7 @@ locals { control_plane_endpoint = var.control_plane_endpoint != "" ? var.control_plane_endpoint : (local.use_load_balancer ? "[${hcloud_load_balancer.control_plane[0].ipv6}]" : "[${module.master[0].ipv6_address}]") + adverise_addresses = var.primary_ip_family == "ipv6" ? module.master.*.ipv6_address : module.master.*.ipv4_address # If using IP as an apiserver endpoint, add also the IPv4 SAN to the TLS certificate apiserver_cert_sans = concat(var.control_plane_endpoint != "" ? [ @@ -57,10 +58,11 @@ resource "null_resource" "cluster_bootstrap" { apiserver_cert_sans = local.apiserver_cert_sans certificate_key = random_id.certificate_key.hex control_plane_endpoint = local.control_plane_endpoint - advertise_address = module.master[0].ipv6_address + advertise_address = local.adverise_addresses[0] pod_cidr_ipv4 = var.pod_cidr_ipv4 service_cidr_ipv4 = var.service_cidr_ipv4 service_cidr_ipv6 = var.service_cidr_ipv6 + primary_ip_family = var.primary_ip_family }) destination = "/root/cluster.yaml" } @@ -97,7 +99,7 @@ resource "null_resource" "master_join" { ssh -i ${var.ssh_private_key_path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ root@${local.kubeadm_host} \ 'echo $(kubeadm token create --print-join-command --ttl=60m) \ - --apiserver-advertise-address ${module.master[count.index].ipv6_address} \ + --apiserver-advertise-address ${local.adverise_addresses[count.index]} \ --control-plane \ --certificate-key ${random_id.certificate_key.hex}' | \ ssh -i ${var.ssh_private_key_path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ @@ -116,4 +118,4 @@ resource "null_resource" "master_join" { "/root/cluster-join.sh", ] } -} \ No newline at end of file +} diff --git a/scripts/cluster-setup.sh b/scripts/install-addons.sh similarity index 100% rename from scripts/cluster-setup.sh rename to scripts/install-addons.sh diff --git a/templates/kubeadm.yaml.tpl b/templates/kubeadm.yaml.tpl index b67232b..c03dc2d 100644 --- a/templates/kubeadm.yaml.tpl +++ b/templates/kubeadm.yaml.tpl @@ -21,9 +21,13 @@ apiServer: %{ endfor ~} networking: podSubnet: "${pod_cidr_ipv4}" +%{ if primary_ip_family == "ipv4" ~} + serviceSubnet: "${service_cidr_ipv4},${service_cidr_ipv6}" +%{ else ~} serviceSubnet: "${service_cidr_ipv6},${service_cidr_ipv4}" +%{ endif ~} controlPlaneEndpoint: "${control_plane_endpoint}:6443" --- kind: KubeProxyConfiguration apiVersion: kubeproxy.config.k8s.io/v1alpha1 -mode: ipvs \ No newline at end of file +mode: ipvs diff --git a/variables.tf b/variables.tf index e6f2609..05607a1 100644 --- a/variables.tf +++ b/variables.tf @@ -111,13 +111,18 @@ variable "apiserver_extra_sans" { } variable "filter_pod_ingress_ipv6" { - description = "Filter out ingress IPv6 traffic directed to pods (default: false)" + description = "Filter out ingress IPv6 traffic directed to pods (default: true)" type = bool default = true } -variable "generate_join_configuration" { - description = "Generate cloud-init user data file for additional workers to join" - type = bool - default = false +variable "primary_ip_family" { + description = "(Optional) Primary IP family for Service resources in cluster (default: ipv6)" + type = string + default = "ipv6" + + validation { + condition = can(regex("^(ipv4|ipv6)$", var.primary_ip_family)) + error_message = "The primary_ip_family value must be a \"ipv6\" or \"ipv4\"." + } }