feat: Make primary IP family configurable (#13)

* feat: Make primary IP family configurable

* fixup

* fixup

.github/workflows/test.yml

@@ -36,7 +36,9 @@ jobs:
       run: terraform apply -auto-approve
       working-directory: test
 
+      # Disable locking of statefile in case previous step was canceled, we
+      # still want to clean up any resources created.
     - name: Terraform Destroy
-      if: ${{ always() }}
-      run: terraform destroy -auto-approve
+      if: ${{ always() }}      
+      run: terraform destroy -auto-approve -lock=false
       working-directory: test

README.md

@@ -6,8 +6,8 @@ Creates a Kubernetes cluster on the [Hetzner cloud](https://registry.terraform.i
 
 - Single or multiple control plane nodes (in [HA configuration with stacked `etcd`](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/high-availability/))
 - containerd for container runtime
-- [Wigglenet](https://github.com/tibordp/wigglenet) as a network plugin
-  - the primary address family for the cluster is IPv6, which is used for control plane communication
+- [Wigglenet](https://github.com/tibordp/wigglenet) for the network plugin
+  - the primary address family for the cluster is configurable, but defaults to IPv6, which is used for control plane communication
   - pods are allocated a private IPv4 address and a public IPv6 from the /64 subnet that Hetzner gives to every node. No masquerading needed for outbound IPv6 traffic! 🎉 (stateful firewall rules are still in place, so direct ingress traffic to pods is blocked by default, prefer to expose workloads through Service)
   - Dual-stack and IPv6-only `Service`s get a private (ULA) IPv6 address
   - A full-mesh dynamic overlay network using Wireguard, so pod-to-pod traffic is encrypted (Hetzner private networks [are not encrypted](https://docs.hetzner.com/cloud/networks/faq#is-traffic-inside-hetzner-cloud-networks-encrypted), just segregated)
@@ -31,6 +31,7 @@ Create a simple Kubernetes cluster:
 ```hcl
 module "k8s" {
   source  = "tibordp/dualstack-k8s/hcloud"
+  version = "0.6.0"
 
   name               = "k8s"
   hcloud_ssh_key     = hcloud_ssh_key.key.id
@@ -92,7 +93,8 @@ First master node is special in that it is used by the provisioning process (e.g
 ```hcl
 module "k8s" {
   source  = "tibordp/dualstack-k8s/hcloud"
- 
+  version = "0.6.0"
+
   ...
  
   kubeadm_host = "<ip address of another master node>"
@@ -143,11 +145,13 @@ Read these notes carefully before using this module in production.
    - Node replacement (see notes above for control plane nodes)
    - Vertical scaling of node (changing the server type)
    - Horizontal scaling (changing node count).
+   - Changing cluster addons settings (Wigglenet firewall settings, Hetzner API token for the Hetzner CCM and CSI).
 - As kube-proxy is configured to use IPVS mode, `load-balancer.hetzner.cloud/hostname: <hostname>` must be set on all `LoadBalancer` services, otherwise healthchecks will fail and the service will not be accessible from outsie the cluster (see [this issue](https://github.com/kubernetes/kubernetes/issues/79783) for more details)
 
 In addition some caveats for dual-stack clusters in general:
 
-- `Services` are single-stack by default. Since IPv6 is the primary IP family of the clusters created with this modules, this means the `ClusterIP` will be IPv6 only, leading to issues for workloads that only bind on IPv4. Pass `ipFamilyPolicy: PreferDualStack` when creating services to assign both IPv4 and IPv6 ClusterIPs.
+- `Services` are single-stack by default. Since IPv6 is the primary IP family of the clusters created with this modules, this means the `ClusterIP` will be IPv6 only, leading to issues for workloads that only bind on IPv4. Pass `ipFamilyPolicy: PreferDualStack` when creating services to assign both IPv4 and IPv6 ClusterIPs. You can use the [prefer-dual-stack-webhook](https://github.com/tibordp/prefer-dual-stack-webhook) admission controller to change the default to `PreferDualStack` for all newly creted services that don't specify IP family policy.
+- the apiserver Service (`kubernetes.default.svc.cluster.local`) has to be single-stack, as `--apiserver-advertise-address` does not support dual-stack yet. The default address family for the cluster can be selected with `primary_ip_family` variable (defaults to `ipv6`).
 
 
 ## Acknowledgements 

setup.tf → addons.tf

@@ -1,4 +1,4 @@
-resource "null_resource" "setup_cluster" {
+resource "null_resource" "install_addons" {
   depends_on = [
     null_resource.cluster_bootstrap
   ]
@@ -24,14 +24,14 @@ resource "null_resource" "setup_cluster" {
   }
 
   provisioner "file" {
-    source      = "${path.module}/scripts/cluster-setup.sh"
-    destination = "/root/cluster-setup.sh"
+    source      = "${path.module}/scripts/install-addons.sh"
+    destination = "/root/install-addons.sh"
   }
 
   provisioner "remote-exec" {
     inline = [
-      "chmod +x /root/cluster-setup.sh",
-      "HCLOUD_TOKEN='${var.hcloud_token}' /root/cluster-setup.sh",
+      "chmod +x /root/install-addons.sh",
+      "HCLOUD_TOKEN='${var.hcloud_token}' /root/install-addons.sh",
     ]
   }
-}
+}

examples/cloud_init.tf

@@ -29,8 +29,6 @@ module "k8s" {
   location           = "hel1"
   master_server_type = "cx31"
   worker_server_type = "cx31"
-
-  generate_join_configuration = true
 }
 
 // After control plane is set up, additional workers can be joined

master.tf

@@ -5,6 +5,7 @@ locals {
 
   control_plane_endpoint = var.control_plane_endpoint != "" ? var.control_plane_endpoint : (local.use_load_balancer ? "[${hcloud_load_balancer.control_plane[0].ipv6}]" : "[${module.master[0].ipv6_address}]")
 
+  adverise_addresses = var.primary_ip_family == "ipv6" ? module.master.*.ipv6_address : module.master.*.ipv4_address
 
   # If using IP as an apiserver endpoint, add also the IPv4 SAN to the TLS certificate
   apiserver_cert_sans = concat(var.control_plane_endpoint != "" ? [
@@ -57,10 +58,11 @@ resource "null_resource" "cluster_bootstrap" {
       apiserver_cert_sans    = local.apiserver_cert_sans
       certificate_key        = random_id.certificate_key.hex
       control_plane_endpoint = local.control_plane_endpoint
-      advertise_address      = module.master[0].ipv6_address
+      advertise_address      = local.adverise_addresses[0]
       pod_cidr_ipv4          = var.pod_cidr_ipv4
       service_cidr_ipv4      = var.service_cidr_ipv4
       service_cidr_ipv6      = var.service_cidr_ipv6
+      primary_ip_family      = var.primary_ip_family
     })
     destination = "/root/cluster.yaml"
   }
@@ -97,7 +99,7 @@ resource "null_resource" "master_join" {
       ssh -i ${var.ssh_private_key_path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
         root@${local.kubeadm_host} \
         'echo $(kubeadm token create --print-join-command --ttl=60m) \
-        --apiserver-advertise-address ${module.master[count.index].ipv6_address} \
+        --apiserver-advertise-address ${local.adverise_addresses[count.index]} \
         --control-plane \
         --certificate-key ${random_id.certificate_key.hex}' | \
       ssh -i ${var.ssh_private_key_path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
@@ -116,4 +118,4 @@ resource "null_resource" "master_join" {
       "/root/cluster-join.sh",
     ]
   }
-}
+}

templates/kubeadm.yaml.tpl

@@ -21,9 +21,13 @@ apiServer:
 %{ endfor ~}
 networking:
   podSubnet: "${pod_cidr_ipv4}"
+%{ if primary_ip_family  == "ipv4" ~}  
+  serviceSubnet: "${service_cidr_ipv4},${service_cidr_ipv6}"
+%{ else ~}
   serviceSubnet: "${service_cidr_ipv6},${service_cidr_ipv4}"
+%{ endif ~}
 controlPlaneEndpoint: "${control_plane_endpoint}:6443"
 ---
 kind: KubeProxyConfiguration
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
-mode: ipvs
+mode: ipvs

variables.tf

@@ -111,13 +111,18 @@ variable "apiserver_extra_sans" {
 }
 
 variable "filter_pod_ingress_ipv6" {
-  description = "Filter out ingress IPv6 traffic directed to pods (default: false)"
+  description = "Filter out ingress IPv6 traffic directed to pods (default: true)"
   type        = bool
   default     = true
 }
 
-variable "generate_join_configuration" {
-  description = "Generate cloud-init user data file for additional workers to join"
-  type        = bool
-  default     = false
+variable "primary_ip_family" {
+  description = "(Optional) Primary IP family for Service resources in cluster (default: ipv6)"
+  type        = string
+  default     = "ipv6"
+
+  validation {
+    condition     = can(regex("^(ipv4|ipv6)$", var.primary_ip_family))
+    error_message = "The primary_ip_family value must be a \"ipv6\" or \"ipv4\"."
+  }
 }