feat: Support dynamic worker node joining with cloud-init (#11)

README.md

@@ -13,6 +13,7 @@ Creates a Kubernetes cluster on the [Hetzner cloud](https://registry.terraform.i
   - A full-mesh dynamic overlay network using Wireguard, so pod-to-pod traffic is encrypted (Hetzner private networks [are not encrypted](https://docs.hetzner.com/cloud/networks/faq#is-traffic-inside-hetzner-cloud-networks-encrypted), just segregated)
 - deploys the [Controller Manager](https://github.com/hetznercloud/hcloud-cloud-controller-manager) so `LoadBalancer` services provision Hetzner load balancers and deleted nodes are cleaned up.
 - deploys the [Container Storage Interface](https://github.com/hetznercloud/csi-driver) for dynamic provisioning of volumes
+- supports dynamic worker node provisioning with cloud-init e.g. for use with [cluster autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/hetzner)
 
 ## Getting Started
 
@@ -54,7 +55,7 @@ terraform output -raw kubeconfig > kubeconfig.conf
 and check the access by viewing the created cluster nodes:
 
 ```cmd
-$ kubectl get nodes --kubeconfig=demo-cluster.conf
+$ kubectl get nodes --kubeconfig=kubeconfig.conf
 NAME           STATUS   ROLES                  AGE   VERSION
 k8s-master-0   Ready    control-plane,master   31m   v1.21.1
 k8s-worker-0   Ready    <none>                 31m   v1.21.1
@@ -124,6 +125,13 @@ provider "kubernetes" {
 }
 ```
 
+## Cloud-init script for joining additional worker nodes
+
+Once control plane is set up, module has an output called `join_user_data` that contains a cloud-init script that
+can be used to join additional worker nodes outside of Terraform (e.g. for use with [cluster autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/hetzner)).
+
+See [example](./examples/cloud_init.tf) for how it can be used to manage worker separately from this module.
+
 ## Caveats
 
 Read these notes carefully before using this module in production.
@@ -144,4 +152,4 @@ In addition some caveats for dual-stack clusters in general:
 
 ## Acknowledgements 
 
-Some parts, including this README, adapted from [JWDobken/terraform-hcloud-kubernetes](https://github.com/JWDobken/terraform-hcloud-kubernetes) by Joost Döbken.
+Some parts, including this README, adapted from [JWDobken/terraform-hcloud-kubernetes](https://github.com/JWDobken/terraform-hcloud-kubernetes) by Joost Döbken.

examples/cloud_init.tf

@@ -0,0 +1,51 @@
+# A simple dual-stack cluster with a single master node
+
+terraform {
+  required_providers {
+    hcloud = {
+      source  = "hetznercloud/hcloud"
+      version = "1.26.0"
+    }
+  }
+}
+
+variable "hetzner_token" {}
+
+provider "hcloud" {
+  token = vars.hetzner_token
+}
+
+resource "hcloud_ssh_key" "key" {
+  name       = "key"
+  public_key = file("~/.ssh/id_rsa.pub")
+}
+
+module "k8s" {
+  source = "tibordp/dualstack-k8s/hcloud"
+
+  name               = "k8s"
+  hcloud_ssh_key     = hcloud_ssh_key.key.id
+  hcloud_token       = vars.hetzner_token
+  location           = "hel1"
+  master_server_type = "cx31"
+  worker_server_type = "cx31"
+
+  generate_join_configuration = true
+}
+
+// After control plane is set up, additional workers can be joined
+// just with user data (can be used for e.g. cluster autoscaler)
+resource "hcloud_server" "instance" {
+  name        = "additional-worker-node"
+  ssh_keys    = [hcloud_ssh_key.key.id]
+  image       = "ubuntu-20.04"
+  location    = "hel1"
+  server_type = "cx31"
+
+  user_data = module.k8s.join_user_data
+}
+
+
+output "simple_kubeconfig" {
+  value = module.k8s.kubeconfig
+}

joinconfig.tf

@@ -0,0 +1,24 @@
+module "join_config" {
+  source     = "matti/resource/shell"
+  depends_on = [null_resource.cluster_bootstrap]
+
+  trigger = null_resource.cluster_bootstrap.id
+
+  command = <<EOT
+    ssh -i ${var.ssh_private_key_path} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
+      root@${local.kubeadm_host} 'kubeadm token create --print-join-command'
+  EOT
+}
+
+data "template_cloudinit_config" "join_config" {
+  gzip          = true
+  base64_encode = true
+
+  part {
+    content_type = "text/x-shellscript"
+    content = join("\n", [
+      file("${path.module}/modules/kubernetes-node/scripts/prepare-node.sh"),
+      module.join_config.stdout
+    ])
+  }
+}

master.tf

@@ -27,9 +27,6 @@ module "master" {
   image          = var.image
   location       = var.location
 
-  pool_index = 1
-  node_index = count.index
-
   labels       = merge(var.labels, { cluster = var.name, role = "master" })
   firewall_ids = var.firewall_ids
 
@@ -57,10 +54,11 @@ resource "null_resource" "cluster_bootstrap" {
 
   provisioner "file" {
     content = templatefile("${path.module}/templates/kubeadm.yaml.tpl", {
-      apiserverCertSans      = local.apiserver_cert_sans
+      apiserver_cert_sans    = local.apiserver_cert_sans
       certificate_key        = random_id.certificate_key.hex
       control_plane_endpoint = local.control_plane_endpoint
       advertise_address      = module.master[0].ipv6_address
+      pod_cidr_ipv4          = var.pod_cidr_ipv4
       service_cidr_ipv4      = var.service_cidr_ipv4
       service_cidr_ipv6      = var.service_cidr_ipv6
     })
@@ -118,18 +116,4 @@ resource "null_resource" "master_join" {
       "/root/cluster-join.sh",
     ]
   }
-
-  provisioner "remote-exec" {
-    connection {
-      host        = local.kubeadm_host
-      type        = "ssh"
-      timeout     = "5m"
-      user        = "root"
-      private_key = file(var.ssh_private_key_path)
-    }
-
-    inline = [
-      "kubectl patch node '${module.master[count.index].node_name}' -p '${jsonencode(module.master[count.index].podcidrs_patch)}'",
-    ]
-  }
 }

modules/kubernetes-node/main.tf

@@ -4,40 +4,13 @@ terraform {
       source  = "hetznercloud/hcloud"
       version = ">= 1.26.0"
     }
+    template = {
+      source  = "hashicorp/template"
+      version = "2.2.0"
+    }
   }
 }
 
-/*
-
-Subnetting plan for IPv4. Fixed cluster CIDR of 10.0.0.0/8
-
-  8 bits - 10.x.x.x. prefix 
-  4 bits for pool index (16 pools)
-  10 bits for node index (1024 nodes per pool)
-  10 bits for pod index (1024 pods per node)
-
-Currently only two pools are used:
-  0 - reserved
-  1 - master nodes
-  2 - worker nodes
-
-First IP in each node subnet is a private address of the node itself (not really used,
-but useful for pinging nodes wia the Wiregard tunnel)
-
-For IPv6, every node just uses the 2nd /80 of its own public /64.
-
-*/
-
-
-locals {
-  pod_subnet_v4 = cidrsubnet(
-    cidrsubnet("10.0.0.0/8", 4, var.pool_index),
-    10, var.node_index
-  )
-  pod_subnet_v6        = cidrsubnet(hcloud_server.instance.ipv6_network, 16, 1)
-  private_ipv4_address = cidrhost(local.pod_subnet_v4, 1)
-}
-
 resource "hcloud_server" "instance" {
   name        = var.name
   ssh_keys    = [var.hcloud_ssh_key]

modules/kubernetes-node/outputs.tf

@@ -8,11 +8,6 @@ output "node_name" {
   value       = var.name
 }
 
-output "private_ipv4_address" {
-  description = "IPv4 address of the server"
-  value       = local.private_ipv4_address
-}
-
 output "ipv4_address" {
   description = "IPv4 address of the server"
   value       = hcloud_server.instance.ipv4_address
@@ -26,27 +21,4 @@ output "ipv6_address" {
 output "ipv6_network" {
   description = "IPv6 network of the server"
   value       = hcloud_server.instance.ipv6_network
-}
-
-output "pod_subnet_v6" {
-  description = "IPv6 address of the server"
-  value       = local.pod_subnet_v6
-}
-
-output "pod_subnet_v4" {
-  description = "IPv6 network of the server"
-  value       = local.pod_subnet_v4
-}
-
-output "podcidrs_patch" {
-  description = "Pod cidrs patch"
-  value = {
-    "spec" = {
-      "podCIDR" = local.pod_subnet_v6,
-      "podCIDRs" = [
-        local.pod_subnet_v6,
-        local.pod_subnet_v4
-      ]
-    }
-  }
 }

modules/kubernetes-node/scripts/prepare-node.sh

@@ -8,55 +8,60 @@ EOF
 
 sudo modprobe overlay
 sudo modprobe br_netfilter
-
 # Setup required sysctl params, these persist across reboots.
 cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
 net.bridge.bridge-nf-call-iptables  = 1
 net.ipv4.ip_forward                 = 1
 net.ipv6.conf.all.forwarding        = 1
 net.bridge.bridge-nf-call-ip6tables = 1
 EOF
-
 sudo sysctl --system
 
-# Install CRI
-sudo apt-get update
-sudo apt-get install -y \
-  apt-transport-https \
-  ca-certificates \
-  curl \
-  gnupg \
-  lsb-release \
-  ipvsadm \
-  wireguard
-
+# Install prerequisites
+sudo apt-get update -qq
+sudo apt-get install -qq apt-transport-https ca-certificates curl gnupg lsb-release ipvsadm wireguard
 curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-echo \
-  "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
-  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
-
 sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg
-echo \
-  "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
+echo  "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | \
+  sudo tee /etc/apt/sources.list.d/docker.list >/dev/null
+echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | \
+  sudo tee /etc/apt/sources.list.d/kubernetes.list
 
-sudo apt-get update
-sudo apt-get install -y containerd.io
+# Install container runtime and Kubernetes
+sudo apt-get update -qq
+sudo apt-get install -qq containerd.io kubelet=1.21.1-00 kubeadm=1.21.1-00 kubectl=1.21.1-00
+apt-mark hold kubelet kubeadm kubectl
 
 # Enable systemd cgroups driver
 sudo mkdir -p /etc/containerd
 containerd config default | \
   perl -i -pe 's/(\s+)(\[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options\])/\1\2\n\1  SystemdCgroup = true/g' | \
-  sudo tee /etc/containerd/config.toml
-
-sudo systemctl restart containerd
-
-# Install Kubernetes
-sudo apt-get install -y kubelet kubeadm kubectl
+  sudo tee /etc/containerd/config.toml > /dev/null
 
-cat <<EOF | sudo tee /etc/systemd/system/kubelet.service.d/20-hcloud.conf
+# Necessary for out-of-tree cloud providers as of 1.21.1 (soon to be deprecated)
+cat <<EOF | sudo tee /etc/systemd/system/kubelet.service.d/20-hcloud.conf > /dev/null
 [Service]
 Environment="KUBELET_EXTRA_ARGS=--cloud-provider=external"
 EOF
 
-kubeadm config images pull
-sudo systemctl restart kubelet
+sudo systemctl daemon-reload
+sudo systemctl restart containerd kubelet
+
+# Determine the IPv6 pod subnet based on the /64 assigned to eth0 interface (take 2nd /80)
+sudo mkdir -p /etc/wigglenet
+sudo python3 <<EOF
+import re
+import os
+import ipaddress
+import itertools
+
+addrs = os.popen("ip -6 addr show eth0 scope global").read()
+addr = re.search(r"inet6 ([^ ]+/64) scope global", addrs, re.MULTILINE).group(1)
+net = ipaddress.IPv6Network(addr, strict=False)
+pod_subnet = next(itertools.islice(net.subnets(16), 1, None))
+
+with open("/etc/wigglenet/cidrs.txt", "w") as f:
+    print(pod_subnet, file=f)
+
+print(f"Pod CIDR is {pod_subnet}")
+EOF

modules/kubernetes-node/variables.tf

@@ -28,16 +28,6 @@ variable "ssh_private_key_path" {
   type        = string
 }
 
-variable "pool_index" {
-  description = "IPv4 node pool index"
-  type        = number
-}
-
-variable "node_index" {
-  description = "IPv4 node pod CIDR index"
-  type        = number
-}
-
 variable "firewall_ids" {
   description = "List of firewalls attached to this server"
   type        = list(number)

outputs.tf

@@ -36,4 +36,9 @@ output "client_key_data" {
 output "kubeconfig" {
   description = "kubeconfig for the cluster"
   value       = module.kubeconfig.stdout
-}
+}
+
+output "join_user_data" {
+  description = "cloud-init user data for additional worker nodes"
+  value       = data.template_cloudinit_config.join_config.rendered
+}

templates/kubeadm.yaml.tpl

@@ -16,14 +16,12 @@ featureGates:
   IPv6DualStack: true
 apiServer:
   certSANs:
-%{ for san in apiserverCertSans ~}
+%{ for san in apiserver_cert_sans ~}
   - "${san}"
 %{ endfor ~}
-controllerManager:
-  extraArgs:
-    allocate-node-cidrs: "false"
 networking:
-  serviceSubnet: ${service_cidr_ipv6},${service_cidr_ipv4}
+  podSubnet: "${pod_cidr_ipv4}"
+  serviceSubnet: "${service_cidr_ipv6},${service_cidr_ipv4}"
 controlPlaneEndpoint: "${control_plane_endpoint}:6443"
 ---
 kind: KubeProxyConfiguration