Merge pull request #343 from vitobotta:system-upgrade-controller-tole…

…rations

Refactor and fix System Upgrade Controller installation

README.md

@@ -159,7 +159,8 @@ schedule_workloads_on_masters: false
 # snapshot_os: microos # optional: specified the os type when using a custom snapshot
 # cloud_controller_manager_manifest_url: "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.19.0/ccm-networks.yaml"
 # csi_driver_manifest_url: "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.6.0/deploy/kubernetes/hcloud-csi.yml"
-# system_upgrade_controller_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.4.0/system-upgrade-controller.yaml"
+# system_upgrade_controller_deployment_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml"
+# system_upgrade_controller_crd_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml"
 datastore:
   mode: etcd # etcd (default) or external
   external_datastore_endpoint: postgres://....

cluster_config.yaml.example

@@ -24,7 +24,8 @@ disable_flannel: false # set to true if you want to install a different CNI
 # snapshot_os: microos # optional: specified the os type when using a custom snapshot
 # cloud_controller_manager_manifest_url: "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.19.0/ccm-networks.yaml"
 # csi_driver_manifest_url: "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.6.0/deploy/kubernetes/hcloud-csi.yml"
-# system_upgrade_controller_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.4.0/system-upgrade-controller.yaml"
+# system_upgrade_controller_deployment_manifest_url: "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml"
+# system_upgrade_controller_crd_manifest_urlL "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml"
 datastore:
   mode: etcd # etcd (default) or external
   external_datastore_endpoint: postgres://....

src/configuration/main.cr

@@ -41,7 +41,8 @@ class Configuration::Main
   getter cluster_dns : String = "10.43.0.10"
   getter cloud_controller_manager_manifest_url : String = "https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/download/v1.19.0/ccm-networks.yaml"
   getter csi_driver_manifest_url : String = "https://raw.githubusercontent.com/hetznercloud/csi-driver/v2.6.0/deploy/kubernetes/hcloud-csi.yml"
-  getter system_upgrade_controller_manifest_url : String = "https://github.com/rancher/system-upgrade-controller/releases/download/v0.4.0/system-upgrade-controller.yaml"
+  getter system_upgrade_controller_deployment_manifest_url : String = "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/system-upgrade-controller.yaml"
+  getter system_upgrade_controller_crd_manifest_url : String = "https://github.com/rancher/system-upgrade-controller/releases/download/v0.13.4/crd.yaml"
   getter disable_flannel : Bool = false
   getter ssh_port : Int32 = 22
   getter datastore : Configuration::Datastore = Configuration::Datastore.new

src/kubernetes/installer.cr

@@ -1,12 +1,14 @@
 require "crinja"
 require "base64"
+require "file_utils"
+
 require "../util"
 require "../util/ssh"
 require "../util/shell"
 require "../hetzner/server"
 require "../hetzner/load_balancer"
 require "../configuration/loader"
-require "file_utils"
+require "./software/system_upgrade_controller"
 
 class Kubernetes::Installer
   MASTER_INSTALL_SCRIPT = {{ read_file("#{__DIR__}/../../templates/master_install_script.sh") }}
@@ -46,7 +48,7 @@ class Kubernetes::Installer
     create_hetzner_cloud_secret
     deploy_cloud_controller_manager
     deploy_csi_driver
-    deploy_system_upgrade_controller
+    Kubernetes::Software::SystemUpgradeController.new(configuration, settings).install
     deploy_cluster_autoscaler unless autoscaling_worker_node_pools.size.zero?
   end
 
@@ -296,23 +298,6 @@ class Kubernetes::Installer
     puts "...CSI Driver deployed"
   end
 
-  private def deploy_system_upgrade_controller
-    puts "\nDeploying k3s System Upgrade Controller..."
-
-    # Run second manifest twice to fix problem with namespace creation
-    command = "kubectl apply -f #{settings.system_upgrade_controller_manifest_url}"
-
-    result = Util::Shell.run(command, configuration.kubeconfig_path, settings.hetzner_token)
-
-    unless result.success?
-      puts "Failed to deploy k3s System Upgrade Controller:"
-      puts result.output
-      exit 1
-    end
-
-    puts "...k3s System Upgrade Controller deployed."
-  end
-
   private def deploy_cluster_autoscaler
     puts "\nDeploying Cluster Autoscaler..."
 

src/kubernetes/resources/deployment.cr

@@ -0,0 +1,10 @@
+require "./deployment/spec"
+
+module Kubernetes::Resources
+  class Deployment
+    include YAML::Serializable
+    include YAML::Serializable::Unmapped
+
+    property spec : Kubernetes::Resources::Deployment::Spec
+  end
+end

src/kubernetes/resources/deployment/spec.cr

@@ -0,0 +1,12 @@
+require "./spec/template"
+
+module Kubernetes::Resources
+  class Deployment
+    class Spec
+      include YAML::Serializable
+      include YAML::Serializable::Unmapped
+
+      property template : Kubernetes::Resources::Deployment::Spec::Template
+    end
+  end
+end

src/kubernetes/resources/deployment/spec/template.cr

@@ -0,0 +1,14 @@
+require "../../pod/spec"
+
+module Kubernetes::Resources
+  class Deployment
+    class Spec
+      class Template
+        include YAML::Serializable
+        include YAML::Serializable::Unmapped
+
+        property spec : Kubernetes::Resources::Pod::Spec
+      end
+    end
+  end
+end

src/kubernetes/resources/deployment/spec/template/spec.cr

@@ -0,0 +1,16 @@
+require "../../../pod/spec/toleration"
+
+module Kubernetes::Resources
+  class Deployment
+    class Spec
+      class Template
+        class Spec
+          include YAML::Serializable
+          include YAML::Serializable::Unmapped
+
+          property tolerations : Array(Kubernetes::Resources::Pod::Spec::Toleration)?
+        end
+      end
+    end
+  end
+end

src/kubernetes/resources/manifest_sections/section_with_spec.cr

@@ -0,0 +1,12 @@
+module Kubernetes::Resources
+  class Deployment
+    class Spec
+      class Template
+        include YAML::Serializable
+        include YAML::Serializable::Unmapped
+
+        property spec : Kubernetes::Resources::Deployment::Spec::Template::Spec
+      end
+    end
+  end
+end

src/kubernetes/resources/pod.cr

@@ -0,0 +1,10 @@
+require "./pod/spec"
+
+module Kubernetes::Resources
+  class Pod
+    include YAML::Serializable
+    include YAML::Serializable::Unmapped
+
+    property spec : Kubernetes::Resources::Pod::Spec
+  end
+end

src/kubernetes/resources/pod/spec.cr

@@ -0,0 +1,12 @@
+require "./spec/toleration"
+
+module Kubernetes::Resources
+  class Pod
+    class Spec
+      include YAML::Serializable
+      include YAML::Serializable::Unmapped
+
+      property tolerations : Array(Kubernetes::Resources::Pod::Spec::Toleration)?
+    end
+  end
+end

src/kubernetes/resources/pod/spec/toleration.cr

@@ -0,0 +1,15 @@
+class Kubernetes::Resources::Pod
+  class Spec
+    class Toleration
+      include YAML::Serializable
+      include YAML::Serializable::Unmapped
+
+      property effect : String?
+      property key : String?
+      property value : String?
+
+      def initialize(@effect, @key, @value)
+      end
+    end
+  end
+end

src/kubernetes/resources/resource.cr

@@ -0,0 +1,8 @@
+module Kubernetes::Resources
+  class Resource
+    include YAML::Serializable
+    include YAML::Serializable::Unmapped
+
+    property kind : String
+  end
+end

src/kubernetes/software/system_upgrade_controller.cr

@@ -0,0 +1,93 @@
+require "../resources/resource"
+require "../resources/deployment"
+require "../resources/pod/spec/toleration"
+require "../../configuration/loader"
+require "../../configuration/main"
+
+class Kubernetes::Software::SystemUpgradeController
+  getter configuration : Configuration::Loader
+  getter settings : Configuration::Main { configuration.settings }
+
+  def initialize(@configuration, @settings)
+
+  end
+
+  def install
+    puts "\n[System Upgrade Controller] Deploying k3s System Upgrade Controller..."
+
+    create_namespace
+    create_crd
+    create_resources
+
+    puts "[System Upgrade Controller] ...k3s System Upgrade Controller deployed."
+  end
+
+  private def create_namespace
+    run_command "kubectl create ns system-upgrade --dry-run=client -o yaml | kubectl apply -f -"
+  end
+
+  private def create_crd
+    run_command "kubectl apply -f #{settings.system_upgrade_controller_crd_manifest_url}"
+  end
+
+  private def create_resources
+    manifest = fetch_resources_manifest
+    resources = YAML.parse_all(manifest)
+    patched_resources = patch_resources(resources)
+    patched_manifest = patched_resources.map(&.to_yaml).join
+    updated_manifest_path = "/tmp/manifest.yaml"
+
+    File.write(updated_manifest_path, patched_manifest)
+
+    run_command "kubectl apply -f #{updated_manifest_path}"
+
+    File.delete(updated_manifest_path)
+  end
+
+  private def fetch_resources_manifest
+    response = Crest.get(settings.system_upgrade_controller_deployment_manifest_url)
+
+    unless response.success?
+      puts "[System Upgrade Controller] Failed to download System Upgrade Controller manifest from #{settings.system_upgrade_controller_deployment_manifest_url}"
+      puts "[System Upgrade Controller] Server responded with status #{response.status_code}"
+      exit 1
+    end
+
+    response.body.to_s
+  end
+
+  private def deployment_with_added_toleration(resource)
+    deployment = Kubernetes::Resources::Deployment.from_yaml(resource.to_yaml)
+    toleration = Kubernetes::Resources::Pod::Spec::Toleration.new(effect: "NoExecute", key: "CriticalAddonsOnly", value: "true")
+
+    if tolerations = deployment.spec.template.spec.tolerations
+      tolerations << toleration
+    else
+      deployment.spec.template.spec.tolerations = [toleration]
+    end
+
+    deployment
+  end
+
+  private def run_command(command)
+    result = Util::Shell.run(command, configuration.kubeconfig_path, settings.hetzner_token, prefix: "System Upgrade Controller")
+
+    unless result.success?
+      puts "[System Upgrade Controller] Failed to deploy k3s System Upgrade Controller:"
+      puts result.output
+      exit 1
+    end
+  end
+
+  private def patch_resources(resources)
+    resources.map do |resource|
+      resource = Kubernetes::Resources::Resource.from_yaml(resource.to_yaml)
+
+      if resource.kind == "Deployment"
+        deployment_with_added_toleration(resource)
+      else
+        resource
+      end
+    end
+  end
+end

src/util/prefixed_io.cr

@@ -6,10 +6,10 @@ class PrefixedIO < IO
   end
 
   def write(slice : Bytes) : Nil
-    @io.print @prefix
-    slice.each do |byte|
-      @io.write_byte byte
-      @io.print @prefix if 10 == byte
+    content = String.new(slice)
+    lines = content.lines
+    lines.each do |line|
+      @io << @prefix << "#{line}\n"
     end
   end
 end

src/util/shell.cr

@@ -12,7 +12,7 @@ module Util
       end
     end
 
-    def self.run(command : String, kubeconfig_path : String, hetzner_token : String) : Result
+    def self.run(command : String, kubeconfig_path : String, hetzner_token : String, prefix : String = "") : Result
       cmd_file_path = "/tmp/cli.cmd"
 
       File.write(cmd_file_path, <<-CONTENT
@@ -26,7 +26,11 @@ module Util
       stdout = IO::Memory.new
       stderr = IO::Memory.new
 
-      all_io_out = IO::MultiWriter.new(STDOUT, stdout)
+      all_io_out = if prefix.blank?
+        IO::MultiWriter.new(STDOUT, stdout)
+      else
+        IO::MultiWriter.new(PrefixedIO.new("[#{prefix}] ", STDOUT), stdout)
+      end
       all_io_err = IO::MultiWriter.new(STDERR, stderr)
 
       env = {