kubevirtci

#!/bin/bash

set -e

export KUBEVIRT_PROVIDER=${KUBEVIRT_PROVIDER:-k8s-1.23}
export CAPK_GUEST_K8S_VERSION=${CAPK_GUEST_K8S_VERSION:-v1.21.0}
export KUBEVIRTCI_TAG=${KUBEVIRTCI_TAG:-2205231118-f12b50e}
export KUBECONFIG=$(cluster-up/cluster-up/kubeconfig.sh)
export KUBEVIRT_DEPLOY_PROMETHEUS=false
export KUBEVIRT_DEPLOY_CDI=false
export KUBEVIRT_NUM_NODES=${KUBEVIRT_NUM_NODES:-1}
export KUBEVIRT_MEMORY_SIZE=${KUBEVIRT_MEMORY_SIZE:-15360M}
export KUBEVIRT_DEPLOY_CDI="true"
export KUBEVIRT_STORAGE="rook-ceph-default"
export METALLB_VERSION="v0.12.1"
export CAPK_RELEASE_VERSION="v0.1.7"
export CLUSTERCTL_VERSION="v1.0.0"
export CALICO_VERSION="v3.21"

_default_bin_path=./hack/tools/bin
_default_tmp_path=./hack/tools/bin/tmp
_default_clusterctl_path=./hack/tools/bin/clusterctl
_default_virtctl_path=./hack/tools/bin/virtctl

export CLUSTERCTL_PATH=${CLUSTERCTL_PATH:-${_default_clusterctl_path}}
export TENANT_CLUSTER_NAME=${TENANT_CLUSTER_NAME:-kvcluster}
export TENANT_CLUSTER_NAMESPACE=${TENANT_CLUSTER_NAMESPACE:-kvcluster}

_kubectl=cluster-up/cluster-up/kubectl.sh
_ssh_infra=cluster-up/cluster-up/ssh.sh

_action=$1
shift


function kubevirtci::usage() {
	echo "Usage:

	./kubevirtci <cammand>

	Commands:

	  up                                Start a cluster and prepare the management/infra cluster and create a tenant on top
	  sync                              Build and deploy current cloud-provider-kubevirt from source
	  down                              Destroy the cluster
	  clean-cache                       Removes all files cached by kubevirtci

	  kubeconfig                        Return the kubeconfig of the cluster
	  kubectl <kubectl options>         Interact with the cluster
	  kubectl-tenant <kubectl options>  Interact with the tenant cluster
	  virtctl <virtctl options>         Run virtctl commands against the cluster
	  clusterctl <clusterctl options>   Run clusterctl commands against the cluster

	  install-capk                      Installs capk from published release manifests
	  install-metallb                   Installs metallb into the infra cluster
	  install-calico                    Installs calico cni into tenant cluster
	  curl-lb <lb name> [lb namespace]  Curls lb service within infra cluster

	  ssh-infra <node name>             SSH into one of the infra nodes (like node01)
	  ssh-tenant <vmi> [vmi namespace]  SSH into one of the guest nodes
	  create-cluster                    Create new kubernetes tenant cluster
	  destroy-cluster                   Destroy the tenant cluster
	  delete                            Uninstall cloud-provider-kubevirt
	  logs                              Print cloud-provider-kubevirt controller logs

	  help                              Print usage
	"
}

function kubevirtci::kubeconfig() {
	cluster-up/cluster-up/kubeconfig.sh
}

function kubevirtci::fetch_kubevirtci() {
	[[ -d cluster-up ]] || git clone https://github.com/kubevirt/kubevirtci.git cluster-up
	(cd cluster-up && git checkout main > /dev/null 2>&1 && git pull > /dev/null && git checkout ${KUBEVIRTCI_TAG} > /dev/null 2>&1)
	mkdir -p ./hack/tools/bin/
	if [ ! -f "${_default_clusterctl_path}" ]; then
		echo >&2 "Downloading clusterctl version ${CLUSTERCTL_VERSION}..."
		curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/${CLUSTERCTL_VERSION}/clusterctl-linux-amd64 -o ${_default_clusterctl_path}
		chmod u+x ${_default_clusterctl_path}
	fi
	if [ ! -f "${_default_virtctl_path}" ]; then
		echo >&2 "Downloading virtctl ..."
		LATEST=$(curl -L https://storage.googleapis.com/kubevirt-prow/devel/release/kubevirt/kubevirt/stable.txt)
		curl -L https://github.com/kubevirt/kubevirt/releases/download/${LATEST}/virtctl-${LATEST}-linux-amd64 -o ${_default_virtctl_path}
		chmod u+x ${_default_virtctl_path}
	fi
}

function kubevirtci::up() {
	make cluster-up -C cluster-up
}

function kubevirtci::prepare() {
	export KUBECONFIG=$(cluster-up/cluster-up/kubeconfig.sh)
	echo "installing kubevirt..."
	LATEST=$(curl -L https://storage.googleapis.com/kubevirt-prow/devel/release/kubevirt/kubevirt/stable.txt)
	${_kubectl} apply -f https://github.com/kubevirt/kubevirt/releases/download/${LATEST}/kubevirt-operator.yaml
	${_kubectl} apply -f https://github.com/kubevirt/kubevirt/releases/download/${LATEST}/kubevirt-cr.yaml
	echo "installing capi..."

	cat << EOF > ${_default_bin_path}/clusterctl_config.yaml
---
cert-manager:
  url: "https://github.com/cert-manager/cert-manager/releases/latest/cert-manager.yaml"
EOF
	$CLUSTERCTL_PATH init -v 4 --config=${_default_bin_path}/clusterctl_config.yaml
	echo "waiting for kubevirt to become ready, this can take a few minutes. You can safely abort this step, the cluster is ready ..."
	${_kubectl} -n kubevirt wait kv kubevirt --for condition=Available --timeout=5m
	kubevirtci::install_capk_release
	kubevirtci::install_metallb
	kubevirtci::create_tenant_namespace
	kubevirtci::create_cluster
	echo "Wait for tenant cluster kubernetes apiserver up"
 	kubevirtci::retry_until_success kubevirtci::kubectl_tenant get pods -n kube-system
	echo "Waiting for worker VMI in tenant cluster namespace"
	kubevirtci::wait_for_vm "${TENANT_CLUSTER_NAME}-md-"
	kubevirtci::install_calico
	echo "Waiting for calico pods rollout"
	while [[ "$(kubevirtci::kubectl_tenant get ds -n kube-system --no-headers | grep -v Found | grep calico-node | wc -l)" -eq 0 ]]; do
		sleep 5
	done
	kubevirtci::kubectl_tenant rollout status ds/calico-node -n kube-system --timeout=2m
}

function kubevirtci::down() {
	make cluster-down -C cluster-up
}

function kubevirtci::ssh_tenant() {
	vmi_name=$1
	vmi_namespace=${2:-$TENANT_CLUSTER_NAMESPACE}

	mkdir -p $_default_tmp_path

	echo "vmi $vmi_name namespace $vmi_namespace"

	${_kubectl} get secret -n $TENANT_CLUSTER_NAMESPACE kvcluster-ssh-keys -o jsonpath='{.data}' | grep key | awk -F '"' '{print $4}' | base64 -d > ${_default_tmp_path}/key.pem

	chmod 600 ${_default_tmp_path}/key.pem

	ssh -o IdentitiesOnly=yes -o "ProxyCommand=$_default_virtctl_path port-forward --stdio=true $vmi_name.$vmi_namespace 22" capk@$vmi_name.$vmi_namespace -i ${_default_tmp_path}/key.pem

	rm ${_default_tmp_path}/key.pem
}

function kubevirtci::destroy_cluster() {
	${_kubectl} delete cluster -n ${TENANT_CLUSTER_NAMESPACE} ${TENANT_CLUSTER_NAME} --ignore-not-found
}

function kubevirtci::create_cluster() {
	export NODE_VM_IMAGE_TEMPLATE=quay.io/capk/ubuntu-container-disk:20.04
	export IMAGE_REPO=k8s.gcr.io
	export CRI_PATH="/var/run/containerd/containerd.sock"
	template="https://github.com/kubernetes-sigs/cluster-api-provider-kubevirt/blob/main/templates/cluster-template.yaml"
	echo "Using cluster template $template"

	$CLUSTERCTL_PATH generate cluster ${TENANT_CLUSTER_NAME} --target-namespace ${TENANT_CLUSTER_NAMESPACE} --kubernetes-version ${CAPK_GUEST_K8S_VERSION} --control-plane-machine-count=1 --worker-machine-count=1 --from $template | ${_kubectl} apply -f -
    ${_kubectl} wait cluster -n kvcluster kvcluster --for=condition=Ready --timeout=5m
}

function kubevirtci::create_tenant_namespace {
	echo "creating namespace ${TENANT_CLUSTER_NAMESPACE}"
	${_kubectl} apply -f - <<EOF
---
apiVersion: v1
kind: Namespace
metadata:
  name: ${TENANT_CLUSTER_NAMESPACE}
EOF
}

function kubevirtci::install_capk_release {
	${_kubectl} apply -f https://github.com/kubernetes-sigs/cluster-api-provider-kubevirt/releases/download/${CAPK_RELEASE_VERSION}/infrastructure-components.yaml

	${_kubectl} wait -n capk-system --for=condition=Available=true deployment/capk-controller-manager --timeout=10m

	echo "capk release $CAPK_RELEASE_VERSION installed!"
}

function kubevirtci::sync() {

	REGISTRY="127.0.0.1:$(cluster-up/cluster-up/cli.sh ports registry)"
	if curl --unix-socket "/run/podman/podman.sock" http://d/v3.0.0/libpod/info >/dev/null 2>&1; then
		cat <<EOF >> /etc/containers/registries.conf
[[registry]]
location = "$REGISTRY"
insecure = true
EOF
	fi
	REGISTRY=$REGISTRY make image push
	$_ssh_infra node01  -- sudo cat  /etc/kubernetes/admin.conf > config/secret/infra-kubeconfig
	$CLUSTERCTL_PATH get kubeconfig ${TENANT_CLUSTER_NAME} -n ${TENANT_CLUSTER_NAMESPACE} > config/secret/kubeconfig
	${_kubectl} kustomize config/overlays/kubevirtci | envsubst | ${_kubectl} apply -f -
    ${_kubectl} delete pod -n kvcluster -l k8s-app=kubevirt-cloud-controller-manager
    ${_kubectl} rollout status -w -n ${TENANT_CLUSTER_NAMESPACE} deployment/kubevirt-cloud-controller-manager
}

function kubevirtci::delete() {
	${_kubectl} kustomize config/overlays/kubevirtci | envsubst | ${_kubectl} delete --ignore-not-found -f -
}

function kubevirtci::install_calico {
	kubevirtci::kubectl_tenant apply -f https://docs.projectcalico.org/${CALICO_VERSION}/manifests/calico.yaml
	kubevirtci::kubectl_tenant wait node --for=condition=Ready --all --timeout=5m
}

function kubevirtci::install_metallb {
	${_kubectl} apply -f https://raw.githubusercontent.com/metallb/metallb/${METALLB_VERSION}/manifests/namespace.yaml
	${_kubectl} apply -f https://raw.githubusercontent.com/metallb/metallb/${METALLB_VERSION}/manifests/metallb.yaml

	echo "waiting for metallb to come online"
	${_kubectl} -n metallb-system wait deployment controller --for condition=Available --timeout=5m

	mkdir -p ${_default_tmp_path}
	local metal_config=${_default_tmp_path}/metallb-config.yaml

	cat << EOF > $metal_config
---
apiVersion: v1
kind: ConfigMap
metadata:
  namespace: metallb-system
  name: config
data:
  config: |
    address-pools:
    - name: default
      protocol: layer2
      addresses:
      - 192.168.66.201-192.168.66.250
EOF

	${_kubectl} apply -f ${metal_config}

	rm $metal_config
	echo "metallb installed!"
}

function kubevirtci::curl_lb {
	mkdir -p ${_default_tmp_path}
	job_yaml=${_default_tmp_path}/curl-test-pod.yaml
	if [ -f ${job_yaml} ]; then
		${_kubectl} delete -f ${job_yaml} --ignore-not-found
	fi

	lb_name=$1
	lb_namespace=${2:-$TENANT_CLUSTER_NAMESPACE}

	$_kubectl get service $lb_name -n $lb_namespace

	lb_ip=$($_kubectl get service $lb_name -n $lb_namespace -o yaml | grep "ip:" | awk '{print $3}')
	lb_port=$($_kubectl get service $lb_name -n $lb_namespace -o yaml | grep "port:" | awk '{print $2}')


	cat << EOF > $job_yaml
---
apiVersion: batch/v1
kind: Job
metadata:
  name: curl-test-job
  namespace: ${lb_namespace}
spec:
  template:
    spec:
      containers:
      - name: fedora
        image: fedora:35
        command:
          - curl
          - "${lb_ip}:${lb_port}"
      restartPolicy: Never
  backoffLimit: 4
EOF

	${_kubectl} create -f $job_yaml
	echo "-----------Waiting for curl job to complete"
	${_kubectl} wait job curl-test-job -n default --for condition=Complete --timeout=5m

	pod_name=$($_kubectl get pods --selector=job-name=curl-test-job --output=jsonpath='{.items[*].metadata.name}')

	echo "-----------CURL LOG FOR POD $pod_name"
	$_kubectl logs -n default $pod_name 2>/dev/null

	${_kubectl} delete -f ${job_yaml} --ignore-not-found > /dev/null 2>&1
}


function kubevirtci::kubectl_tenant {
	vms_list=$(${_kubectl} get vm -n ${TENANT_CLUSTER_NAMESPACE} --no-headers -o custom-columns=":metadata.name")
	for vm in $vms_list; do
		if [[ "$vm" == ${TENANT_CLUSTER_NAME}-control-plane* ]]; then
			control_plane_vm_name=$vm
		fi
	done
	if [ -n "${control_plane_vm_name}" ]; then
		echo "Found control plane VM: ${control_plane_vm_name} in namespace ${TENANT_CLUSTER_NAMESPACE}"
	else
		echo "control-plane vm is not found in namespace ${TENANT_CLUSTER_NAMESPACE} (looking for regex ${TENANT_CLUSTER_NAME}-control-plane*)"
		exit 1
	fi
	${_default_virtctl_path} port-forward -n ${TENANT_CLUSTER_NAMESPACE} vm/${control_plane_vm_name} 64443:6443 > /dev/null 2>&1 &
	trap 'kill $(jobs -p) > /dev/null 2>&1' EXIT
	rm -f .${TENANT_CLUSTER_NAME}-kubeconfig
	$CLUSTERCTL_PATH get kubeconfig ${TENANT_CLUSTER_NAME} -n ${TENANT_CLUSTER_NAMESPACE} > .${TENANT_CLUSTER_NAME}-kubeconfig
	sleep 0.1
	kubectl --kubeconfig .${TENANT_CLUSTER_NAME}-kubeconfig --insecure-skip-tls-verify --server https://localhost:64443 "$@"
}

function kubevirtci::retry_until_success {
    set +e
    while true; do
        sleep 5
        $@
        ret=$?
        if [[ "$ret" == "0" ]]; then
            break
        fi
    done
    set -e
}

function kubevirtci::wait_for_vm {
    local vm_name=$1
    while true ; do
		vms_list=$(${_kubectl} get vm -n ${TENANT_CLUSTER_NAMESPACE} --no-headers -o custom-columns=":metadata.name")
		for vm in $vms_list ; do
			if [[ "$vm" =~ .*"$vm_name".* ]]; then
				tenant_vm=$vm
			fi
		done
		if [[ "${tenant_vm}" == "" ]]; then
			echo "tenant VM not found"
			sleep 10
			continue
		fi
		break
	done
}

function kubevirtci::logs {
	${_kubectl} logs -n ${TENANT_CLUSTER_NAMESPACE} -l k8s-app=kubevirt-cloud-controller-manager 
}

kubevirtci::fetch_kubevirtci

case ${_action} in
	"up")
		kubevirtci::up
		kubevirtci::prepare
		;;
	"down")
		kubevirtci::down
		;;
	"prepare")
		kubevirtci::prepare
		;;
	"sync")
		kubevirtci::sync
		;;
	"install-capk")
		kubevirtci::install_capk_release
		;;
	"install-metallb")
		kubevirtci::install_metallb
		;;
	"install-calico")
		kubevirtci::install_calico
		;;
	"install-cpk")
		kubevirtci::install_cpk
		;;
	"curl-lb")
		kubevirtci::curl_lb "$@"
		;;
	"kubeconfig")
		kubevirtci::kubeconfig
		;;
	"kubectl")
		${_kubectl} "$@"
		;;
	"kubectl-tenant")
		kubevirtci::kubectl_tenant "$@"
		;;
	"virtctl")
		${_default_virtctl_path} "$@"
		;;
	"ssh-infra")
		$_ssh_infra "$@"
		;;
	"ssh-tenant")
		kubevirtci::ssh_tenant "$@"
		;;
	"clusterctl")
		$CLUSTERCTL_PATH "$@"
		;;
	"create-cluster")
		kubevirtci::create_tenant_namespace
		kubevirtci::create_cluster
		;;
	"destroy-cluster")
		kubevirtci::destroy_cluster
		;;
	"delete")
		kubevirtci::delete
		;;
	"clean-cache")
		rm ${_default_clusterctl_path}
		rm ${_default_virtctl_path}
		rm -rf ${_default_tmp_path}
		;;
	"logs")
		kubevirtci::logs
		;;
	"help")
		kubevirtci::usage
		;;
	*)
		echo "Error: Unknown kubevirtci command"
		echo ""
		kubevirtci::usage
		exit 1
		;;
esac