test(performance): status-access-vms (#44)

This is part of the performance tests. Check via ansible ssh access to vm, also added yandex-tank for emulate workload for apis.

---------
Signed-off-by: Nikita Korolev <[email protected]>
Co-authored-by: Tishkov Pavel <[email protected]>

.gitignore

@@ -34,10 +34,22 @@ __pycache__/
 *$py.class
 .pytest_cache/
 
+tmp/
 tmp/*
 
 # Kubernetes
 .kube
 
 # direnv
 .envrc
+
+# logs
+log/
+logs/
+*.log
+
+# ansible inventory
+inventory/
+
+# generated directories
+retry/

tests/performance/.helmignore

@@ -0,0 +1,7 @@
+# Match any file or path named .helmignore
+.helmignore
+
+# Match any file or path named .git
+.git
+
+status-access-vms/

tests/performance/Taskfile.yaml

@@ -8,6 +8,14 @@ includes:
   shatal:
     taskfile: ./shatal
     dir: ./shatal
+  tank:
+    taskfile: status-access-vms/tank/Taskfile.tank.yaml
+    dir: status-access-vms/tank
+    optional: true
+  ansible:
+    taskfile: status-access-vms/ansible/Taskfile.ansible.yaml
+    dir: status-access-vms/ansible
+    optional: true
 
 vars:
   COUNT: '{{ .COUNT | default "1" }}'

tests/performance/ssh/id_ed

@@ -4,4 +4,4 @@ QyNTUxOQAAACBcXFx5sGhpyfLHCWhDeUc5JQT2aVUonOBnWgLCo0KHgAAAAKDCANDUwgDQ
 1AAAAAtzc2gtZWQyNTUxOQAAACBcXFx5sGhpyfLHCWhDeUc5JQT2aVUonOBnWgLCo0KHgA
 AAAED/iI2D9QTc70eISkYFC/TrXG3JpHYLu5FqQhGCTxveElxcXHmwaGnJ8scJaEN5Rzkl
 BPZpVSic4GdaAsKjQoeAAAAAFnlvdXJfZW1haWxAZXhhbXBsZS5jb20BAgMEBQYH
------END OPENSSH PRIVATE KEY-----
+-----END OPENSSH PRIVATE KEY-----

tests/performance/status-access-vms/ansible/Taskfile.ansible.yaml

@@ -0,0 +1,21 @@
+# https://taskfile.dev
+
+version: "3"
+
+silent: true
+
+vars:
+  SSK_KEY: '{{.SSK_KEY | default "../../ssh/id_ed"}}'
+  ANSIBLE_CFG: '{{.ANSIBLE_CFG | default ".ansible.cfg"}}'
+  INVENTORY_FILE: '{{.INVENTORY_FILE | default "inventory/hosts.yml"}}'
+
+tasks:
+  help:
+    desc: "Help about run_tank"
+    cmds:
+      - ./run.sh -h
+
+  run:
+    desc: "Start endless fact-gathering and checks vms availability (need pass namespace like 'run -- -n testnamespace')"
+    cmds:
+      - ./run.sh {{.CLI_ARGS}}

tests/performance/status-access-vms/ansible/ansible.cfg

@@ -0,0 +1,29 @@
+[defaults]
+ansible_managed=Ansible managed: {file} modified on %Y-%m-%d %H:%M:%S
+inventory=$PWD/inventory
+interpreter_python=python3
+
+forks=25
+transport=smart
+host_key_checking=false
+# fact_caching=jsonfile
+# fact_caching_connection=$PWD/tmp
+# fact_caching_timeout=3600
+
+# gathering=smart
+
+bin_ansible_callbacks=true
+deprecation_warnings=false
+
+# retry_files_enabled=true
+# retry_files_save_path=./retry
+
+remote_user=cloud
+ansible_ssh_user=cloud
+private_key_file=../../ssh/id_ed
+
+
+[ssh_connection]
+pipelining=true
+ssh_args = -o ProxyCommand='d8 v port-forward --stdio=true %h %p'
+ansible_ssh_args = -o StrictHostKeyChecking=no -o ControlMaster=auto -o ControlPersist=2m -o ConnectionAttempts=50 -o UserKnownHostsFile=/dev/null

tests/performance/status-access-vms/ansible/playbook.yaml

@@ -0,0 +1,7 @@
+- name: Run
+  hosts: all
+
+  tasks:
+    - name: Print hostname and ip address
+      ansible.builtin.debug:
+        msg: "{{ ansible_host }} | {{ ansible_hostname }} - {{ ansible_default_ipv4.address }}"

tests/performance/status-access-vms/ansible/run.sh

@@ -0,0 +1,152 @@
+#!/bin/bash
+
+SSK_KEY="../../ssh/id_ed"
+ANSIBLE_CFG="./ansible.cfg"
+INVENTORY_FILE="inventory/hosts.yml"
+
+function Help() {
+# Display Help
+   cat <<EOF
+
+Syntax: scriptTemplate [-s|n|h]:
+options:
+n     Set namespace with VirtualMachines
+s     Path to ssh private key, default ../../ssh/id_ed
+u     Enable or disable ansible unreachable host file (Only 'true' or 'false' required; -u false )
+h     Print this help
+   
+EOF
+}
+
+while getopts "s:n:u:h" opt; do
+  case $opt in
+    s) SSK_KEY=$OPTARG ;;
+    n) NAMESPACE=$OPTARG ;;
+    u) UNREACHABLE_HOST_FILE=$OPTARG ;;
+    h) Help
+       exit 0;;
+    \?) echo "Error: Invalid option -$OPTARG" >&2
+        Help
+        exit 1 ;;
+  esac
+done
+
+function enable_unreachable_host {
+    local ENABLE=$1
+
+    if $ENABLE; then
+        echo "Enable write to file unreachable host"
+        sed -i -E "s|^# retry_files_enabled=true|retry_files_enabled=true|" $ANSIBLE_CFG
+        sed -i -E "s|^# retry_files_save_path=./retry|retry_files_save_path=./retry|" $ANSIBLE_CFG
+    else
+        echo "Disable write to file unreachable host"
+        sed -i -E "s|^retry_files_enabled=true|# retry_files_enabled=true|" $ANSIBLE_CFG
+        sed -i -E "s|^retry_files_save_path=./retry|# retry_files_save_path=./retry|" $ANSIBLE_CFG
+    fi
+}
+
+function darwin_sed {
+    if $ENABLE; then
+        echo "Enable write to file unreachable host"
+        sed -i '' -E "s|^# retry_files_enabled=true|retry_files_enabled=true|" $ANSIBLE_CFG
+        sed -i '' -E "s|^# retry_files_save_path=./retry|retry_files_save_path=./retry|" $ANSIBLE_CFG
+    else
+        echo "Disable write to file unreachable host"
+        sed -i '' -E "s|^retry_files_enabled=true|# retry_files_enabled=true|" $ANSIBLE_CFG
+        sed -i '' -E "s|^retry_files_save_path=./retry|# retry_files_save_path=./retry|" $ANSIBLE_CFG
+    fi
+}
+
+function enable_unreachable_host_file {
+    local ENABLE=$1
+
+    if [ "$(uname)" = "Darwin" ]; then
+        darwin_sed $ENABLE
+    elif [ "$(uname)" = "Linux" ]; then
+        enable_unreachable_host $ENABLE
+    else
+        echo "unknown OS"
+        echo "try linux"
+        enable_unreachable_host $ENABLE
+    fi
+
+}
+
+function prepare_ssh_key {
+    chmod 600 $SSK_KEY
+    if [ "$(uname)" = "Darwin" ]; then
+        sed -i '' -E "s|private_key_file=.+|private_key_file=${SSK_KEY}|" $ANSIBLE_CFG
+    elif [ "$(uname)" = "Linux" ]; then
+        sed -i -E "s|private_key_file=.+|private_key_file=${SSK_KEY}|" $ANSIBLE_CFG
+    else
+        echo "unknown OS"
+        echo "try linux"
+        sed -i -E "s|private_key_file=.+|private_key_file=${SSK_KEY}|" $ANSIBLE_CFG
+    fi
+
+}
+
+exit_handler() {
+    echo "Exit"
+    exit 0
+}
+trap 'exit_handler' EXIT
+
+if [ -z $NAMESPACE ]; then echo "Namespace must be defined"; exit 1;fi
+
+if [[ $UNREACHABLE_HOST_FILE == true ]] ; then
+    enable_unreachable_host_file true
+else
+    enable_unreachable_host_file false
+fi
+
+function generate_inventory {
+    VMS=$(kubectl -n $NAMESPACE get vm -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')
+    mkdir -p inventory
+    echo "---
+all:
+  hosts:" > $INVENTORY_FILE
+
+    while IFS= read -r VM_NAME; do
+        echo "    ${VM_NAME}.${NAMESPACE}:" >> $INVENTORY_FILE
+    done <<< "$VMS"
+}
+
+function main {
+    prepare_ssh_key
+    ANSIBLE_REPORT_FILE=play_report.log
+
+    while true
+    do
+        echo "Generate inventory"
+        generate_inventory
+
+        echo "Try to access all hosts from inventory $(date); Namespace: $NAMESPACE"
+        ansible-playbook playbook.yaml | sed -n '/PLAY RECAP/,$p' > $ANSIBLE_REPORT_FILE
+        while [ ! -f $ANSIBLE_REPORT_FILE ]; do sleep 2; done
+
+        echo $(wc -l $ANSIBLE_REPORT_FILE)
+
+        if [ "$(uname)" = "Darwin" ]; then
+            HOSTS_TOTAL=$(( $(wc -l $ANSIBLE_REPORT_FILE | grep -Eo '\d{1,7}') - 2 ))
+        elif [ "$(uname)" = "Linux" ]; then
+            HOSTS_TOTAL=$(( $(wc -l $ANSIBLE_REPORT_FILE | cut -d ' ' -f1) - 2 ))
+        fi
+
+
+        HOSTS_OK=$(( $(grep -E 'ok=[1-9]+' $ANSIBLE_REPORT_FILE | wc -l) ))
+        HOSTS_UNREACHABLE=$(( $(grep -E 'unreachable=[1-9]+' $ANSIBLE_REPORT_FILE | wc -l) ))
+        OK_PCT=$(bc -l <<< "scale=2; $HOSTS_OK/$HOSTS_TOTAL*100")
+
+        if [[ $HOSTS_UNREACHABLE -ne 0 ]]; then
+            grep -E 'unreachable=[1-9]+' $ANSIBLE_REPORT_FILE
+        fi
+
+        echo "OK hosts count:$HOSTS_OK pct.:$OK_PCT% | Unreachable hosts $HOSTS_UNREACHABLE | Total hosts $HOSTS_TOTAL"
+        echo "Wait 2 sec"
+        echo -e "---\n"
+        sleep 2
+    done
+}
+
+main

tests/performance/status-access-vms/ansible/vmops/vmops_restart.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+VMS_UNREACHBLE_FILE="../ansible/retry/playbook.retry"
+
+function Help() {
+# Display Help
+   cat <<EOF
+
+Syntax: scriptTemplate [-n|u|h]:
+options:
+n     Set namespace with VirtualMachines
+u     File with list of unreacheble VirtualMachines for VMOPS (default "../ansible/retry/playbook.retry")
+h     Print this help
+   
+EOF
+}
+
+while getopts "n:u:h" opt; do
+  case $opt in
+    n) NAMESPACE=$OPTARG ;;
+    u) VMS_UNREACHBLE_FILE=$OPTARG ;;
+    h) Help
+       exit 0;;
+    \?) echo "Error: Invalid option -$OPTARG" >&2
+        Help
+        exit 1 ;;
+  esac
+done
+
+exit_handler() {
+    echo "Exit"
+    exit 0
+}
+trap 'exit_handler' EXIT
+
+if [ ! -f $VMS_UNREACHBLE_FILE ]; then echo "File does not exist"; exit 1;fi
+if [ -z $NAMESPACE ]; then echo "Namespace must be defined"; exit 1;fi
+
+VMS_UNREACHBLE=( $(cut -d '.' -f1 $VMS_UNREACHBLE_FILE) )
+
+echo "Generate VirtualMachineOperation for vm in namespace $NAMESPACE"
+for vm in "${VMS_UNREACHBLE[@]}"; do
+
+    kubectl apply -f -<<EOF
+apiVersion: virtualization.deckhouse.io/v1alpha2
+kind: VirtualMachineOperation
+metadata:
+  name: restart-$vm
+  namespace: $NAMESPACE
+spec:
+  virtualMachineName: $vm
+  type: Restart
+  force: true
+EOF
+
+done
+
+echo "Sleep 10 sec"
+sleep 10
+echo "Clear all vmops"
+kubectl -n $NAMESPACE delete vmops --all

tests/performance/status-access-vms/tank/Taskfile.tank.yaml

@@ -0,0 +1,24 @@
+# https://taskfile.dev
+
+version: "3"
+
+silent: true
+
+vars:
+  TANK_TEST_RUN: '{{ .TANK_TEST_RUN | default "-t ya.ru -c load.yaml" }}'
+
+tasks:
+  help:
+    desc: Help about run_tank
+    cmds:
+      - ./run_tank.sh -h
+
+  run:
+    desc: Start stress testing for site or ip (use run -- -t site.com -c load.yaml)
+    cmds:
+      - ./run_tank.sh {{.CLI_ARGS | default .TANK_TEST_RUN}}
+
+  test_run:
+    desc: Start stress testing for ya.ru
+    cmds:
+      - ./run_tank.sh {{.TANK_TEST_RUN}}

tests/performance/status-access-vms/tank/load.yaml

@@ -0,0 +1,17 @@
+phantom:
+  address: ya.ru
+  header_http: "1.1"
+  headers:
+    - "[Host: ya.ru]"
+    - "[Connection: close]"
+    - "[User-Agent: Tank]"
+  uris:
+    - /
+  load_profile:
+    load_type: rps
+    schedule: line(5, 500, 10m) const(50,2m) const(500,10m)
+  instances: 30
+console:
+  enabled: true
+telegraf:
+  enabled: false