temp: add pitr logic

addons/mogdb/dataprotection/brm-common.sh

@@ -0,0 +1,117 @@
+# brm variables setting
+: ${BRM_CONFIG:=/etc/brm.yaml}
+: ${BACKUP_ROOTDIR:=/backupdata}
+: ${BACKUP_DIR:=mogdb}
+: ${PGDATA:=/var/lib/mogdb/data}
+: ${LAST_INFO:=last_info.log}
+: ${ARCHIVE_DIR:=/var/lib/mogdb/archives}
+
+
+function brm_backup_dir() {
+    echo "${BACKUP_ROOTDIR}/${BACKUP_DIR}"
+}
+
+function brm_instance_backup_dir() {
+    echo $(brm_backup_dir)/backups/${KB_CLUSTER_NAME}
+}
+
+function brm_instance_wal_dir() {
+    echo $(brm_backup_dir)/wal/${KB_CLUSTER_NAME}
+}
+
+function init_brm_required_vars() {
+    BRM_BACKUP_DIR=$(brm_backup_dir)
+    BRM_INST_BACKUP_DIR=$(brm_instance_backup_dir)
+    BRM_INST_WAL_DIR=$(brm_instance_wal_dir)
+}
+
+function brm_cleanup_instance() {
+    local inst="$1"
+
+    if [ -n "$inst" ]; then
+        rm -rf $(brm_instance_backup_dir)
+        rm -rf $(brm_instance_wal_dir)
+    fi
+}
+
+function is_first_backup() {
+    # TODO: simple judgment currently, enhance this logic in the future
+    local inst_backup_dir=$(brm_instance_backup_dir)
+
+    for dir in $(ls $inst_backup_dir); do
+        if [ -d "$inst_backup_dir/$dir" ] && [[ "$dir" =~ [A-Z0-9]+ ]]; then
+            return 1
+        fi
+    done
+
+    return 0
+}
+
+function setup_brm_configure() {
+    which yq >/dev/null || apt_get_install_package yq
+
+    local brm_dir=$(home_directory omm)/brm
+    test -d ${brm_dir} || {
+        echo "Making brm directory.."
+        mkdir -p ${brm_dir}
+        chown -R omm:omm ${brm_dir}
+    }
+
+    yq -i ".backup_home = \"$(brm_backup_dir)\"" ${BRM_CONFIG}
+    yq -i ".log_file = \"${brm_dir}/brm.log\"" ${BRM_CONFIG}
+    yq -i ".lock_directory = \"/var/run\"" ${BRM_CONFIG}
+}
+
+function brm_add_server() {
+    local inst_backup_dir=$(brm_instance_backup_dir)
+
+    local cmd=(gosu omm brm)
+    if [ ! -e "$inst_backup_dir" ]; then
+        cmd+=(add-server)
+    else
+        cmd+=(set-server)
+    fi
+
+    ${cmd[@]} \
+    -i ${KB_CLUSTER_NAME} \
+    -D ${PGDATA} \
+    --pghost=${DP_DB_HOST} \
+    --pguser=${DP_DB_USER}  \
+    --pgpassword=\'${DP_DB_PASSWORD}\' \
+    --remote-user=omm --remote-host=${DP_DB_HOST} \
+    -p ${PGPORT:-26000}
+}
+
+
+function brm_set_server_wal() {
+    brm set-server -i ${KB_CLUSTER_NAME} --archive-dir=${ARCHIVE_DIR}
+}
+
+
+function brm_analysis_log() {
+    # last_info.log
+    local last_line=$(tail -n 1 ${BRM_INST_BACKUP_DIR}/${LAST_INFO})
+
+    if [[ "$last_line" =~ "Backup "([A-Z0-9]+)" completed" ]]; then
+        local backup_id=${BASH_REMATCH[1]}
+    fi
+
+    echo "{\"backup_id\":\"$backup_id\"}"
+}
+
+function brm_backup_info() {
+    local backup_id="$1"
+    local cmd=(brm show-backup)
+
+    if [ -n "$backup_id" ]; then
+        cmd+=(-b $backup_id -i ${KB_CLUSTER_NAME} -f JSON)
+    fi
+
+    local output=$(${cmd[@]})
+
+    if [[ $output =~ \[.*\] ]]; then
+        echo ${BASH_REMATCH[0]}
+    else
+        echo "[]"
+    fi
+}

addons/mogdb/dataprotection/brm-full-backup.sh

@@ -0,0 +1,39 @@
+#set -e
+#set -o pipefail
+export PATH="$PATH:$DP_DATASAFED_BIN_PATH"
+export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH"
+export PGPASSWORD=${DP_DB_PASSWORD}
+
+trap handle_exit EXIT
+
+START_TIME=`get_current_time`
+BACKUP_TMPDIR="/tmp"
+
+function brm_backup() {
+    gosu omm brm backup -i ${KB_CLUSTER_NAME} --debug=info | tee $(brm_instance_backup_dir)/${LAST_INFO}
+}
+
+echo "Init brm required vars.."
+init_brm_required_vars
+
+# setup brm configuration
+echo "Setuping brm configuration.."
+setup_brm_configure
+
+echo "Configure ssh.."
+setup_ssh_configure
+
+# add brm server
+echo "Adding brm server.."
+brm_add_server
+
+# backup using brm
+echo "Backuping.."
+brm_backup
+
+# stat and save the backup information
+stat_and_save_backup_info $START_TIME
+
+# temporarily add for test
+# TODO: remove it in the future
+# sleep infinity

addons/mogdb/dataprotection/brm-info-collector.sh

@@ -0,0 +1,36 @@
+function get_current_time() {
+  curr_time=$(gsql -U ${DP_DB_USER} -h ${DP_DB_HOST} -W ${DP_DB_PASSWORD} -d postgres -t -c "SELECT now() AT TIME ZONE 'UTC'")
+  echo $curr_time
+}
+
+function stat_and_save_backup_info() {
+    local start_time="$1"
+    local stop_time="$2"
+
+    if [ -z $stop_time ]; then
+        stop_time=$(get_current_time)
+    fi
+
+    start_time=$(date -d "${start_time}" -u '+%Y-%m-%dT%H:%M:%SZ')
+    stop_time=$(date -d "${stop_time}" -u '+%Y-%m-%dT%H:%M:%SZ')
+
+    local analysis=$(brm_analysis_log)
+    local backup_id=$(jq -r .backup_id <<< $analysis)
+
+    local backup_info=$(brm_backup_info "$backup_id")
+    local size_bytes=$(jq -r '.[0].backups[0]."data-bytes"' <<< ${backup_info})
+    local human_size=$(human_format $size_bytes)
+
+    echo "{\"totalSize\":\"$human_size\",\"timeRange\":{\"start\":\"${start_time}\",\"end\":\"${stop_time}\"}}" >"${DP_BACKUP_INFO_FILE}"
+}
+
+# if the script exits with a non-zero exit code, touch a file to indicate that the backup failed,
+# the sync progress container will check this file and exit if it exists
+function handle_exit() {
+  exit_code=$?
+  if [ $exit_code -ne 0 ]; then
+    echo "failed with exit code $exit_code"
+    touch "${DP_BACKUP_INFO_FILE}.exit"
+    exit 1
+  fi
+}

addons/mogdb/dataprotection/brm-ptrack-backup.sh

@@ -0,0 +1,34 @@
+START_TIME=`get_current_time`
+
+function brm_ptrack_backup() {
+    if is_first_backup; then
+        gosu omm brm backup -i ${KB_CLUSTER_NAME} --debug=info | tee $(brm_instance_backup_dir)/${LAST_INFO}
+    else
+        gosu omm brm backup -i ${KB_CLUSTER_NAME} -b PTRACK --debug=info | tee $(brm_instance_backup_dir)/${LAST_INFO}
+    fi
+}
+
+
+echo "Init brm required vars.."
+init_brm_required_vars
+
+# setup brm configuration
+echo "Setuping brm configuration.."
+setup_brm_configure
+
+echo "Configure ssh.."
+setup_ssh_configure
+
+# add brm server
+echo "Adding brm server.."
+brm_add_server
+
+echo "Starting ptrack backup.."
+brm_ptrack_backup
+
+# stat and save the backup information
+stat_and_save_backup_info $START_TIME
+
+# temporarily add for test
+# TODO: remove it in the future
+# sleep infinity

addons/mogdb/dataprotection/brm-wal-backup.sh

@@ -0,0 +1,43 @@
+# brm backup-wal -i test --debug=info
+
+START_TIME=`get_current_time`
+
+function brm_ptrack_backup() {
+    brm backup-wal -i ${KB_CLUSTER_NAME} --debug=info | tee ${BRM_INST_WAL_DIR}/${LAST_INFO}
+}
+
+
+function save_backup_status() {
+    echo
+}
+
+
+echo "Init brm required vars.."
+init_brm_required_vars
+
+# setup brm configuration
+echo "Setuping brm configuration.."
+setup_brm_configure
+
+echo "Configure ssh.."
+setup_ssh_configure
+
+# add brm server
+echo "Adding brm server.."
+brm_set_server_wal
+
+# trap term signal
+trap "echo 'Terminating...' && sync && exit 0" TERM
+DP_log "start to archive wal logs"
+while true; do
+    brm_ptrack_backup
+
+    save_backup_status
+
+    sleep ${LOG_ARCHIVE_SECONDS}
+
+    # temporarily add for test
+    # TODO: remove it in the future
+    sleep infinity
+done
+

addons/mogdb/dataprotection/ssh.sh

@@ -0,0 +1,16 @@
+function setup_ssh_configure() {
+    if [ ! -f /etc/ssh/ssh_config ]; then
+        echo "SSH client config not exist!"
+        exit 1
+    fi
+
+    {
+        echo "StrictHostKeyChecking no"
+    } >> /etc/ssh/ssh_config
+
+    mkdir -p /home/omm/.ssh
+    echo -n "$SSH_RSA" > /home/omm/.ssh/id_rsa
+
+    chown -R omm:omm /home/omm/.ssh
+    chmod 0600 /home/omm/.ssh/id_rsa
+}

addons/mogdb/reloader/update-parameters.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+set -ex
+
+OB_SERVICE_PORT=${COMP_MYSQL_PORT:-2881}
+OB_CLI="/kb_tools/obtools"
+
+paramName="${1:?missing config}"
+paramValue="${2:?missing value}"
+
+$OB_CLI --host 127.0.0.1 -uroot -P ${OB_SERVICE_PORT} var-update --set "${paramName}=${paramValue}"