#!/bin/sh
system_subr="/usr/share/veil/system.subr"
deb_subr="/usr/share/veil/deb.subr"
backup_subr="/usr/share/veil/backup.subr"
color_subr="/usr/share/veil/color.subr"
utils_subr="/usr/share/veil/utils.subr"

EXTRA_MAIN_CMD="numfmt psql rsync ssh"

set -e
. ${color_subr}
. ${system_subr}
. ${deb_subr}
. ${utils_subr}
. ${backup_subr}
. /etc/veil/backup.conf
set +e

DEST_DIR="/var/cache/spacevm/pgstats"
DEST_STATS="${DEST_DIR}/pgstats.conf"
DEST_RSTATS="${DEST_DIR}/rpgstats.conf"
# todo: use facts instead of hard-code
PG_VER="13"
PG_DATA_DIR="/var/lib/postgresql/17/main"
DEB_LOGFILE="/var/log/veil/controller/veil-controller.log"
REPLICATION_LOG="/var/log/veil/controller/replication.log"
CONTROLLER_PATH="/var/lib/ecp-veil/controller"
NODE_PATH="/var/lib/ecp-veil/node"
NODE_CRT_PATH="${NODE_PATH}/local_data/node.crt"
NODE_KEY_PATH="${NODE_PATH}/local_data/node.key"
PYTHON_PATH="${CONTROLLER_PATH}/env/bin/python3"
MANAGE_PY_PATH="${CONTROLLER_PATH}/app/manage.py shell"
CHANGING_ROLE_FLAG="/var/lib/ecp-veil/run/replication_role_changing"

#if pg_is_in_recovery; then
#       echo "its slave"
#fi

controller_services_action()
{
  # менеджмент службами ведем через методы в veilrpc, т.к. задейсвуется puppet
  action=$1
  for _service in "controller-engine" "controller-web-proxy" "controller-web-uploader" "websockify" "controller-web-api" "controller-log-sender"; do
    /var/lib/ecp-veil/cli/env/bin/python3 -c "from veilrpc.states.services import services_action; services_action('${_service}', '${action}', timeout=120)"
  done
}

pg_is_in_recovery()
{
	local _res= _ret=

	_res=$( 2>>${DEB_LOGFILE} ${SUDO_CMD} -u postgres env PGCONNECT_TIMEOUT=3 ${PSQL_CMD} -tc "select pg_is_in_recovery();" | ${AWK_CMD} '{printf $1}' )
	ret=$?
	[ ${ret} -ne 0 ] && err 1 "unable to get database role via psql: ${_res}"

	case "${_res}" in
		f)
			return 1
			;;
		t)
			return 0
			;;
		*)
			# spacevm cli always assumes code 0 and does not proxy the message if the code is non-zero 
			err 0 "$0 error: unknown pg_is_in_recovery status: ${_res}"
			;;
	esac
}

sql_query()
{
  ${PSQL_CMD} -t postgresql://${PGSQL_DB_USER}:${PGSQL_DB_PASSWORD}@${LOCALHOST}:${PGSQL_DB_PORT}/${PGSQL_DB_NAME}?sslmode=disable << EOF
$1
EOF
}


create_replication_slot()
{
  ${ECHO} "Creating replication slot ..."
  (sql_query "SELECT * FROM pg_create_physical_replication_slot('spacevm_replica');" 2>&1) > /dev/null
  (sql_query "SELECT * FROM pg_replication_slots;" 2>&1 > /dev/null && ${ECHO} "Created successful.") || ${ECHO} "Replication slot not created."
}


drop_replication_slot()
{
  #  Если реплика активна, нужно остановить на ней постгрес, иначе мы не сможем прибить на мастере процесс и удалить слот
  if [ -r ${CONTROLLER_SLAVE_IP_FILE} ]; then
    . ${CONTROLLER_SLAVE_IP_FILE}
    (${SSH_CMD} ${CONTROLLER_SSH_OPT} root@${controller_slave_ip} '[ -r '${CONTROLLER_MASTER_IP_FILE}' ] && systemctl stop postgresql')
  fi
  _pid=$(sql_query 'SELECT active_pid from pg_replication_slots;' || echo )
  (sql_query "SELECT pg_terminate_backend(${_pid});" && echo "Terminated slot pid:${_pid}") || echo "Can't terminate pid:${_pid}"
  (sql_query "SELECT pg_drop_replication_slot('spacevm_replica');" && echo "Dropped slot") || echo "Can't drop slot."
  (${SSH_CMD} ${CONTROLLER_SSH_OPT} root@${controller_slave_ip} '[ -r '${CONTROLLER_MASTER_IP_FILE}' ] && systemctl start postgresql')
}

# getyesno var
# Show $1 strings and get answer in infinity loop
# return 0 when YES/1 and 1 when not
# return 3 if not interactive ($inter=0) (or INTER=0 env exist)
# if ALWAYS_YES=1 and inter=0, always return 0
# if ALWAYS_NO=1 and inter=0, always return 1
# $1 - pre-defined return: 1 - yes, 0 - no
getyesno()
{
	if [ -n "${2}" ]; then
		case "${2}" in
		1)
			return 0
			;;
		0)
			return 1
			;;
		esac
	fi

	if [ "${inter}" = "0" -o -n "${NOINTER}" ]; then
		[ "${ALWAYS_YES}" = "1" ] && return 0
		[ "${ALWAYS_NO}" = "1" ] && return 1
		return 3
	fi

	${ECHO} "${BOLD}${1}${N0_COLOR}"
	answ=
	while [ x$answ = x ]; do
		${ECHO} "[${N2_COLOR}yes(1) or no(0)${N0_COLOR}]"
		read reply leftover
		case $reply in
			# "yes", "true", "on", or "1"
			Y*|y*|1)
				return 0
				;;
			# "no", "false", "off", or "0"
			N*|n*|0)
				return 1
				;;
		esac
	done
}

conv2human()
{
	local S="$1"
	printf %s\\n ${S} | ${NUMFMT_CMD} --to=iec-i
}

# if is_number 5; then
#   echo "number"
# fi
is_number()
{
	case "${1}" in
		''|*[!0-9]*) return 1 ;;
		*) return 0 ;;
	esac

	return 0
}

# export MGMT_INTERFACE ip in $myip
get_myip()
{
	myip=
  NETWORK_UUID=$( nectl network show | ${GREP_CMD} -B1 Mgmt | ${GREP_CMD} UUID | ${AWK_CMD} '{print $3}' )
	myip=$( nectl network mgmt show "${NETWORK_UUID}" | ${GREP_CMD} 'Address' | ${SED_CMD} -e 's/\// /' | ${AWK_CMD} '{print $2}' )
	[ -z "${myip}" ] && err 1 "${H5_COLOR}Unable to get management ip${N0_COLOR}"
}

remote_ssh()
{
	_status=$( current_status_int )
	if [ ${_status} -eq 10 -o ${_status} -ne 11 -o ${_status} -ne 20 -o ${_status} -ne 21 ]; then
		_rip=$( ${CAT_CMD} ${REMOTE_IP_FILE} | ${AWK_CMD} '{ printf $1}' )
		${ECHO} "${H5_COLOR}Connecting to: ${H3_COLOR}${_rip}${N0_COLOR}"
		echo
		${SSH_CMD} ${CONTROLLER_SSH_OPT} root@${_rip}
		return 0
	else
		${ECHO} "${N1_COLOR}No remote controller, please init controller via 'controller add <IP> first'${N0_COLOR}"
		return 1
	fi
}

redis_stats()
{
	role="alone"
	# Master
	connected_slaves=0
	master_repl_offset=0
	# Slave
	master_host=
	master_link_status=0
	slave_repl_offset=0

	OIFS="${IFS}"
	IFS=":"
	eval $( redis-cli -a bazalt info replication | ${EGREP_CMD} -E "^role|^master|^connected|^slave" | ${TR_CMD} -d \\r" " | while read _param _val; do
		${ECHO} "${_param}=\"${_val}\""
	done )
	IFS="${OIFS}"

	${ECHO} "Redis role: ${role}"

	if [ "${role}" = "master" ]; then
		${ECHO} "Redis connected_slaves: ${connected_slaves}"
		${ECHO} "Redis master_repl_offset: ${master_repl_offset}"
	elif [ "${role}" = "slave" ]; then
		${ECHO} "Redis master_link_status: ${master_link_status}"
		${ECHO} "Redis master_host: ${master_host}"
		${ECHO} "Redis slave_repl_offset: ${slave_repl_offset}"
	fi
}

# return:
#  0 - not initialized
#
#  10 - master, in sync
#  11 - master, sync lost
#  12 - master without slave
#
#  20 - slave, in sync
#  21 - slave, sync lost
#  22 - slave without master
current_status_int()
{
	local _ret
	controller_master_ip=${LOCALHOST}
	[ -r ${CONTROLLER_MASTER_IP_FILE} ] && . ${CONTROLLER_MASTER_IP_FILE}
	controller_slave_ip=${LOCALHOST}
	[ -r ${CONTROLLER_SLAVE_IP_FILE} ] && . ${CONTROLLER_SLAVE_IP_FILE}

	if [ -r ${REMOTE_IP_FILE} ]; then
		_rnode=$( ${CAT_CMD} ${REMOTE_IP_FILE} |awk '{ printf $1}' )
		test_connection > /dev/null 2>&1
		_rstate=$( ${CAT_CMD} ${REMOTE_STATUS_FILE} )
	else
		_ret=0
	fi

	if [ -z "${controller_master_ip}" -o "${controller_master_ip}" = ${LOCALHOST} ]; then
		if [ -z "${controller_slave_ip}" -o "${controller_slave_ip}" = ${LOCALHOST} ]; then
			_ret=12
		else
			if [ "${_rstate}" = 'active' ]; then
				_ret=10
			else
				_ret=11
			fi
		fi
	else

		if [ -z "${controller_master_ip}" -o "${controller_master_ip}" = ${LOCALHOST} ]; then
			_ret=22
		else
			if [ "${_rstate}" = 'active' ]; then
				_ret=20
			else
				_ret=21
			fi
		fi
	fi

	echo ${_ret}
}

# Обновляем конфиг pgsql для применения изменений с обновлением
reload_pg_config()
{
	local _remote_ip=
	[ ! -r /etc/veil/remote_ip.conf ] && return 0
	_remote_ip=$( ${GREP_CMD} \. /etc/veil/remote_ip.conf | ${AWK_CMD} '{printf $1}' )
	iptype ${_remote_ip}
	_ret=$?

	if [ ${_ret} -ne 1 ]; then
		echo "${0}: check_remote_config: /etc/veil/remote_ip.conf does not contain IPv4: ${_remote_ip}" >> ${DEB_LOGFILE}
		return 0
	fi

	${SED_CMD} -e "s#%%SLAVE%%#${_remote_ip}#g" \
		/var/lib/ecp-veil/controller/other/pgsql-master.yaml > /var/lib/ecp-veil/data/local_modules/pgsql.yaml

	${CHOWN_CMD} veil-controller:veil /var/lib/ecp-veil/data/local_modules/pgsql.yaml

	/usr/local/sbin/veil-role -a pgsql

}

current_status()
{
	local _ret=
	local _inactive=0

	/var/lib/ecp-veil/controller/other/pgsql/pgstats.sh > /dev/null 2>&1
	if [ ! -r "${DEST_STATS}" ]; then
		${ECHO} "${N1_COLOR}${0} error: stats not found: ${N2_COLOR}${DEST_STATS}${N0_COLOR}"
		[ -r /var/log/veil/controller/pgstats.log ] && ${CAT_CMD} /var/log/veil/controller/pgstats.log

		# spacevm cli always assumes code 0 and does not proxy the message if the code is non-zero
		exit 0
	fi
	pg_data_dir_bytes=
	pg_partition_free_bytes=
	pg_role=
	pg_database_size_bytes=
	pg_last_xact_replay=

	test_connection > /dev/null 2>&1
	. ${DEST_STATS}

	controller_master_ip=${LOCALHOST}
	[ -r ${CONTROLLER_MASTER_IP_FILE} ] && . ${CONTROLLER_MASTER_IP_FILE}
	controller_slave_ip=${LOCALHOST}
	[ -r ${CONTROLLER_SLAVE_IP_FILE} ] && . ${CONTROLLER_SLAVE_IP_FILE}

	is_master=0
	is_slave=0

	current_node_status=

	if [ -r ${REMOTE_IP_FILE} ]; then
		_rnode=$( ${CAT_CMD} ${REMOTE_IP_FILE} | ${AWK_CMD} '{ printf $1}' )
		_rstate=$( ${CAT_CMD} ${REMOTE_STATUS_FILE} )
		if [ "${_rstate}" = 'active' ]; then
			status="Initialized for remote controller: ${_rnode}, state: ${N2_COLOR}${_rstate}${N0_COLOR}"
		else
			status="Initialized for remote controller: ${_rnode}, state: ${W2_COLOR}${_rstate}${N0_COLOR}"
		fi
	else
		status="Not initialized yet"
		_ret=0
	fi

	if [ -z "${controller_master_ip}" -o "${controller_master_ip}" = ${LOCALHOST} ]; then
		is_master=1

		if [ -z "${controller_slave_ip}" -o "${controller_slave_ip}" = ${LOCALHOST} ]; then
			current_node_status="${N2_COLOR}master${N0_COLOR} (no slave)"
			_ret=12
		else
			current_node_status="${N2_COLOR}master${N0_COLOR} (slave: ${controller_slave_ip})"
			if [ "${_rstate}" = 'active' ]; then
				_ret=10
			else
				_ret=11
			fi
		fi
	elif [ -z "${controller_slave_ip}" -o "${controller_slave_ip}" = ${LOCALHOST} ]; then
		is_slave=1

		if [ -z "${controller_master_ip}" -o "${controller_master_ip}" = ${LOCALHOST} ]; then
			current_node_status="${N2_COLOR}slave${N0_COLOR} (no master)"
			_ret=22
		else
			current_node_status="${N2_COLOR}slave${N0_COLOR} (master: ${controller_master_ip})"
			if [ "${_rstate}" = 'active' ]; then
				_ret=20
			else
				_ret=21
			fi
		fi
	fi

	${ECHO} "Status: ${status}"
	${ECHO} "Current node status: ${current_node_status}"

	if [ ${is_master} -eq 1 ] || [ ${is_slave} -eq 1 ]; then
		set -e
		. /etc/veil/controller.conf
		set +e

		if [ ${is_master} -eq 1 ]; then
			# Master status
			if pgrep -f ".*walsender.*" >/dev/null; then
				${ECHO} "Postgresql replication process: ${H1_COLOR}active${N0_COLOR}"
			else
				${ECHO} "Postgresql replication process: ${W1_COLOR}inactive${N0_COLOR}"
				_inactive=1
			fi

			# Postgres info
			pgsql_stats=$( 2>/dev/null ${PSQL_CMD} -t  postgresql://${PGSQL_DB_USER}:${PGSQL_DB_PASSWORD}@${LOCALHOST}:${PGSQL_DB_PORT}/${PGSQL_DB_NAME}?sslmode=disable <<EOF
SELECT client_addr AS client, state, sync_state, \
(pg_wal_lsn_diff(pg_current_wal_lsn(),sent_lsn) / 1024)::INTEGER as pending, \
(pg_wal_lsn_diff(sent_lsn,write_lsn) / 1024)::INTEGER AS write, \
(pg_wal_lsn_diff(write_lsn,flush_lsn) / 1024)::INTEGER AS flush, \
(pg_wal_lsn_diff(flush_lsn,replay_lsn) / 1024)::INTEGER AS replay, \
(pg_wal_lsn_diff(pg_current_wal_lsn(),replay_lsn) / 1024)::INTEGER AS total_lag from pg_stat_replication;
EOF
)

			# pgsql_stats=$( echo ${pgsql_stats} | ${TR_CMD} -d '|' )
			# ${ECHO} "${pgsql_stats}"
		elif [ ${is_slave} -eq 1 ]; then
			# Slave status

			if pgrep -f ".*walreceiver.*" >/dev/null; then
				${ECHO} "Postgresql replication process: ${H1_COLOR}active${N0_COLOR}"
				pgsql_lag=$( 2>/dev/null ${PSQL_CMD} -t  postgresql://${PGSQL_DB_USER}:${PGSQL_DB_PASSWORD}@${LOCALHOST}:${PGSQL_DB_PORT}/${PGSQL_DB_NAME}?sslmode=disable <<EOF
SELECT CASE WHEN pg_last_wal_receive_lsn() = pg_last_wal_receive_lsn() THEN 0 ELSE EXTRACT (EPOCH FROM now() - pg_last_xact_replay_timestamp())::INTEGER END AS replication_lag;
EOF
)
				pgsql_lag=$( echo ${pgsql_lag} | ${AWK_CMD} '{printf $1}' )
				${ECHO} "Postgres lag: ${pgsql_lag}"
				# TODO Проверка превышения pgsql_lag. Выполнена, но число 2000 выбрано почти наугад
				[ ${pgsql_lag} -gt 2000 ] && ${ECHO} "${H3_COLOR}High Postgres lag. Check resources.${N0_COLOR}"
			else
				${ECHO} "Postgresql replication process: ${W1_COLOR}inactive${N0_COLOR}"
				_inactive=1
			fi
		fi
	fi

	if [ -d /var/lib/ecp-veil/run/witness ] && [ "$( ls -A /var/lib/ecp-veil/run/witness )" ]; then
		old_dir=$(pwd)
		cd /var/lib/ecp-veil/run/witness
		for file in *; do
			${ECHO} "Witness id ${H5_COLOR}${file}${N0_COLOR} was seen at $( stat --format %y ${file} )"
		done
		cd "$old_dir"
	fi

	if ! is_number "${pg_data_dir_bytes}"; then
		pg_data_dir_bytes=1
	fi

	if ! is_number "${pg_partition_free_bytes}"; then
		pg_partition_free_bytes=1
	fi

	if ! is_number "${pg_database_size_bytes}"; then
		pg_database_size_bytes=1
	fi

	PG_DATA_COLOR="${N2_COLOR}"

	if [ ${pg_database_size_bytes} -gt 3000000000 ]; then
		x=$(( pg_data_dir_bytes / pg_database_size_bytes ))
		if [ ${x} -gt 2 ]; then
			PG_DATA_COLOR="${W1_COLOR}"
		fi
	fi

	PG_PARTITION_COLOR="${N2_COLOR}"
	x=$(( pg_partition_free_bytes / pg_data_dir_bytes ))

	if [ ${x} -lt 2 ]; then
		PG_DATA_COLOR="${W1_COLOR}"
		PG_PARTITION_COLOR="${W1_COLOR}"
	fi

	pg_data_dir=$( conv2human ${pg_data_dir_bytes} )
	${ECHO} "Pg_data_dir: ${PG_DATA_COLOR}${pg_data_dir}${N0_COLOR}"

	pg_partition_free=$( conv2human ${pg_partition_free_bytes} )
	${ECHO} "Pg_partition_free: ${PG_PARTITION_COLOR}${pg_partition_free}${N0_COLOR}"

	${ECHO} "pg_role: ${pg_role}"

	pg_database_size=$( conv2human ${pg_database_size_bytes} )
	${ECHO} "Pg_database_size: ${PG_DATA_COLOR}${pg_database_size}${N0_COLOR}"

	# no updates in 10 minutes?
	if [ ${pg_last_xact_replay} -gt 610 ]; then
		${ECHO} "Pg_last_xact_replay: ${W1_COLOR}${pg_last_xact_replay}${N0_COLOR}"
	else
		${ECHO} "Pg_last_xact_replay: ${pg_last_xact_replay}"
	fi

	if [ "${pg_last_xact_replay}" != "3000" ]; then
		cur_time=$( ${DATE_CMD} +%s )
		# check if pg_last_xact_replay is number ?
		last_xact=$(( cur_time - pg_last_xact_replay ))
		last_xact_date=$( ${DATE_CMD} -d @${last_xact} 2>/dev/null )
		[ -n "${last_xact_date}" ] && ${ECHO} "Pg_last_xact_replay date: ${last_xact_date}"
	else
		last_xact=0
	fi

	# tail logs when slave inactive
	if [ "${pg_role}" = "slave" -a "${_inactive}" = "1" ]; then
		if [ -r /var/log/postgresql/postgresql-17-main.log ]; then
			echo "last postgresql.log entries"
			${TAIL_CMD} -n10 /var/log/postgresql/postgresql-17-main.log | ${GREP_CMD} FATAL
		fi
	fi

	exit 0
}

current_status_json()
{
  # Чтоб не страдать фигней в свидетеле вычисляя нужную позиции строки и не парсить текст
  # отдаем ему удобный JSON, плюс можно накидать еще параметров, которые не пойдут в вебку или еще куда

	controller_master_ip=${LOCALHOST}
	[ -r ${CONTROLLER_MASTER_IP_FILE} ] && . ${CONTROLLER_MASTER_IP_FILE}
	controller_slave_ip=${LOCALHOST}
	[ -r ${CONTROLLER_SLAVE_IP_FILE} ] && . ${CONTROLLER_SLAVE_IP_FILE}

	is_master=0
	is_slave=0

	current_node_status=
  echo '{'
	if [ -r ${REMOTE_IP_FILE} ]; then
		_rnode=$( ${CAT_CMD} ${REMOTE_IP_FILE} | ${AWK_CMD} '{ printf $1}' )
		test_connection > /dev/null 2>&1
		[ $? -eq 2 ] && ssh_state="false" || ssh_state="true"
		_rstate=$( ${CAT_CMD} ${REMOTE_STATUS_FILE} )
		status='{"rnode": "'${_rnode}'", "state": "'${_rstate}'"},'
	else
		ssh_state="null"
		status="null,"
	fi

	if [ -z "${controller_master_ip}" -o "${controller_master_ip}" = ${LOCALHOST} ]; then
		is_master=1

		if [ -z "${controller_slave_ip}" -o "${controller_slave_ip}" = ${LOCALHOST} ]; then
			current_node_status='{"role": "master"},'
		else
			current_node_status='{"role": "master", "remote": "'${controller_slave_ip}'"},'
		fi
	else
		is_slave=1

		if [ -z "${controller_master_ip}" -o "${controller_master_ip}" = ${LOCALHOST} ]; then
			current_node_status='{"role": "slave"},'
		else
			current_node_status='{"role": "slave", "remote": "'${controller_master_ip}'"},'
		fi
	fi

	${ECHO} '"status": '${status}
	${ECHO} '"node_status": '${current_node_status}
	if [ ${is_master} -eq 1 ] || [ ${is_slave} -eq 1 ]; then
		set -e
		. /etc/veil/controller.conf
		set +e

		if [ ${is_master} -eq 1 ]; then
			# Master status
			if pgrep -f ".*walsender.*" >/dev/null; then
				${ECHO} '"replication_active": true,'
			else
				${ECHO} '"replication_active": false,'
			fi
		elif [ ${is_slave} -eq 1 ]; then
			# Slave status
			if pgrep -f ".*walreceiver.*" >/dev/null; then
				pgsql_lag=$( sql_query "SELECT CASE WHEN pg_last_wal_receive_lsn() = pg_last_wal_receive_lsn() THEN 0 ELSE EXTRACT (EPOCH FROM now() - pg_last_xact_replay_timestamp())::INTEGER END AS replication_lag;" 2>/dev/null)
				pgsql_lag=$( echo ${pgsql_lag} | ${AWK_CMD} '{printf $1}' )
				${ECHO} '"replication_active": true,'
				${ECHO}  '"psql_lag": '${pgsql_lag}','
			else
				${ECHO} '"replication_active": false,'
			fi
		fi
	fi
	_in_role_since="null"
	for file in "$CONTROLLER_MASTER_IP_FILE" "$CONTROLLER_SLAVE_IP_FILE"; do
	  (ls $file > /dev/null 2>&1) && _in_role_since='"'$( stat --format %w ${file} )'"' && break
	done;
	${ECHO} '"in_role_since": '${_in_role_since}','
	role_changing=$( [ -r ${CHANGING_ROLE_FLAG} ] && echo true  || echo false )
	validation=$( (2>&1 checks_for_slave)  > /dev/null && echo true || echo false )
	${ECHO} '"role_changing": '${role_changing}','
	${ECHO} '"validation": '${validation}','
	${ECHO} '"ssh_check": '${ssh_state}
	${ECHO} '}'

	exit 0
}

getpw()
{
	local oldmodes=$( stty -g )
	NEW_CONTROLLER_PASSWORD=""

	trap "stty ${oldmodes}" HUP INT ABRT BUS TERM EXIT

	printf "${BOLD}Enter password of remote controller root user ${N2_COLOR}${node}${N0_COLOR}${BOLD}:${N0_COLOR} "
	while [ -z "${NEW_CONTROLLER_PASSWORD}" ]; do
		stty -echo
		set -e
		read NEW_CONTROLLER_PASSWORD
		set +e
	done

	stty ${oldmodes}
	echo
}

init_key()
{
	local _mykey

	# already exist ?
	if [ ! -r ${L_DEST_DIR}/id_rsa -o ! -r ${L_DEST_DIR}/id_rsa.pub ]; then
		[ -d ${R_DEST_DIR} ] && ${RM_CMD} -rf ${R_DEST_DIR}
		[ ! -d ${R_DEST_DIR} ] && ${MKDIR_CMD} -p ${R_DEST_DIR}
		[ ! -d ${L_DEST_DIR} ] && ${MKDIR_CMD} ${L_DEST_DIR}

		${ECHO} "* ${H5_COLOR}${MY_APPLICATION}: ${WHITE}generate ${L_DEST_DIR}/id_rsa,id_rsa.pub pairs${N0_COLOR}"
		ssh-keygen -m PEM -b 2048 -t rsa -q -N "" -f ${L_DEST_DIR}/id_rsa
		[ ! -d /root/.ssh ] && ${MKDIR_CMD} /root/.ssh && ${CHMOD_CMD} 0700 /root/.ssh
	fi

	# copy/append pubkey
	_mykey=$( ${CAT_CMD} ${L_DEST_DIR}/id_rsa.pub )
	if ! ${GREP_CMD} -q "${_mykey}" 2>/dev/null /root/.ssh/authorized_keys; then
		${ECHO} "${_mykey}" >> /root/.ssh/authorized_keys
	fi
}

del_key()
{
	local _mykey _rootdir

	_rootdir="/root/.ssh"

	if [ -r ${L_DEST_DIR}/id_rsa.pub ]; then
		_mykey=$( ${CAT_CMD} ${L_DEST_DIR}/id_rsa.pub )
		if ${GREP_CMD} -v "${_mykey}" ${_rootdir}/authorized_keys > ${_rootdir}/tmp; then
			${CAT_CMD} ${_rootdir}/tmp > ${_rootdir}/authorized_keys && ${RM_CMD} ${_rootdir}/tmp;
		fi
	fi

	[ -d ${R_DEST_DIR} ] && ${RM_CMD} -rf ${R_DEST_DIR}
	[ -d ${L_DEST_DIR} ] && ${RM_CMD} -rf ${L_DEST_DIR}
}

add_controller()
{
	local _res
	local _myip

	[ -z "${NEW_CONTROLLER_IP}" ] && err 1 "${H5_COLOR}Empty IP, please use via 'controller add <IP>'${N0_COLOR}"
	[ "${NEW_CONTROLLER_IP}" = "${myip}" ] && err 1 "${H5_COLOR}Please use IP of another controller${N0_COLOR}"
	[ -z "${NEW_CONTROLLER_PASSWORD}" ] && getpw
	[ -z "${NEW_CONTROLLER_PASSWORD}" ] && err 1 "${H5_COLOR}Empty password, please use -p <password>${N0_COLOR}"

	[ -z "${NEW_CONTROLLER_IP}" ] && err 1 "${N1_COLOR}Empty node${N0_COLOR}"
	[ -z "${port}" ] && port=22
	[ -z "${rootkeyfile}" ] && rootkeyfile="/root/.ssh/id_rsa"

	iptype ${NEW_CONTROLLER_IP} >/dev/null 2>&1
	_ret=$?

	case ${_ret} in
		1|2)
			# ip4 or ip6
			_myip="${NEW_CONTROLLER_IP}"
			;;
		*)
			err 1 "${N1_COLOR}Please use IP address instead of: ${N2_COLOR}${NEW_CONTROLLER_IP}${N0_COLOR}"
			;;
	esac

	iptype ${_myip} >/dev/null 2>&1
	_ret=$?

	# if natip is not valid IPv4, assume it is NIC variable.
	# so try to find out first IPv4 for aliasing
	case ${_ret} in
		1)
			VEIL_SSH_CMD="veilssh"
			VEIL_SFTP_CMD="veilsftp"
			proto="IPv4"
			;;
		2)
			VEIL_SSH_CMD="veilssh6"
			VEIL_SFTP_CMD="veilsftp6"
			proto="IPv6"
			;;
		*)
			err 1 "${N1_COLOR}Unknown IP type: ${N2_COLOR}${_myip}${N0_COLOR}"
		;;
	esac

	${ECHO} "${H5_COLOR}Connecting to ${NEW_CONTROLLER_IP} via ${proto}...${N0_COLOR}"
	NODENAME=$( timeout 12 ${VEIL_SSH_CMD} ${NEW_CONTROLLER_IP} ${port} ${veiluser} ${NEW_CONTROLLER_PASSWORD} hostname )
	code=$?

	case ${code} in
		0)
			${ECHO} "${H5_COLOR}${NEW_CONTROLLER_IP} has nodename: ${N2_COLOR}${NODENAME}${N0_COLOR}"
			;;
		2)
			err 1 "${N1_COLOR}Bad password or system user${N0_COLOR}"
			;;
		*)
			err 1 "${N1_COLOR}Connection problem (code ${code}): ${N2_COLOR}${NEW_CONTROLLER_IP}${N0_COLOR}"
			;;
	esac

	[ -z "${NODENAME}" ] && err 1 "${N1_COLOR}No nodename found. Check remote veil settings${N0_COLOR}"

	${ECHO} "${H5_COLOR}Init key for remote node: ${N2_COLOR}${_myip}${N0_COLOR}"
	init_key >/dev/null 2>&1
	[ ! -d ${R_DEST_DIR} ] && ${MKDIR_CMD} -p ${R_DEST_DIR}

	${ECHO} "${H5_COLOR}Add key to remote node: ${N2_COLOR}${_myip}${N0_COLOR}"
	${VEIL_SFTP_CMD} ${_myip} ${port} ${veiluser} ${NEW_CONTROLLER_PASSWORD} ${L_DEST_DIR}/id_rsa ${R_DEST_DIR}/id_rsa
	RESULT=$?

	case ${RESULT} in
		0)
			${CHMOD_CMD} 0400 /root/rctl/id_rsa
			${ECHO} "${H5_COLOR}Key has been added successful: ${N2_COLOR}${_myip}${N0_COLOR}"
			${ECHO} "${_myip}" > ${REMOTE_IP_FILE}
			${ECHO} "pending" > ${REMOTE_STATUS_FILE}
			;;
		1)
			err 1 "${N1_COLOR}Bad user or password.${N0_COLOR}"
			;;
		2)
			err 2 "${N1_COLOR}No such key on remote host:${N2_COLOR} Please init controller via 'controller add ${myip}${N0_COLOR}'"
			;;
		*)
			err 3 "${N1_COLOR}Unknown error.${N0_COLOR}"
			;;
	esac

	test_connection > /dev/null 2>&1

	return 0
}

del_controller()
{
	_status=$( current_status_int )
	# Нельзя удалять контроллер при активной роли репликации
	if [ ${_status} -ne 12 -a ${_status} -ne 22 ]; then
		${ECHO} "${N1_COLOR}Before deleting controller change role to alone via 'controller role alone'${N0_COLOR}"
		exit 0
	fi
	del_key
	[ -r ${REMOTE_STATUS_FILE} ] && ${RM_CMD} -f ${REMOTE_STATUS_FILE}
	[ -r ${REMOTE_IP_FILE} ] &&  ${RM_CMD} -f ${REMOTE_IP_FILE}

	/usr/bin/veil-sendrole full > /dev/null 2>&1
	${ECHO} "${H5_COLOR}Deleted remote controller from configuration.${N0_COLOR}"
	return 0
}

test_connection()
{
	local _rnode
	[ ! -r ${REMOTE_IP_FILE} ] && echo "${N1_COLOR}No remote controller, please init controller via: ${N2_COLOR}controller add <IP>${N0_COLOR}" && return 1
	_rnode=$( ${CAT_CMD} ${REMOTE_IP_FILE} | ${AWK_CMD} '{ printf $1}' )
	if [ -z "${_rnode}" ]; then
		${ECHO} "${N1_COLOR}Empty IP address or remote controller not initialized, please init controller via: ${N2_COLOR}controller add <IP>${N0_COLOR}"
		return 1
	fi
	[ -r ${DEST_RSTATS} ] && ${RM_CMD} -f ${DEST_RSTATS}

	rpg_data_dir_bytes=
	rpg_partition_free_bytes=
	rpg_role=
	rpg_database_size_bytes=
	rpg_last_xact_replay=

	${TRUNCATE_CMD} -s0 ${DEB_LOGFILE}.ssh
	2>>${DEB_LOGFILE}.ssh ${SSH_CMD} ${CONTROLLER_SSH_OPT} root@${_rnode} "test -r ${DEST_STATS} && ${CAT_CMD} ${DEST_STATS}" > ${DEST_RSTATS}
	_ret=$?
	case ${_ret} in
		0)
			if [ -r ${DEST_RSTATS} ]; then
				. ${DEST_RSTATS}
				rpg_data_dir_bytes="${pg_data_dir_bytes}"
				rpg_partition_free_bytes="${pg_partition_free_bytes}"
				rpg_role="${pg_role}"
				rpg_database_size_bytes="${pg_database_size_bytes}"
				rpg_last_xact_replay="${pg_last_xact_replay}"
				# restore main stats
				. ${DEST_STATS}
				if [ -n "${rpg_role}" ]; then
					${ECHO} "active" > ${REMOTE_STATUS_FILE}
				else
					_ret=1
					${ECHO} "pending (ssh ready, waiting for config)" > ${REMOTE_STATUS_FILE}
					echo "pending (ssh ready, waiting for config)" >> ${DEB_LOGFILE}.ssh
				fi
			else
				${ECHO} "pending (ssh ready, waiting for config)" > ${REMOTE_STATUS_FILE}
				echo "pending (ssh ready, waiting for config)" >> ${DEB_LOGFILE}.ssh
				_ret=1
			fi
			;;
		1)
			;;
		*)
			${ECHO} "pending" > ${REMOTE_STATUS_FILE}
			_ret=2
			;;
	esac

	# 0 - all good, 1 - config problem, 2 - ssh failed
	return ${_ret}
}

slave_sync()
{
	_status=$( current_status_int )
	if [ ${_status} -gt 19 ]; then
		if [ ! -r "${SLAVE_HTML_FILE}" ]; then
			${CP_CMD} ${CONTROLLER_CONFIG_DIR}/slave.html ${SLAVE_HTML_FILE}
		fi
	else
		[ -r ${SLAVE_HTML_FILE} ] && ${RM_CMD} -f ${SLAVE_HTML_FILE}
	fi

	return 0
}

set_ntp_from_base() {
  # for new master we form ntp.conf by getting ntp servers from DB or set defaults
NTP_SERVERS=$( ${PSQL_CMD} -t postgresql://${PGSQL_DB_USER}:${PGSQL_DB_PASSWORD}@${LOCALHOST}:${PGSQL_DB_PORT}/${PGSQL_DB_NAME}?sslmode=disable << EOF
SELECT ntp_servers FROM controller_controller;
EOF
)
  if [ -z "$NTP_SERVERS" ] || [ "$( echo ${NTP_SERVERS} | sed -e 's/^[\ ]+//g')" = "[]" ]; then
    ${ECHO} "${N2_COLOR}NTP servers not set in DB (${H3_COLOR}${NTP_SERVERS}${N2_COLOR}), setting defaults${N0_COLOR}"
    SRV_LIST="0.ru.pool.ntp.org,1.ru.pool.ntp.org,2.ru.pool.ntp.org,3.ru.pool.ntp.org"
  else
    ${ECHO} "${N2_COLOR}NTP servers from DB: ${H3_COLOR}${NTP_SERVERS}${N0_COLOR}"
    SRV_LIST=$(${ECHO} "${NTP_SERVERS}" | sed 's/[^a-zA-Z0-9\-\.\,]//g')
  fi
  /usr/local/sbin/veil-change-ntp "${SRV_LIST}" &
  return $!
}

change_nodes_master_ip()
{
	local first_pids=""

	node_ips=$( 2>/dev/null ${PSQL_CMD} -t postgresql://${PGSQL_DB_USER}:${PGSQL_DB_PASSWORD}@${LOCALHOST}:5432/${PGSQL_DB_NAME}?sslmode=disable <<EOF
SELECT address FROM networking_mgmt;
EOF
)
	if [ -z "${node_ips}" ]; then
		${ECHO} "No nodes for updating"
	else

	sql_ret=$( 2>/dev/null ${PSQL_CMD} -t postgresql://${PGSQL_DB_USER}:${PGSQL_DB_PASSWORD}@${LOCALHOST}:5432/${PGSQL_DB_NAME}?sslmode=disable <<EOF
UPDATE public.node_node SET status = 'SERVICE' WHERE status='ACTIVE';
EOF
)
	case $? in
		0)
			${ECHO} "Successfully update nodes to state 'SERVICE'."
			;;
		*)
			${ECHO} "Failed to update nodes to state 'SERVICE'."
			;;
	esac

	node_ips=$( echo ${node_ips} | ${TR_CMD} -d '|' )

	for _rip in ${node_ips}; do
		if [ "${_rip}" = "${myip}" ]; then
		  if [ x"$1" = "x1" ]; then  # when called script with '-y' parameter since else we process own ntp addresses earlier
  			self_pid=set_ntp_from_base
	  		first_pids="$first_pids $self_pid"
		  	${ECHO} "${H5_COLOR}Created ntp.conf on new master (${_rip}).${N0_COLOR}"
		  fi
		else
			${ECHO} "${H5_COLOR}Update ntp for node, connecting to: ${H3_COLOR}${_rip}${N0_COLOR}"
			_rip_name=$( ${SSH_CMD} ${NODE_SSH_OPT} root@${_rip} hostname )
			_ret=$?
			case ${_ret} in
				0)
					${ECHO} "Successfully connected to node ${_rip}, start updating ntp..."
					${SSH_CMD} ${NODE_SSH_OPT} root@${_rip} /usr/local/sbin/veil-change-ntp ${myip} &
					first_pids="$first_pids $!"
					;;
				*)
					${ECHO} "${N1_COLOR}Failed to connect to node ${_rip}${N0_COLOR}"
					;;
			esac
		 fi
	done
	wait $first_pids

	for _rip in ${node_ips}; do
		${ECHO} "${H5_COLOR}Update master for node, connecting to: ${H3_COLOR}${_rip}${N0_COLOR}"

		_rip_name=$( ${SSH_CMD} ${NODE_SSH_OPT} root@${_rip} hostname )
		_ret=$?
		TMP_DIR="/tmp/keys"
		[ -d $TMP_DIR ] || mkdir $TMP_DIR
		case ${_ret} in
			0)
				${ECHO} "Successfully connected to node ${_rip}, start updating master..."
				${SSH_CMD} ${NODE_SSH_OPT} root@${_rip} /usr/local/sbin/veil-change-master-ip ${myip}
				# Пересоздаем grpc ключи во избежании доступа к узлам старым контроллером
				cd "${CONTROLLER_PATH}/app" & (${ECHO} "from veilrpc.utils.cert import grpc_key_generation as gen_key; gen_key('${_rip_name}', '${TMP_DIR}', ssh=False); gen_key('${_rip_name}', '${TMP_DIR}', ssh=False, ext=True)" | ${PYTHON_PATH} ${MANAGE_PY_PATH}) > /dev/null 2>&1
				for ext in "_ext.crt" "_ext.key" ".crt" ".key"; do
					scp ${NODE_SSH_OPT} "${TMP_DIR}/${_rip_name}${ext}" root@${_rip}:${NODE_PATH}/local_data/node${ext}
					rm "${TMP_DIR}/${_rip_name}${ext}" 2>/dev/null
				done
				${SSH_CMD} ${NODE_SSH_OPT} root@${_rip} "systemctl restart node_supervisor.service prometheus-node-exporter &" > /dev/null
				;;
			*)
				${ECHO} "${N1_COLOR}Failed to connect to node ${_rip}${N0_COLOR}"
				;;
		esac
	done
	# Рестарт контроллера с новыми сертификатами
	systemctl restart controller-supervisor prometheus-node-exporter postgres_exporter grafana-server prometheus

	psql_ret=$( 2>/dev/null ${PSQL_CMD} -t  postgresql://${PGSQL_DB_USER}:${PGSQL_DB_PASSWORD}@${LOCALHOST}:5432/${PGSQL_DB_NAME}?sslmode=disable <<EOF
UPDATE public.node_node SET status = 'ACTIVE' WHERE status='SERVICE';
EOF
)
	case $? in
		0)
			${ECHO} "Successfully update nodes to state 'ACTIVE'."
			;;
		*)
			${ECHO} "Failed to update nodes to state 'ACTIVE'."
			;;
		esac
	 fi

	/usr/local/sbin/veil-hosts
}

update_slave()
{
	[ -r ${SLAVE_HTML_FILE} ] && ${RM_CMD} -f ${SLAVE_HTML_FILE}
	local _rnode

	if [ ! -r ${REMOTE_IP_FILE} ]; then
		${ECHO} "${N1_COLOR}No remote controller, please init controller via 'controller add <IP> first'${N0_COLOR}"
		return 1
	fi

	_rnode=$( ${CAT_CMD} ${REMOTE_IP_FILE} | ${AWK_CMD} '{ printf $1}' )

	if [ -z "${_rnode}" ]; then
		${ECHO} "${N1_COLOR}Empty IP address or remote controller not initialized, please init controller via 'controller add <IP> first'${N0_COLOR}"
		return 1
	fi

	controller_master_ip=${LOCALHOST}
	[ -r ${CONTROLLER_MASTER_IP_FILE} ] && . ${CONTROLLER_MASTER_IP_FILE}
	controller_slave_ip=${LOCALHOST}
	[ -r /etc/veil/controller_slave_ip.conf ] && . /etc/veil/controller_slave_ip.conf

	is_master=0
	remote_is_master=$(ssh root@${controller_slave_ip} '(/usr/local/sbin/veil-controller -m | grep "status: master" > /dev/null && echo 1) || echo 0')

	if [ -r ${REMOTE_IP_FILE} ]; then
		_rnode=$( ${CAT_CMD} ${REMOTE_IP_FILE} | ${AWK_CMD} '{ printf $1}' )
		test_connection > /dev/null 2>&1
		_rstate=$( ${CAT_CMD} ${REMOTE_STATUS_FILE} )
		status="initialized for remote node: ${_rnode}, state: ${_rstate}"
	else
		status="not initialized yet"
	fi

	if [ -z "${controller_master_ip}" -o "${controller_master_ip}" = ${LOCALHOST} ]; then
		is_master=1
	fi

	if [ -z "${controller_slave_ip}" -o "${controller_slave_ip}" = ${LOCALHOST} ]; then
		err 1 "${N1_COLOR}Slave controller not initialized${N0_COLOR}"
	fi

	[ ${is_master} -eq 0 ] && err 0 "${N1_COLOR}Skip for update: not master${N0_COLOR}"
	[ ${remote_is_master} -eq 1 ] && err 0 "${N1_COLOR}Skip for update: remote is master${N0_COLOR}"
	# TODO: from conf or backup.subr
	# TODO Убран /var/lib/redis
	_dirs="/var/lib/ecp-veil/.users/veil-controller/.ssh /etc/puppetveil/code/environments/production/data/modules /etc/puppetveil/code/environments/production/data/modules-defaults /etc/puppetveil/code/environments/production/data/nodes /etc/puppetveil/code/environments/production/data/role /var/lib/ecp-veil/data/modules /etc/hosts.d"
	_files="/etc/openvswitch/system-id.conf /etc/puppetveil/code/environments/production/data/common.yaml"
	for file in /var/lib/ecp-veil/controller/*.key; do
		_files="${_files} ${file}"
	done
	for i in ${_dirs}; do
		if [ ! -d "${i}" ]; then
			${ECHO} "warning: no such dir: [$i]"
			continue
		fi
		${ECHO} "Sync for dir: $i"
		${RSYNC_CMD} -az --recursive --delete -e "${SSH_CMD} ${CONTROLLER_SSH_OPT}" ${i}/ root@${controller_slave_ip}:${i}/
		_err=$?
		case ${_err} in
			0|6|24|25)
				;;
			*)
				${ECHO} "warning: sync files error for slave: ${controller_slave_ip}"
				;;
		esac
	done

	for i in ${_files}; do
		if [ ! -r "${i}" ]; then
			${ECHO} "warning: no such file: [$i]"
			continue
		fi
		_spath=$( ${DIRNAME_CMD} ${i} )
		_sdir=$( ${BASENAME_CMD} ${i} )
		${ECHO} "Sync for file: $i"
		${RSYNC_CMD} -az -e "${SSH_CMD} ${CONTROLLER_SSH_OPT}" ${i} root@${controller_slave_ip}:${i}
		_err=$?
		case ${_err} in
			0|6|24|25)
				;;
			*)
				${ECHO} "warning: sync files error for slave: ${controller_slave_ip}"
				;;
		esac
	done

	return 0
}

checks_for_slave()
{
  if [ ! -r ${REMOTE_IP_FILE} ]; then
    exit 0
  fi

  _rnode=$( ${CAT_CMD} ${REMOTE_IP_FILE} | ${AWK_CMD} '{ printf $1}' )

  /var/lib/ecp-veil/controller/other/pgsql/pgstats.sh > /dev/null
  test_connection > /dev/null 2>&1

  if [ ${_ret} -ne 0 ]; then
    [ -r "${DEB_LOGFILE}.ssh" ] && ${CAT_CMD} ${DEB_LOGFILE}.ssh
    err 0 ""
  fi

  rpg_data_dir=$( conv2human ${rpg_data_dir_bytes} )
  pg_partition_free=$( conv2human ${pg_partition_free_bytes} )

  # +3GB extra
  _rec_space_bytes=$(( rpg_data_dir_bytes + 3000000000 ))
  _rec_space_free=$( conv2human ${_rec_space_bytes} )

  if [ ${_rec_space_bytes} -ge ${pg_partition_free_bytes} ]; then
    err 1 "FAILED"
  else
    ${ECHO} "PASSED"
  fi
}

release_flag_and_exit()
{
  [ -r ${CHANGING_ROLE_FLAG} ] && rm $CHANGING_ROLE_FLAG 2>/dev/null
  exit $1 || exit 1
}

# меняем роль:
# slave  - запускаем basebackup
# master - удаляем recovery.conf и перезапускаем slave в качестве master
# reset  - только для slave, удаление репликации, данных и останов postgresql
change_role()
{
	local _ret=
	local _rec_space_bytes=
	local _rec_space_free=

	${ECHO} "[ $(date) ] Begin role changing."

	[ -r ${CHANGING_ROLE_FLAG} ] && err 0 'Role changing process already in progress.'
	touch $CHANGING_ROLE_FLAG

	case "${NEW_ROLE}" in
		slave)
			if [ ! -r ${REMOTE_IP_FILE} ]; then
				${ECHO} "${N1_COLOR}No such remote ip, please init node via 'controller add <IP>' first${N0_COLOR}"
				release_flag_and_exit 0
			fi

			_rnode=$( ${CAT_CMD} ${REMOTE_IP_FILE} | ${AWK_CMD} '{ printf $1}' )
			test_connection > /dev/null 2>&1
			_ret=$?
			if [ ${_ret} -ne 0 ]; then
				${ECHO} "${N1_COLOR}remote host (${_rnode}) not ready${N0_COLOR}"
				[ -r "${DEB_LOGFILE}.ssh" ] && ${CAT_CMD} ${DEB_LOGFILE}.ssh
				release_flag_and_exit 0
			fi

			rpg_data_dir=$( conv2human ${rpg_data_dir_bytes} )
			pg_partition_free=$( conv2human ${pg_partition_free_bytes} )

			${ECHO} -n "${N1_COLOR}Check for datasize (${N2_COLOR}free:${pg_partition_free}/used:${rpg_data_dir}${N1_COLOR}): ${N0_COLOR}"

			# +3GB extra
			_rec_space_bytes=$(( rpg_data_dir_bytes + 3000000000 ))
			_rec_space_free=$( conv2human ${_rec_space_bytes} )

			if [ ${_rec_space_bytes} -ge ${pg_partition_free_bytes} ]; then
				${ECHO} "${W1_COLOR}failed${N0_COLOR}"
				${ECHO} "${N1_COLOR}Your partition must have at least ${N2_COLOR}${_rec_space_free}${N1_COLOR} (current: ${N2_COLOR}${pg_partition_free}${N1_COLOR})${N0_COLOR}"
				${ECHO} "${N1_COLOR}Please increase the volume of the partition or reduce the size of the database first: ${N2_COLOR}${_rnode}:${PG_DATA_DIR}${N0_COLOR}"
				# spacevm cli always assumes code 0 and does not proxy the message if the code is non-zero
				release_flag_and_exit 0
			else
				${ECHO} "${N2_COLOR}passed${N0_COLOR}"
			fi

			${ECHO} "${N1_COLOR}Change local controller role to: ${N2_COLOR}slave${N0_COLOR}"

			/usr/local/sbin/veil-db-backup || release_flag_and_exit 0

			# replace controller_ip in /var/lib/ecp-veil/node/local_data/heartbeat.json
			# -> redis get node_master_ip
			/usr/local/sbin/veil-change-master-ip ${_rnode}

			${ECHO} "controller_master_ip=\"${_rnode}\"" > ${CONTROLLER_MASTER_IP_FILE}

			# Очищаем директорию domains при переходе с master на slave
			if [ -d "/var/lib/ecp-veil/controller/domains" ]; then
				${ECHO} "${N1_COLOR}Cleaning up domains xml directory${N0_COLOR}"
				${RM_CMD} -rf /var/lib/ecp-veil/controller/domains/*
			fi

			if [ -r ${CONTROLLER_SLAVE_IP_FILE} ]; then
				# Значит, мы были master до этого момента.
				${RM_CMD} -f ${CONTROLLER_SLAVE_IP_FILE}
			fi
			_rstate=$( ${CAT_CMD} ${REMOTE_STATUS_FILE} )
			if [ "${_rstate}" = 'active' ]; then
				${ECHO} "Change remote controller role to master"
				${ECHO} "${N4_COLOR}Remote controller log <<<"
				${SSH_CMD} ${CONTROLLER_SSH_OPT} root@${_rnode} ${RM_CMD} -f ${CONTROLLER_MASTER_IP_FILE}
				${SSH_CMD} ${CONTROLLER_SSH_OPT} root@${_rnode} /usr/local/sbin/veil-controller -r master 2>&1 | sed 's/\x1b\[[0-9;]*m//g' # sed убирает цвета текста
				${ECHO} ">>>${N0_COLOR}"
			fi
			${ECHO} "${H5_COLOR} * Setup Postgresql replication...${N0_COLOR}"

			# Сделаем слот, если он там еще не готов
			${SSH_CMD} ${CONTROLLER_SSH_OPT} root@${_rnode} /usr/local/sbin/veil-controller -o || true

			/usr/local/sbin/veil-pgsql-replication
			if [ $? -ne 0 ]; then
				${ECHO} "Replication not started"
				release_flag_and_exit 1
			fi

			${ECHO} "Stopping SpaceVM services..."
			controller_services_action "stop"

			# ${ECHO} "${H5_COLOR} * Setup Redis replication...${N0_COLOR}"
			# cp -a ${HIERA_DEFAULTS_DIR}/redis.yaml ${HIERA_CUSTOM_DIR}/redis.yaml
			# /usr/bin/veil-sendrole redis > /dev/null 2>&1

			${ECHO} "${H5_COLOR} * Sync from master controller...${N0_COLOR}"
			/usr/local/sbin/veil-change-ntp ${_rnode}
			slave_sync

			/usr/bin/veil-sendrole full > /dev/null 2>&1
			;;
		master)
			if [ ! -r ${REMOTE_IP_FILE} ]; then
				${ECHO} "${N1_COLOR}No such remote ip, please init node via 'controller add <IP>' first${N0_COLOR}"
				release_flag_and_exit 0
			fi

			test_connection > /dev/null 2>&1
			_rstate=$( ${CAT_CMD} ${REMOTE_STATUS_FILE} )

			${ECHO} "Change local controller role to master"

			/usr/local/sbin/veil-db-backup || release_flag_and_exit 0

			_rnode=$( ${CAT_CMD} ${REMOTE_IP_FILE} | ${AWK_CMD} '{ printf $1}' )
			${ECHO} "controller_slave_ip=\"${_rnode}\"" > ${CONTROLLER_SLAVE_IP_FILE}
			if [ -r ${CONTROLLER_MASTER_IP_FILE} ]; then
				# Значит, мы были slave до этого момента.
				${RM_CMD} -f ${CONTROLLER_MASTER_IP_FILE}
			fi

			if [ "${_rstate}" = 'active' ]; then
				${ECHO} "Change remote controller role to slave"
				${ECHO} "${N4_COLOR}Remote controller log <<<"
				${SSH_CMD} ${CONTROLLER_SSH_OPT} root@${_rnode} /usr/local/sbin/veil-controller -v 2>&1 > /dev/null
				if [ $? -eq 0 ]; then
					${SSH_CMD} ${CONTROLLER_SSH_OPT} root@${_rnode} ${RM_CMD} -f ${CONTROLLER_SLAVE_IP_FILE}
					${SSH_CMD} ${CONTROLLER_SSH_OPT} root@${_rnode} /usr/local/sbin/veil-controller -r slave 2>&1 | sed 's/\x1b\[[0-9;]*m//g'  # sed убирает цвета текста
					${ECHO} ">>>${N0_COLOR}"
				else
					${ECHO} "Can't change remote role to slave."
					fi
			fi

			# sed "s/redis::bind:.*/redis::bind: 0.0.0.0/g" ${HIERA_DEFAULTS_DIR}/redis.yaml > ${HIERA_CUSTOM_DIR}/redis.yaml
			# /usr/bin/veil-sendrole redis > /dev/null 2>&1
			${ECHO} "Stopping services."
			stop_services nginx controller-supervisor gunicorn-proxy gunicorn-uploader websockify gunicorn-django daphne-django gunicorn-logger gunicorn-apiclient
			stop_services postgresql postgresql@17-main
			[ -r ${PG_RECOVERY_FILE} ] && ${RM_CMD} -f ${PG_RECOVERY_FILE}
			[ -r ${PG_STANDBY_FILE} ] && ${RM_CMD} -f ${PG_STANDBY_FILE}

			${ECHO} "Starting PostgreSQL."
			start_services postgresql postgresql@17-main

			create_replication_slot
			set_ntp_from_base
			change_nodes_master_ip
			/usr/local/sbin/veil-prometheus

			[ -r ${SLAVE_HTML_FILE} ] && ${RM_CMD} -f ${SLAVE_HTML_FILE}
			${ECHO} "Starting SpaceVM services."
			systemctl start nginx
			controller_services_action "start"
			/usr/bin/veil-sendrole full > /dev/null 2>&1
			${ECHO} "Controller role changed to master."
			;;
		alone)
			(drop_replication_slot 2>&1) > /dev/null
			if [ ! -r ${CONTROLLER_MASTER_IP_FILE} ] && [ ! -r ${CONTROLLER_SLAVE_IP_FILE} ]; then
				${ECHO} "${N1_COLOR}Controller role is not slave or master.${N0_COLOR}"
				release_flag_and_exit 0
			fi

			${ECHO} "Change local controller role to alone"

			[ -r ${CONTROLLER_MASTER_IP_FILE} ] && ${RM_CMD} -f ${CONTROLLER_MASTER_IP_FILE}
			[ -r ${CONTROLLER_SLAVE_IP_FILE} ] && ${RM_CMD} -f ${CONTROLLER_SLAVE_IP_FILE}

			stop_services nginx controller-supervisor gunicorn-proxy gunicorn-uploader websockify gunicorn-django daphne-django gunicorn-logger gunicorn-apiclient

			# ${ECHO} "${H5_COLOR} * Disabling Redis replication...${N0_COLOR}"
			# cp -a ${HIERA_DEFAULTS_DIR}/redis.yaml ${HIERA_CUSTOM_DIR}/redis.yaml
			# /usr/bin/veil-sendrole redis > /dev/null 2>&1
			# TODO Stop postgres replication maybe by killing process
			${ECHO} "${H5_COLOR} * Disabling Postgresql replication...${N0_COLOR}"
			stop_services postgresql postgresql@17-main
			[ -r ${PG_RECOVERY_FILE} ] && ${RM_CMD} -f ${PG_RECOVERY_FILE}
			[ -r ${PG_STANDBY_FILE} ] && ${RM_CMD} -f ${PG_STANDBY_FILE}
			start_services postgresql postgresql@17-main

			[ -r ${SLAVE_HTML_FILE} ] && ${RM_CMD} -f ${SLAVE_HTML_FILE}
			start_services nginx
			controller_services_action "start"
			[ -r ${SLAVE_HTML_FILE} ] && ${RM_CMD} -f ${SLAVE_HTML_FILE}

			/usr/bin/veil-sendrole full > /dev/null 2>&1
			;;
		reset)
			if ! pg_is_in_recovery; then
				# spacevm cli always assumes code 0 and does not proxy the message if the code is non-zero
				${ECHO} "postgres is not a slave. This operation is intended for slave only"
				release_flag_and_exit 0
			fi
			if getyesno "This operation will destroy postgresql data and replication config. Are you sure?"; then
				stop_services postgresql postgresql@17-main
				# todo: factory-reset script: /var/lib/ecp-veil/controller/other/reset/reset_postgresql ?
				[ -d "${PG_DATA_DIR}" ] && ${RM_CMD} -rf ${PG_DATA_DIR}
				${SUDO_CMD} -u postgres /usr/lib/postgresql/17/bin/initdb -E 'UTF-8' --lc-collate='en_US.UTF-8' --lc-ctype='en_US.UTF-8' -D /var/lib/postgresql/17/main
				start_services postgresql postgresql@17-main
				# wait service for a moment
				[ -r ${CONTROLLER_MASTER_IP_FILE} ] && ${RM_CMD} -f ${CONTROLLER_MASTER_IP_FILE}
				[ -r ${CONTROLLER_SLAVE_IP_FILE} ] && ${RM_CMD} -f ${CONTROLLER_SLAVE_IP_FILE}
				[ -r ${REMOTE_STATUS_FILE} ] && ${RM_CMD} -f ${REMOTE_STATUS_FILE}
				[ -r ${REMOTE_IP_FILE} ] && ${RM_CMD} -f ${REMOTE_IP_FILE}
				sleep 5
				/usr/local/sbin/veil-role -a pgsql
				# re-run migrate script
				/var/lib/ecp-veil/controller/other/init_controller_db
				# reset old stats/refresh
				[ -r /var/log/veil/controller/pgstats.log ] && ${RM_CMD} /var/log/veil/controller/pgstats.log
				echo "re-run /var/lib/ecp-veil/controller/other/pgsql/pgstats.sh..."
				timeout 60 /var/lib/ecp-veil/controller/other/pgsql/pgstats.sh > /dev/null 2>&1
				release_flag_and_exit 0
			fi
			release_flag_and_exit 0
			;;
		*)
			${ECHO} "Unknown role. Please set: 'slave', 'master', 'alone' or 'reset'"
			release_flag_and_exit 1

			;;
	esac
  [ -r ${CHANGING_ROLE_FLAG} ] && rm $CHANGING_ROLE_FLAG 2>/dev/null
  ${ECHO} "[ $(date) ] End role changing."
	return 0
}


usage()
{
	${ECHO} "veil-controller [-a <remote_controller_ip] [-i init rsa_key] [-p <remote_controler_password>] [-r change role: master,slave,alone,reset] [-l current_status] [-u update slave] [-c slave_sync]"
	exit 0
}


set_witness()
{
	witness_id=$(echo $1 | awk '{print $1}')
	witness_interval=$(echo $1 | awk '{print $2}')
	[ -d /var/lib/ecp-veil/run/witness ] || ${MKDIR_CMD} /var/lib/ecp-veil/run/witness
	COUNT=$(ls -l /var/lib/ecp-veil/run/witness | grep -v $witness_id | wc -l)
	COUNT=$(( COUNT - 1 ))
	if [ $COUNT -eq 0 ]; then
	  echo $witness_interval > /var/lib/ecp-veil/run/witness/"${witness_id}"
	else
	  err 1 "Cluster is managed by another witness. To remove deprecated witness use CLI (controller remove-witness)"
	fi
}

## MAIN
add_ctl=0
del_ctl=0
change_role=0
NEW_CONTROLLER_IP=
NEW_CONTROLLER_PASSWORD=
NEW_ROLE=
LOCALHOST="127.0.0.1"
veiluser="root"
SLAVE_HTML_FILE="/tmp/slave.html"
PG_RECOVERY_FILE="/var/lib/postgresql/17/main/postgresql.auto.conf"
PG_STANDBY_FILE="/var/lib/postgresql/17/main/standby.signal"
HIERA_DEFAULTS_DIR="/etc/puppetveil/code/environments/production/data/modules-defaults"
HIERA_CUSTOM_DIR="/etc/puppetveil/code/environments/production/data/modules"
NODE_SSH_OPT="-q -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oBatchMode=yes -oConnectTimeout=5 -oVerifyHostKeyDNS=no -oServerAliveInterval=15  -oIdentitiesOnly=yes -i /var/lib/ecp-veil/.users/veil-controller/.ssh/id_rsa"
R_DEST_DIR="/root/rctl"
L_DEST_DIR="/root/lctl"

${TRUNCATE_CMD} -s0 ${DEB_LOGFILE}

[ -z "$*" ] && usage
init_color

get_myip

# -t just test for update available
while getopts "a:biglmonp:r:cdtsuyjvw:" opt; do
	case "${opt}" in
		a)
			# Добавление ключа соседнего контроллера
			NEW_CONTROLLER_IP="${OPTARG}"
			add_ctl=1
			;;
		b)
			# Show TOP/bloat tables
			${ECHO} "${N1_COLOR}Wasted/Bloat stats:${N0_COLOR}"
			2>/dev/null  ${SUDO_CMD} -u postgres env PGCONNECT_TIMEOUT=3 ${PSQL_CMD} -t veildb < /var/lib/ecp-veil/controller/other/pgsql/wasted-bloat.sql
			echo
			${ECHO} "${N1_COLOR}Top tables by size:${N0_COLOR}"
			2>/dev/null ${SUDO_CMD} -u postgres env PGCONNECT_TIMEOUT=3 ${PSQL_CMD} -t veildb < /var/lib/ecp-veil/controller/other/pgsql/top-tables.sql
			exit 0
			;;
		c)
			# Авто синхронизация на slave контроллере
			slave_sync
			exit $?
			;;
		d)
			# Удаление ключа соседнего контроллера
			del_ctl=1
			;;
		i)
			# Для тестов
			init_key
			exit 0
			;;
		l)
			# Выводит статус репликации
			current_status
			exit 0
			;;
		m)
			# Выводит статус репликации для ui без спецсимволов
			current_status 2>/dev/null | sed 's/\x1b\[[0-9;]*m//g' # sed убирает цвета текста
			exit 0
			;;
		j)
			# Выводит статус репликации в JSON формате
			current_status_json | jq
			exit 0
			;;
		o)
			# Создание слота репликации на мастере
			create_replication_slot
			exit 0
			;;
		n)
			# Выводит статус репликации int (для скрипта обновления)
			echo $(current_status_int)
			exit 0
			;;
		p)
			# root ssh пароль
			NEW_CONTROLLER_PASSWORD="${OPTARG}"
			;;
		r)
			# Смена роли контроллера на master, slave, alone, reset
			NEW_ROLE="${OPTARG}"
			change_role=1
			;;
		t)
			# Для тестов. Проверка связи
			test_connection
			exit 0
			;;
		s)
			# remote ssh
			remote_ssh
			exit $?
			;;
		u)
			# Авто синхронизация slave из cron на master
			update_slave
			exit $?
			;;
		y)
			# Смена ip адреса мастера на узлах
			change_nodes_master_ip 1
			exit $?
			;;
		v)
			# Валидации перед переключением в слейва
			checks_for_slave
			exit $?
			;;
		w)
			# Статус с указанием id свидетеля
			set_witness "${OPTARG}"
			current_status_json
			exit 0
			;;
		g)
			# Обновляем конфиг pgsql для применения изменений с обновлением
			reload_pg_config
			exit 0
			;;
		*)
			usage
			;;
	esac
done
shift $(($OPTIND - 1))

[ ${add_ctl} -eq 1 -a ${del_ctl} -eq 1 ] && err 1 "${H5_COLOR}Error: -a and -d mutually exclusive options"

if [ ${add_ctl} -eq 1 ]; then
	add_controller
	exit $?
fi

if [ ${del_ctl} -eq 1 ]; then
	del_controller
	exit $?
fi

if [ ${change_role} -eq 1 ];then
	[ ! -d "${HIERA_CUSTOM_DIR}" ] && ${MKDIR_CMD} -p ${HIERA_CUSTOM_DIR}
	${ECHO} "${N1_COLOR}update stats...${N0_COLOR}"
	timeout 60 /var/lib/ecp-veil/controller/other/pgsql/pgstats.sh > /dev/null 2>&1
	[ ! -r ${DEST_STATS} ] && err 0 "no such stats file"
	. ${DEST_STATS}
	change_role 2>&1 | tee -a $REPLICATION_LOG
	_ret=$?
	#re-run stats
	/var/lib/ecp-veil/controller/other/pgsql/pgstats.sh > /dev/null 2>&1

	exit ${_ret}
fi
