#!/bin/sh

set -e
#set -x

RED="\033[1;31m"
NO_COL="\033[0m"
GREEN="\033[1;32m"
YELLOW="\033[1;33m"
green_echo () {
	echo ${GREEN}${1}${NO_COL}
}
red_echo () {
	echo ${RED}${1}${NO_COL}
}

yellow_echo () {
	echo ${YELLOW}${1}${NO_COL}
}

ALL_SERVICES_CTRL="aodh-api aodh-evaluator aodh-expirer aodh-listener aodh-notifier barbican-api barbican-keystone-listener barbican-worker ceilometer-agent-central ceilometer-agent-notification ceilometer-polling cinder-api cinder-backup cinder-scheduler cinder-volume cloudkitty-api cloudkitty-processor glance-api gnocchi-api gnocchi-metricd gnocchi-statsd heat-api-cfn heat-api heat-engine keystone magnum-api magnum-conductor neutron-api neutron-dhcp-agent neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent neutron-rpc-server nova-api nova-api-metadata nova-compute nova-conductor nova-novncproxy nova-scheduler octavia-api octavia-health-manager octavia-housekeeping octavia-worker panko-api placement-api neutron-api neutron-rpc-server"
ALL_SERVICES_VOL="cinder-backup cinder-volume"
ALL_SERVICES_NET="neutron-dhcp-agent neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent"
ALL_SERVICES_COMP="ceilometer-agent-compute cinder-backup cinder-volume neutron-dhcp-agent neutron-l3-agent neutron-metadata-agent neutron-metering-agent neutron-openvswitch-agent nova-compute"
ALL_SERVICES_MSG="gnocchi-api gnocchi-metricd gnocchi-statsd cloudkitty-api cloudkitty-processor"

# Prior calling, CLUSTER_NAME must be set.
fetch_cluster_info () {
	green_echo "---> Fetching cluster info"
	TMP_CLUSTER_SHOW=$(mktemp -t ${ME}-cluster-show.XXXXXX)
	ocicli -csv cluster-show ${CLUSTER_NAME} >${TMP_CLUSTER_SHOW}

	DOMAIN_NAME=$(cat ${TMP_CLUSTER_SHOW} | grep Domain: | cut -d, -f2 | sed -e 's#"##g')
	HAS_RABBIT=$(cat ${TMP_CLUSTER_SHOW} | grep "Install RabbitMQ:" | cut -d, -f2 | sed -e 's#"##g')
	HAS_BARBICAN=$(cat ${TMP_CLUSTER_SHOW} | grep "Install Barbican:" | cut -d, -f2 | sed -e 's#"##g')
	HAS_HEAT=$(cat ${TMP_CLUSTER_SHOW} | grep "Install Heat:" | cut -d, -f2 | sed -e 's#"##g')
	HAS_DESIGNATE=$(cat ${TMP_CLUSTER_SHOW} | grep "Install designate:" | cut -d, -f2 | sed -e 's#"##g')
	HAS_OCTAVIA=$(cat ${TMP_CLUSTER_SHOW} | grep "Install octavia:" | cut -d, -f2 | sed -e 's#"##g')
	HAS_TELEMETRY=$(cat ${TMP_CLUSTER_SHOW} | grep "Install telemetry:" | cut -d, -f2 | sed -e 's#"##g')
	HAS_CLOUDKITTY_DASHBOARD=$(cat ${TMP_CLUSTER_SHOW} | grep "Install cloudkitty dashboard:" | cut -d, -f2 | sed -e 's#"##g')
	USE_EXTERNAL_KEYSTONE=$(cat ${TMP_CLUSTER_SHOW} | grep "External keystone activate:" | cut -d, -f2 | sed -e 's#"##g')
	USE_QUORUM_QUEUES=$(cat ${TMP_CLUSTER_SHOW} | grep "RabbitMQ enable quorum queues:" | cut -d, -f2 | sed -e 's#"##g')

	# Make a survey of all the hosts we have in the cluster
	green_echo "---> Fetching nodes info"
	TMP_MACHINE_LIST=$(mktemp -t ${ME}-machine-list.XXXXXX)
	ocicli -csv machine-list -a --filter cluster_name=${CLUSTER_NAME} >${TMP_MACHINE_LIST}

	# Since we're using --filter cluster_name=, no need for a WHERE cluster='${CLUSTER_NAME}'
	ONE_CTRL=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='controller' AND status='installed' ORDER BY hostname LIMIT 1" | tr '\n' ' ')
	ONE_CTRL_IP=$(echo ${ONE_CTRL} | cut -d, -f1 | sed -e 's#"##g')
	WHERE_IS_API_VIP=$(sshi -x root@${ONE_CTRL_IP} "crm resource locate openstack-api-vip | sed -e 's/resource openstack-api-vip is running on: //' | awk '{print \$1}'")
	ONE_CTRL=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='controller' AND status='installed' AND hostname='${WHERE_IS_API_VIP}' ORDER BY hostname LIMIT 1" | tr '\n' ' ')

	ALL_CTRL=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='controller' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
	OTHER_CTRL=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='controller' AND status='installed' AND hostname NOT LIKE '${WHERE_IS_API_VIP}' ORDER BY hostname" | tr '\n' ' ')
	ONE_CTRL_IP=$(echo ${ONE_CTRL} | cut -d, -f1 | sed -e 's#"##g')
	ONE_CTRL_HOST=$(echo ${ONE_CTRL} | cut -d, -f2 | sed -e 's#"##g')
	ALL_COMP=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='compute' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
	ALL_VOL=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='volume' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
	ALL_NET=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='network' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
	ALL_MSG=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='messaging' AND status='installed' ORDER BY hostname" | tr '\n' ' ')

	ONE_MSG=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='messaging' AND status='installed' ORDER BY hostname LIMIT 1" | tr '\n' ' ')
	if [ -n "${ONE_MSG}" ] ; then
		ONE_MSG_IP=$(echo ${ONE_MSG} | cut -d, -f1 | sed -e 's#"##g')
		ONE_MSG_HOST=$(echo ${ONE_MSG} | cut -d, -f2 | sed -e 's#"##g')
		WHERE_IS_MSG_VIP=$(sshi -x root@${ONE_MSG_IP} "crm resource locate openstack-messaging-vip | sed -e 's/resource openstack-messaging-vip is running on: //' | awk '{print \$1}'")
		ONE_MSG=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='messaging' AND status='installed' AND hostname='${WHERE_IS_MSG_VIP}' ORDER BY hostname LIMIT 1" | tr '\n' ' ')
		ONE_MSG_IP=$(echo ${ONE_MSG} | cut -d, -f1 | sed -e 's#"##g')
		ONE_MSG_HOST=$(echo ${ONE_MSG} | cut -d, -f2 | sed -e 's#"##g')
		OTHER_MSG=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='messaging' AND status='installed' AND hostname NOT LIKE '${WHERE_IS_MSG_VIP}' ORDER BY hostname" | tr '\n' ' ')
	fi

	ALL_SWIFTPRX=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='swiftproxy' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
	ALL_SWIFTSTR=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='swiftstore' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
	ALL_NODES=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE status='installed' ORDER BY hostname" | tr '\n' ' ')
	ALL_NODES_BUT_CEPH=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE status='installed' AND role NOT LIKE 'ceph%' AND role NOT LIKE 'bill%' ORDER BY hostname" | tr '\n' ' ')
	ALL_TEMPEST=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='tempest' AND status='installed' ORDER BY hostname" | tr '\n' ' ')

	ALL_SQL=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='sql' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
	ALL_SQLMSG=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='sqlmsg' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
	ALL_DNS=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='dns' AND status='installed' ORDER BY hostname" | tr '\n' ' ')

	ALL_BILLMON=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='billmon' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
	ALL_BILLOSD=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='billosd' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
	ONE_BILLMON=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='billmon' AND status='installed' ORDER BY hostname LIMIT 1" | tr '\n' ' ')
	ALL_CEPHMON=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='cephmon' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
	ALL_CEPHOSD=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='cephosd' AND status='installed' ORDER BY hostname" | tr '\n' ' ')
	ONE_CEPHMON=$(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT Cur_ip,hostname FROM - WHERE role='cephmon' AND status='installed' ORDER BY hostname LIMIT 1" | tr '\n' ' ')

	# Get to know if we have specific nodes for:
	# - volume
	# - network
	# - cephosd
	if [ $(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT COUNT(serial) FROM - WHERE role='cephosd'") = "0" ] ; then
		HAS_OSD="no"
	else
		HAS_OSD="yes"
	fi

	if [ $(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT COUNT(serial) FROM - WHERE role='volume'") = "0" ] ; then
		HAS_VOL="no"
	else
		HAS_VOL="yes"
	fi

	if [ $(cat ${TMP_MACHINE_LIST} | q -H -d, "SELECT COUNT(serial) FROM - WHERE role='network'") = "0" ] ; then
		HAS_NET="no"
	else
		HAS_NET="yes"
	fi

	green_echo "---> Getting rabbitmq monitoring credentials"
	RMQ_PASS_ID=$(ocicli -csv password-list --filter service=rabbitmq,passtype=monitoring | q -H -d, "SELECT id FROM -")
	RMQ_MONITORING_PASS=$(ocicli password-show ${RMQ_PASS_ID} | grep Pass: | awk '{print $2}')

	rm -f ${TMP_MACHINE_LIST}

	green_echo "---> Checking if cluster uses self-signed cert"
	if [ $(ocicli cluster-show ${CLUSTER_NAME} | grep "Self signed API cert:" | awk '{print $5}')"" = "yes" ] ; then
		SELF_SIGNED_API_CERT=yes
	else
		SELF_SIGNED_API_CERT=no
	fi

	rm -f ${TMP_CLUSTER_SHOW}
}

display_cluster_info () {
	green_echo "---> Cluster info summary"
	echo "HAS_RABBIT: ${HAS_RABBIT}"
	echo "HAS_BARBICAN: ${HAS_BARBICAN}"
	echo "HAS_HEAT: ${HAS_HEAT}"
	echo "HAS_DESIGNATE: ${HAS_DESIGNATE}"
	echo "HAS_OCTAVIA: ${HAS_OCTAVIA}"
	echo "HAS_TELEMETRY: ${HAS_TELEMETRY}"
	echo "HAS_CLOUDKITTY_DASHBOARD: ${HAS_CLOUDKITTY_DASHBOARD}"
	echo "USE_EXTERNAL_KEYSTONE: ${USE_EXTERNAL_KEYSTONE}"
	sleep 5
}

rabbitmq_msg_num () {
	local RMQ_USER RMQ_PASS RMQ_IP NUM_VERSIONNED_NOTIF NUM_NORMAL_NOTIF NUM_VERSIONNED_NOTIF_ERROR NUM_NORMAL_NOTIF_ERROR NUM_TOTAL_NOTIF OTHER_MESSAGES
	RMQ_USER=${1}
	RMQ_PASS=${2}
	RMQ_IP=${3}
	TMP=$(mktemp)
	curl -k -u${RMQ_USER}:${RMQ_PASS} https://${RMQ_IP}:15671/api/queues 2>/dev/null >${TMP}
	NUM_VERSIONNED_NOTIF=$(cat ${TMP} | jq '.[] | select(.name == "versioned_notifications.info") | .messages')
	NUM_NORMAL_NOTIF=$(cat ${TMP} | jq '.[] | select(.name == "notifications.info") | .messages')
	NUM_VERSIONNED_NOTIF_ERROR=$(cat ${TMP} | jq '.[] | select(.name == "versioned_notifications.error") | .messages')
	NUM_NORMAL_NOTIF_ERROR=$(cat ${TMP} | jq '.[] | select(.name == "notifications.error") | .messages')
	if [ -z "${NUM_VERSIONNED_NOTIF}" ] ; then
		NUM_VERSIONNED_NOTIF=0
	fi
	if [ -z "${NUM_NORMAL_NOTIF}" ] ; then
		NUM_NORMAL_NOTIF=0
	fi
	if [ -z "${NUM_VERSIONNED_NOTIF_ERROR}" ] ; then
		NUM_VERSIONNED_NOTIF_ERROR=0
	fi
	if [ -z "${NUM_NORMAL_NOTIF_ERROR}" ] ; then
		NUM_NORMAL_NOTIF_ERROR=0
	fi
	NUM_TOTAL_NOTIF=$(( ${NUM_VERSIONNED_NOTIF} + ${NUM_NORMAL_NOTIF} + ${NUM_VERSIONNED_NOTIF_ERROR} + ${NUM_NORMAL_NOTIF_ERROR} ))
	OTHER_MESSAGES=$(( ${NUM_MESSAGE} - ${NUM_TOTAL_NOTIF} ))
	rm ${TMP}
	RET=${OTHER_MESSAGES}
}

#########################
### GENERIC FUNCTIONS ###
#########################

sshi () {
	unset SSH_AUTH_SOCK
	ssh -i /etc/openstack-cluster-installer/id_rsa -o ConnectTimeout=10 $@
}

scpi () {
	scp -i /etc/openstack-cluster-installer/id_rsa $@
}

wait_for_ssh () {
	local COUNT CYCLES OTCI_CAN_SSH SSH_HOST
	SYSUSERNAME=root
	# This is 15 minutes
	COUNT=900
	CYCLES=0
	OTCI_CAN_SSH=no
	SSH_HOST=${1}
	echo -n "${GREEN}---> Waiting for ssh to be up for ${SSH_HOST}:${NO_COL}"
	ssh-keygen -f ~/.ssh/known_hosts -R ${SSH_HOST} 1>/dev/null 2>/dev/null || true
	while [ "${OTCI_CAN_SSH}" != "yes" ] && [ ${COUNT} != 0 ] ; do
		if ssh -i /etc/openstack-cluster-installer/id_rsa -o "ConnectTimeout 2" ${SYSUSERNAME}@${SSH_HOST} 'echo -n ""' 2>/dev/null ; then
			OTCI_CAN_SSH=yes
			echo "${GREEN}ok.${NO_COL}"
		else
			COUNT=$(( ${COUNT} - 1 ))
			CYCLES=$(( ${CYCLES} + 1 ))
			sleep 2
			echo -n "."
		fi
	done
	if [ ${OTCI_CAN_SSH} = "yes" ] ; then
		sleep 1
	else
		echo "${RED}timeout.${NO_COL}"
		exit 1
	fi
}

disable_puppet () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)

	green_echo "-> ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl stop puppet"
	sshi root@${HOST_IP} "systemctl disable puppet"
	sshi root@${HOST_IP} "systemctl mask puppet"
	sshi root@${HOST_IP} "rm -f /usr/sbin/policy-rc.d"
}

enable_puppet () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)

	green_echo "-> ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl unmask puppet"
	sshi root@${HOST_IP} "systemctl enable puppet"
	sshi root@${HOST_IP} "systemctl start puppet"
}

switch_to_release () {
	local HOST HOST_IP HOST_NAME FROM_DEB_RELEASE FROM_OS_RELEASE TO_DEB_RELEASE TO_OS_RELEASE
	HOST=${1}
	FROM_DEB_RELEASE=${2}
	FROM_OS_RELEASE=${3}
	TO_DEB_RELEASE=${4}
	TO_OS_RELEASE=${5}

	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)

	green_echo "-> Switching ${HOST_NAME} / ${HOST_IP} to ${TO_DEB_RELEASE}-${TO_OS_RELEASE}"
	sshi root@${HOST_IP} "if [ -e /etc/apt/sources.list.d/${FROM_DEB_RELEASE}-${FROM_OS_RELEASE}.list ] ; then rm /etc/apt/sources.list.d/${FROM_DEB_RELEASE}-${FROM_OS_RELEASE}.list ; fi"

	TMP_OSBPO_REPO_FILE=$(mktemp -t oci-repo-file-XXXXXX)
	echo "Types: deb deb-src
URIs: http://osbpo.debian.net/debian
Suites: ${TO_DEB_RELEASE}-${TO_OS_RELEASE}-backports ${TO_DEB_RELEASE}-${TO_OS_RELEASE}-backports-nochange
Components: main
Signed-By: /etc/oci/oci-repository-key.asc
" >${TMP_OSBPO_REPO_FILE}
	scpi ${TMP_OSBPO_REPO_FILE} root@${HOST_IP}:/etc/apt/sources.list.d/osbpo.debian.net.sources
	rm -f ${TMP_OSBPO_REPO_FILE}

	scpi /etc/openstack-cluster-installer/oci-repository-key.asc root@${HOST_IP}:/etc/oci/

	if [ "${FROM_DEB_RELEASE}" != "${TO_DEB_RELEASE}" ] ; then
		sshi root@${HOST_IP} "sed -i \"s/${FROM_DEB_RELEASE}/${TO_DEB_RELEASE}/g\" /etc/apt/sources.list"
		if [ "${TO_DEB_RELEASE}" = "bookworm" ] ; then
			sshi root@${HOST_IP} "if ! grep -q non-free-firmware /etc/apt/sources.list ; then sed -i 's/non-free/non-free non-free-firmware/g' /etc/apt/sources.list ; fi"
		fi
	fi
	sshi root@${HOST_IP} "apt-get update"
}

# Example call:
# iterate_on_hosts "192.168.101.2,z-controller-1 192.168.101.3,z-controller-2" switch_to_release buster rocky buster stein
# Example transformation for the first host:
# switch_to_release 192.168.101.2,z-controller-1 buster rocky buster stein
iterate_on_hosts () {
	local HOST_LIST FUNCTION USE_PARALLEL
	if [ "${1}" = "-p" ] || [ "${1}" = "--parallel" ] ; then
		USE_PARALLEL=yes
		shift
	else
		USE_PARALLEL=no
	fi
	HOST_LIST=${1}
	FUNCTION=${2}
	shift
	shift

	if [ -z "${ITERATE_ON_HOST_MAX_PARALLEL}" ] ; then
		ITERATE_ON_HOST_MAX_PARALLEL=8
	fi

	if [ "${USE_PARALLEL}" = "yes" ] ; then
		CNT=1
		for i in ${HOST_LIST} ; do
			${FUNCTION} ${i} $@ &
			CNT=$(( ${CNT} + 1))
			if [ ${CNT} -gt ${ITERATE_ON_HOST_MAX_PARALLEL} ] ; then
				CNT=1
				wait
			fi
		done
		wait
	else
		for i in ${HOST_LIST} ; do
			${FUNCTION} ${i} $@
		done
	fi
}

#####################
### OCI FUNCTIONS ###
#####################

switch_oci_to_release () {
	local FROM_DEB_RELEASE FROM_OS_RELEASE TO_DEB_RELEASE TO_OS_RELEASE
	FROM_DEB_RELEASE=${1}
	FROM_OS_RELEASE=${2}
	TO_DEB_RELEASE=${3}
	TO_OS_RELEASE=${4}

	green_echo "===> Switch OCI to use ${TO_OS_RELEASE}"
	echo "Types: deb deb-src
URIs: http://osbpo.debian.net/debian
Suites: ${TO_DEB_RELEASE}-${TO_OS_RELEASE}-backports ${TO_DEB_RELEASE}-${TO_OS_RELEASE}-backports-nochange
Components: main
Signed-By: /etc/oci/oci-repository-key.asc
" >/etc/apt/sources.list.d/osbpo.debian.net.sources

	if [ -e /etc/apt/sources.list.d/${FROM_DEB_RELEASE}-${FROM_OS_RELEASE}.list ] ; then
		rm /etc/apt/sources.list.d/${FROM_DEB_RELEASE}-${FROM_OS_RELEASE}.list
	fi
	# That one is used in the PoC
	if [ -e /etc/apt/sources.list.d/openstack.list ] ; then
		rm /etc/apt/sources.list.d/openstack.list
	fi
	apt-get update
	DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::=--force-confold --allow-downgrades
	if [ "${FROM_OS_RELEASE}" = "stein" ] ; then
		DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold puppet-module-placement
	fi
	if [ -e /lib/systemd/system/puppetserver.service ] ; then
		service puppetserver restart
	else
		/etc/init.d/puppet-master restart
	fi
	sed -i s/openstack_release=${FROM_OS_RELEASE}/openstack_release=${TO_OS_RELEASE}/ /etc/openstack-cluster-installer/openstack-cluster-installer.conf
}

########################
### Common functions ###
########################
drain_backend () {
	local HOST HOST_IP HOST_NAME BACKEND
	HOST=${1}
	BACKEND=${2}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Draining ${HOST_NAME} backend from ${BACKEND}"
	sshi root@${ONE_CTRL_IP} "hapc drain-server --backend ${BACKEND} --server ${HOST_NAME} --verbose --wait"
}

enable_backend () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	BACKEND=${2}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Enabling ${HOST_NAME} in ${BACKEND} backend"
	sshi root@${ONE_CTRL_IP} "hapc enable-server --backend ${BACKEND} --server ${HOST_NAME} --verbose"
}

stop_service () {
	local HOST HOST_IP HOST_NAME SERVICE
	HOST=${1}
	SERVICE=${2}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Stopping ${SERVICE} on ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl stop ${SERVICE} || true ; systemctl disable ${SERVICE} || true ; systemctl mask ${SERVICE} || true"
}

start_service () {
	local HOST HOST_IP HOST_NAME SERVICE
	HOST=${1}
	SERVICE=${2}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Starting ${SERVICE} on ${HOST_NAME}"
	sshi root@${HOST_IP} "systemctl unmask ${SERVICE} || true ; systemctl enable ${SERVICE} || true ; systemctl start ${SERVICE} || true"
}

upgrade_package () {
	local HOST HOST_IP HOST_NAME PACKAGE
	HOST=${1}
	PACKAGE=${2}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Upgrading ${PACKAGE} on ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold ${PACKAGE}"
}

# This is a generic fonction that upgrades a project in the controllers.
# The project must have a unique API backend and one or multiple services
# running on the controllers.
cluster_upgrade_project () {
	local PROJECT_NICE_NAME PROJECT_BACKEND PROJECT_SERVICE_LIST PROJECT_PACKAGE
	PROJECT_NICE_NAME=${1}
	PROJECT_BACKEND=${2}
	PROJECT_SERVICE_LIST=${3}
	PROJECT_PACKAGE=${4}
	PROJECT_UPGRADE_DB_CMD=${5}

	green_echo "===> Upgrading ${PROJECT_NICE_NAME}"
	drain_backend ${ONE_CTRL_IP},${ONE_CTRL_HOST} ${PROJECT_BACKEND}
	stop_service ${ONE_CTRL_IP},${ONE_CTRL_HOST} "${PROJECT_SERVICE_LIST}"
	upgrade_package ${ONE_CTRL_IP},${ONE_CTRL_HOST} "${PROJECT_PACKAGE}"
	if [ -n "${PROJECT_UPGRADE_DB_CMD}" ] ; then
		green_echo "-> Running: ${PROJECT_UPGRADE_DB_CMD}"
		sshi root@${ONE_CTRL_IP} "${PROJECT_UPGRADE_DB_CMD}"
	fi
	start_service ${ONE_CTRL_IP},${ONE_CTRL_HOST} "${PROJECT_SERVICE_LIST}"
	enable_backend ${ONE_CTRL_IP},${ONE_CTRL_HOST} ${PROJECT_BACKEND}

	green_echo "===> Upgrading ${PROJECT_NICE_NAME} on all other controllers"
	iterate_on_hosts "${OTHER_CTRL}" drain_backend ${PROJECT_BACKEND}
	for SERVICE in ${PROJECT_SERVICE_LIST} ; do
		iterate_on_hosts "${OTHER_CTRL}" stop_service ${SERVICE}
	done
	iterate_on_hosts "${OTHER_CTRL}" upgrade_package ${PROJECT_PACKAGE}
	for SERVICE in ${PROJECT_SERVICE_LIST} ; do
		iterate_on_hosts "${OTHER_CTRL}" start_service ${SERVICE}
	done
	iterate_on_hosts "${OTHER_CTRL}" enable_backend ${PROJECT_BACKEND}
}

upgrade_package_list_on_host () {
	local HOST HOST_IP HOST_NAME PKG_LIST
	HOST=${1}
	PKG_LIST="${2}"
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	TMPF=$(sshi root@${HOST_IP} mktemp -t oci-upgrade-pkg-list-XXXXXX)
	green_echo "-> Calculating package upgrade list"

	PACKAGES_LIST=$(sshi root@${HOST_IP} "TMPF=\$(mktemp -t upgrade_package_list_on_host.XXXXXX) ; for PKG in ${PKG_LIST} ; do if dpkg-query -W \$PKG 1>/dev/null 2>/dev/null ; then INSTALLED=\$(apt-cache policy \$PKG | grep Installed | awk '{print \$2}') ; CANDIDATE=\$(apt-cache policy \$PKG | grep Candidate | awk '{print \$2}') ; if dpkg --compare-versions \$CANDIDATE gt \$INSTALLED ; then echo -n \"\$PKG \" >>\$TMPF ; fi ; fi ; done ; cat \$TMPF ; rm -f \$TMPF")

	if [ -z "${PACKAGES_LIST}" ] ; then
		green_echo "-> Nothing to upgrade on ${HOST_NAME}"
	else
		green_echo "-> Upgrading packages on ${HOST_NAME}: ${PACKAGES_LIST}"
		sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold --allow-downgrades ${PACKAGES_LIST}"
	fi
}
################
### FINALIZE ###
################
distupgrade_everyone () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Dist-upgrading ${HOST_NAME}"
	sshi root@${HOST_IP} "DEBIAN_FRONTEND=noninteractive apt-get dist-upgrade -y -o Dpkg::Options::=--force-confold --allow-downgrades"
}

run_puppet_on_node () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Running puppet on ${HOST_NAME}"
	sshi root@${HOST_IP} "oci-puppet || true"
}

restart_all_services_on_node () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	shift
	for i in $@ ; do
		green_echo "-> Restarting ${i} on ${HOST_NAME}"
		sshi root@${HOST_IP} "if [ -e /etc/init.d/${i} ] ; then /etc/init.d/${i} stop ; sleep 1 ; /etc/init.d/${i} start ; fi"
	done
}

cluster_upgrade_finilize () {
# This is a bad idea...
#	green_echo "===> Distupgrading everyone"
#	iterate_on_hosts "${ALL_NODES}" distupgrade_everyone

# We don't switch the initial-cluster-setup, because this triggers the dbsync,
# which restarts the services.
# In Xena, we're moving from puppet.conf to admin-clouds.yaml, though
# it should be fine since this is always re-written, even with initial-cluster-setup
# to no.
#	if [ "${FROM_OS_RELEASE}" != "xena" ] ; then
#		green_echo "===> Activating initial setup"
#		ocicli cluster-set ${CLUSTER_NAME} --initial-cluster-setup yes
#	fi

	if [ "${PARTIAL_UPGRADE}" = "yes" ] ; then
		green_echo "===> Run puppet on all nodes, but Ceph nodes, networks and computes (partial upgrade)."
		green_echo "-> Run puppet on all controllers"
		iterate_on_hosts -p "${ALL_CTRL}" run_puppet_on_node
		green_echo "-> Run puppet on all messaging"
		iterate_on_hosts -p "${ALL_MSG}" run_puppet_on_node
		green_echo "-> Run puppet on all swift proxies"
		iterate_on_hosts -p "${ALL_SWIFTPRX}" run_puppet_on_node
		green_echo "-> Run puppet on all swift stores"
		iterate_on_hosts -p "${ALL_SWIFTSTR}" run_puppet_on_node
		green_echo "-> Run puppet on all DNS"
		iterate_on_hosts -p "${ALL_DNS}" run_puppet_on_node
		green_echo "-> Run puppet on all tempest nodes"
		iterate_on_hosts -p "${ALL_TEMPEST}" run_puppet_on_node
		green_echo "-> Run puppet on all volume nodes"
		iterate_on_hosts -p "${ALL_VOL}" run_puppet_on_node
	else
		green_echo "===> Run puppet on all nodes (but Ceph nodes)"
		iterate_on_hosts -p "${ALL_NODES_BUT_CEPH}" run_puppet_on_node
	fi

# See above.
#	if [ "${FROM_OS_RELEASE}" != "xena" ] ; then
#		green_echo "===> Removing initial setup"
#		ocicli cluster-set ${CLUSTER_NAME} --initial-cluster-setup no
#	fi

# Not needed anymore
#	green_echo "===> Restarting all services everywhere"
#	iterate_on_hosts "${ALL_CTRL}" restart_all_services_on_node ${ALL_SERVICES_CTRL}
#	iterate_on_hosts -p "${ALL_COMP}" restart_all_services_on_node ${ALL_SERVICES_COMP}
#	iterate_on_hosts -p "${ALL_VOL}" restart_all_services_on_node ${ALL_SERVICES_VOL}
#	iterate_on_hosts "${ALL_NET}" restart_all_services_on_node ${ALL_SERVICES_NET}
#	iterate_on_hosts "${ALL_MSG}" restart_all_services_on_node ${ALL_SERVICES_MSG}

	if [ "${PARTIAL_UPGRADE}" = "yes" ] ; then
		green_echo "===> Enabling puppet on all nodes, but Ceph nodes, networks and computes (partial upgrade)."
		green_echo "-> Run puppet on all controllers"
		iterate_on_hosts -p "${ALL_CTRL}" enable_puppet
		green_echo "-> Run puppet on all messaging"
		iterate_on_hosts -p "${ALL_MSG}" enable_puppet
		green_echo "-> Run puppet on all swift proxies"
		iterate_on_hosts -p "${ALL_SWIFTPRX}" enable_puppet
		green_echo "-> Run puppet on all swift stores"
		iterate_on_hosts -p "${ALL_SWIFTSTR}" enable_puppet
		green_echo "-> Run puppet on all DNS"
		iterate_on_hosts -p "${ALL_DNS}" enable_puppet
		green_echo "-> Run puppet on all tempest nodes"
		iterate_on_hosts -p "${ALL_TEMPEST}" enable_puppet
		green_echo "-> Run puppet on all volume nodes"
		iterate_on_hosts -p "${ALL_VOL}" enable_puppet
	else
		green_echo "===> Enabling puppet everywhere (but Ceph nodes)"
		iterate_on_hosts -p "${ALL_NODES_BUT_CEPH}" enable_puppet
	fi
}

cluster_check_status () {
	if [ "${USE_EXTERNAL_KEYSTONE}" != "yes" ] ; then
		sshi root@${ONE_CTRL_IP} "keystone-status upgrade check"
	fi
	if [ "${HAS_BARBICAN}" = "yes" ] ; then
		sshi root@${ONE_CTRL_IP} "barbican-status upgrade check"
	fi
	if [ "${HAS_DESIGNATE}" = "yes" ] ; then
		sshi root@${ONE_CTRL_IP} "designate-status upgrade check"
	fi
	if [ "${HAS_HEAT}" = "yes" ] ; then
		sshi root@${ONE_CTRL_IP} "heat-status upgrade check"
	fi
	if [ -n "${ALL_COMP}" ] ; then
		sshi root@${ONE_CTRL_IP} "glance-status upgrade check" 
		sshi root@${ONE_CTRL_IP} "nova-status upgrade check"
		sshi root@${ONE_CTRL_IP} "cinder-status upgrade check || true"
		sshi root@${ONE_CTRL_IP} "neutron-status upgrade check"
		sshi root@${ONE_CTRL_IP} "placement-status upgrade check"
		if [ "${HAS_TELEMETRY}" = "yes" ] ; then
			sshi root@${ONE_CTRL_IP} "aodh-status upgrade check"
			sshi root@${ONE_CTRL_IP} "ceilometer-status upgrade check"
			# This fails if using storage v1
			sshi root@${ONE_CTRL_IP} "cloudkitty-status upgrade check || true"
		fi
	fi
#	sshi root@${ONE_CTRL_IP} "octavia-status upgrade check"
}

oci_restart_all_services_on_host () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	green_echo "-> Restarting all services on ${HOST_NAME}"
	sshi root@${HOST_IP} "oci-restart-all-services || true"
}

oci_cluster_restart_all_services () {
	iterate_on_hosts "${ALL_CTRL}" oci_restart_all_services_on_host
        if [ -n "${ALL_MSG}" ] ; then
                iterate_on_hosts "${ALL_MSG}" oci_restart_all_services_on_host
        fi
        if [ -n "${ALL_NET}" ] ; then
                iterate_on_hosts -p "${ALL_NET}" oci_restart_all_services_on_host
        fi
        if [ -n "${ALL_VOL}" ] ; then
                iterate_on_hosts -p "${ALL_VOL}" oci_restart_all_services_on_host
        fi
        if [ -n "${ALL_COMP}" ] ; then
                iterate_on_hosts -p "${ALL_COMP}" oci_restart_all_services_on_host
        fi
        if [ -n "${ALL_SWIFTPRX}" ] ; then
                iterate_on_hosts -p "${ALL_SWIFTPRX}" oci_restart_all_services_on_host
        fi
        if [ -n "${ALL_SWIFTSTR}" ] ; then
                iterate_on_hosts -p "${ALL_SWIFTSTR}" oci_restart_all_services_on_host
        fi
}

oci_check_node_alive () {
	local HOST HOST_IP HOST_NAME
	HOST=${1}
	HOST_IP=$(echo ${HOST} | cut -d, -f1)
	HOST_NAME=$(echo ${HOST} | cut -d, -f2)
	echo -n "${GREEN}-> Checking that $HOST_NAME pings: ${NO_COL}"
	if ! ping -c 1 ${HOST_IP} 2>/dev/null 1>/dev/null ; then
		red_echo "NO"
		exit 1
	else
		green_echo "ok"
	fi
}

oci_check_all_nodes_alive () {
	iterate_on_hosts "${ALL_NODES}" oci_check_node_alive
}