#!/bin/sh

set -e

if ! [ -r /etc/oci-poc/openstack-ci.conf ] ; then
	echo "Cannot read /etc/oci-poc/openstack-ci.conf: exiting."
	exit 1
fi
. /etc/oci-poc/openstack-ci.conf

# Load OpenStack credentials
# Do set +x to make sure we aren't showing the credentials in the logs
set +x
. ${OPENSTACK_CREDENTIALS}

MY_SCRIPT_NAME=$(basename $0)

set_cluster_variables () {
	PXE_SERVER_NAME=${CLUSTER_NAME}-oci
	KEYPAIR_NAME=${CLUSTER_NAME}-oci-keypair

	RANDOM_SUFFIX=$(shuf -i100-199 -n1)
	NET_PRIV_NAME=${CLUSTER_NAME}-oci-boot-network
	SUBNET_PRIV_NAME=${CLUSTER_NAME}-oci-boot-subnet
	SUBNET_PRIV_PREFIX=192.168.${RANDOM_SUFFIX}
	SUBNET_PRIV_RANGE=${SUBNET_PRIV_PREFIX}.0
	SUBNET_PRIV_ROUTER_NAME=${CLUSTER_NAME}-oci-boot-router
	SUBNET_PRIV_ROUTER_PORT=${CLUSTER_NAME}-oci-router-privnet-port
	SUBNET_PRIV_ROUTER_PORT_IP=${SUBNET_PRIV_PREFIX}.10
	EXT_NETWORK_PREFIX=192.168.${RANDOM_SUFFIX}

	OCI_PXE_IP=${SUBNET_PRIV_PREFIX}.${OCI_PXE_SERVER_IP_LAST_DIGIT}
	OCI_PXE_PORT_NAME=${CLUSTER_NAME}-${OCI_PXE_PORT_NAME}
}

### Command line parsing ###
for i in $@ ; do
	case "${1}" in
	"--enable-compute")
		OCI_COMPUTE_NODES=yes
		OCI_NETWORK_NODES=yes
		shift
	;;
	"--enable-volume")
		OCI_VOLUME_NODES=yes

		shift
	;;
	"--enable-ceph")
		OCI_CEPH_NODES=yes
		shift
	;;
	"--enable-swift")
		OCI_SWIFT_NODES=yes
		shift
	;;
	"--enable-billing")
		OCI_MESSAGING_NODES=yes
		OCI_BILLCEPH_NODES=yes
		shift
	;;
	"--cluster-name")
		CLUSTER_NAME="cl${2}"
		shift
		shift
		set_cluster_variables
	;;
	"--random-cluster-name")
		CLUSTER_NAME="cl$(openssl rand -hex 4)"
		set_cluster_variables
		shift
	;;
	"--teardown-after-only")
		TEARDOWN_AFTER_ONLY=yes
		shift
	;;
	*)
	;;
	esac
done


calc_node_list () {
	LST_CTRL_NODES="${CLUSTER_NAME}-controller-1 ${CLUSTER_NAME}-controller-2 ${CLUSTER_NAME}-controller-3"
	if [ "${OCI_COMPUTE_NODES}" = "yes" ] ; then
		LST_COMP_NODES="${CLUSTER_NAME}-compute-1 ${CLUSTER_NAME}-compute-2"
	fi
	if [ "${OCI_NETWORK_NODES}" = "yes" ] ; then
		LST_NET_NODES="${CLUSTER_NAME}-network-1"
	fi
	if [ "${OCI_SWIFT_NODES}" = "yes" ] ; then
		LST_SWPROX_NODES=${CLUSTER_NAME}-swiftproxy-1
		LST_SWSTOR_NODES=${CLUSTER_NAME}-swiftstore-1 ${CLUSTER_NAME}-swiftstore-2 ${CLUSTER_NAME}-swiftstore-3
		LST_SWIFT_NODES="${LST_SWPROX_NODES} ${LST_SWSTOR_NODES}"
	fi
	if [ "${OCI_CEPH_NODES}" = "yes" ] ; then
		LST_CEPHMON_NODES="${CLUSTER_NAME}-cephmon-1 ${CLUSTER_NAME}-cephmon-2 ${CLUSTER_NAME}-cephmon-3"
		LST_CEPHOSD_NODES="${CLUSTER_NAME}-cephosd-1 ${CLUSTER_NAME}-cephosd-2 ${CLUSTER_NAME}-cephosd-3"
		LST_CEPH_NODES="${LST_CEPHMON_NODES} ${LST_CEPHOSD_NODES}"
	fi
	if [ "${OCI_VOLUME_NODES}" = "yes" ] ; then
		LST_VOL_NODES="${CLUSTER_NAME}-volume-1 ${CLUSTER_NAME}-volume-2"
	fi
	if [ "${OCI_MESSAGING_NODES}" = "yes" ] ; then
		LST_MSG_NODES="${CLUSTER_NAME}-messaging-1 ${CLUSTER_NAME}-messaging-2 ${CLUSTER_NAME}-messaging-3"
	fi
	if [ "${OCI_BILLCEPH_NODES}" = "yes" ] ; then
		LST_BILLMON_NODES="${CLUSTER_NAME}-billmon-1 ${CLUSTER_NAME}-billmon-2 ${CLUSTER_NAME}-billmon-3"
		LST_BILLOSD_NODES="${CLUSTER_NAME}-billosd-1 ${CLUSTER_NAME}-billosd-2 ${CLUSTER_NAME}-billosd-3"
		LST_CEPHBILL_NODES="${LST_BILLMON_NODES} ${LST_BILLOSD_NODES}"
	fi
	if [ "${OCI_TEMPEST_NODES}" = "yes" ] ; then
		LST_TEMPEST_NODES="${CLUSTER_NAME}-tempest-1"
	fi
	LST_ALL_NODES="${LST_CTRL_NODES} ${LST_NET_NODES} ${LST_COMP_NODES} ${LST_SWIFT_NODES} ${LST_CEPH_NODES} ${LST_VOL_NODES} ${LST_MSG_NODES} ${LST_CEPHBILL_NODES} ${LST_TEMPEST_NODES}"
}

# Params: $1 id or name of VM
#         $2 STATUS to wait for
#         $3 timeout (optional, default to 120, ie 2 mins)
wait_for_vm () {
	local VM_ID_OR_NAME STATUS_TO_WAIT_FOR TIMEOUT STAT
	VM_ID_OR_NAME=$1
	STATUS_TO_WAIT_FOR=$2
	TIMEOUT=$3

	if [ -z "${TIMEOUT}" ] ; then
		TIMEOUT=120
	fi

	END_TIMESTAMP=$(( $(date +%s) + ${TIMEOUT} ))

	echo -n "-> Waiting for ${VM_ID_OR_NAME} to have status ${STATUS_TO_WAIT_FOR}:"
	STAT=$(openstack server show ${VM_ID_OR_NAME} --format value -c status)
	while [ "${STAT}" != "${STATUS_TO_WAIT_FOR}" ] && [ "${END_TIMESTAMP}" -gt $(date +%s) ] ; do
		sleep 2
		STAT=$(openstack server show ${VM_ID_OR_NAME} --format value -c status)
		echo -n "."
	done
	if [ "${STAT}" != "${STATUS_TO_WAIT_FOR}" ] ; then
		echo "timeout."
		return 1
	else
		echo "ok."
		return 0
	fi
}


provision_priv_net () {
	echo "===> Creating private network: ${NET_PRIV_NAME}"
	# Check if network exists, create it otherwise
	NET_PRIV_ID=$(openstack network list --name ${NET_PRIV_NAME} --format value -c ID)
	if [ -z "${NET_PRIV_ID}" ] ; then
		openstack network create ${NET_PRIV_NAME}
		NET_PRIV_ID=$(openstack network list --name ${NET_PRIV_NAME} --format value -c ID)
		echo "-> Created."
	else
		echo "-> Already exists."
	fi
	echo "===> Creating private subnet: ${SUBNET_PRIV_NAME}"
	SUBNET_PRIV_ID=$(openstack subnet list --name ${SUBNET_PRIV_NAME} --format value -c ID)
	if [ -z "${SUBNET_PRIV_ID}" ] ; then
		openstack subnet create --network ${NET_PRIV_NAME} --subnet-range ${SUBNET_PRIV_RANGE}/${SUBNET_PRIV_CIDR} --dns-nameserver ${SUBNET_PRIV_DNS_SERVER} ${SUBNET_PRIV_NAME}
		SUBNET_PRIV_ID=$(openstack subnet list --name ${SUBNET_PRIV_NAME} --format value -c ID)
		echo "-> Created."
	else
		echo "-> Already exists."
	fi

	echo "===> Creating router"
	ROUTER_NAME=$(openstack router list --format value -c Name | grep ${CLUSTER_NAME})
	if [ -z "${ROUTER_NAME}" ] ; then
		openstack router create ${SUBNET_PRIV_ROUTER_NAME}
		echo "-> Created."
	else
		echo "-> Already exists."
	fi

	echo "===> Creating router default gw"
	EXTGWINFO=$(openstack router show ${SUBNET_PRIV_ROUTER_NAME} --format value -c external_gateway_info)
	if [ "${EXTGWINFO}" = "None" ] ; then
		openstack router set --external-gateway ${ROUTER_GW_PUB_NAME} ${SUBNET_PRIV_ROUTER_NAME}
		echo "-> External gw is now set."
	else
		echo "-> External gw already set."
	fi

#	echo "===> Creating router privnet port"
#	PORT_NAME=$(openstack port list --name ${SUBNET_PRIV_ROUTER_PORT} --format value -c Name)
#	if [ "${PORT_NAME}" != ${SUBNET_PRIV_ROUTER_PORT} ] ; then
#		openstack port create --network ${NET_PRIV_NAME} --fixed-ip subnet=${SUBNET_PRIV_NAME},ip-address=${SUBNET_PRIV_ROUTER_PORT_IP} ${SUBNET_PRIV_ROUTER_PORT}
#		echo "-> Created."
#	else
#		echo "-> Already exists."
#	fi
#
#	# TODO: Add creating of router port on fixed IP and add interface for it.
#	echo "===> Adding private subnet port to router"
#	IF_INFO=$(openstack router show --format json -c interfaces_info ${SUBNET_PRIV_ROUTER_NAME} | jq '.["interfaces_info"][]' -r)
#	if [ -z "${IF_INFO}" ] ; then
#		openstack router add port ${SUBNET_PRIV_ROUTER_NAME} ${SUBNET_PRIV_ROUTER_PORT}
#		echo "-> Added."
#	else
#		echo "-> Already added."
#	fi

	echo "===> Adding subnet to router"
	IF_INFO=$(openstack router show --format json -c interfaces_info ${SUBNET_PRIV_ROUTER_NAME} | jq '.["interfaces_info"][]' -r)
	if [ -z "${IF_INFO}" ] ; then
		openstack router add subnet ${SUBNET_PRIV_ROUTER_NAME} ${SUBNET_PRIV_NAME}
		echo "-> Added."
	else
		echo "-> Already added."
	fi
}

upload_pubkey () {
	echo "===> Creating ssh keypair"
	if ! [ -e ${AUTHORIZED_KEY_PATH} ] ; then
		cp ${SSH_KEY_PATH}.pub ${AUTHORIZED_KEY_PATH}
	fi
	SSHKEY_NAME=$(openstack keypair list --format csv | q -H -d, "SELECT Name FROM - WHERE Name='${KEYPAIR_NAME}'")
	if [ -z "${SSHKEY_NAME}" ] ; then
		openstack keypair create --public-key ${SSH_KEY_PATH}.pub ${KEYPAIR_NAME}
		echo "-> Created."
	else
		echo "-> Already exists."
	fi
}

# Params: none
# Return: $PXE_SERVER_ID
start_oci_pxe_server () {
	echo "===> Creating OCI PXE server"
	PXE_SERVER_ID=$(openstack server list --name ${PXE_SERVER_NAME} --format value -c ID)
	if [ -z "${PXE_SERVER_ID}" ] ; then
		openstack server create --image ${PXE_SERVER_IMAGE_NAME} --flavor ${FLAVOR_PXE} --key-name ${KEYPAIR_NAME} --nic net-id=${NET_PUB_NAME} ${PXE_SERVER_NAME}
		PXE_SERVER_ID=$(openstack server list --name ${PXE_SERVER_NAME} --format value -c ID)
		echo "-> Created."
	else
		echo "-> Already exists."
	fi
}

create_one_security_group_and_rule (){
	local SECGROUP_NAME PROTO PORTRANGE
	SECGROUP_NAME=$1
	PROTO=$2
	PORTRANGE=$3

	echo "===> Creating ${SECGROUP_NAME} security group"
	SSH_SECGROUP_ID=$(openstack security group list --format csv | q -H -d, "SELECT ID FROM - WHERE Name='${SECGROUP_NAME}'")
	if [ -z "${SSH_SECGROUP_ID}" ] ; then
		openstack security group create ${SECGROUP_NAME}
		echo "-> Created secgroup ${SECGROUP_NAME}."
	else
		echo "-> Secgroup ${SECGROUP_NAME} already exists."
	fi

	echo "===> Adding ${SECGROUP_NAME} proto to secgroup"
	if [ "${PROTO}" = "icmp" ] ; then
		SECGROUP_RULE=$(openstack security group rule list ${SECGROUP_NAME} --format csv | q -d, -H "SELECT ID FROM - WHERE \"IP Protocol\"='icmp'")
		if [ -z "${SECGROUP_RULE}" ] ; then
			openstack security group rule create --protocol ${PROTO} ${SECGROUP_NAME}
			echo "-> Created rule for secgroup ${SECGROUP_NAME}, proto ${PROTO}."
		else
			echo "-> Rule for ${PROTO} already exists."
		fi
	else
		SECGROUP_RULE=$(openstack security group rule list ${SECGROUP_NAME} --format csv | q -d, -H "SELECT ID FROM - WHERE \"IP Protocol\"='${PROTO}' AND \"Port Range\"='${PORTRANGE}'")
		if [ -z "${SECGROUP_RULE}" ] ; then
			openstack security group rule create --protocol ${PROTO} --dst-port ${PORTRANGE} ${SECGROUP_NAME}
			echo "-> Created rule for secgroup ${SECGROUP_NAME}, proto ${PROTO} and port range ${PORTRANGE}."
		else
			echo "-> Rule for ${PROTO} and port ${PORTRANGE} already exists."
		fi
	fi
}

create_security_groups_and_rules () {
	create_one_security_group_and_rule ${CLUSTER_NAME}-ssh tcp 22:22
	create_one_security_group_and_rule ${CLUSTER_NAME}-http tcp 80:80
	create_one_security_group_and_rule ${CLUSTER_NAME}-https tcp 443:443
	create_one_security_group_and_rule ${CLUSTER_NAME}-puppet tcp 8140:8140
	create_one_security_group_and_rule ${CLUSTER_NAME}-pxe udp 67:69
	create_one_security_group_and_rule ${CLUSTER_NAME}-ping icmp
}

add_secgroup_to_oci_server () {
	TMPFILE=$(mktemp -t ${MY_SCRIPT_NAME}-secgrp-list.XXXXXX)
	openstack server show ${PXE_SERVER_NAME} --format json -c security_groups | jq '.["security_groups"][]["name"]' -r >$TMPFILE
	echo "===> Adding security group to OCI server"
	for SECR in ${CLUSTER_NAME}-ssh ${CLUSTER_NAME}-http ${CLUSTER_NAME}-https ${CLUSTER_NAME}-pxe ${CLUSTER_NAME}-puppet ${CLUSTER_NAME}-ping ; do
		if ! cat ${TMPFILE} | grep -q -E '^'${SECR}'$' ; then
			openstack server add security group ${PXE_SERVER_NAME} ${SECR}
			echo "-> Added ${SECR}"
		else
			echo "-> ${SECR} already added."
		fi
	done
	rm -f ${TMPFILE}
}

get_vm_ip () {
	local SERVER_NAME
	SERVER_NAME=$1
	echo "===> Getting OCI server IP"
	RET=$(openstack server show ${PXE_SERVER_NAME} --format json -c addresses | jq '.["addresses"]["'"${NET_PUB_NAME}"'"][]' -r)
	echo "-> $RET"
}

# Start a VM
# Param: $1 flavor
#start_vm () {
#}

wait_for_ssh () {
	local COUNT CYCLES OTCI_CAN_SSH SSH_HOST
	COUNT=120
	CYCLES=0
	OTCI_CAN_SSH=no
	SSH_HOST=${1}
	echo -n "-> Waiting for ssh: "
	while [ "${OTCI_CAN_SSH}" != "yes" ] && [ ${COUNT} != 0 ] ; do
		if timeout 5 ssh -i ${SSH_KEY_PATH} -o "StrictHostKeyChecking no" -o "ConnectTimeout 2" root@${SSH_HOST} 'echo -n ""' 1>/dev/null 2>/dev/null ; then
			OTCI_CAN_SSH=yes
		else
			echo -n "."
			COUNT=$(( ${COUNT} - 1 ))
			CYCLES=$(( ${CYCLES} + 1 ))
			sleep 1
		fi
	done
	echo "ok."
	ssh -i ${SSH_KEY_PATH} -o "StrictHostKeyChecking no" -o "ConnectTimeout 2" root@${SSH_HOST} 'echo -n ""' 1>/dev/null 2>/dev/null
}

upload_ipxe_iso () {
	echo "===> Uploading ipxe.iso"
	IPXE_IMAGE_ID=$(openstack image list --name ${CLUSTER_NAME}-ipxe.iso --format value -c ID)
	if [ -z "${IPXE_IMAGE_ID}" ] ; then
		# Note: hw:boot_menu=True enables the seabios boot menu
		openstack image create \
			--container-format bare \
			--disk-format iso \
			--property hw_disk_bus=scsi \
			--property hw_scsi_model=virtio-scsi \
			--property os_type=linux \
			--property os_distro=ipxe \
			--property hw:boot_menu=True \
			--file /usr/lib/ipxe/ipxe.iso \
			${CLUSTER_NAME}-ipxe.iso
		IPXE_IMAGE_ID=$(openstack image list --name ${CLUSTER_NAME}-ipxe.iso --format value -c ID)
		echo "-> Created."
	else
		echo "-> Already exist."
	fi
}

ssh_cmd () {
	SSH_AUTH_SOCK= ssh -i ${SSH_KEY_PATH} -o "StrictHostKeyChecking no" -o "GlobalKnownHostsFile /dev/null" -o "UserKnownHostsFile /dev/null" -o "ConnectTimeout 2" root@${PXE_SERVER_IP} $@
}
scp_cmd () {
	SSH_AUTH_SOCK= scp -i ${SSH_KEY_PATH} -o "StrictHostKeyChecking no" -o "GlobalKnownHostsFile /dev/null" -o "UserKnownHostsFile /dev/null" -o "ConnectTimeout 2" ${1} root@${PXE_SERVER_IP}:${2}
}

install_package_in_oci_server (){
	local PACKAGE_NAME
	PACKAGE_NAME=$1
	if ! ssh_cmd "dpkg-query -W --showformat '\${Status}\n' ${PACKAGE_NAME} 2>/dev/null | grep -q 'install ok'" ; then
		ssh_cmd "DEBIAN_FRONTEND=noninteractive apt-get install ${PACKAGE_NAME} -y"
	fi
}

get_oci_privnet_ifname () {
	if ssh_cmd "LANG=C ip link show | grep BROADCAST | grep -q eth1" ; then
		PRIVNET_IFNAME=eth1
	elif ssh_cmd "LANG=C ip link show | grep BROADCAST | grep -q enp0s6" ; then
		PRIVNET_IFNAME=enp0s6
	elif ssh_cmd "LANG=C ip link show | grep BROADCAST | grep -q ens6" ; then
		PRIVNET_IFNAME=ens6
	else
		echo "Error: Cannot find OCI VM privnet port name."
		exit 1
	fi
}

fix_oci_directive () {
	local SECTION DIRECTIVE_NAME VALUE
	SECTION=$1
	DIRECTIVE_NAME=$2
	VALUE=$3
	ssh_cmd ". /usr/share/openstack-pkg-tools/pkgos_func ; pkgos_inifile set /etc/openstack-cluster-installer/openstack-cluster-installer.conf ${SECTION} ${DIRECTIVE_NAME} ${VALUE}" 2>/dev/null
	ssh_cmd "sed -i 's#${DIRECTIVE_NAME} = #${DIRECTIVE_NAME}=#' /etc/openstack-cluster-installer/openstack-cluster-installer.conf" 2>/dev/null
}

setup_debian_repo () {
	echo "===> Installing openstack_${OS_RELEASE} repository"
	# Without extrepo: using bullseye-${OS_RELEASE}.debian.net repos
	ssh_cmd "apt-get update"
	install_package_in_oci_server gnupg2
	ssh_cmd "wget http://bullseye-${OS_RELEASE}.debian.net/debian/dists/pubkey.gpg ; apt-key add pubkey.gpg ; rm pubkey.gpg"
	ssh_cmd "echo 'deb http://bullseye-${OS_RELEASE}.debian.net/debian bullseye-${OS_RELEASE}-backports main' >/etc/apt/sources.list.d/bullseye-${OS_RELEASE}.list"
	ssh_cmd "echo 'deb http://bullseye-${OS_RELEASE}.debian.net/debian bullseye-${OS_RELEASE}-backports-nochange main' >>/etc/apt/sources.list.d/bullseye-${OS_RELEASE}.list"
	ssh_cmd "apt-get update"
	# With extrepo: using osbpo.debian.net
#	if ! ssh_cmd "test -e /etc/apt/sources.list.d/extrepo_openstack_xena.sources" ; then
#		ssh_cmd "apt-get update && apt-get install extrepo -y && extrepo enable openstack_${OS_RELEASE} && apt-get update"
#	fi
}

install_oci () {
	for PKG in php-cli iptables mariadb-server tftp-hpa isc-dhcp-server openstack-pkg-tools openstack-cluster-installer ; do
		echo "===> Installing $PKG"
		install_package_in_oci_server $PKG
	done
	echo "===> Creating OCI DB."
	PASSWORD=$(openssl rand -hex 8)
	ssh_cmd "mysql --execute 'CREATE DATABASE IF NOT EXISTS oci;'" 2>/dev/null
	ssh_cmd "mysql --execute \"GRANT ALL PRIVILEGES ON oci.* TO 'oci'@'localhost' IDENTIFIED BY '${PASSWORD}';\"" 2>/dev/null
	echo "===> Setting-up connection= directive"
	ssh_cmd ". /usr/share/openstack-pkg-tools/pkgos_func ; pkgos_inifile set /etc/openstack-cluster-installer/openstack-cluster-installer.conf database connection mysql+pymysql://oci:${PASSWORD}@localhost:3306/oci" 2>/dev/null
	ssh_cmd "sed -i 's#connection = #connection=#' /etc/openstack-cluster-installer/openstack-cluster-installer.conf" 2>/dev/null
	echo "===> Populating DB."
	ssh_cmd "cd /usr/share/openstack-cluster-installer ; php db_sync.php" 2>/dev/null
	echo "===> oci-root-ca-gen"
	ssh_cmd "if ! [ -e /var/lib/oci/ssl/ca/oci-pki-oci-ca.pem ] ; then oci-root-ca-gen ; fi" 2>/dev/null
	echo "===> Fixing mirror addresses"
	fix_oci_directive network debian_mirror ${DEBIAN_MIRROR_ADDRESS}
	fix_oci_directive network debian_security_mirror ${DEBIAN_SECURITY_MIRROR_ADDRESS}
	echo "===> Fix a few OCI configuration directives"
	fix_oci_directive network OCI_IP ${SUBNET_PRIV_PREFIX}.${OCI_PXE_SERVER_IP_LAST_DIGIT}
	fix_oci_directive releasenames openstack_release ${OS_RELEASE}
	fix_oci_directive releasenames use_debian_dot_net_backport yes
	echo "===> Creating live image"
	ssh_cmd "if ! [ -e /var/lib/openstack-cluster-installer/filesystem.squashfs ] ; then mkdir -p /root/live ; cd /root/live ; openstack-cluster-installer-build-live-image --pxe-server-ip ${SUBNET_PRIV_PREFIX}.${OCI_PXE_SERVER_IP_LAST_DIGIT} ; fi" 2>/dev/null
	echo "===> Copying boot files"
	ssh_cmd "cp -auxf /var/lib/openstack-cluster-installer/tftp/* /usr/share/openstack-cluster-installer" 2>/dev/null
	echo "===> Adding OCI user"
	OCI_PASS=$(openssl rand -hex 8)
	ssh_cmd "if oci-userdb -l | grep -q poc ; then oci-userdb -d poc ; oci-userdb -a poc ${OCI_PASS} ; else oci-userdb -a poc ${OCI_PASS} ; fi" 2>/dev/null
	echo "===> Configuring .bashrc shell in include color and OCI login"
	ssh_cmd "echo \"export LS_OPTIONS='--color=auto'\" >/root/.bashrc" 2>/dev/null
	ssh_cmd "echo 'eval \"\$(dircolors)\"' >>/root/.bashrc" 2>/dev/null
	ssh_cmd "echo \"alias ls='ls \$LS_OPTIONS'\" >>/root/.bashrc" 2>/dev/null
	ssh_cmd "echo \"export OCI_API_URL='http://127.0.0.1/oci/api.php?'\" >>/root/.bashrc" 2>/dev/null
	ssh_cmd "echo \"export OCI_LOGIN=poc\" >>/root/.bashrc" 2>/dev/null
	ssh_cmd "echo \"export OCI_PASS=${OCI_PASS}\" >>/root/.bashrc" 2>/dev/null
	echo "===> Restarting Apache2"
	ssh_cmd "systemctl restart apache2" 2>/dev/null
	echo "===> Configuring tftp-hpa"
	ssh_cmd "sed -i 's#TFTP_DIRECTORY=.*#TFTP_DIRECTORY=\"/var/lib/openstack-cluster-installer/tftp\"#' /etc/default/tftpd-hpa" 2>/dev/null
	ssh_cmd "systemctl restart tftpd-hpa.service" 2>/dev/null
#	echo "===> Configuring DHCPd"
#	ssh_cmd "sed -i 's/INTERFACESv4=\".*\"/INTERFACESv4=\"${PRIVNET_IFNAME}\"/' /etc/default/isc-dhcp-server" 2>/dev/null
#	echo "allow booting;
#allow bootp;
#default-lease-time 600;
#max-lease-time 7200;
#ddns-update-style none;
#authoritative;
#ignore-client-uids On;
#
#subnet ${SUBNET_PRIV_PREFIX}.0 netmask 255.255.255.0 {
#	range ${SUBNET_PRIV_PREFIX}.200 ${SUBNET_PRIV_PREFIX}.250;
#	option domain-name \"example.com\";
#	option domain-name-servers 9.9.9.9;
#	option routers ${SUBNET_PRIV_PREFIX}.1;
#	option subnet-mask 255.255.255.0;
#	option broadcast-address ${SUBNET_PRIV_PREFIX}.255;
#	next-server ${SUBNET_PRIV_PREFIX}.1;
#	if exists user-class and option user-class = \"iPXE\" {
#		filename \"http://${SUBNET_PRIV_PREFIX}.1/oci/ipxe.php\";
#	} else {
#		filename \"pxelinux.0\";
#	}
#}" >/etc/oci-poc/dhcpd.conf
#	scp_cmd /etc/oci-poc/dhcpd.conf /etc/dhcp/dhcpd.conf
#	ssh_cmd "systemctl restart isc-dhcp-server.service"
	echo "===> Configuring puppet-master"
	ssh_cmd ". /usr/share/openstack-pkg-tools/pkgos_func ; pkgos_add_directive /etc/puppet/puppet.conf master \"external_nodes = /usr/bin/oci-puppet-external-node-classifier\" \"# Path to enc\" ; pkgos_inifile set /etc/puppet/puppet.conf master external_nodes /usr/bin/oci-puppet-external-node-classifier ; pkgos_add_directive /etc/puppet/puppet.conf master \"node_terminus = exec\" \"# Tell what type of ENC\" ; pkgos_inifile set /etc/puppet/puppet.conf master node_terminus exec" 2>/dev/null
	ssh_cmd "systemctl restart puppet-master" 2>/dev/null
}

add_privnet_to_oci () {
	echo "===> Creating port"
	PORT_ID=$(openstack port list --name ${OCI_PXE_PORT_NAME} --format value -c ID)
	if [ -z "${PORT_ID}" ] ; then
		openstack port create --network ${NET_PRIV_NAME} --fixed-ip subnet=${SUBNET_PRIV_NAME},ip-address=${SUBNET_PRIV_PREFIX}.${OCI_PXE_SERVER_IP_LAST_DIGIT} ${OCI_PXE_PORT_NAME}
		PORT_ID=$(openstack port list --name ${OCI_PXE_PORT_NAME} --format value -c ID)
		echo "-> Created."
	else
		echo "-> Already exists."
	fi
	# Check device_owner to see if the port is attached.
	echo "===> Adding port to VM"
	PORT_DEV_OWNER=$(openstack port show ${PORT_ID} --format value -c device_owner)
	if [ -z "${PORT_DEV_OWNER}" ] ; then
		openstack server add port ${PXE_SERVER_NAME} ${OCI_PXE_PORT_NAME}
		PORT_DEV_OWNER=$(openstack port show ${PORT_ID} --format value -c device_owner)
		echo "-> Added."
	else
		echo "-> Already added."
	fi
	echo -n "===> Waiting until port appears in VM:"
	while [ ""$(openstack server show ${PXE_SERVER_NAME} --format json -c addresses | jq '.["addresses"]["'"${NET_PRIV_NAME}"'"][]' -r) != "${SUBNET_PRIV_PREFIX}.${OCI_PXE_SERVER_IP_LAST_DIGIT}" ] ; do
		echo "."
	done
	echo "ok."
}

configure_privnet_interface () {
	# Make it work for both eth1 or enos6
	ssh_cmd "ip addr add ${SUBNET_PRIV_PREFIX}.${OCI_PXE_SERVER_IP_LAST_DIGIT}/24 dev ${PRIVNET_IFNAME} || true"
	ssh_cmd "sed -i 's/iface eth1 inet dhcp/iface ${PRIVNET_IFNAME} inet static\n\taddress ${SUBNET_PRIV_PREFIX}.${OCI_PXE_SERVER_IP_LAST_DIGIT}\/24/' /etc/network/interfaces"
}

# This isn't really needed, but it's simply easier this way
# to get all the time the same IP for all machines (we can
# edit /etc/hosts somewhere with it, for example).
get_machine_ip_suffix () {
	local MACHINE_NAME
	MACHINE_NAME=${1}
	case ${MACHINE_NAME} in
	oci)
		RET=4
	;;
	${CLUSTER_NAME}-controller-1)
		RET=100
	;;
	${CLUSTER_NAME}-controller-2)
		RET=101
	;;
	${CLUSTER_NAME}-controller-3)
		RET=102
	;;
	${CLUSTER_NAME}-network-1)
		RET=103
	;;
	${CLUSTER_NAME}-network-2)
		RET=104
	;;
	${CLUSTER_NAME}-compute-1)
		RET=105
	;;
	${CLUSTER_NAME}-compute-2)
		RET=106
	;;
	${CLUSTER_NAME}-volume-1)
		RET=107
	;;
	${CLUSTER_NAME}-volume-2)
		RET=108
	;;
	${CLUSTER_NAME}-swiftproxy-1)
		RET=109
	;;
	${CLUSTER_NAME}-swiftstore-1)
		RET=110
	;;
	${CLUSTER_NAME}-swiftstore-2)
		RET=111
	;;
	${CLUSTER_NAME}-swiftstore-3)
		RET=112
	;;
	${CLUSTER_NAME}-cephmon-1)
		RET=113
	;;
	${CLUSTER_NAME}-cephmon-2)
		RET=114
	;;
	${CLUSTER_NAME}-cephmon-3)
		RET=115
	;;
	${CLUSTER_NAME}-cephosd-1)
		RET=116
	;;
	${CLUSTER_NAME}-cephosd-2)
		RET=117
	;;
	${CLUSTER_NAME}-cephosd-3)
		RET=118
	;;
	${CLUSTER_NAME}-messaging-1)
		RET=119
	;;
	${CLUSTER_NAME}-messaging-2)
		RET=120
	;;
	${CLUSTER_NAME}-messaging-3)
		RET=121
	;;
	${CLUSTER_NAME}-billmon-1)
		RET=122
	;;
	${CLUSTER_NAME}-billmon-2)
		RET=123
	;;
	${CLUSTER_NAME}-billmon-3)
		RET=124
	;;
	${CLUSTER_NAME}-billosd-1)
		RET=125
	;;
	${CLUSTER_NAME}-billosd-2)
		RET=126
	;;
	${CLUSTER_NAME}-billosd-3)
		RET=127
	;;
	${CLUSTER_NAME}-tempest-1)
		RET=130
	;;
	*)
		echo "We dont know this machine, please add it here."
		exit 1
	;;
	esac
}

bootup_to_install_vms () {
	echo "===> Creating ports"
	TMPFILE=$(mktemp -t ${MY_SCRIPT_NAME}.port-list.XXXXXX)
	openstack port list --format value -c Name | grep ${CLUSTER_NAME} >${TMPFILE}

	for MACHINE in ${LST_ALL_NODES} ; do
		echo "-> $MACHINE"
		get_machine_ip_suffix ${MACHINE}
		MACHINE_FIXED_IP_SUFFIX=$RET
		if ! grep -q -E "^${MACHINE}\$" ${TMPFILE} ; then
			openstack port create ${MACHINE} --network ${NET_PRIV_NAME} \
				--extra-dhcp-option name=tftp-server,value=${OCI_PXE_IP},ip-version=4 \
				--extra-dhcp-option name=bootfile-name,value=http://${OCI_PXE_IP}/oci/ipxe.php,ip-version=4 \
				--fixed-ip subnet=${SUBNET_PRIV_NAME},ip-address=${SUBNET_PRIV_PREFIX}.${MACHINE_FIXED_IP_SUFFIX}
		fi
	done
	rm -f ${TMPFILE}

	echo "===> Starting up VMs"
	TMPFILE=$(mktemp -t ${MY_SCRIPT_NAME}.vm-list.XXXXXX)
	openstack server list --format value -c Name | grep ${CLUSTER_NAME} >${TMPFILE}
	for MACHINE in ${LST_ALL_NODES} ; do
		echo "-> $MACHINE"
		if ! grep -q -E "^${MACHINE}\$" ${TMPFILE} ; then
			MACHINE_ROLE=$(echo ${MACHINE} | cut -d- -f2)
			case ${MACHINE_ROLE} in
			controller) USE_FLAVOR=${FLAVOR_CTRL} ;;
			messaging)  USE_FLAVOR=${FLAVOR_CTRL} ;;
			cephmon)    USE_FLAVOR=${FLAVOR_MON}  ;;
			cephosd)    USE_FLAVOR=${FLAVOR_OSD}  ;;
			volume)     USE_FLAVOR=${FLAVOR_VOL}  ;;
			network)    USE_FLAVOR=${FLAVOR_NET}  ;;
			compute)    USE_FLAVOR=${FLAVOR_COMP} ;;
			*)          USE_FLAVOR=${FLAVOR_CTRL} ;;
			esac
			openstack server create --flavor ${USE_FLAVOR} --image ipxe.iso --key-name ${KEYPAIR_NAME} --port ${MACHINE} ${MACHINE}
		fi
	done
	rm -f ${TMPFILE}

	echo "===> Creating volumes"
	TMPFILE=$(mktemp -t ${MY_SCRIPT_NAME}.volume-list.XXXXXX)
	openstack volume list --format value -c Name | grep ${CLUSTER_NAME} >${TMPFILE}
	for MACHINE in ${LST_ALL_NODES} ; do
		echo "-> $MACHINE"
		if ! grep -q -E "^${MACHINE}\$" ${TMPFILE} ; then
			MACHINE_ROLE=$(echo ${MACHINE} | cut -d- -f2)
			case ${MACHINE_ROLE} in
			controller) SYSVOL=${SYSVOL_CTRL} ; DATAVOL=${SYSVOL_CTRL} ;;
			messaging)  SYSVOL=${SYSVOL_MSG}  ; DATAVOL=${SYSVOL_MSG}  ;;
			cephmon)    SYSVOL=${SYSVOL_MON}  ; DATAVOL=${SYSVOL_MON}  ;;
			cephosd)    SYSVOL=${SYSVOL_OSD}  ; DATAVOL=${SYSVOL_OSD}  ;;
			volume)     SYSVOL=${SYSVOL_VOL}  ; DATAVOL=${SYSVOL_VOL}  ;;
			network)    SYSVOL=${SYSVOL_NET}  ; DATAVOL=${SYSVOL_NET}  ;;
			compute)    SYSVOL=${SYSVOL_COMP} ; DATAVOL=${SYSVOL_COMP} ;;
			swiftstore) SYSVOL=${SYSVOL_SWP}  ; DATAVOL=${SYSVOL_SWP}  ;;
			swiftproxy) SYSVOL=${SYSVOL_SWS}  ; DATAVOL=${SYSVOL_SWS}  ;;
			*)          SYSVOL=${SYSVOL_COMP} ; DATAVOL=${SYSVOL_COMP} ;;
			esac
			echo "-> Creating system volume for ${MACHINE} with size ${SYSVOL}"
			openstack volume create --size ${SYSVOL} ${MACHINE}
			if [ "${DATAVOL}" = "0" ] ; then
				echo "-> No data volume for ${MACHINE}"
			else
				echo "-> Creating system data for ${MACHINE} with size ${DATAVOL}"
				openstack volume create --size ${DATAVOL} ${MACHINE}-data
			fi
		fi
	done
	rm -f ${TMPFILE}

	echo "===> Attaching volumes"
	TMPFILE=$(mktemp -t ${MY_SCRIPT_NAME}.volume-attach-list.XXXXXX)
	openstack volume list  --format json -c Name -c 'Attached to' >${TMPFILE}
	for MACHINE in ${LST_ALL_NODES} ; do
		echo "-> $MACHINE system disk"
		if [ -z ""$(cat ${TMPFILE} | jq '.[] | select(.Name == "'${MACHINE}'") | .["Attached to"][]' -r) ] ; then
			openstack server add volume ${MACHINE} ${MACHINE}
		fi

		MACHINE_ROLE=$(echo ${MACHINE} | cut -d- -f2)
		case ${MACHINE_ROLE} in
		controller) DATAVOL=${SYSVOL_CTRL} ;;
		messaging)  DATAVOL=${SYSVOL_MSG}  ;;
		cephmon)    DATAVOL=${SYSVOL_MON}  ;;
		cephosd)    DATAVOL=${SYSVOL_OSD}  ;;
		volume)     DATAVOL=${SYSVOL_VOL}  ;;
		network)    DATAVOL=${SYSVOL_NET}  ;;
		compute)    DATAVOL=${SYSVOL_COMP} ;;
		swiftstore) DATAVOL=${SYSVOL_SWP}  ;;
		swiftproxy) DATAVOL=${SYSVOL_SWS}  ;;
		*)          DATAVOL=${SYSVOL_COMP} ;;
		esac

		if [ "${DATAVOL}" = "0" ] ; then
			echo "-> $MACHINE does not have a data disk"
		else
			echo "-> $MACHINE data disk"
			openstack server add volume ${MACHINE} ${MACHINE}-data
		fi
	done
	rm -f ${TMPFILE}
}

configure_oci_networking () {
	echo "===> Configuring networks in OCI"
	ssh_cmd "ocicli cluster-create ${CLUSTER_NAME} ${DOMAIN_NAME}" 2>/dev/null
	# This tells to use Qemu as virt, and force a MAC in the br-ex OVS bridges.
	ssh_cmd "ocicli cluster-set ${CLUSTER_NAME} --nested-openstack yes --neutron-use-dvr no" 2>/dev/null
	ssh_cmd "ocicli swift-region-create swift-region1" 2>/dev/null
	ssh_cmd "ocicli swift-region-create pub-region" 2>/dev/null
	ssh_cmd "ocicli location-create reg-1-zone-1 swift-region1" 2>/dev/null
	ssh_cmd "ocicli location-create pub-zone pub-region" 2>/dev/null
	ssh_cmd "ocicli network-create reg1-zone1-net1 ${SUBNET_PRIV_PREFIX}.0 24 reg-1-zone-1 no" 2>/dev/null
	ssh_cmd "ocicli network-set reg1-zone1-net1 --mtu 1400 --first-ip ${SUBNET_PRIV_PREFIX}.${OCI_CLUSTER_SUFFIX_START} --last-ip ${SUBNET_PRIV_PREFIX}.199" 2>/dev/null
	ssh_cmd "ocicli network-add reg1-zone1-net1 ${CLUSTER_NAME} all eth0 none" 2>/dev/null
	ssh_cmd "ocicli network-create pub-net ${SUBNET_PRIV_PREFIX}.0 24 pub-zone yes" 2>/dev/null
	ssh_cmd "ocicli network-set pub-net --first-ip ${SUBNET_PRIV_PREFIX}.${OCI_API_SUFIX_START} --last-ip ${SUBNET_PRIV_PREFIX}.99" 2>/dev/null
	ssh_cmd "ocicli network-add pub-net ${CLUSTER_NAME} all eth0 none" 2>/dev/null
}

tear_down_secgroup () {
	local SECGROUP_NAME
	SECGROUP_NAME=$1
	echo "===> Delete security group ${SECGROUP_NAME}"
	for RULE in $(openstack security group rule list ${SECGROUP_NAME} --ingress --format value -c ID || true) ; do
		openstack security group rule delete ${RULE} || true
	done
	openstack security group delete ${SECGROUP_NAME} || true
	
}

tear_down_all () {
	echo "===> Delete ${CLUSTER_NAME}-ipxe.iso"
	openstack image delete ${CLUSTER_NAME}-ipxe.iso || true

	TMP_SRV_LIST=$(mktemp -t ${MY_SCRIPT_NAME}-srv-list.XXXXXX)
	openstack server list --name ${CLUSTER_NAME} --format value -c Name >${TMP_SRV_LIST}
	for VM in ${LST_ALL_NODES} ; do
		echo "===> Deleting server $VM"
		if grep -q -E "^${VM}\$" ${TMP_SRV_LIST} ; then
			openstack server delete $VM || true
			echo "-> Deleted."
		else
			echo "-> Already deleted."
		fi
	done
	rm -f ${TMP_SRV_LIST}

	TMP_VOL_LIST=$(mktemp -t ${MY_SCRIPT_NAME}-vol-list.XXXXXX)
	openstack volume list --format value -c Name | grep ${CLUSTER_NAME} >${TMP_VOL_LIST}
	for MACHINE in ${LST_ALL_NODES} ; do
		echo "===> Deleting system volume $MACHINE"
		if grep -q -E "^${MACHINE}\$" ${TMP_VOL_LIST} ; then
			openstack volume delete $MACHINE || true
			echo "-> Deleted."
		else
			echo "-> Already deleted."
		fi

		MACHINE_ROLE=$(echo ${MACHINE} | cut -d- -f2)
		case ${MACHINE_ROLE} in
		controller) SYSVOL=${SYSVOL_CTRL} ; DATAVOL=${SYSVOL_CTRL} ;;
		messaging)  SYSVOL=${SYSVOL_MSG}  ; DATAVOL=${SYSVOL_MSG}  ;;
		cephmon)    SYSVOL=${SYSVOL_MON}  ; DATAVOL=${SYSVOL_MON}  ;;
		cephosd)    SYSVOL=${SYSVOL_OSD}  ; DATAVOL=${SYSVOL_OSD}  ;;
		volume)     SYSVOL=${SYSVOL_VOL}  ; DATAVOL=${SYSVOL_VOL}  ;;
		network)    SYSVOL=${SYSVOL_NET}  ; DATAVOL=${SYSVOL_NET}  ;;
		compute)    SYSVOL=${SYSVOL_COMP} ; DATAVOL=${SYSVOL_COMP} ;;
		swiftstore) SYSVOL=${SYSVOL_SWP}  ; DATAVOL=${SYSVOL_SWP}  ;;
		swiftproxy) SYSVOL=${SYSVOL_SWS}  ; DATAVOL=${SYSVOL_SWS}  ;;
		*)          SYSVOL=${SYSVOL_COMP} ; DATAVOL=${SYSVOL_COMP} ;;
		esac

		if [ "${DATAVOL}" != 0 ] ; then
			echo "===> Deleting data volume $MACHINE-data"
			if grep -q -E "^${MACHINE}-data\$" ${TMP_VOL_LIST} ; then
				openstack volume delete ${MACHINE}-data || true
				echo "-> Deleted."
			else
				echo "-> Already deleted."
			fi
		fi
	done
	rm -f ${TMP_VOL_LIST}

	echo "===> Delete port ${OCI_PXE_PORT_NAME}"
	PORT_ID=$(openstack port list --name ${OCI_PXE_PORT_NAME} --format value -c ID)
	if [ -n "${PORT_ID}" ] ; then
		openstack port delete ${PORT_ID}
	fi

	echo "===> Delete all port in subnet"
	TMP_PORT_LIST=$(mktemp -t ${MY_SCRIPT_NAME}-port-list.XXXXXX)
	openstack port list --format value -c Name | grep ${CLUSTER_NAME} >${TMP_PORT_LIST}
	for i in ${LST_ALL_NODES} ; do
		if grep -q -E "^${i}\$" ${TMP_PORT_LIST} ; then
			echo "-> $i"
			openstack port delete $i || true
		fi
	done
	rm -f ${TMP_PORT_LIST}

	echo "===> Delete ${PXE_SERVER_NAME}"
	openstack server delete ${PXE_SERVER_NAME} || true

	echo "===> Deleting security groups"
	for SECGRP in $(openstack security group list --format value -c Name | grep -v default | grep ${CLUSTER_NAME}) ; do
		echo "-> $SECGRP"
		tear_down_secgroup ${SECGRP}
	done

	echo "===> Removing router"
	openstack router remove port ${SUBNET_PRIV_ROUTER_NAME} ${SUBNET_PRIV_ROUTER_PORT} || true
	openstack router remove subnet ${SUBNET_PRIV_ROUTER_NAME} ${SUBNET_PRIV_NAME} || true
	openstack port delete ${SUBNET_PRIV_ROUTER_PORT} || true
	openstack router unset --external-gateway ${SUBNET_PRIV_ROUTER_NAME} || true
	openstack router delete ${SUBNET_PRIV_ROUTER_NAME} || true

	echo "===> Delete subnet ${SUBNET_PRIV_NAME}"
	SUBNET_PRIV_ID=$(openstack subnet list --name ${SUBNET_PRIV_NAME} --format value -c ID)
	if [ -n "${SUBNET_PRIV_ID}" ] ; then
		openstack subnet delete ${SUBNET_PRIV_ID}
	fi

	echo "===> Delete network ${NET_PRIV_NAME}"
	NET_PRIV_ID=$(openstack network list --name ${NET_PRIV_NAME} --format value -c ID)
	if [ -n "${NET_PRIV_ID}" ] ; then
		openstack network delete ${NET_PRIV_ID}
	fi

	echo "===> Delete keypair"
	for i in $(openstack keypair list --format value -c Name) ; do
		if [ "${i}" = "${KEYPAIR_NAME}" ] ; then
			openstack keypair delete ${KEYPAIR_NAME}
		fi
	done
}

set_oci_hostname_and_ipforward () {
	echo "===> Fix OCI VM hostname"
	ssh_cmd "echo ${PXE_SERVER_NAME}.${DOMAIN_NAME} >/etc/hostname ; hostname ${PXE_SERVER_NAME}.${DOMAIN_NAME}"  2>/dev/null
#	TMPFILE=$(mktemp -t ${MY_SCRIPT_NAME}-ipforward.XXXXXX)
#	echo "#!/bin/sh
#set -e
#
#echo 1 > /proc/sys/net/ipv4/ip_forward
#iptables -t nat -A POSTROUTING -s 192.168.96.0/19 ! -d 192.168.96.0/19 -j SNAT --to-source ${PXE_SERVER_IP}
#" >${TMPFILE}
#	scp_cmd ${TMPFILE} /usr/bin/oci-ip-forward
#	rm -f ${TMPFILE}
#
#	TMPFILE=$(mktemp -t ${MY_SCRIPT_NAME}-ipforward.XXXXXX)
#	echo "[Unit]
#Description=OCI IP forward
#After=frr.service networking.service chrony.service
#
#[Service]
#User=root
#Group=root
#ExecStart=/usr/bin/oci-ip-forward
#Type=oneshot
#
#[Install]
#WantedBy=multi-user.target
#" >${TMPFILE}
#	scp_cmd ${TMPFILE} /lib/systemd/system/oci-ip-forward.service
#	ssh_cmd "chmod 644 /lib/systemd/system/oci-ip-forward.service"
#	rm -f ${TMPFILE}
	ssh_cmd "sed -i 's/127\.0\.1\.1.*/${PXE_SERVER_IP} ${PXE_SERVER_NAME}.${DOMAIN_NAME} ${PXE_SERVER_NAME}/' /etc/hosts" 2>/dev/null
	ssh_cmd "sed -i '/- update_etc_hosts/d' /etc/cloud/cloud.cfg" 2>/dev/null
	ssh_cmd "sed -i '/- update_hostname/d' /etc/cloud/cloud.cfg" 2>/dev/null
	ssh_cmd "sed -i '/- set_hostname/d' /etc/cloud/cloud.cfg" 2>/dev/null
	echo "===> Rebooting OCI VM"
	ssh_cmd "reboot" 2>/dev/null || true
	sleep 5
	wait_for_ssh ${PXE_SERVER_IP}
}

# Check that all servers are up, if not, attempt to reboot the OpenStack VM.
# Returns NOT_SEEN_VM: list of OpenStack VM IDs that aren't up
# or an empty list if all are up.
check_all_servers_are_up_or_reboot () {
	END_TIMESTAMP=$(( $(date +%s) + 60 ))

	TMP_SERIAL_LIST=$(mktemp -t ${MY_SCRIPT_NAME}-serial-list.XXXXXX)
	openstack server list --format csv -c ID -c Name | q -d, -H "SELECT ID FROM - WHERE Name!='${PXE_SERVER_NAME}' AND Name LIKE '%${CLUSTER_NAME}%'" >${TMP_SERIAL_LIST}

	NOT_SEEN_VM="bla"
	while [ -n "${NOT_SEEN_VM}" ] && [ "${END_TIMESTAMP}" -gt $(date +%s) ] ; do
		NOT_SEEN_VM=""
		for SERIAL in $(cat ${TMP_SERIAL_LIST}) ; do
			if [ ""$(ssh_cmd "ocicli -csv machine-list | q -H -d, \"SELECT serial FROM - WHERE serial='${SERIAL}'\"" 2>/dev/null) != ${SERIAL} ] ; then
				NOT_SEEN_VM="${NOT_SEEN_VM} ${SERIAL}"
			fi
		done
		echo -n "."
	done
   	if [ -n "${NOT_SEEN_VM}" ] ; then
   		NOT_SEEN_VM=""
		for SERIAL in $(cat ${TMP_SERIAL_LIST}) ; do
			if [ ""$(ssh_cmd "ocicli -csv machine-list | q -H -d, \"SELECT serial FROM - WHERE serial='${SERIAL}'\"" 2>/dev/null) != ${SERIAL} ] ; then
				openstack server reboot --hard ${SERIAL}
				NOT_SEEN_VM="${NOT_SEEN_VM} ${SERIAL}"
			fi
		done
	fi
	rm -f ${TMP_SERIAL_LIST}
}

add_servers_to_cl1_cluster (){
	echo -n "===> Checking VMs are seen within 10x60 seconds:"
	# We check 5 times...
	NOT_SEEN_VM="bla"
	SECOND_LEVEL_END_TIMESTAMP=$(( $(date +%s) + 600 ))
	while [ -n "${NOT_SEEN_VM}" ] && [ "${SECOND_LEVEL_END_TIMESTAMP}" -gt $(date +%s) ] ; do
		check_all_servers_are_up_or_reboot
	done

	if [ -n "${NOT_SEEN_VM}" ] ; then
		echo "error: machine ${NOT_SEEN_VM} are not seen in OCI."
		exit 1
	fi

	echo "ok."

	echo "===> Adding machines to ${CLUSTER_NAME} cluster"
	for MACHINE in ${LST_ALL_NODES} ; do
		echo "-> ${MACHINE}"
		get_machine_ip_suffix ${MACHINE}
		MACHINE_IP="${SUBNET_PRIV_PREFIX}.${RET}"
		MACHINE_SERIAL=$(openstack server show --format value -c id ${MACHINE})
		ROLE=$(echo ${MACHINE} | cut -d- -f2)
		ssh_cmd "ocicli machine-add --fixed-ip ${MACHINE_IP} ${MACHINE_SERIAL} ${CLUSTER_NAME} ${ROLE} reg-1-zone-1"
	done

	echo "===> Setting-up br-ex MAC addresses"
	for MACHINE in ${LST_COMP_NODES} ${LST_NET_NODES} ; do
		echo "-> ${MACHINE}"
		MACHINE_MAC=$(openstack port show --format value -c mac_address ${MACHINE})
		ssh_cmd "ocicli machine-set ${MACHINE}.${DOMAIN_NAME} --brex-macaddr ${MACHINE_MAC}"
	done
}

api_ip_as_allowed_address (){
	echo "===> Allowing the API ip address on controllers"
	for CTRLNAME in ${LST_CTRL_NODES} ; do
		echo "-> ${CTRLNAME}"
		PAIR=$(openstack port show ${CTRLNAME} --format json -c allowed_address_pairs | jq '.["allowed_address_pairs"][]' -r)
		if [ -z "${PAIR}" ] ; then
			openstack port set --allowed-address ip-address=${SUBNET_PRIV_PREFIX}.${OCI_API_SUFIX_START} ${CTRLNAME}
			echo "-> Added."
		else
			echo "-> Already added."
		fi
	done
}

create_open_sec_group (){
	SECGROUP_ID=$(openstack security group list --format csv | q -H -d, "SELECT ID FROM - WHERE Name='${CLUSTER_NAME}-allopen'")
	if [ -z "${SECGROUP_ID}" ] ; then
		openstack security group create ${CLUSTER_NAME}-allopen
		echo "-> Created."
	else
		echo "-> Already created."
	fi

	echo "===> Creating ${CLUSTER_NAME}-allopen rules to secgroup"
	echo "-> tcp"
	SECGROUP_RULE=$(openstack security group rule list --protocol tcp --ingress ${CLUSTER_NAME}-allopen --format csv | q -d, -H "SELECT ID FROM - WHERE \"Port Range\"='1:65300' AND Ethertype='IPv4'")
	if [ -z "${SECGROUP_RULE}" ] ; then
		openstack security group rule create --protocol tcp ${CLUSTER_NAME}-allopen
		echo "-> Created rule for secgroup ${CLUSTER_NAME}-allopen, proto tcp."
	else
		echo "-> Rule for tcp already exists."
	fi
	echo "-> udp"
	SECGROUP_RULE=$(openstack security group rule list --protocol udp --ingress ${CLUSTER_NAME}-allopen --format csv | q -d, -H "SELECT ID FROM - WHERE \"Port Range\"='1:65300' AND Ethertype='IPv4'")
	if [ -z "${SECGROUP_RULE}" ] ; then
		openstack security group rule create --protocol udp ${CLUSTER_NAME}-allopen
		echo "-> Created rule for secgroup ${CLUSTER_NAME}-allopen, proto udp."
	else
		echo "-> Rule for udp already exists."
	fi
	echo "-> icmp"
	SECGROUP_RULE=$(openstack security group rule list --protocol icmp --ingress ${CLUSTER_NAME}-allopen --format csv | q -d, -H "SELECT ID FROM -")
	if [ -z "${SECGROUP_RULE}" ] ; then
		openstack security group rule create --protocol icmp ${CLUSTER_NAME}-allopen
		echo "-> Created rule for secgroup ${CLUSTER_NAME}-allopen, proto icmp."
	else
		echo "-> Rule for icmp already exists."
	fi

	echo "-> tcp v6"
	SECGROUP_RULE=$(openstack security group rule list --protocol tcp --ingress ${CLUSTER_NAME}-allopen --format csv | q -d, -H "SELECT ID FROM - WHERE \"Port Range\"='1:65300' AND Ethertype='IPv6'")
	if [ -z "${SECGROUP_RULE}" ] ; then
		openstack security group rule create --protocol tcp --ethertype IPv6 ${CLUSTER_NAME}-allopen
		echo "-> Created rule for secgroup ${CLUSTER_NAME}-allopen, proto tcp v6."
	else
		echo "-> Rule for tcp v6 already exists."
	fi
	echo "-> udp v6"
	SECGROUP_RULE=$(openstack security group rule list --protocol udp --ingress ${CLUSTER_NAME}-allopen --format csv | q -d, -H "SELECT ID FROM - WHERE \"Port Range\"='1:65300' AND Ethertype='IPv6'")
	if [ -z "${SECGROUP_RULE}" ] ; then
		openstack security group rule create --protocol udp --ethertype IPv6 ${CLUSTER_NAME}-allopen
		echo "-> Created rule for secgroup ${CLUSTER_NAME}-allopen, proto udp v6."
	else
		echo "-> Rule for udp v6 already exists."
	fi
	echo "-> icmp v6"
	SECGROUP_RULE=$(openstack security group rule list --protocol icmp --ingress ${CLUSTER_NAME}-allopen --format csv | q -d, -H "SELECT ID FROM -  WHERE Ethertype='IPv6'")
	if [ -z "${SECGROUP_RULE}" ] ; then
		openstack security group rule create --protocol icmp --ethertype IPv6 ${CLUSTER_NAME}-allopen
		echo "-> Created rule for secgroup ${CLUSTER_NAME}-allopen, proto icmp v6."
	else
		echo "-> Rule for icmp v6 already exists."
	fi

}

add_open_sec_to_vms () {
	echo "===> Adding ${CLUSTER_NAME}-allopen secgroup to VMs"
	for i in ${LST_ALL_NODES} ; do
		echo "-> $i"
		if ! [ ""$(openstack server show --format json -c security_groups $i | jq '.["security_groups"][] | select(.name == "'${CLUSTER_NAME}'-allopen") | .[]' -r) = "${CLUSTER_NAME}-allopen" ] ; then
			openstack server add security group $i ${CLUSTER_NAME}-allopen
		fi
	done
}

install_cl1_cluster () {
	ssh_cmd "ocicli cluster-install ${CLUSTER_NAME}"
}

prepare_vm_image_in_oci () {
	echo "===> Preparing cirros image in OCI"
	echo '-> Checking if image is here'
	if ! [ -e /root/${VM_IMAGE_FILENAME} ] ; then
		wget ${VM_IMAGE_URL} -O /root/${VM_IMAGE_FILENAME}
	fi
	echo '-> Checking if image is in OCI'
	if ! ssh_cmd "ls ${VM_IMAGE_FILENAME}" 1>/dev/null 2>/dev/null; then
		echo '-> Uploading to OCI'
		scp_cmd /root/${VM_IMAGE_FILENAME} /root/${VM_IMAGE_FILENAME}
	else
		echo '-> Already in OCI'
	fi
	echo '-> Fixing glance_image directives in OCI'
	fix_oci_directive glance_image image_name ${VM_IMAGE_FILENAME}
	fix_oci_directive glance_image image_ssh_user ${IMAGE_SSH_USER}
}

upload_vm_image_to_cluster () {
	echo "===> Uploading image to controller-1"
	get_machine_ip_suffix ${CLUSTER_NAME}-controller-1
	CTRL_1_IP=${SUBNET_PRIV_PREFIX}.${RET}
	echo "-> Checking if image is in ${CLUSTER_NAME}-controller-1"
	if ! ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} 'ls /root/${VM_IMAGE_FILENAME} 1>/dev/null 2>/dev/null'" ; then
		echo '-> Not present: scp...'
		ssh_cmd "scp -i /etc/openstack-cluster-installer/id_rsa /root/${VM_IMAGE_FILENAME} root@${CTRL_1_IP}:"
	else
		echo '-> Already there'
	fi
	echo '-> Checking if image has been uploaded to OpenStack'
	if ! [ ""$(ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} '. oci-openrc ; openstack image list --name ${VM_IMAGE_FILENAME} --format value -c Name'") = "${VM_IMAGE_FILENAME}" ] ; then
		echo '-> Not present: uploading'
		ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} '. oci-openrc ; openstack image create --container-format bare --disk-format qcow2 --property hw_disk_bus=scsi --property hw_scsi_model=virtio-scsi --property os_type=linux --property os_distro=cirros --property os_admin_user=cirros --public --file ${VM_IMAGE_FILENAME} ${VM_IMAGE_FILENAME}'"
	else
		echo '-> Already uploaded'
	fi
}

wait_for_tempest_install () {
	ssh_cmd "ocicli machine-wait-for-install ${CLUSTER_NAME}-tempest-1.${DOMAIN_NAME}"
}

wait_for_tempest_puppet () {
	ssh_cmd "ocicli machine-wait-for-puppet ${CLUSTER_NAME}-tempest-1.${DOMAIN_NAME}"
}

upload_image_to_tempest_vm () {
	echo "===> Uploading image to tempest-1"
	get_machine_ip_suffix ${CLUSTER_NAME}-tempest-1
	TEMPEST_1_IP=${SUBNET_PRIV_PREFIX}.${RET}
	echo '-> Checking if image is already there'
	if ! ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${TEMPEST_1_IP} 'ls /root/${VM_IMAGE_FILENAME} 1>/dev/null 2>/dev/null'" ; then
		echo '-> Not present: scp'
		ssh_cmd "scp -i /etc/openstack-cluster-installer/id_rsa /root/${VM_IMAGE_FILENAME} root@${TEMPEST_1_IP}:"
	else
		echo '-> Already there'
	fi
}

run_script_in_node () {
	local SERVER_IP SCRIPT_PATH OCI_SERVER_TMP TARGET_TMP
	SERVER_IP=${1}
	SCRIPT_PATH=${2}

	OCI_SERVER_TMP=$(ssh_cmd "mktemp -t ${MY_SCRIPT_NAME}-run-script-oci-server.XXXXXX" 2>/dev/null)
	TARGET_TMP=$(ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${SERVER_IP} 'mktemp -t ${MY_SCRIPT_NAME}-run-script-target-server.XXXXXX'" 2>/dev/null)
	scp_cmd ${SCRIPT_PATH} ${OCI_SERVER_TMP} 1>/dev/null 2>/dev/null
	ssh_cmd "scp -i /etc/openstack-cluster-installer/id_rsa ${OCI_SERVER_TMP} root@${SERVER_IP}:${TARGET_TMP}" 2>/dev/null
	echo "-> Running ${SCRIPT_PATH} on target ${SERVER_IP}"
	if ! ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${SERVER_IP} 'chmod +x ${TARGET_TMP} ; ${TARGET_TMP}'" 2>/dev/null ; then
		echo "------> Failed to run script on ${SERVER_IP}:"
		cat ${SCRIPT_PATH}
		exit 1
	fi
	ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${SERVER_IP} 'rm -f ${TARGET_TMP}'" 2>/dev/null
	ssh_cmd "rm -f ${OCI_SERVER_TMP}" 2>/dev/null
}

# This installs a package on a given target node,
# using the OCI server as jump host
install_package_in_node () {
	local SERVER_IP PACKAGE_NAME INSTALL_PKG_TMP_FILE OCI_SERVER_TMP
	SERVER_IP=${1}
	PACKAGE_NAME=${2}

	echo "-> Installing ${PACKAGE_NAME} in ${SERVER_IP}"
	INSTALL_PKG_TMP_FILE=$(mktemp -t ${MY_SCRIPT_NAME}-install-pkg.XXXXXX)
	echo "#!/bin/sh

if ! dpkg-query -W --showformat '\${Status}\\\n' ${PACKAGE_NAME} 2>/dev/null | grep -q 'install ok' ; then
	DEBIAN_FRONTEND=noninteractive apt-get install ${PACKAGE_NAME} -y
fi

exit 0" >${INSTALL_PKG_TMP_FILE}
	run_script_in_node ${SERVER_IP} ${INSTALL_PKG_TMP_FILE}
	rm -f ${INSTALL_PKG_TMP_FILE}
	echo "-> done installing."
}

scp_file_to_node () {
	local SERVER_IP SOURCE_FILE DEST_FILE OCI_SERVER_TMP
	SERVER_IP=${1}
	SOURCE_FILE=${2}
	DEST_FILE=${3}

	echo "-> Installing ${DEST_FILE} in ${SERVER_IP}"
	OCI_SERVER_TMP=$(ssh_cmd "mktemp -t ${MY_SCRIPT_NAME}-scp_file_to_node.XXXXXX" 2>/dev/null)
	scp_cmd ${SOURCE_FILE} ${OCI_SERVER_TMP} 1>/dev/null 2>/dev/null
	ssh_cmd "scp -i /etc/openstack-cluster-installer/id_rsa ${OCI_SERVER_TMP} root@${SERVER_IP}:${DEST_FILE}" 2>/dev/null
	ssh_cmd "rm -f ${OCI_SERVER_TMP}" 2>/dev/null
}

setup_ocici_internal_bridge () {
	echo "===> Install OVS in OCI"
	install_package_in_oci_server openvswitch-switch

	OVS_TUNNEL_NODE_LIST="${LST_NET_NODES} ${LST_TEMPEST_NODES}"

	echo "===> Install and configure the OVS br-oci bridge in the tempest node"
	echo "-> Get tempest node IP"
	get_machine_ip_suffix ${CLUSTER_NAME}-tempest-1
	TEMPEST_NODE_IP_SUFFIX=${RET}
	TEMPEST_NODE_IP=${SUBNET_PRIV_PREFIX}.${RET}

	echo "-> Get network-1 node IP"
	get_machine_ip_suffix ${CLUSTER_NAME}-network-1
	NETWORK_NODE_IP_SUFFIX=${RET}
	NETWORK_NODE_IP=${SUBNET_PRIV_PREFIX}.${RET}

	echo "-> Install OVS"
	install_package_in_node ${TEMPEST_NODE_IP} openvswitch-switch

	echo "-> Setup the br-oci bridge in the tempest node"
	RUNME=$(mktemp -t ${MY_SCRIPT_NAME}-run-script.XXXXXX)
			echo "#!/bin/sh
set -e

if ! ip link show dev br-oci >/dev/null ; then
	ovs-vsctl --may-exist add-br br-oci
fi

ip addr add ${EXT_NETWORK_PREFIX}.${TEMPEST_NODE_IP_SUFFIX}/24 dev br-oci || true
ovs-vsctl --may-exist add-port br-oci vxlan130 -- set interface vxlan130 type=vxlan options:local_ip=${TEMPEST_NODE_IP} options:remote_ip=${NETWORK_NODE_IP}
ip link set up br-oci
" >${RUNME}
	run_script_in_node ${TEMPEST_NODE_IP} ${RUNME}

	echo "-> Bridge network node br-ex to tempest br-oci in the network node"
			echo "#!/bin/sh

set -e

ip addr add ${EXT_NETWORK_PREFIX}.1/24 dev br-ex || true
ovs-vsctl --may-exist add-port br-ex vxlan130 -- set interface vxlan130 type=vxlan options:local_ip=${NETWORK_NODE_IP} options:remote_ip=${TEMPEST_NODE_IP}
ip link set up br-ex
" >${RUNME}
	run_script_in_node ${NETWORK_NODE_IP} ${RUNME}
	rm -f ${RUNME}
}

setup_stuff_in_openstack () {
	get_machine_ip_suffix ${CLUSTER_NAME}-controller-1
	set -x
	CTRL_1_IP=${SUBNET_PRIV_PREFIX}.${RET}
	if [ "${OCI_COMPUTE_NODES}" = "yes" ] ; then
		echo "===> Running nova-manage cell_v2 discover_hosts"
		ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} 'su nova -s /bin/sh -c \"nova-manage cell_v2 discover_hosts\"'"
		echo "===> Enabling hypervisors"
		for HYPERVISOR in ${LST_COMP_NODES} ; do
			echo "-> ${HYPERVISOR}"
			ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} '. oci-openrc ; openstack compute service set --enable ${HYPERVISOR}.${DOMAIN_NAME} nova-compute'"
		done
		echo "===> Creating flavor"
		if ! [ ""$(ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} '. oci-openrc ; openstack flavor show demo-flavor --format value -c name'") = "demo-flavor" ] ; then
			ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} '. oci-openrc ; openstack flavor create --ram 2048 --disk 5 --vcpus 1 demo-flavor'"
			echo "-> Created."
		else
			echo "-> Already exist."
		fi
		echo "===> Creating private network"
		if ! [ ""$(ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} '. oci-openrc ; openstack network show --format value -c name demo-network'") = "demo-network" ] ; then
			ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} '. oci-openrc ; openstack network create demo-network'"
			echo "-> Created."
		else
			echo "-> Already exist."
		fi
		echo "===> Creating private subnet"
		if ! [ ""$(ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} '. oci-openrc ; openstack subnet show --format value -c name demo-subnet'") = "demo-subnet" ] ; then
			ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} '. oci-openrc ; openstack subnet create demo-subnet --network demo-network --subnet-range 192.168.105.0/24'"
			echo "-> Created."
		else
			echo "-> Already exist."
		fi
		echo "===> Creating ext-net1"
		if ! [ ""$(ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} '. oci-openrc ; openstack network show --format value -c name ext-net1'") = "ext-net1" ] ; then
			ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} '. oci-openrc ; openstack network create --external --provider-physical-network external --provider-network-type flat ext-net1'"
			echo "-> Created."
		else
			echo "-> Already exists."
		fi
		echo "===> Creating ext-net1 subnet"
		if ! [ ""$(ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} '. oci-openrc ; openstack subnet show --format value -c name ext-subnet'") = "ext-subnet" ] ; then
			ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${CTRL_1_IP} '. oci-openrc ; openstack subnet create --network ext-net1 --allocation-pool start=${EXT_NETWORK_PREFIX}.${EXT_NETWORK_FIRST_IP},end=${EXT_NETWORK_PREFIX}.${EXT_NETWORK_LAST_IP} --dns-nameserver ${SUBNET_PRIV_DNS_SERVER} --gateway ${EXT_NETWORK_PREFIX}.1 --subnet-range ${EXT_NETWORK_PREFIX}.0/24 --no-dhcp ext-subnet'"
			echo "-> Created."
		else
			echo "-> Already exists."
		fi
	fi
}

run_tempest_tests () {
	echo "===> Running tempest tests"
	ALL_TESTS="identity-test-list"

	if [ "${OCI_COMPUTE_NODES}" = "yes" ] ; then
		ALL_TESTS="${ALL_TESTS} compute-test-list network-test-list orchestration-test-list"
	fi
	if [ "${OCI_VOLUME_NODES}" = "yes" ] || [ "${OCI_CEPH_NODES}" = "yes" ] ; then
		ALL_TESTS="${ALL_TESTS} volume-test-list"
	fi
	if [ "${OCI_SWIFT_NODES}" = "yes" ] ; then
		ALL_TESTS="${ALL_TESTS} object-test-list"
	fi

	TEST_RUN_SUCCESS=yes
	for TESTLIST in ${ALL_TESTS} ; do
		echo "-> Copying ${TESTLIST} to OCI VM"
		if ! ssh_cmd "ls /root/${TESTLIST} 1>/dev/null 2>/dev/null" ; then
			echo '-> scp'
			scp_cmd /etc/oci-poc/tests/${TESTLIST} /root/${TESTLIST}
		else
			echo '-> Already there'
		fi

		get_machine_ip_suffix ${CLUSTER_NAME}-tempest-1
		TEMPEST_1_IP=${SUBNET_PRIV_PREFIX}.${RET}

		echo "-> Copying ${TESTLIST} to tempest VM"
		if ! ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${TEMPEST_1_IP} 'ls /root/${TESTLIST} 1>/dev/null 2>/dev/null'" ; then
			echo '-> scp'
			ssh_cmd "scp -i /etc/openstack-cluster-installer/id_rsa /root/${TESTLIST} root@${TEMPEST_1_IP}:"
		else
			echo '-> Already there'
		fi

		echo "-> Running no object test list from refstack"
		if ! ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${TEMPEST_1_IP} 'cd /var/lib/tempest ; tempest run --config-file /etc/tempest/tempest.conf --load-list /etc/tempest/platform.2021.11-test-list.txt | tee /var/log/${TESTLIST}-refstack.log'" ; then
			TEST_RUN_SUCCESS=no
		fi

		echo "-> Running ${TESTLIST}"
		if ! ssh_cmd "ssh -i /etc/openstack-cluster-installer/id_rsa root@${TEMPEST_1_IP} 'cd /var/lib/tempest ; tempest run --load-list /root/${TESTLIST} | tee /var/log/${TESTLIST}.log'" ; then
			TEST_RUN_SUCCESS=no
		fi
	done
	if [ "${TEST_RUN_SUCCESS}" = "no" ] ; then
		echo "***** FAILURE *****"
		echo "Some tests were not successful."
		echo "Cluster was not tear down."
		exit 1
	else
		echo "***** SUCCESS *****"
		echo "All tests executed with success."
		echo "Starting to tear down..."
		tear_down_all
		exit 0
	fi
}

calc_node_list

cleanup() {
	local exit_error
	exit_error=$?
	echo "Failed CI run: tearing down..."
	tear_down_all
	exit $exit_error
}

if [ "${TEARDOWN_AFTER_ONLY}" = "yes" ] ; then
	tear_down_all
	trap "cleanup" EXIT TERM INT
fi


upload_ipxe_iso
provision_priv_net
upload_pubkey
start_oci_pxe_server
wait_for_vm ${PXE_SERVER_NAME} ACTIVE
get_vm_ip ${PXE_SERVER_NAME}
PXE_SERVER_IP=${RET}
create_security_groups_and_rules
add_secgroup_to_oci_server
wait_for_ssh ${PXE_SERVER_IP}
add_privnet_to_oci
get_oci_privnet_ifname
configure_privnet_interface
setup_debian_repo
set_oci_hostname_and_ipforward
install_oci
configure_oci_networking
bootup_to_install_vms
add_servers_to_cl1_cluster
api_ip_as_allowed_address
create_open_sec_group
add_open_sec_to_vms
prepare_vm_image_in_oci
install_cl1_cluster
upload_vm_image_to_cluster
setup_stuff_in_openstack
wait_for_tempest_install
upload_image_to_tempest_vm
wait_for_tempest_puppet
setup_ocici_internal_bridge
run_tempest_tests