#!/bin/bash
#
# common functions for ovs based plugin
# -------------------------------------

#
# start/stop service
#

function start_ovs_dpdk {
    if [ -e /etc/init.d/ovs-dpdk ]; then
        sudo service ovs-dpdk start
    fi
}

function stop_ovs_dpdk {
    if [ -e /etc/init.d/ovs-dpdk ]; then
        sudo service ovs-dpdk stop
    fi
}

#
# ovs initialisation
#

function set_agent_type {
    if [ "$OVS_DPDK_MODE" != "controller_ovs" ]; then
        # for not controller modes it makes sense to specify agent_type in ml2
        # this is valid also when one forget to fill OVS_DPDK_MODE
        iniset /$Q_PLUGIN_CONF_FILE agent agent_type "DPDK OVS Agent"
    fi
}


function ovs_dpdk_configure_bridge_datapath {
    local bridge=$1
    if [ "$OVS_DATAPATH_TYPE" != "" ]; then
        sudo ovs-vsctl --no-wait set Bridge $bridge datapath_type=${OVS_DATAPATH_TYPE}
    fi
}

function ovs_dpdk_add_bridge {
    local bridge=$1
    sudo ovs-vsctl --no-wait -- --may-exist add-br $bridge
    ovs_dpdk_configure_bridge_datapath $bridge
    sudo ovs-vsctl --no-wait br-set-external-id $bridge bridge-id $bridge
}


function ovs_dpdk_setup_bridge {
    local bridge=$1
    neutron-ovs-cleanup
    ovs_dpdk_add_bridge $bridge
}


function ovs_dpdk_create_kvm_wrapper {

    install_qemu_kvm

    if [ -e /usr/bin/kvm ]; then
        KVM_CMD="/usr/bin/kvm"
    elif [ -e /usr/bin/qemu-kvm ]; then
        KVM_CMD="/usr/bin/qemu-kvm"
    elif [ -e /usr/libexec/qemu-kvm ]; then
        KVM_CMD="/usr/libexec/qemu-kvm"
    fi

    sudo mv $KVM_CMD $KVM_CMD.orig

cat << 'EOF' | sudo tee  $KVM_CMD
#!/bin/bash -
VIRTIO_OPTIONS="csum=off,gso=off,guest_tso4=off,guest_tso6=off,guest_ecn=off"
VHOST_FORCE="vhostforce"
SHARE="share=on"
add_mem=False
i=0
while [ $# -gt 0 ]; do
    case "$1" in
    -device)
        args[i]="$1"
        (( i++ ))
        shift
        if [[ "$1" =~ "vhost-user" ]]; then
                args[i]=${1},${VHOST_FORCE}
                (( i++))
                shift

        fi
        if [[ $1 == virtio-net-pci* ]]; then
                args[i]=${1},${VIRTIO_OPTIONS}
                (( i++))
                shift

        fi
        ;;
    -object)
        args[i]="$1"
        (( i++ ))
        shift
        if [[ "$1" =~ "memory-backend-file" ]]; then
                args[i]=${1},${SHARE}
                (( i++))
                shift
        fi
        ;;
    *)
         args[i]="$1"
         (( i++ ))
         shift ;;
    esac
done
echo "qemu ${args[@]}"  > /tmp/qemu.orig
if [ -e /usr/bin/qemu-system-x86_64 ]; then
    exec /usr/bin/qemu-system-x86_64  "${args[@]}"
elif [ -e /usr/libexec/qemu-kvm.orig ]; then
    exec /usr/libexec/qemu-kvm.orig  "${args[@]}"
fi
EOF

    sudo chmod +x $KVM_CMD

}

#
# ovs cleanup
#

function ovs_dpdk_db_cleanup {
    # remove all OVS ports that look like Neutron created ports
    for port in $(sudo ovs-vsctl list port | grep -o -e tap[0-9a-f\-]* -e q[rg]-[0-9a-f\-]*); do
        sudo ovs-vsctl --no-wait del-port ${port}
    done

    # remove all OVS bridges created by Neutron
    for bridge in $(sudo ovs-vsctl list-br | grep -o -e ${OVS_BRIDGE} -e ${PUBLIC_BRIDGE}); do
        sudo ovs-vsctl --no-wait del-br ${bridge}
    done
    if [ -e /usr/bin/kvm ]; then
        KVM_CMD="/usr/bin/kvm"
    elif [ -e /usr/bin/qemu-kvm ]; then
        KVM_CMD="/usr/bin/qemu-kvm"
    fi
    if [ -e "$KVM_CMD.orig" ]; then
        sudo mv $KVM_CMD.orig $KVM_CMD
    fi
}

function ovs_dpdk_clean(){
    sudo rm -f /usr/bin/ovs*
    sudo rm -f $OVS_DB_CONF_DIR/*
    sudo rm -f $OVS_DB_SOCKET_DIR/*
    sudo rm -f ${OVS_DIR}/BUILD_COMPLETE
}

#
# ovs installation
#

function clone_ovs_dpdk(){
    OFFLINE=$(trueorfalse False OFFLINE)
    RECLONE=$(trueorfalse False RECLONE)
    if [[ "$OFFLINE" != True && ( "$RECLONE" == True || ! -e ${OVS_DIR}/BUILD_COMPLETE ) ]]; then
        if [ ! -d ${OVS_DIR} ] || [ "$RECLONE" == True ]; then
            if [ -d ${OVS_DIR} ]; then
                rm -rf ${OVS_DIR}
            fi
            if [ "$OVS_GIT_TAG" != "" ]; then
                git_clone ${OVS_GIT_REPO} ${OVS_DIR} $OVS_GIT_TAG
            else
                git_clone ${OVS_GIT_REPO} ${OVS_DIR}
            fi
        fi

        if [ ! -d "${OVS_DPDK_DIR}" ] || [ "$RECLONE" == True ]; then
            git_clone ${OVS_DPDK_GIT_REPO} ${OVS_DPDK_DIR} ${OVS_DPDK_GIT_TAG}
        fi
        if [ "$OVS_PATCHES" != "" ]; then
            local patches=( $OVS_PATCHES )
            pushd  ${OVS_DIR}
            git clean -f -x -d
            git reset --hard $OVS_GIT_TAG
            for url in "${patches[@]}"; do
                curl $url | patch -p1
            done
            popd
        fi
        if [ "$OVS_DPDK_PATCHES" != "" ]; then
            local patches=( $OVS_DPDK_PATCHES )
            pushd  ${OVS_DPDK_DIR}
            git clean -f -x -d
            git reset --hard ${OVS_DPDK_GIT_TAG}
            for url in "${patches[@]}"; do
                curl $url | patch -p1
            done
            popd
        fi
    fi
}

function build_ovs_dpdk(){
    if [ -e "${OVS_DIR}/BUILD_COMPLETE" ]; then
        echo "Build alread done."
        cd ${OVS_DIR}
        sudo make install
        return
    fi
    cd ${OVS_DPDK_DIR}
    make config T=${RTE_TARGET}
    if [ -e "${OVS_DPDK_DIR}/${RTE_TARGET}" ]; then
       rm -rf $RTE_TARGET
    fi
    ln -s -f build $RTE_TARGET

    OVS_DPDK_RTE_LIBRTE_VHOST=$(trueorfalse True OVS_DPDK_RTE_LIBRTE_VHOST)
    if [ "$OVS_DPDK_RTE_LIBRTE_VHOST" == "True" ]; then
        OVS_DPDK_RTE_LIBRTE_VHOST='y'
    else
        OVS_DPDK_RTE_LIBRTE_VHOST='n'
    fi

    OVS_DPDK_VHOST_USER_DEBUG=$(trueorfalse False OVS_DPDK_VHOST_USER_DEBUG)
    if [ "$OVS_DPDK_VHOST_USER_DEBUG" == "True" ]; then
        OVS_DPDK_VHOST_USER_DEBUG='y'
    else
        OVS_DPDK_VHOST_USER_DEBUG='n'
    fi

    sed "s/CONFIG_RTE_BUILD_COMBINE_LIBS=n/CONFIG_RTE_BUILD_COMBINE_LIBS=y/" -i ${OVS_DPDK_DIR}/build/.config
    sed "s/CONFIG_RTE_MAX_MEMSEG=.*$/CONFIG_RTE_MAX_MEMSEG=${OVS_DPDK_MEM_SEGMENTS}/" -i ${OVS_DPDK_DIR}/build/.config
    sed "s/CONFIG_RTE_LIBRTE_VHOST=.*$/CONFIG_RTE_LIBRTE_VHOST=${OVS_DPDK_RTE_LIBRTE_VHOST}/" -i ${OVS_DPDK_DIR}/build/.config
    sed "s/CONFIG_RTE_LIBRTE_VHOST_DEBUG=.*$/CONFIG_RTE_LIBRTE_VHOST_DEBUG=${OVS_DPDK_VHOST_USER_DEBUG}/" -i ${OVS_DPDK_DIR}/build/.config
    sed "s/CONFIG_RTE_LIBRTE_KNI=.*$/CONFIG_RTE_LIBRTE_KNI=n/" -i ${OVS_DPDK_DIR}/build/.config

    make -j $(nproc)
    sudo cp ${OVS_DPDK_DIR}/build/lib/*dpdk.* /lib
    cd ${OVS_DIR}
    ./boot.sh
    ./configure --with-dpdk=${OVS_DPDK_DIR}/${RTE_TARGET} --prefix=/usr --with-rundir=$OVS_DB_SOCKET_DIR
    make -j $(nproc) CFLAGS='-O3 -march=native'
    touch ${OVS_DIR}/BUILD_COMPLETE
    sudo make install
}

function ovs_dpdk_write_conf {
    sudo cp $NETWOKING_OVS_DPDK_DIR/devstack/ovs-dpdk/ovs-dpdk-conf /etc/default/ovs-dpdk

    OVS_ALLOCATE_HUGEPAGES=$(trueorfalse True OVS_ALLOCATE_HUGEPAGES)

    sudo sed "s#RTE_SDK=.*#RTE_SDK=$OVS_DPDK_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#RTE_TARGET=.*#RTE_TARGET=$RTE_TARGET#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_SRC_DIR=.*#OVS_SRC_DIR=$OVS_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_NUM_HUGEPAGES=.*#OVS_NUM_HUGEPAGES=$OVS_NUM_HUGEPAGES#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_HUGEPAGE_MOUNT=.*#OVS_HUGEPAGE_MOUNT=$OVS_HUGEPAGE_MOUNT#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_HUGEPAGE_MOUNT_PAGESIZE=.*#OVS_HUGEPAGE_MOUNT_PAGESIZE=$OVS_HUGEPAGE_MOUNT_PAGESIZE#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_DB_SOCKET_DIR=.*#OVS_DB_SOCKET_DIR=$OVS_DB_SOCKET_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_DB_CONF_DIR=.*#OVS_DB_CONF_DIR=$OVS_DB_CONF_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_DB_SOCKET=.*#OVS_DB_SOCKET=$OVS_DB_SOCKET#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_DB_CONF=.*#OVS_DB_CONF=$OVS_DB_CONF#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_SOCKET_MEM=.*#OVS_SOCKET_MEM=$OVS_SOCKET_MEM#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_MEM_CHANNELS=.*#OVS_MEM_CHANNELS=$OVS_MEM_CHANNELS#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_CORE_MASK=.*#OVS_CORE_MASK=$OVS_CORE_MASK#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_PMD_CORE_MASK=.*#OVS_PMD_CORE_MASK=$OVS_PMD_CORE_MASK#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_ALLOCATE_HUGEPAGES=.*#OVS_ALLOCATE_HUGEPAGES=$OVS_ALLOCATE_HUGEPAGES#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_LOG_DIR=.*#OVS_LOG_DIR=$OVS_LOG_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_LOCK_DIR=.*#OVS_LOCK_DIR=$OVS_LOCK_DIR#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_INTERFACE_DRIVER=.*#OVS_INTERFACE_DRIVER=$OVS_INTERFACE_DRIVER#" -i /etc/default/ovs-dpdk
    sudo sed "s#OVS_TUNNEL_CIDR_MAPPING=.*#OVS_TUNNEL_CIDR_MAPPING=$OVS_TUNNEL_CIDR_MAPPING#" -i /etc/default/ovs-dpdk

    if [ "$OVS_SOCKET_MEM" == "auto" ]; then
        for d in /sys/devices/system/node/node? ; do
            if [ "$OVS_SOCKET_MEM" == "auto" ]; then
                OVS_SOCKET_MEM=2048
            else
                OVS_SOCKET_MEM=$OVS_SOCKET_MEM,2048
            fi
        done
    fi

    sudo sed -e "s#OVS_SOCKET_MEM=.*#OVS_SOCKET_MEM=$OVS_SOCKET_MEM#g" -i /etc/default/ovs-dpdk

    # Creates an array of pci addres to interface names delimeted by # e.g. <pci_address>#<interface name>
    PAIRS=( `ls -al /sys/class/net/* | awk '$0 ~ /pci|virtual/ {n=split($NF,a,"/"); if (a[4] == "virtual") { a[n-2] = "virtual."NR}; print a[n-2] "#" a[n] }' ` )
    # Populates OVS_BRIDGE_MAPPINGS if $PHYSICAL_NETWORK and $OVS_PHYSICAL_BRIDGE are used instead.
    if [[ "$OVS_DATAPATH_TYPE" != "" ]] && [[ "$OVS_BRIDGE_MAPPINGS" = "" ]] && [[ "$PHYSICAL_NETWORK" != "" ]] && [[ "$OVS_PHYSICAL_BRIDGE" != "" ]]; then
        OVS_BRIDGE_MAPPINGS=$PHYSICAL_NETWORK:$OVS_PHYSICAL_BRIDGE
    fi

    if [[ -z "$OVS_DPDK_PORT_MAPPINGS" ]]; then
        OVS_BRIDGES=${OVS_BRIDGE_MAPPINGS//,/ }
        ARRAY=( $OVS_BRIDGES )
        for net in "${ARRAY[@]}"; do
            bridge="${net##*:}"
            nic=${bridge/br-/}
            if [[ -z "$OVS_DPDK_PORT_MAPPINGS" ]]; then
                OVS_DPDK_PORT_MAPPINGS=$nic:$bridge
            else
                OVS_DPDK_PORT_MAPPINGS=$OVS_DPDK_PORT_MAPPINGS,$nic:$bridge
            fi
        done
    fi

    # replace bonds with nic's
    # first collect nic's into associate array BONDS
    declare -A BONDS
    PORTS=${OVS_BOND_PORTS//,/ }
    PORTS_ARRAY=( $PORTS )

    for pair in "${PORTS_ARRAY[@]}"; do
        name="${pair%%:*}"
        nic="${pair##*:}"
        if [[ ${BONDS[$name]} == "" ]]; then
            BONDS[$name]="($nic)"
        else
            BONDS[$name]=${BONDS[$name]},"($nic)"
        fi
    done

    # adding nics from bonds into OVS_DPDK_PORT_MAPPINGS
    for k in "${!BONDS[@]}"; do
        if [[ ${OVS_DPDK_PORT_MAPPINGS} =~ .*$k.* ]]; then
            replace=""
            bridge=$(echo $OVS_DPDK_PORT_MAPPINGS | sed -e "s/\(.*\)$k:\([^,]*\).*/\2/g")
            ports=${BONDS[$k]//,/ }
            ports_mdf=$(echo $ports | sed -e "s/(//g" | sed -e "s/)//g")
            ports_mdf_array=( $ports_mdf )
            for nic in "${ports_mdf_array[@]}"; do
                if [ -z "$replace" ]; then
                    replace="$k:$bridge,$nic:$bridge"
                else
                    replace="$replace,$nic:$bridge"
                fi
            done
            OVS_DPDK_PORT_MAPPINGS=$(echo $OVS_DPDK_PORT_MAPPINGS | sed -e "s/$k:\([^,]*\)/$replace/g")
        else
            # potential misconfiguration
            echo "warning: bond $k specified but bridge not found in OVS_DPDK_PORT_MAPPINGS"
        fi
    done

    MAPPINGS=${OVS_DPDK_PORT_MAPPINGS//,/ }

    ARRAY=( $MAPPINGS )
    NICS=""
    for net in "${ARRAY[@]}"; do
         nic="${net%%:*}"
         bridge="${net##*:}"
         printf "%s in %s\n" "$nic" "$bridge"
         for pair in "${PAIRS[@]}"; do
            if [[ $nic == `echo $pair | cut -f 2 -d "#"` ]]; then
                if [[ $NICS == "" ]]; then
                    NICS=$pair
                else
                    NICS=$NICS,$pair
                fi
            fi
        done
    done
    sudo sed "s/OVS_PCI_MAPPINGS=.*/OVS_PCI_MAPPINGS=$NICS/" -i /etc/default/ovs-dpdk
    sudo sed "s/OVS_BRIDGE_MAPPINGS=.*/OVS_BRIDGE_MAPPINGS=$OVS_BRIDGE_MAPPINGS/" -i /etc/default/ovs-dpdk
    sudo sed "s/OVS_DPDK_PORT_MAPPINGS=.*/OVS_DPDK_PORT_MAPPINGS=$OVS_DPDK_PORT_MAPPINGS/" -i /etc/default/ovs-dpdk
    sudo sed "s/OVS_BOND_PORTS=.*/OVS_BOND_PORTS=$OVS_BOND_PORTS/" -i /etc/default/ovs-dpdk
    sudo sed "s/OVS_BOND_MODE=.*/OVS_BOND_MODE=$OVS_BOND_MODE/" -i /etc/default/ovs-dpdk
}

function uninstall_libvirt_CentOS {

    if [ "$os_VENDOR" == CentOS ] ; then

        sudo rm /usr/libexec/qemu-kvm
        sudo rm /usr/libexec/qemu-kvm.orig

        sudo yum remove -y libvirt* qemu* libcard* glusterfs*
    fi
}

function install_qemu_kvm {

    if [ "$os_VENDOR" == CentOS ] ; then
        echo "Configuring libvirt for CentOS"
        sudo service libvirtd stop

        sudo yum remove -y libvirt* qemu* libcacard* glusterfs*

        if [ ! -d /opt/stack/rpms ]; then
            sudo mkdir -m 777 -p /opt/stack/rpms
        fi

        RPMS=/opt/stack/rpms

        wget_cmd=" wget -nc -P"

        if [ "$RECLONE" == True ] ; then
            sudo rm -f $RPMS/*
            wget_cmd=" wget -P"
        fi

        for file in ${CentOS_libvirt_repo[@]} ; do
            $wget_cmd $RPMS $file
        done

        for file in ${CentOS_qemu_repo[@]} ; do
            $wget_cmd $RPMS $file
        done

        sudo yum install -y $RPMS/*

        sudo service libvirtd start
    else
        install_package qemu-kvm
    fi

}

function install_ovs_dpdk {
    # Install deps
    set +o errexit
    if is_ubuntu; then
        stop_service openvswitch-switch
        uninstall_package openvswitch-switch openvswitch-datapath-dkms openvswitch-common
        install_package autoconf libtool libfuse-dev
    else
        stop_service openvswitch
        uninstall_package openvswitch
        install_package pciutils autoconf libtool fuse-devel
    fi
    # This function exits on an error so that errors don't compound and you see
    # only the first error that occurred.
    sudo rmmod openvswitch
    set -o errexit

    build_ovs_dpdk

    sudo cp $NETWOKING_OVS_DPDK_DIR/devstack/ovs-dpdk/ovs-dpdk-init /etc/init.d/ovs-dpdk
    ovs_dpdk_write_conf
    ovs_dpdk_create_kvm_wrapper

    sudo chmod +x /etc/init.d/ovs-dpdk
    sudo service ovs-dpdk init

    # Create integration bridge
    ovs_dpdk_add_bridge $OVS_BRIDGE
}

function ovs_dpdk_configure_debug_command {
    Q_USE_PROVIDERNET_FOR_PUBLIC=$(trueorfalse False Q_USE_PROVIDERNET_FOR_PUBLIC)
    if [ "$Q_USE_PROVIDERNET_FOR_PUBLIC" == "True" ]; then
        iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT external_network_bridge ""
    else
        iniset $NEUTRON_TEST_CONFIG_FILE DEFAULT external_network_bridge $PUBLIC_BRIDGE
    fi
}

function ovs_dpdk_configure_firewall_driver {
    iniset /$Q_PLUGIN_CONF_FILE securitygroup firewall_driver neutron.agent.firewall.NoopFirewallDriver
}

function ovs_dpdk_configure_l3_agent {
    Q_USE_PROVIDERNET_FOR_PUBLIC=$(trueorfalse False Q_USE_PROVIDERNET_FOR_PUBLIC)
    Q_USE_PUBLIC_VETH=$(trueorfalse False Q_USE_PUBLIC_VETH)
    if [ "$Q_USE_PROVIDERNET_FOR_PUBLIC" == "True" ]; then
        iniset $Q_L3_CONF_FILE DEFAULT external_network_bridge ""
    else
        iniset $Q_L3_CONF_FILE DEFAULT external_network_bridge $PUBLIC_BRIDGE
    fi
    if [ "$Q_USE_PUBLIC_VETH" == "True" ]; then
        ip link show $Q_PUBLIC_VETH_INT > /dev/null 2>&1 ||
        sudo ip link add $Q_PUBLIC_VETH_INT type veth peer name $Q_PUBLIC_VETH_EX
        sudo ip link set $Q_PUBLIC_VETH_INT up
        sudo ip link set $Q_PUBLIC_VETH_EX up
        sudo ip addr flush dev $Q_PUBLIC_VETH_EX
    else
        ovs_dpdk_add_bridge $PUBLIC_BRIDGE
    fi
}

function set_vcpu_pin_set {
    OVS_CORE_MASK=$(echo $OVS_CORE_MASK | sed 's/^0x//')
    OVS_PMD_CORE_MASK=$(echo $OVS_PMD_CORE_MASK | sed 's/^0x//')
    BAD_CORES=$((`echo $((16#${OVS_CORE_MASK}))` | `echo $((16#${OVS_PMD_CORE_MASK}))`))
    TOTAL_CORES=`nproc`
    vcpu_pin_set=""

    for cpu in $(seq 0 `expr $TOTAL_CORES - 1`);do
        tmp=`echo 2^$cpu | bc`
        if [ $(($tmp & $BAD_CORES)) -eq 0 ]; then
            vcpu_pin_set+=$cpu","
        fi
    done
    vcpu_pin_set=${vcpu_pin_set::-1}

    if is_service_enabled nova; then
        iniset $NOVA_CONF DEFAULT vcpu_pin_set $vcpu_pin_set
    fi
}

function update_ovs_pmd_core_mask {
    OVS_CORE_MASK=$(echo $OVS_CORE_MASK | sed 's/^0x//')
    OVS_PMD_CORE_MASK=$(echo $OVS_PMD_CORE_MASK | sed 's/^0x//')

    if [ $OVS_PMD_CORE_MASK -eq 4 ]; then
        #default value, check for siblings in case of hyperthreading enabled
        SIBLINGS=""
        RESULT=0
        FILE="/sys/devices/system/cpu/cpu3/topology/thread_siblings_list"
        if [ -e $FILE ]; then
            SIBLINGS=`cat $FILE`
        else
            echo "warning: don't know how to check siblings"
            SIBLINGS=3
        fi

        for SIBLING in $(echo $SIBLINGS | sed -n 1'p' | tr ',' '\n'); do
            SIBLING_CORE=`echo "obase=10;$((1<<($SIBLING-1)))" | bc`
            RESULT=$(($RESULT | $SIBLING_CORE))
        done

        OVS_PMD_CORE_MASK=`printf "%x" $RESULT`
    fi
}