#!/usr/bin/python -u

import os
import subprocess
import re
import stat
import logging
import socket
import argparse
from os import environ
from novaclient.v1_1 import Client
from time import sleep, time, strftime, localtime, gmtime
import paramiko

class TestNetperf() :
    """
    Will attempt to launch some instances and then run netperf in them
    Leverages heavily from scripts pointed-at by Gavin Brebner
    """

    def __init__(self) :
        self.verbose = 1
        self.sshopts = " -o StrictHostKeyChecking=no -o HashKnownHosts=no -o ConnectTimeout=30 "

        self.os = None
        self.ip_pool = []
        self.vm_pool = []
        self.adminPasses = dict()
        self.suitable_images = []
        self.chosen_image = None
        self.keypair = None
        self.start = time()
        self.this_run_started = strftime("%Y-%m-%d-%H-%M",gmtime()) + "/"

    def fail(self,message) :
        """
        print out a final utterance and die
        """
        logging.critical(message)
        print strftime("%s, %d, %b %Y %H:%M:%S", localtime())+" FAILURE "+message
        os._exit(1)


    def do_in_shell(self, command, may_fail = False) :
        """
        do a task in the shell. by default it must succeed
        """
        p=subprocess.Popen(command, 
                           shell=True,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT)
        (dout, eout)=p.communicate()
        if (eout == None) :
            eout=""

        if (p.returncode != 0) and (may_fail == False):
            logging.debug("Shell command '%s' failed with eout '%s' and dout '%s'",
                          command, eout, dout)
            raise RuntimeError

        return dout

    # we will need to handle IDs that are UUIDs at some point
    def find_suitable_images(self) :
        """ 
        Try to find a suitable image to use for the testing
        """
        image_regex = None
        image_id = None
        if self.args.image :
            img = self.args.image
        else:
            try:
                img = environ['IMAGE']
            except:
                img = "netperf"

        try:
            image_id = int(img)
        except:
            image_regex =  img

        for image in self.os.images.list() :
            if ((image_regex and re.search(image_regex,image.name))
                or
                (image_id and (image_id == int(image.id)))) :
                self.suitable_images.append(image)


    def await_server_pool_gone(self,time_limit) :
        """
        Keep checking the status of the server pool until it is gone or
        our patience has run-out.  Alas, novaclient.v1_1 is not sophisticated
        enough to return "ENOEND" or somesuch as a status when the specified
        server is well and truly gone.  So we get status until there is an 
        exception raise, which we interpret as "server gone"
        """
        waiting_started = time()
        now = time()
        sleeptime = 1

        while ((len(self.vm_pool) > 0) and
               ((now - waiting_started) < time_limit)) :
            waiter = self.vm_pool.pop()
            try: 
                waiter = self.os.servers.get(waiter.id)

                if (re.search(r"ERROR",waiter.status)) :
                    self.clean_up_overall()
                    self.fail("Instance id " + str(waiter.id) + " name " + str(waiter.name) + " errored during deletion " + str(waiter.status))

                self.vm_pool.append(waiter)
                sleep(sleeptime)
                sleeptime *= 2
                now = time()
            except :
                sleeptime = 1
                continue

        if (len(self.vm_pool) > 0) :
            self.clean_up_overall()
            self.fail(str(len(self.vm_pool)) + " failed to achieve 'deleted' status within " + str(time_limit) + " seconds")

    def associate_public_ips(self) :
        """
        Associate our public IPs with instances
        """
        for (vm, ip) in zip(self.vm_pool, self.ip_pool) :
            vm.add_floating_ip(ip.ip)


    def disassociate_public_ips(self) :
        """
        Break the association between an instance and its floating IP
        """
        for (vm, ip) in zip(self.vm_pool, self.ip_pool) :
            vm.remove_floating_ip(ip.ip)

    def extract_public_ip(self, vm) :
        """
        Find the public IP of a VM
        """
        if "rackspace" in self.env('OS_AUTH_SYSTEM',default=[]):
            ip = vm.accessIPv4
        else:
            try:
                #print "trying networks['hpcloud']"
                iplist = vm.networks['hpcloud']
            except Exception as e2 :
                #print "exception 2"
                try :
                    #print "trying networks['private']"
                    iplist = vm.networks['private']
                except Exception as e3 :
                    #print "exception 3"
                    logging.warning("Unable to find a public IP for %s (%s:%s:$s)",
                                    vm.name, e, e2, e3)
                    raise RuntimeError
            ip = iplist[1]

        #print "returning %s" % ip
        return ip

    def deallocate_server_pool(self) :
        """
        Issue delete requests on any and all existing servers
        """
#        print "Length of server pool %d" %len(self.vm_pool)

        for server in self.vm_pool :
            try:
                try:
                    # try to remove the public ip for this server from
                    # the known hosts file but don't get too hung up
                    # about it if we cannot.  If we happen to
                    # encounter this IP again in another test we will
                    # rely on its being removed before first use to
                    # cover things
                    public_ip = self.extract_public_ip(server)
                    self.known_hosts_ip_remove(public_ip)
                except:
                    pass
                self.os.servers.delete(server)
            except Exception as e:
                logging.warning("Unable to delete server %s. Error: %s",
                                server.name, e)

    def allocate_server_pool(self, count, flavor, image) :
        """
        Allocate the number of requested servers of the specified 
        flavor and image, and with the specified key name
        """

        #print "before server_pool opening warning"
        keyname = None
        secgroup = None
        if self.keypair:
            keyname = self.keypair.name
        if self.security_group:
            secgroup = [self.security_group.name]

        logging.warning("Allocating %d servers of flavor '%s' with image name '%s'",
                        count, flavor.name, image.name)
        #print "after same"

        # instance number (filled-in later) time flavorname imagename
        basename = "netperftest%s_%s_%s_%s" % ("%d",str(self.start),flavor.name,image.name)
        for i in xrange(count) :
            thisname = basename % i
            try:
                newvm = self.os.servers.create(name = thisname,
                                               image = image,
                                               flavor = flavor,
                                               key_name = keyname,
                                               security_groups = secgroup)
                #print "adminPass is %s" % newvm.adminPass
                self.adminPasses[newvm.id]=newvm.adminPass
                self.vm_pool.append(newvm)

            except Exception as e :
                logging.warning("Error allocating instance: %s" , e)
                raise RuntimeError



    def await_server_pool_ready(self, time_limit) :
        """
        Wait until all the servers in the server pool are in an "ACTIVE" state.
        If they happen to come-up in an "ERROR" state we are toast.  If it 
        takes too long for them to transition to the "ACTIVE" state we are
        similarly toast
        """
        waiters = list(self.vm_pool)
        actives = []
        waiting_started = time()
        deadline = waiting_started + time_limit
        sleeptime = 1

        #print "awaiting server pool active"
        while (len(waiters) > 0) :

            waiter = waiters.pop()
            waiter = self.os.servers.get(waiter.id)
            print "elapsed %d server %s state %s" % ((time() - waiting_started), waiter.name, waiter.status)
            if (waiter.status == "ACTIVE") :
                #print "Extracting public IP"
                ip_address = self.extract_public_ip(waiter)
                #print "got IP address from extract"
                #print "IP address is %s" % ip_address
                self.known_hosts_ip_remove(ip_address)
                actives.append(waiter)
                sleeptime = 1
                continue
            if (re.search(r"ERROR",waiter.status)) :
                logging.warning("Instance id %s name %s encountered an error during instantiation of %s",
                                waiter.id, waiter.name, waiter.status)
                raise RuntimeError

            if (time() >= deadline):
                break

            waiters.append(waiter)
            sleep(sleeptime)
            sleeptime *= 2

            sleeptime = min(120, deadline - time(), sleeptime)


        if (len(waiters) > 0) :
            logging.warning("%d waiters failed to acheive ACTIVE status within %d seconds",
                            len(waiters), time_liit)
            raise RuntimeError

        # I wonder what the right way to do this is
        self.vm_pool = []
        self.vm_pool = list(actives)

    def print_ip_pool(self) :
        """
        Display the current contents of the IP pool
        """
        logging.debug("Contents of IP pool:")
        for ip in self.ip_pool :
            logging.debug("IP: %s  ID: %s",
                          ip.ip, ip.id)

    def known_hosts_ip_remove(self, ip_address) :
        """
        Remove an IP address from the known_hosts file
        """
        cmd = "ssh-keygen -f ~/.ssh/known_hosts -R " + str(ip_address) + " > /dev/null"
        logging.debug("Removing %s from ~/.ssh/known_hosts using command '%s'",
                      ip_address, cmd)
        self.do_in_shell(cmd,True)


    def deallocate_ip_pool(self) :
        """
        Return the contents of self.ip_pool to the sea
        """
        while (len(self.ip_pool) > 0) :
            ip = self.ip_pool.pop()
            try:
                self.os.floating_ips.delete(ip.id)
            except Exception as e:
                logging.warning("Unable to deallocate IP: %s  ID: %s  Error: %s",
                                ip.ip, ip.id, e)

    def allocate_ip_pool(self, count) :
        """
        Allocate a pool of public IPs to be used when perf testing
        """

        for i in xrange(count) :
            try :
                ip = self.os.floating_ips.create()
                self.ip_pool.append(ip)
            except Exception as e:
                self.deallocate_ip_pool()
                self.fail("Error while allocating %d IPs '%s'" %
                          (count, str(e)))
            self.known_hosts_ip_remove(str(ip.ip))

    def clean_vestigial_keypairs(self) :
        """
        Seek-out and delete vestigial keypairs matching our magic
        regular expression
        """

        for keypair in self.os.keypairs.list() :
            if (re.match("net[0-9]+perf[0-9]+dot[0-9]+", str(keypair.name))) :
                logging.info("Deleting vestigial keypair named %s",
                             keypair.name)
                try :
                    self.os.keypairs.delete(keypair)
                    os.remove(str(keypair.name) + ".pem")
                except Exception as e:
                    logging.warning("Exception while purging keypair %s '%s'",
                                    keypair.name,e)
                    continue

    def create_security_group(self):

        #print "Creating security group"

        self.security_group = self.os.security_groups.create("netperftesting " + str(self.start),"A rather open security group for netperf testing debugging")

        self.os.security_group_rules.create(parent_group_id = self.security_group.id,
                                            ip_protocol = 'tcp',
                                            from_port = 1,
                                            to_port = 65535)

        self.os.security_group_rules.create(parent_group_id = self.security_group.id,
                                            ip_protocol = 'udp',
                                            from_port = 1,
                                            to_port = 65535)
        
        self.os.security_group_rules.create(parent_group_id = self.security_group.id,
                                            ip_protocol = 'icmp',
                                            from_port = -1,
                                            to_port = -1)

    def allocate_key(self) :
        """
        Allocate the private key to be used with instances for this
        session of testing, and write it to a file
        """

        pid = self.do_in_shell("echo $$")

        self.uniquekeyname = re.sub(r"\n", "", "net" + str(pid) + 
                                    "perf" + str(self.start))
        self.uniquekeyname = re.sub(r"\.", "dot", self.uniquekeyname)

        #print "Allocating key %s" % self.uniquekeyname

        self.keypair = self.os.keypairs.create(self.uniquekeyname)

        self.keyfilename = self.keypair.name + ".pem"

        keyfile = None
        try :
            keyfile = open(self.keyfilename, 'w')
        except Exception as e:
            if (keyfile != None) :
                keyfile.close()
            self.fail("Error opening " + self.keyfilename +
                      "for writing: " + str(e))

        try :
            keyfile.write(self.keypair.private_key)
        except Exception as e:
            self.fail("Error writing to " + self.keyfilename + str(e))
        finally:
            keyfile.close()

        os.chmod(self.keyfilename, stat.S_IRWXU)

    # lifted quite liberally from python-novaclient/novaclient/utils.py
    def env(self, *args, **kwargs):
        """
        returns the first environment variable set
        if none are non-empty, defaults to '' or keyword arg default
        """
        for arg in args:
            value = os.environ.get(arg, None)
            if value:
                return value

        if kwargs.get('failure',None):
            print "found failure"
            self.fail(kwargs.get('failure'))

        return kwargs.get('default', '')
        
    def setup_parser(self) :
        parser = argparse.ArgumentParser()
        parser.add_argument("-f", "--flavor",
                            help="Specify the flavor to test")
        parser.add_argument("--image",
                            help="Specify the image type to test")
        parser.add_argument("--region",
                            default=self.env('OS_REGION_NAME',
                                             'NOVA_REGION_NAME',
                                             failure="A Nova region name must be specified via either --region, OS_REGION_NAME or NOVA_REGION_NAME"),
                            help="Specify the region in which to test")
        parser.add_argument("--username",
                            default=self.env('OS_USERNAME',
                                             'NOVA_USERNAME',
                                             failure="A username for Nova access must be specified either via --username, OS_USERNAME, or NOVA_USERNAME"),
                            help="The name of the user to use")
        parser.add_argument("--password",
                            default=self.env('OS_PASSWORD',
                                             "NOVA_PASSWORD",
                                             failure="A password for Nova access must be specified via either --password, OS_PASSWORD or NOVA_PASSWORD"),
                            help="The password for the Nova user")
        parser.add_argument("--url",
                            default=self.env('OS_AUTH_URL',
                                             'NOVA_URL',
                                             failure="A URL for authorization must be provided via either --url, OS_AUTH_URL or NOVA_URL"),
                            help="The URL for Nova")
        parser.add_argument("--project",
                            default=self.env('OS_TENANT_NAME',
                                             'NOVA_PROJECT_ID',
                                             failure="You must provide a project via --project, OS_TENANT_NAME or NOVA_PROJECT_ID"),
                            help="The ID for the Nova project")

        parser.add_argument("--archivepath",
                            help="The path to where the archive is to be kept")
        parser.add_argument("--collectdsockname",
                            help="The name of the collectd socket.")
        parser.add_argument("--instanceuser",
                            default=self.env('INSTANCE_USERNAME', default='ubuntppppu'),
                            help="The name of the user on the instance(s)")

        return parser

    def get_flavor(self) :
        flav = None
        if self.args.flavor :
            flav = self.args.flavor
        else:
            try:
                self.flavor_name = environ['FLAVOR']
            except:
                return

        try:
            self.flavor_id = int(flav)
        except:
            self.flavor_name = flav
                    
    def get_collectd_sock(self) :
        self.collectd_socket = None
        if self.args.collectdsockname :
            self.collectd_socket_name = self.args.collectdsockname

        try:
            self.collectd_socket_name = environ['COLLECTD_SOCK_NAME']
        except:
#            self.collectd_socket_name = "/opt/collectd/collectd-socket"
            return

        if self.collectd_socket_name:
            try:
                self.collectd_socket = socket.socket(socket.AF_UNIX,
                                                     socket.SOCK_STREAM)
                self.collectd_socket.connect(self.collectd_socket_name)
            except Exception as e:
                if self.collectd_socket:
                    self.collectd_socket.close()
                self.collectd_socket = None
                logging.warning("Unable to open %s error %s ",
                                self.collectd_socket_name, e)


    def get_archive_path(self) :
        if self.args.archivepath :
            self.archive_path = self.args.archivepath
        else:
            try:
                self.archive_path = environ['ARCHIVE_PATH']
            except:
                self.archive_path = "Archive"

        self.archive_path = os.path.join(self.archive_path,
                                         self.args.region,
                                         self.this_run_started)
        logging.debug("Asking to make %s", self.archive_path)
        try:
            os.makedirs(self.archive_path)
        except Exception as e:
            if os.path.isdir(self.archive_path):
                # already there
                pass
            else:
                logging.warning("Archive path %s could not be created %s",
                                self.archive_path,e)
                raise RuntimeError

        return

    def initialize(self) :
        self.flavor_name = None
        self.flavor_id = None
        self.archive_path = None
        self.collectd_sock_name = None

        parser = self.setup_parser()

        self.args = parser.parse_args()

        self.get_flavor()
        self.get_collectd_sock()
        self.get_archive_path()

        self.os = Client(self.args.username,
                         self.args.password,
                         self.args.project,
                         self.args.url,
                         region_name = self.args.region)

        if (self.os == None) :
            self.fail("OpenStack API connection setup unsuccessful!")

#        self.clean_vestigial_keypairs()

        if not "rackspace" in self.env('OS_AUTH_SYSTEM',default=[]):
            self.allocate_key()
            self.create_security_group()
        else:
            self.security_group = None
            self.keypair = None

        self.flavor_list = self.os.flavors.list()

#        self.ip_count = 3
#        self.allocate_ip_pool(self.ip_count)
#        self.print_ip_pool()

        self.find_suitable_images()
        # for now just take the first one

        if self.suitable_images:
            self.chosen_image = self.suitable_images.pop()
            logging.warning("The chosen image is %s",
                         self.chosen_image.name)
        else:
            self.fail("Unable to find a suitable image to test")

    def create_remote_hosts(self) :
        """
        Create the remote_hosts file that will be used by the
        runemomniaggdemo.sh script.  if the script is ever enhanced to
        make sure it does not run a loopback test, we can just dump
        all the IP addresses in there.
        """
        first = True
        index = 0

        try :
            fl = open("remote_hosts", "w")
            for server in self.vm_pool :
                if not first :
                    public_ip = self.extract_public_ip(server)
                    fl.write("REMOTE_HOSTS["+str(index)+"]="+str(public_ip)+"\n")
                    index += 1
                else :
                    first = False
            fl.write("NUM_REMOTE_HOSTS="+str(index)+"\n")
        except Exception as e:
#            self.clean_up_instances()
            logging.warning("Unable to create/write remote_hosts file. %s ",
                            e)
            raise RuntimeError
        finally:
            fl.close()

    def scp_binaries(self) :
        """
        Scp all the binaries to the nodes.  Unlke the ssh_check, we
        will bail on any failures.  in the name of simplicity/laziness
        we will copy everything to everyone.
        """
        upload_files = ("netperf", "netserver", "runemomniaggdemo.sh", "find_max_burst.sh", "remote_hosts")
        for node in self.vm_pool :
            publicip = self.extract_public_ip(node)
            try :
                transport = paramiko.Transport((publicip,22))
                if self.keypair:
                    mykey = paramiko.RSAKey.from_private_key_file(os.path.abspath(self.keyfilename))
                    transport.connect(username=self.args.instanceuser,
                                      pkey = mykey)
                else:
                    transport.connect(username=self.args.instanceuser,
                                      password=self.adminPasses[node.id])

                sftp = paramiko.SFTPClient.from_transport(transport)
                for file in upload_files:
                    sftp.put(file,file)
                    mode = sftp.stat(file).st_mode
                    # I would really rather not have to do this, but
                    # SFTP, unlike scp, does not seem to preserve file
                    # permissions.
                    sftp.chmod(file, mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
                sftp.close()
                transport.close()
            except Exception as e :
                logging.warning("The copying of files to %s failed.",
                                node.name, e)
                raise RuntimeError

    def start_netservers(self) :
        """
        Start netservers on every node.
        """

        for node in self.vm_pool :
            publicip = self.extract_public_ip(node)
            sudo = "sudo"
            if self.args.instanceuser == 'root':
                sudo = ""

            cmd = "%s ./netserver -p 12865" % sudo

            try :
                ssh = paramiko.SSHClient()
                ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
                if self.keypair:
                    ssh.connect(hostname=publicip,
                                username=self.args.instanceuser,
                                key_filename=os.path.abspath(self.keyfilename),
                                timeout=30.0)
                else:
                    ssh.connect(hostname=publicip,
                                username=self.args.instanceuser,
                                password=self.adminPasses[node.id],
                                timeout=30.0)

                stdin, stdout, stderr = ssh.exec_command(cmd)
#                print stdout.readlines()
#                print stderr.readlines()
                ssh.close()

            except Exception as e:
                logging.warning("Starting netserver processes was unsuccessful on %s at IP %s",
                                node.name, publicip)
                raise RuntimeError

    def run_netperf(self) :
        """
        Actually run the runemomniaggdemo.sh script on the first node
        """
        try :
            publicip = self.extract_public_ip(self.vm_pool[0])
            cmd = "export PATH=$PATH:. ; ./runemomniaggdemo.sh | tee overall.log "
            ssh = paramiko.SSHClient()
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            if self.keypair:
                ssh.connect(hostname=publicip,
                            username=self.args.instanceuser,
                            key_filename=os.path.abspath(self.keyfilename),
                            timeout=30.0)
            else:
                ssh.connect(hostname=publicip,
                            username=self.args.instanceuser,
                            password=self.adminPasses[self.vm_pool[0].id],
                            timeout=30.0)

            stdin, stdout, stderr = ssh.exec_command(cmd)
            # if would seem that unless one does something with the
            # output the exec_command will complete rather quickly,
            # which is not what we want right here so we will await
            # the exit status of the underlying channel, whatever that
            # is :) but it is what I found in a web search :)
            status = stdout.channel.recv_exit_status()
            ssh.close()
        except Exception as e:
            logging.warning("Could not run the netperf script on %s at IP %s %s",
                            self.vm_pool[0].name, publicip,str(e))
            raise RuntimeError

    def copy_back_results(self, destination) :
        try :
            publicip = self.extract_public_ip(self.vm_pool[0])
            logging.warning("Copying-back results")
            transport = paramiko.Transport((publicip,22))
            if self.keypair:
                mykey = paramiko.RSAKey.from_private_key_file(os.path.abspath(self.keyfilename))
                transport.connect(username=self.args.instanceuser,
                                  pkey = mykey)
            else:
                transport.connect(username=self.args.instanceuser,
                                  password=self.adminPasses[self.vm_pool[0].id])
                
            sftp = paramiko.SFTPClient.from_transport(transport)
            for file in sftp.listdir():
                # one of these days I should read-up on how to make
                # that one regular expression
                if (re.match(r".*\.log",file) or
                    re.match(r".*\.out",file) or
                    re.match(r".*\.txt",file)):
                    sftp.get(file,destination+file)
            sftp.close()
        except Exception as e:
           logging.exception("Could not retrieve results from %s at IP %s",
                             self.vm_pool[0].name, publicip)
           raise RuntimeError

    def ssh_check(self, time_limit=300) :
        """
        SSH into the VMs and report on progress. we ass-u-me that the
        VMs are already in a happy, running state, or will be within
        the time_limit.  We give-up only if there is a problem after
        our time_limit.  So long as the ssh's remain error-free we
        will keep going as long as it takes to get through the list.
        """
        sshers = list(self.vm_pool)
        starttime = time()
        deadline = starttime + time_limit
        sleeptime = 1

        while (len(sshers) > 0) :
            ssher = sshers.pop()
            publicip = self.extract_public_ip(ssher)
            sudo = "sudo"
            if self.args.instanceuser == 'root':
                sudo = ""

            cmd = "hostname; uname -a; date; %s sysctl -w net.core.rmem_max=1048576 net.core.wmem_max=1048576" % sudo

            try:
                ssh = paramiko.SSHClient()
                ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
                # paper or plastic?
                if self.keypair:
                    ssh.connect(hostname=publicip,
                                username=self.args.instanceuser,
                                key_filename=os.path.abspath(self.keyfilename),
                                timeout=30.0)
                else:
                    #print "ssh.connect to %s user %s pass %s" % (publicip,self.args.instanceuser,self.adminPasses[ssher.id])
                    ssh.connect(hostname=publicip,
                                username=self.args.instanceuser,
                                password=self.adminPasses[ssher.id],
                                timeout=30.0)

                stdin, stdout, stderr = ssh.exec_command(cmd)
#                print stdout.readlines()
#                print stderr.readlines()
                ssh.close()
                sleeptime = 1
            except Exception as e :
                now = time()
                #print "Attempt to ssh failed with %s" % str(e)
                if (now < deadline) :
                    sleep(sleeptime)
                    sleeptime *= 2
                    sleeptime = min(sleeptime, 120, deadline - now)
                    sshers.append(ssher)
                else :
                    logging.warning("The ssh check to %s (IP:%s ID:%s) using command '%s' failed after the %d second limit (%s)",
                                    ssher.name, publicip, ssher.id, cmd, time_limit,str(e))
                    raise RuntimeError

    def extract_min_avg_max(self, flavor, name, postresults) :

        avg = min = max = end = start ="0"
        units = "unknown"

        logging.warning("Post-processing script output")

        for line in postresults.splitlines() :
            if (re.match("Average of peak interval is",line)) :
                toks = line.split(" ",11)
                avg = toks[5]
                start = toks[8]
                end= toks[10]
                logging.debug("Found Average %s from %s to %s",
                              avg, start, end)

            if (re.match("Minimum of peak interval is",line)) :
                toks = line.split(" ",11)
                min = toks[5]
                logging.debug("Found Minimum of peak interval %s ", min)
            if (re.match("Maximum of peak interval is",line)) :
                toks = line.split(" ",11)
                max = toks[5]
                logging.debug("Found Maximum of peak interval %s", max)
        try:

            # I suppose one could split name on '_' to find the testtype?
            testtype="unknown"
            if (re.search("tps",name)) :
                testtype="tps"
                units="TPS"
            if (re.search("sync",name)):
                testtype="synctps"
                units="TPS"
            if (re.search("inbound",name)) :
                testtype="inbound"
                units="BPS"
            if (re.search("outbound",name)):
                testtype="outbound"
                units="BPS"
            if (re.search("bidirectional",name)):
                testtype="bidirectional"
                units="BPS"

            if (self.collectd_socket != None):
                self.collectd_socket.send("PUTVAL \"netperf-"+self.args.region+"/unixsock-"+flavor.name+"-"+testtype+"/min\" interval=900 "+end+":"+min+"\n")
                self.collectd_socket.send("PUTVAL \"netperf-"+self.args.region+"/unixsock-"+flavor.name+"-"+testtype+"/avg\" interval=900 "+end+":"+avg+"\n")
                self.collectd_socket.send("PUTVAL \"netperf-"+self.args.region+"/unixsock-"+flavor.name+"-"+testtype+"/max\" interval=900 "+end+":"+max+"\n")
                logging.warning("Wrote results for "+self.args.region+" "+flavor.name+" "+testtype+" to collectd")


            prefix = self.args.region + ":" + self.chosen_image.name + ":" + flavor.name + ":" + testtype
            suffix = ":" + units + ":" + end
            logging.warning(prefix + ":min:" + min + suffix)
            logging.warning(prefix + ":avg:" + avg + suffix)
            logging.warning(prefix + ":max:" + max + suffix)

        except Exception as e:
            logging.warning("Post process fubar %s",e)
            raise RuntimeError

    def post_proc_flavor(self,flavor) :
        """
        Post process the results of a given flavor
        """
        logging.info("Post-processing flavor: %s ", flavor.name)
        archive_location = os.path.join(self.archive_path,
                                        str(flavor.name),
                                        "")

        for root, dirs, files in os.walk(archive_location, topdown=False):
            for name in files:
                if (re.match("netperf_.*\.log",name)) :
                    fullname = os.path.join(root,name)
                    cmd = 'PATH=$PATH:. post_proc.py "%s"' % fullname
                    logging.debug("post processing via command '%s'", cmd)
                    try :
                        postresults = self.do_in_shell(cmd,True)
                        # so, look at vrules to get the times
                        self.extract_min_avg_max(flavor, name, postresults)
                    except Exception as e:
                        logging.warning("Unable to post-process flavor %s %s results",
                                        flavor.name, fullname)
                        raise RuntimeError


    def test_flavor(self, flavor) :
        """
        Start the right number of instances of the specified flavor
        """

        logging.warning("Testing flavor: %s in region %s",
                     flavor.name, self.args.region)
        archive_location = os.path.join(self.archive_path,
                                        str(flavor.name),
                                        "")
        os.makedirs(archive_location)

        #print "Calling for server pool allocation"
        self.allocate_server_pool(5,
                                  flavor,
                                  self.chosen_image)

        logging.warning("Awaiting server pool active")
        self.await_server_pool_ready(600)
#        self.associate_public_ips()
        logging.warning("Server pool active. Verifying up and running via ssh")
        self.ssh_check()
        logging.warning("ssh_check complete")
        self.create_remote_hosts()
        logging.warning("remote_hosts file creation complete")
        self.scp_binaries()
        logging.warning("transfer of binaries complete.")
        self.start_netservers()
        logging.warning("netservers started. about to start the script")
        self.run_netperf()
        logging.warning("netperf run complete. copying back results")
        self.copy_back_results(archive_location)
        logging.warning("cleaning up vms")
#       self.disassociate_public_ips()
        self.deallocate_server_pool()
        self.await_server_pool_gone(600)
        logging.warning("Server pool deallocated")



    def test_flavors(self) :
        """
        Iterate through all the available flavors and test them if
        there is a match on either the name or id, or if neither have
        been specified
        """
        for flavor in self.flavor_list :
            #print "Checking flavor name %s id %s" % (flavor.name, flavor.id)
            try:
                if ((not self.flavor_name and not self.flavor_id)
                    or
                    (self.flavor_name and re.search(self.flavor_name,flavor.name))
                    or
                    (self.flavor_id and (self.flavor_id == int(flavor.id)))) :

                    self.test_flavor(flavor)
                    # we could move the post-processing inside
                    # test_flavor to overlap it with the deletion of
                    # the VMs but for the time being we will leave it
                    # here
                    self.post_proc_flavor(flavor)
            except:
                self.clean_up_instances()
                logging.warning("Test of flavor '%s' (%s) unsuccessful",
                                flavor.name,
                                flavor.id)


    def clean_up_instances(self) :

        import inspect

        curframe = inspect.currentframe()
        calframe = inspect.getouterframes(curframe, 2)
        logging.warning("Asked by %s to clean-up instances with pool len %d",
                        calframe[1][3],
                        len(self.vm_pool))
        if (self.vm_pool != []):
            self.deallocate_server_pool()
            self.await_server_pool_gone(600)

    def clean_up_overall(self) :
        """
        Clean some of the things which were created before and not
        flavor-specific
        """

        logging.info("Asked to clean-up everything left")

        if (self.keypair != None) :
            os.remove(self.keypair.name + ".pem")
            self.os.keypairs.delete(self.keypair)
            self.keypair = None


        if (self.ip_pool != []) :
            self.deallocate_ip_pool()
                          
        self.clean_up_instances()

        if self.security_group :
            for rule in self.security_group.rules :
                self.os.security_group_rules.delete(rule['id'])

            self.os.security_groups.delete(self.security_group.id)
            self.security_group = None

if __name__ == '__main__' :
    print "Hello World! Let us run some netperf shall we?"

    logging.basicConfig(level=logging.WARNING, format="%(asctime)s %(message)s")
    tn = TestNetperf()
    try:
        tn.initialize()
    except Exception as e:
        logging.exception("Initialization failed:")
        tn.clean_up_overall()
        tn.fail("Terminating in reponse to initialization failure")

    print "testing flavors"
    tn.test_flavors()
    tn.clean_up_overall()