#!/usr/bin/python3

# swantest: run kvmpluto and dockerpluto tests
#
# Copyright (C) 2016-2017 Antony Antony
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version.  See <https://www.gnu.org/licenses/gpl2.txt>.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
# or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
# for more details.

import threading
import time
import os
import subprocess
import getopt
import sys
import sys
import re
import json
import os
import sys
import socket
import shutil
import logging
import random
import subprocess
import concurrent.futures
import collections
import datetime
import csv
import operator
import signal
import socket

from os import listdir
from os.path import isfile, join, islink
from multiprocessing import Process, Queue
from json import encoder
from contextlib import contextmanager

try:
    import argparse
    import pexpect
    #import setproctitle
except ImportError as e:
    module = str(e)[16:]
    sys.exit("we require the python module %s" % module)

STOP_TESTS_NOW = 'stop-tests-now'

# hosts = ['east', 'west', 'road', 'nic', 'north']
r_init = threading.Event()
i_ran = threading.Event()
n_init = threading.Event()
result_file_lock = threading.Lock()

logger = logging.getLogger()
ch = logging.StreamHandler()
# modify the logger not to log log level and user name
# formatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
formatter = logging.Formatter('%(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

tq = Queue()
tp = Queue()  # test progress queue

dqstart = Queue()
dqstop = Queue()

parent_pid = os.getpid()


def handler(signum, frame):
    ppid = os.getpid()
    pids = subprocess.getoutput("ps --ppid %d -o \"pid=\"" % ppid).split()
    for p in pids:
        try:
            logging.info("terminate child %d and part %d , %d", int(p), int(ppid),
                         int(parent_pid))
            os.kill(int(p), signal.SIGKILL)
            os.kill(int(p), 0)
        except:
            pass
    if parent_pid == ppid:
        logging.info("parent process stop the loop %d", ppid)
        sys.exit(1)
    else:
        logging.info("child process shutdown done %d", ppid)
        sys.exit(1)


class PipeworkEroor():

    """General exception for pipework errors."""
    pass


def grep_4_known_errors(test):
    for host in test["test_hosts"]:
        tpath = os.getcwd()

        plutolog = tpath + '/OUTPUT/' + host + '.pluto.log'
        if not os.path.exists(plutolog):
            plutolog = plutolog + '.gz'
        conslelog = tpath + '/OUTPUT/' + host + '.console.txt'
        if not os.path.exists(conslelog):
            conslelog = conslelog + '.gz'

        conslelog_verbose = tpath + '/OUTPUT/' + host + '.console.verbose.txt'
        if not os.path.exists(conslelog_verbose):
            conslelog_verbose = conslelog_verbose + '.gz'

        if os.path.exists(plutolog):
            grep_n_add(test, host, plutolog, 'ASSERTION FAILED', "ASSERT")
            grep_n_add(test, host, plutolog, 'EXPECTATION FAILED', "EXPECT")

        if os.path.exists(conslelog):
            grep_n_add(test, host, conslelog, 'segfault', "SEGFAULT")
            grep_n_add(test, host, conslelog, 'general protection', "GPFAULT")

        if os.path.exists(conslelog_verbose):
            grep_n_add(test, host, conslelog_verbose, "^CORE FOUND", "CORE")


def grep_n_add(test, host, filename, pattern, note):
    if not os.path.exists(filename):
        return

    fixed = '-F '
    fixed = ''
    key = "%s-status" % host
    status = ''

    cmd = "zgrep " + fixed + "'" + pattern + "'  " + filename
    match = subprocess.getoutput(cmd)
    logging.debug("%s" % cmd)
    if match:
        logging.info("%s %s" % (host, match))
        logging.debug(("%s %s %s" % (host, cmd, match)))
        if key in test:
            test[key] += " " + note
        else:
            test[key] = note


class guest (threading.Thread):

    def __init__(self, hostname, role, testname, args, prefix=None, install=None, compile=None, ipsecstop=None):
        threading.Thread.__init__(self)
        self.hostname = hostname
        self.dname = hostname
        if prefix:
            self.dname = prefix + hostname
        self.role = role
        self.testname = testname
        self.status = "NEW"
        self.bootwait = args.bootwait
        self.stoponerror = args.stoponerror
        self.testingdir = args.testingdir
        self.reboot = True 
        if args.noreboot:
            self.reboot = None
        self.ipsecstop = ipsecstop

    def run(self):
        try:
            self.start = time.time()
            e = self.boot_n_grab_console()
            self.status = "INIT"
            if e:
                self.clean_abort()
            else:
                e = self.run_test()

                if not e:
                    e = "end"
            self.log_line('OUTPUT/RESULT', e)
            logging.info("%s done %s ran %.2fs %s", self.dname, self.testname,
                         time.time() - self.start, e)
        except Exception as x:
            # XXXX: Bad, but easiest way to get rid of a deadlock when
            # things go wrong
            self.clean_abort()
            raise x

    def boot_n_grab_console(self):
        e = self.connect_to_kvm()
        if e:
            self.clean_abort()
            return e

    def clean_abort(self):
        # we are aborting: set all waiting events end of show.
        logging.debug("%s set all events to abort", self.dname)
        r_init.set()
        n_init.set()
        i_ran.set()

    def run_test(self):

        # now on we MUST match the entire prompt,
        # or else we end up sending too soon and getting mangling!

        prompt = "\[root@%s %s\]# " % (self.hostname, self.testname)
        logging.debug("%s role %s running test %s prompt %s",
                      self.dname, self.role, self.testname, prompt.replace("\\", ""))

        child = self.child
        timer = 120
        ret = True

        cmd = "cd %s/pluto/%s" % (self.testingdir, self.testname)
        print("sending command: " + cmd + " expecting prompt: " + prompt)
        child.sendline(str(cmd))
        try:
            child.expect(prompt, searchwindowsize=100, timeout=timer)
        except:
            err = "failed [%s] test directory on %s" % (cmd, self.dname)
            logging.error("%s", err)
            return err

        if self.role == "initiator":
            start = time.time()
            logging.info("%s wait for responder and maybe nic to initialize",
                         self.hostname)
            n_init.wait()
            logging.debug("%s wait for responder to initialize", self.hostname)
            r_init.wait()
            logging.info("%s initiator is ready (waited %.2fs)", self.hostname,
                         time.time() - start)

        output_file = "OUTPUT/%s.console.verbose.txt" % (self.hostname)
        f = open(output_file, 'w')
        child.logfile = f

        self.status = "INIT-RUN"
        cmd = "./%sinit.sh" % (self.hostname)
        e = read_exec_shell_cmd(child, cmd, prompt, timer, self.hostname)

        if (self.role == "responder"):
            r_init.set()

        if (self.role == "nic"):
            n_init.set()

        if e:
            f.close
            return e

        cmd = "./%srun.sh" % (self.hostname)
        if os.path.exists(cmd):
            e = read_exec_shell_cmd(child, cmd, prompt, timer, self.hostname)
            i_ran.set()
            if e:
                f.close
                return e
        else:
            start = time.time()
            print(self.hostname + " waiting for initiator to finish run")
            i_ran.wait()
            print(("%s initiator finished after %.2fs" %
                   (self.hostname, time.time() - start)))

        cmd = "./final.sh"
        if os.path.exists(cmd):
            e = read_exec_shell_cmd(child, cmd, prompt, timer, self.hostname)
            # expirment, give other side time to get ipsec look output
            time.sleep(2)
            if e:
                f.close
                return e

        if self.ipsecstop:
            if os.path.exists(cmd):
                cmd = "../bin/ipsecstop.sh"
                read_exec_shell_cmd(child, cmd, prompt, timer, self.hostname)
                if os.path.exists("OUTPUT/%s.core.txt" % (self.hostname)):
                    cmd = "cat OUTPUT/%s.core.txt" %(self.hostname)
                    read_exec_shell_cmd(child, cmd, prompt, timer, self.hostname)
        f.close

    def log_line(self, filename, msg):

        if not self.testname:
            return

        logline = dict()
        # output_file = "OUTPUT/RESULT"
        logline["epoch"] = int(time.time())
        logline["hostname"] = self.hostname
        logline["testname"] = self.testname
        logline["msg"] = msg
        logline["runtime"] = round((time.time() - self.start), 2)
        logline["time"] = time.strftime("%Y-%m-%d %H:%M", time.localtime())

        result_file_lock.acquire()
        f = open(filename, 'a')
        f.write(json.dumps(logline, ensure_ascii=True))
        f.write("\n")
        f.close
        result_file_lock.release()

    def connect_to_kvm(self):

        prompt = "\[root@%s " % (self.hostname)

        vmlist = subprocess.getoutput("sudo virsh list")
        running = False
        for line in vmlist.split("\n")[2:]:
            try:
                num, host, state = line.split()
                if host == self.dname and state == "running":
                    running = True
                    print(("Found %s running already" % self.hostname))
                    continue
            except:
                pass

        bootwait = self.bootwait
        pause = bootwait

        if bootwait > 15:
            pause = 15

        if not running:
            done = False
            v_start = ''
            tries = bootwait
            while not done and tries != 0:
                if os.path.isfile("OUTPUT/stop-tests-now"):
                    return "aborting: found OUTPUT/stop-tests-now"

                print(("Booting %s %s/%s" % (self.dname, tries, bootwait)))
                v_start = subprocess.getoutput(
                    "sudo virsh start %s" % self.dname)
                logging.info(v_start)
                re_e = re.search(r'error:', v_start, re.I)
                if re_e:
                    if(re.search(r'Domain not found', v_start, re.I)):
                        break
                    tries -= 1
                    time.sleep(1)
                else:
                    done = True

                    # just abort this test
            if not done:
                v_start = "KVMERROR %s " % self.dname + v_start
                logging.error(v_start)
                self.log_line('OUTPUT/stop-tests-now', v_start)
                if self.stoponerror:
                    # the whole show ends here
                    self.log_line('../stop-tests-now', v_start)
                    return v_start

            time.sleep(pause)
        elif self.reboot:
            subprocess.getoutput("sudo virsh reboot %s" % self.dname)
            print(("Rebooting %s - pausing %s seconds" %
                   (self.dname, pause)))
            time.sleep(pause)

        print(("Taking %s console by force" % self.dname))
        cmd = "sudo virsh console --force %s" % self.dname
        timer = 120
        child = pexpect.spawnu(cmd)
        child.delaybeforesend = 0.1
        self.child = child
        # child.logfile = sys.stdout
        # don't match full prompt, we want it to work regardless cwd

        done = False
        tries = bootwait - pause + 1

        print(("Waiting on %s login: %s" % (self.dname, prompt)))
        while not done and tries != 0:
            if os.path.isfile("OUTPUT/stop-tests-now"):
                return "aborting: found OUTPUT/stop-tests-now"
            try:
                child.sendline('')
                print(("%s [%s] waiting on login: or %s" % (self.dname,
                                                            tries, prompt)))
                res = child.expect(['login: ', prompt], timeout=3)
                if res == 0:
                    print(("%s sending login name root" % self.dname))
                    child.sendline('root')
                    print(
                        ("%s found, expecting password prompt" % self.dname))
                    child.expect('Password:', timeout=1)
                    print(("%s found, sending password" % self.dname))
                    child.sendline('swan')
                    print(
                        ("%s waiting on root shell prompt %s" % (self.dname, prompt)))
                    child.expect('root.*', timeout=1)
                    print(("got prompt %s" % prompt.replace("\\", "")))
                    done = True
                elif res == 1:
                    print('----------------------------------------------')
                    print(
                        (' Already logged in as root on %s' % prompt.replace("\\", "")))
                    print('----------------------------------------------')
                    done = True
            except:
                tries -= 1
                time.sleep(1)

        if not done:
            err = 'KVMERROR console is not answering: abort test'
            logging.error("%s %s %s", self.dname, err, self.testname)
            self.log_line('OUTPUT/stop-tests-now', err)

            if self.stoponerror:
                logging.error("stop")
                self.log_line('../stop-tests-now', err)
            return err

        remote = Remote(child, self.dname)
        remote.run('TERM=dumb; export TERM; unset LS_COLORS')
        child.setecho(False)  # this does not seems to work
        remote.run("stty sane")
        remote.run("stty -onlcr")
        remote.run("stty -echo")

# end of class

TIMEOUT = 10
SEARCH_WINDOW_SIZE = 100


class Remote:

    def __init__(self, child, hostname, prompt="\[root@[^ ]+ [^\]]+\]# "):
        self.child = child
        self.hostname = hostname
        self.prompt = prompt
        self.logging = False

    def run(self, command, timeout=TIMEOUT):
        # "swantest" just, sometimes, prints the prompt/command to the
        # console.  An alternative would be to use fab.tee.Tee so that
        # everything is written to both the file, and stdout.
        if self.logging:
            # The re.replace() strips the \ from \[.  The result is
            # pretty but misleading.
            print(("%s: %s" % (self.prompt.replace("\\", ""), command)))
        self.child.sendline(command)
        self.child.expect(
            self.prompt, timeout=timeout, searchwindowsize=SEARCH_WINDOW_SIZE)

    def cd(self, directory):
        self.prompt = "\[root@%s %s\]# " % (
            self.hostname, os.path.basename(directory))
        self.run("cd %s" % directory)
        return self.prompt

    # Read the contents of a local file and run each line.  assumes
    # that all lines contain simple shell commands.
    #
    # For anything more complicated, use run() above to run the script
    # remotely.
    def read_file_run(self, filename, timeout=TIMEOUT):
        f_cmds = None
        try:
            f_cmds = open(filename, "r")
            for line in f_cmds:
                line = line.strip()
                # We need the lines with # for the cut --- tuc
                # sections if line and not line[0] == '#':
                if line:
                    self.run(line, timeout)
        finally:
            if f_cmds:
                f_cmds.close

    # Capture and return last command's exit status.
    #
    # Unfortunately, as side effect, this eats the exit code.  It
    # would be nice if run, above, directly captured the exit code.
    def exit_status(self):
        self.child.sendline("echo status=$?")
        self.child.expect('status=([0-9]+)\s*' + self.prompt, timeout=TIMEOUT)
        status = int(self.child.match.group(1))
        # child.sendline("(exit %s)" % status)
        return status

    def exit_if_error(self):
        status = self.exit_status()
        if status:
            # anything non-zero, pass it out
            sys.exit(status)

# end of class


class TestList:

    def __iter__(self):
        return self

    def __init__(self, testlist, args, trpath='./', verbose=True):
        self.verbose = verbose
        self.args = args
        self.file = open(testlist, 'r')
        self.trpath = trpath

    def __next__(self):
        tdir = ''
        for line in self.file:
            line = line.strip()
            if not line:
                logging.debug("skip blank lines")
                continue
            if line[0] == '#':
                logging.debug("skip comment: %s", line)
                continue
            try:
                tokens = []
                tokens = line.split()
                if len(tokens) == 3 or (len(tokens) > 3 and tokens[3][0] == '#'):
                    testtype = tokens[0]
                    testdir = tokens[1]
                    testexpect = tokens[2]
            except:
                if re.search(r'#', line):
                    continue
                else:
                    # This is serious
                    logging.error("****** malformed line: %s", line)
                    continue
            tdir = self.trpath + '/' + testdir
            if not os.path.exists(tdir):
                # This is serious
                if self.verbose:
                    logging.error(
                        "****** invalid test %s: directory %s not found", testdir, tdir)
                    continue
            reason = self.skip_test(testtype, testdir, testexpect)
            if reason and self.verbose:
                TestList.log_skip(testdir, reason)

            if reason:
                continue

            logging.debug("read: %s %s %s", testtype, testdir, testexpect)
            return (testtype, testdir, testexpect)

        self.file.close
        raise StopIteration

    def log_skip(testdir, reason):
        # At error level so it it always appear, verbose above can
        # suppress it.
        logging.warning("****** skipping test %s: %s", testdir, reason)

    def skip_test(self, testtype, testdir, testexpect):

        if testtype == "skiptest":
            return "type is skiptest"

        if testexpect == "skiptest":
            return "type is skiptest"

        if testtype != "dockerplutotest" and testtype != "kvmplutotest":
            return "type %s yet to be migrated to kvm style" % testtype

        if self.args.include:
            if not self.args.include.search(testtype) and \
               not self.args.include.search(testdir) and \
               not self.args.include.search(testexpect):
                return "does not match '--include %s' regular expression" % self.args.include.pattern

        if self.args.exclude:
            if self.args.exclude.search(testtype) or \
               self.args.exclude.search(testdir) or \
               self.args.exclude.search(testexpect):
                return "matches '--exclude %s' regular expression" % self.args.exclude.pattern

        return None

# end of class


class scantests:

    def __init__(self, args, stdir=None, to_append=False):
        self.args = args
        self.stdir = stdir  # summarize a this single test run directory
        self.to_append = to_append
        self.htmlsums = {"columns": ["Dir", "Passed", "Failed", "Tests", "Run Time(Hours)"], "runDir": "/results/%s" % (
            self.args.node), "suffix": "", "rows": list()}
        self.graphsums = list()

    def scan_test_runs(self):
        if not os.path.isdir(self.args.resultsdir):
            e = "%s is not a directory" % self.args.resultsdir
            logging.info(e)
            return e

        npath = self.args.resultsdir

        if self.args.node:
            npath += '/' + self.args.node

        if not os.path.isdir(npath):
            e = "%s is not a directory" % npath
            logging.info(e)
            return e

        self.npath = npath
        logging.debug("npath %s" % self.npath)

        self.tdirs = list()
        if self.stdir:
            self.tdir = self.stdir
            self.tdirs.append(self.stdir)
            logging.info("scan a single directory %s" % self.stdir)
            self.d_to_scan = 1
        else:
            tdirs = list()
            tdirs = sorted(listdir(self.npath), reverse=True)
            if not tdirs:
                logging.error("nothing here in the directory %s" % self.npath)
                return("nothing here in the directory %s" % self.npath)
            i = 0
            j = 0
            for d in tdirs:
                if os.path.isdir(self.npath + '/' + d):
                    i += 1
                    self.tdirs.append(d)
                    logging.debug("tdir %s/%s" % (self.npath, d))
                else:
                    j += 1
                    logging.debug("ignore %s . not a directory?" % d)

            logging.info("%s directories to scan for test results, ignored %s in npath %s" % (
                i, j, self.npath))
            self.d_to_scan = i

        i = 0
        for d in sorted(self.tdirs):
            i += 1
            logging.info("scan directory %s for tests  %s/%s" % (d, i,
                                                                 self.d_to_scan))

            self.tdir = d
            if self.scan_trun():
                logging.error("scan directory %s", d)
                continue
            self.scan_test_results()

    def append_node_sum(self, g_sum='', h_sum=''):

        graphsums = None
        htmlsums = None

        if not self.to_append:
            self.graphsums.append(g_sum)
            self.htmlsums["rows"].append(h_sum)
            return

        gfile = self.args.resultsdir + '/' + 'graph.json'
        logging.debug("reading file %s" % gfile)
        with opened_w_error(gfile) as (f, err):
            if err:
                logging.error("can't read %s %s" % (gfile, err))
            else:
                lines = f.read().replace("\n", "").replace("\t", "")
                graphsums = json.loads(lines)

        self.graphsums = list()
        if graphsums:
            g = dict()
            for o in graphsums:
                logging.debug("directory %s" % o["dir"])
                if o["dir"] != g_sum["dir"] and (not o["dir"] in g):
                    g[o["dir"]] = True
                    self.graphsums.append(o)
                    logging.debug("append to graphsum %s" % o["dir"])

        hfile = self.args.resultsdir + '/' + 'table.json'
        logging.debug("reading file %s" % hfile)
        with opened_w_error(hfile) as (f, err):
            if err:
                logging.error("can't read %s %s" % (gfile, err))
            else:
                lines = f.read().replace("\n", "").replace("\t", "")
                htmlsums = json.loads(lines)

        self.htmlsums["rows"] = list()
        if htmlsums:
            h = dict()
            for o in htmlsums["rows"]:
                if (not o[0] in h) and (o[0] != h_sum[0]):
                    h[o[0]] = True
                    self.htmlsums["rows"].append(o)

        self.graphsums.append(g_sum)
        self.graphsums = sorted(self.graphsums, key=lambda k: k['dir'])

        self.htmlsums["rows"].append(h_sum)
        self.htmlsums["rows"] = sorted(
            self.htmlsums["rows"], key=lambda k: k[0], reverse=True)

    def write_node_sum(self):
        gfile = self.npath + '/' + 'graph.json'
        logging.info("writing file %s" % gfile)
        with open(gfile, 'w') as jsonfile:
            jsonfile.write(
                json.dumps(self.graphsums, ensure_ascii=True, indent=2))

        gfile = self.args.resultsdir + '/' + 'graph.json'
        logging.info("writing file %s" % gfile)
        with open(gfile, 'w') as jsonfile:
            jsonfile.write(
                json.dumps(self.graphsums, ensure_ascii=True, indent=2))

        hfile = self.args.resultsdir + '/' + 'table.json'
        logging.info("writing file %s" % hfile)
        with open(hfile, 'w') as jsonfile:
            jsonfile.write(
                json.dumps(self.htmlsums, ensure_ascii=True, indent=2))

        hfile = self.npath + '/' + 'table.json'
        logging.info("writing file %s" % hfile)
        with open(hfile, 'w') as jsonfile:
            jsonfile.write(
                json.dumps(self.htmlsums, ensure_ascii=True, indent=2))

    def scan_trun(self):
        self.nopath = ''
        if self.stdir:
            trpath = self.stdir
            if self.args.resanitize:
                self.nopath = setup_result_dir(self.args, True)
        else:
            trpath = self.npath + '/' + self.tdir

        if not os.path.isdir(trpath):
            e = "test run directory %s is not a directory" % trpath
            logging.debug(e)
            return True

        self.trpath = trpath
        logging.debug("trpath %s" % self.trpath)

        if not self.read_testlist(self.trpath):
            logging.info(
                "cannot read %s/TESTLIST will try other locations", self.trpath)
            if self.args.resanitize or self.args.scancwd:
                # we need the TESTLIST in the self.trpath
                return True
            if not self.read_testlist(self.npath):
                if not self.read_testlist('./'):
                    logging.error(
                        "skip this dir %s . cannot read %s/TESTLIST , %s/TESTLIST , ./TESTLIST abort", trpath, trpath, self.npath)
                    return True

        if self.nopath and self.args.resanitize:
            rsync_file(
                (self.trpath + '/' + self.args.testlist), (self.nopath + '/TESTLIST'))

    def read_testlist(self, resultsdir=''):
        if resultsdir:
            r = resultsdir + '/' + self.args.testlist
        else:
            r = self.args.testlist

        if not os.path.exists(r):
            return None

        logging.debug("TESTLIST %s" % r)

        i = 0
        self.tests = collections.OrderedDict()
        self.tests.clear()

        logging.debug("Reading testlist file %s", r)
        for testtype, testdir, testexpect in TestList(r, self.args, trpath=self.trpath, verbose=False):
            self.tests[testdir] = {}
            self.tests[testdir]["name"] = testdir
            self.tests[testdir]["type"] = testtype
            self.tests[testdir]["expect"] = testexpect
            i += 1

        logging.debug("red %s tests from %s" % (i, r))
        return True

    def scan_single_test_result(self, tname=''):

        if not os.path.isdir(self.trpath + '/' + tname):
            return

        if not tname in self.tests:
            self.tests[tname] = {}
        self.tests[tname]["name"] = tname
        self.tests[tname]["type"] = "unknown"

        r = self.trpath + '/' + tname + '/OUTPUT/RESULT'
        if not os.path.exists(r):
            e = "missing %s" % r
            r = self.trpath + '/' + tname + '/OUTPUT'
            if not os.path.exists(r):
                e = "missing %s" % r
                self.tsum['missing OUTPUT'] += 1
                self.tests[tname]["output"] = 'missing OUTPUT'
            else:
                self.tsum['missing RESULT'] += 1
                self.tests[tname]["output"] = 'missing RESULT'

            logging.info(e)
            return

        if not "expect" in self.tests[tname]:
            self.tests[tname]["expect"] = "not in TESTLIST"

        self.testname = tname
        self.tpath = self.trpath + '/' + self.testname
        logging.debug("tpath %s" % self.tpath)

        ocwd = os.getcwd()
        os.chdir(self.tpath)
        tcpmdump_cmds, test_hosts = orient(self.args, self.tests[tname])

        if self.args.resanitize:
            s = sanitize(self.args.sanitizer)
            re_write_result(sanity=s)

        os.chdir("..")
        os.chdir(ocwd)

        if self.nopath:
            rsync_file_to_dir((os.getcwd() + '/' + tname), self.nopath)

        if not test_hosts:
            return

        f = open(r, 'r')
        for line in f:
            try:
                x = json.loads(line)
            except TypeError:
                logging.error(
                    "TypeError cannot convert '%s' to json", line)
                continue
            if not "result" in x:
                continue
            if not "testname" in x:
                continue
            if not x["result"]:
                continue
            x["result"] = x["result"].lower()
            self.tests[tname]["result"] = x["result"]
            self.tests[tname]["runtime"] = round(x["runtime"], 2)
            self.tsum["runtime"] += x["runtime"]
            self.tsum[x["result"]] += 1
            logging.debug("test %s", self.tests[tname])
            self.diffstat_test()
            self.grep_4_known_errors()

    def scan_test_results(self):
        i = 0

        self.tsum = collections.OrderedDict()
        self.tsum.clear()

        self.tsum["Total"] = 0
        self.tsum['passed'] = 0
        self.tsum['failed'] = 0
        self.tsum['abort'] = 0
        self.tsum['missing baseline'] = 0
        self.tsum['missing console output'] = 0
        self.tsum['missing OUTPUT'] = 0
        self.tsum['missing RESULT'] = 0

        self.tsum['ASSERT'] = 0
        self.tsum['CORE'] = 0
        self.tsum['EXPECT'] = 0
        self.tsum['GPFAULT'] = 0
        self.tsum['SEGFAULT'] = 0

        self.tsum["date"] = "0000-00-00"
        self.tsum["dir"] = ''

        self.tsum["runtime"] = 0.0

        for tname, test in self.tests.items():
            i += 1
            self.scan_single_test_result(tname=tname)

        e = self.create_test_run_table()
        if e:
            return
        if self.stdir and not self.to_append:
            return

        row = [self.tsum["dir"], self.tsum["passed"], self.tsum["failed"],
               self.tsum["Total"], self.tsum["runtime_str"]]
        logging.info("append the graph file")
        self.append_node_sum(g_sum=self.tsum, h_sum=row)

        self.write_node_sum()

    def create_test_run_table(self):

        if self.stdir and self.to_append:
            runDir = os.path.basename(self.tdir)
        else:
            runDir = self.tdir

        table = {
            "columns": ["Test", "Expected", "Result", "Run time", "Responder", "Initiator"], "runDir": "/results/%s/%s" % (self.args.node, runDir), "suffix": "/OUTPUT", "rows": list()
        }
        for key, test in self.tests.items():
            row = []
            row.append(test["name"])
            if "expect" in test:
                row.append(test["expect"])
            elif "output" in test:
                row.append(test["output"])

            if "result" in test:
                row.append(test["result"])
            else:
                row.append("missing")
            if "runtime" in test:
                row.append(test["runtime"])
            else:
                row.append(0)
            if "responder" in test:
                key = "%s-status" % test["responder"]
                if key in test:
                    row.append(test[key])
                else:
                    row.append("-")
            else:
                row.append("-")
            if "initiator" in test:
                key = "%s-status" % test["initiator"]
                if key in test:
                    row.append(test[key])
                else:
                    row.append("-")
            else:
                row.append("-")

            table["rows"].append(list(row))

        self.tsum["Total"] = self.tsum['failed'] + self.tsum['passed'] + \
            self.tsum['missing OUTPUT'] + self.tsum['missing RESULT']

        if not self.tsum["Total"]:
            e = "no tests in %s" % self.trpath
            logging.info(e)
            return e

        if self.tsum["runtime"]:
            runtime = self.tsum["runtime"]
            ho = runtime // 3600
            mi = (runtime % 3600) // 60
            se = (runtime % 60)
            self.tsum["runtime_str"] = "%02d:%02d:%02d" % (ho, mi, se)

        if self.stdir and self.to_append:
            self.tsum["dir"] = os.path.basename(self.tdir)
        else:
            self.tsum["dir"] = self.tdir
        match = re.search(r'(\d+-\d+-\d+)', self.tdir)
        logging.info("date %s", match)
        if match:
            self.tsum["date"] = match.group(0)
        elif not self.args.resanitize:
            logging.info(
                "warning missing date in tdir %s. It does not start with date <d+-d+-d+>" % self.tdir)

        table["summary"] = self.tsum
        with open(self.trpath + '/' + 'table.json', 'w') as jsonfile:
            jsonfile.write(json.dumps(table, ensure_ascii=True, indent=2))

        self.write_txt(table)

        i3html = "../../i3.html"
        if not os.path.exists(self.trpath + '/' + "index.html"):
            try:
                os.symlink(i3html, self.trpath + '/' + "index.html")
            except:
                pass

    def write_txt(self, table):

        csvfile = self.trpath + '/' + 'table.txt'
        logging.info("write text summary file %s" % csvfile)

        with open(csvfile, 'w') as csvfile:
            line = ''
            for key in table["summary"].keys():
                if key == "runtime":
                    continue
                line += key + " %s, " % table["summary"][key]
            csvfile.write(line)
            csvfile.write("\n")
            for name, test in self.tests.items():
                line = self.make_test_line(test)
                csvfile.write(line)
                csvfile.write("\n")

    def make_test_line(self, test):
        line = ''
        note = ' & '
        st1 = ' '
        st2 = ' '
        if "initiator" in test:
            if "%s-status" % test["initiator"] in test:
                st1 = test["%s-status" % test["initiator"]]
        if "responder" in test:
            if "%s-status" % test["responder"] in test:
                st2 = test["%s-status" % test["responder"]]

        if "result" in test:
            if "expect" in test:
                if (test["result"] == "passed") and (test["expect"] == "good"):
                    line = "%s \t%s" % (
                        test["result"], test["name"])
                elif test["result"] == "missing baseline":
                    line = "%s, %s\t%s" % (
                        test["result"], test["expect"], test["name"])
                else:
                    line = "%s, %s\t%s, %s, %s" % (
                        test["result"], test["expect"], test["name"], st1, st2)
            else:
                line = "%s %s\t%s, %s, %s" % (
                    test["result"], "unknown", test["name"], st1, st2)
        else:
            line = "%s no result" % test["name"]
        return line

    def write_csv(self, table):
        csvfile = self.trpath + '/' + 'table.csv'
        logging.info("write csv file %s" % csvfile)

        with open(csvfile, 'w') as csvfile:
            csv_h = csv.writer(csvfile)
            csv_h.writerow(["#Total %s" % table["summary"]["Total"],
                            "passed %s" % table["summary"]["passed"],
                            "failed %s" % table["summary"]["failed"],
                            "missing baseline %s" % table[
                                "summary"]["missing baseline"],
                            "nooutput %s" % table["summary"][
                                "missing console output"],
                            "ASSERT %s" % table["summary"]["ASSERT"],
                            "CORE %s" % table["summary"]["CORE"],
                            "EXPECT %s" % table["summary"]["EXPECT"],
                            "GPFAULT %s" % table["summary"]["GPFAULT"],
                            "SEGFAULT %s" % table["summary"]["SEGFAULT"],
                            "runtime %s" % table["summary"]["runtime_str"]]
                           )
            csv_h.writerow(table["columns"])
            csv_h.writerows(table["rows"])

    def diffstat_test(self):
        self.diffstat("initiator")
        self.diffstat("responder")

    def diffstat(self, role):
        host = self.tests[self.testname][role]
        diffr = ''
        goodc = self.tpath + '/' + host + ".console.txt"
        newc = self.tpath + '/OUTPUT/' + host + '.console.txt'

        if os.path.exists(newc) and os.path.getsize(newc) > 0:
            if os.path.exists(goodc) and os.path.getsize(goodc) > 0:
                diffr = self.do_diffstat(goodc, newc)
            else:
                diffr += ' missing baseline'
                self.tsum["missing baseline"] += 1
                self.tests[self.testname]["result"] = "missing baseline"
        else:
            diffr = " missing OUTPUT/" + host + '.console.txt'
            self.tsum["missing console output"] += 1
            self.tests[self.testname]["result"] = "missing console output"

        key = "%s-status" % host
        if key in self.tests[self.testname]:
            self.tests[self.testname]["%s-status" % host] += diffr
        else:
            self.tests[self.testname]["%s-status" % host] = host + diffr

    def do_diffstat(self, goodc, newc):

        diffcmd = 'diff  -N -u'

        diffr = ''
        cmd = diffcmd + ' ' + goodc + ' ' + newc + ' | diffstat -f 0'
        ds = subprocess.getoutput(cmd)
        lines = ds.split("\n")[1:]
        if len(lines):
            for line in ds.split("\n")[1:]:
                c = re.sub(r'\d+ file changed,', '', line)
                if c:
                    diffr += c
                    logging.debug("change%s" % c)
                else:
                    diffr += ' ' + "passed"
        else:
            diffr += ' ' + "passed"

        return diffr

    def grep_4_known_errors(self):
        for role in ["initiator", "responder"]:
            host = self.tests[self.testname][role]
            plutolog = self.tpath + '/OUTPUT/' + host + '.pluto.log'
            if not os.path.exists(plutolog):
                plutolog = plutolog + '.gz'
            conslelog = self.tpath + '/OUTPUT/' + host + '.console.txt'
            if not os.path.exists(conslelog):
                conslelog = conslelog + '.gz'
            conslelog_verbose = self.tpath + '/OUTPUT/' + host + '.console.verbose.txt'
            if not os.path.exists(conslelog_verbose):
                conslelog_verbose = conslelog_verbose + '.gz'

            if os.path.exists(plutolog):
                self.grep_n_add(role, plutolog, 'ASSERTION FAILED', "ASSERT")
                self.grep_n_add(role, plutolog, 'EXPECTATION FAILED', "EXPECT")

            if os.path.exists(conslelog):
                self.grep_n_add(role, conslelog, 'segfault', "SEGFAULT")
                self.grep_n_add(
                    role, conslelog, 'general protection', "GPFAULT")

            if os.path.exists(conslelog_verbose):
                self.grep_n_add(role, conslelog_verbose, "^CORE FOUND", "CORE")


    def grep_n_add(self, role, filename, pattern, note):
        if not os.path.exists(filename):
            return

        fixed = '-F '
        fixed = ''
        key = "%s-status" % self.tests[self.testname][role]
        status = ''

        cmd = "zgrep " + fixed + "'" + pattern + "'  " + filename
        match = subprocess.getoutput(cmd)
        logging.debug("%s" % cmd)
        if match:
            print(("%s %s" % (cmd, match)))
            self.tsum[note] += 1
            if key in self.tests[self.testname]:
                self.tests[self.testname][key] += " " + note
            else:
                self.tests[self.testname][key] = self.tests[
                    self.testname][role] + " " + note


@contextmanager
def opened_w_error(filename, mode="r"):
    try:
        f = open(filename, mode)
    except IOError as err:
        yield None, err
    else:
        try:
            yield f, None
        finally:
            f.close()


def rsync_file(src, dst):
    cmd = "/usr/bin/rsync --delete -q -aP %s %s" % (src, dst)
    logging.debug("%s", cmd)
    try:
        os.system(cmd)
    except:
        logging.exception("EXCEPTION ? cmd %s , cwd %s", os.getcwd(), cmd)


def rsync_file_to_dir(src, dst):
    cmd = "/usr/bin/rsync --delete -q -aP %s %s/" % (src, dst)
    logging.debug("%s", cmd)
    try:
        os.system(cmd)
    except:
        logging.exception("EXCEPTION ? cmd %s , cwd %s", os.getcwd(), cmd)


def get_results(r):
    if not os.path.exists(r):
        return
    f = open(r, 'r')
    for line in f:
        try:
            x = json.loads(line)
        except TypeError:
            logging.error(
                "TypeError cannot convert '%s' to json", line)
            continue
        if "result" in x and "testname" in x:
            return x

def sent_virsh_cmd(args, all_hosts, c="shutdown"):

    running = []

    vmlist = subprocess.getoutput("sudo virsh list")
    for line in vmlist.split("\n")[2:]:
        try:
            num, host, state = line.split()
            for h in all_hosts:
                if h == host:
                    running.append(host)
                    cmd = "sudo virsh %s %s" % (c, host)
                    logging.debug("Found %s %s send %s", host, state, cmd)
                    shut = subprocess.getoutput(cmd)
        except:
            pass

    tries = args.shutdownwait
    while len(running) and tries != 0:
        logging.info(
            "Found %s guests [%s] running. Wait up to %d seconds to shutdown", len(
                running),
            ' '.join(map(str, running)), tries)

        del running[:]
        try:
            vmlist = subprocess.getoutput("sudo virsh list")
            for line in vmlist.split("\n")[2:]:
                try:
                    num, host, state = line.split()
                    for h in all_hosts:
                        if h == host:
                            running.append(host)
                except:
                    pass
        except:
            pass
        tries -= 1
        time.sleep(1)

    return running

def shut_down_hosts(args, test_hosts, prefix=None):

    running = []
    all_hosts = list(DEFAULTCONFIG['swanhosts'])
    all_hosts.extend(DEFAULTCONFIG['regualrhosts'])

    if prefix:
        for i, host in enumerate(all_hosts):
            all_hosts[i] = prefix + host

    if args.noreboot:
        logging.debug(
            "all hosts [%s] shutdown list", ' '.join(map(str, all_hosts)))
        logging.debug(
            "remove [%s] from shutdown list", ' '.join(map(str, test_hosts)))
        for t in test_hosts:
            for h in all_hosts:
                if h is t:
                    all_hosts.remove(t)

    logging.debug("shutdown list [%s]", ' '.join(map(str, all_hosts)))

    running =  sent_virsh_cmd(args, all_hosts, c="shutdown")

    if len(running):
       # try a bigger hammer to shut down
        running =  sent_virsh_cmd(args, all_hosts, c="destroy")
        if len(running):
            e = "KVMERROR not able to shutdown %s guests: [%s] abort" % (
                len(running), ' '.join(map(str, running)))
            logging.error(e)
            return e

def find_other_hosts(test, test_hosts):

    cwd = os.getcwd()
    onlyfiles = []
    for f in listdir(cwd):
        if isfile(join(cwd, f)) or islink(join(cwd, f)):
            onlyfiles.append(f)

    iinit = "%sinit.sh" % (test["initiator"])
    rinit = "%sinit.sh" % (test["responder"])
    if "nic" in test:
        ninit = "%sinit.sh" % (test["nic"])

    for f in onlyfiles:
        host = re.search(r'(.*)init.sh', f)
        if not host:
            continue
        elif iinit == f:
            continue
        elif rinit == f:
            continue
        elif "nic" in test and ninit == f:
            continue

        if not "others" in test:
            test["others"] = []

        test_hosts.append(host.group(1))
        test["others"].append(host.group(1))
        logging.debug("add others %s", host.group(1))


def orient(args, test, prefix=None):
    test_hosts = []
    log_line = ''
    if os.path.exists("eastrun.sh"):
        logging.error(
            "ABORT didn't expect eastrun.sh in %s something is wrong", os.getcwd())
        return None, None

    if os.path.exists("eastinit.sh"):
        test["responder"] = "east"
    else:
        logging.error(
            "ABORT can't identify RESPONDER: no %s/eastinit.sh" % os.getcwd())
        return None, None

    if os.path.exists("nicinit.sh"):
        test["nic"] = "nic"
        test_hosts.append(test["nic"])
        log_line = "and nic"

    if prefix:
        tcpdump_devs = ["-w OUTPUT/%s -i %s" %
                        ("swan12.pcap", prefix + "192_1_2")]
        tcpdump_dev_192_1_3 = "-w OUTPUT/%s -i %s" % (
            "swan13.pcap", prefix + "192_1_3")
    else:
        tcpdump_devs = ["-w OUTPUT/%s -i %s" % ("swan12.pcap", "swan12")]
        tcpdump_dev_192_1_3 = "-w OUTPUT/%s -i %s" % ("swan13.pcap", "swan13")

    if os.path.exists("westrun.sh"):
        if os.path.exists("westinit.sh"):
            test["initiator"] = "west"
        else:
            logging.error(
                "ABORT can't identify INITIATOR: missing %s/westinit.sh but there is westrun.sh" % os.getcwd())
    elif os.path.exists("roadrun.sh"):
        if os.path.exists("roadinit.sh"):
            test["initiator"] = "road"
            tcpdump_devs.append(tcpdump_dev_192_1_3)
        else:
            logging.error(
                "ABORT can't identify INITIATOR: no %s/roadinit.sh, but there is roadrun.sh" % os.getcwd())
    elif os.path.exists("northrun.sh"):
        if os.path.exists("northinit.sh"):
            test["initiator"] = "north"
            tcpdump_devs.append(tcpdump_dev_192_1_3)
        else:
            logging.error(
                "ABORT can't identify INITIATOR: no %s/northinit.sh, but there is northrun.sh" % os.getcwd())
    else:
        logging.error(
            "ABORT can't identify INITIATOR in directory %s" % os.getcwd())

    if not ("initiator" in test and "responder" in test):
        logging.error(
            "ABORT can't identify INITIATOR and RESPONDER in directory %s" % os.getcwd())
        return None, None

    test_hosts.append(test["initiator"])
    test_hosts.append(test["responder"])

    cmds = []

    if test["type"] == 'kvmplutotest':
        for iface in tcpdump_devs:
            pcap_file = 'OUTPUT/' + iface + '.pcap'
            # cmd = "/sbin/tcpdump -s 0 -w %s -n -i %s %s &" % (pcap_file,
            # iface, tcpdump_filter)
            cmd = args.tcpdump + " " + args.tcpdumpfilter + " " + iface
            logging.debug(cmd)
            cmds.append(cmd)

        cmds = sorted(set(cmds))

    find_other_hosts(test, test_hosts)

    if "others" in test:
        log_line = log_line + " also " + " ".join(test["others"])

    logging.debug("test hosts are initiator %s responder %s %s",
                  test["initiator"], test["responder"], log_line)

    test["test_hosts"] = []
    test["test_hosts"].extend(test_hosts)

    return cmds, test_hosts


def read_exec_shell_cmd(ex, filename, prompt, timer, hostname=""):

    if os.path.exists(filename):
        logging.debug("%s execute commands from file %s", hostname, filename)
        f_cmds = open(filename, "r")
        for line in f_cmds:
            if os.path.isfile("OUTPUT/stop-tests-now"):
                return "aborting: found OUTPUT/stop-tests-now"

            line = line.strip()
            # We need the lines with # for the cut --- tuc sections
            # if line and not line[0] == '#':
            if line:
                print(("%s: %s" % (prompt.replace("\\", ""), line)))
                ex.sendline(str(line))
                try:
                    ex.expect(prompt, timeout=timer, searchwindowsize=100)
                except:
                    err = "#%s timedout send line: %s" % (prompt, line)
                    logging.error("%s try sending CTRL+c and continue", err)
                    ex.sendcontrol('c')
                    ex.sendline(str(err))
                    # in the old days the function would return here.
                    # f_cmds.close
                    # return err
                    err = ''

        f_cmds.close

    else:
        # not a file name but a command: send it as it is.
        print(filename)
        ex.sendline(str(filename))
        try:
            ex.expect(prompt, timeout=timer, searchwindowsize=100)
        except:
            err = "%s failed to send command: %s" % (prompt, filename)
            logging.debug("%s %s", hostname, err)
            return err

# kill tcpdump from this run only
def kill_tcpdump(test, signal=1):
    if "tpids" in test:
        for tpid in test["tpids"]:
            gentle_kill(tpid, "tcpdump", signal)

# kill any lingering tcpdumps for the entire KVM runs.
def kill_zombie_tcpdump(signal=1):
    pids = subprocess.getoutput("pidof tcpdump")
    for pid in (pids.split()):
        gentle_kill(pid, "tcpdump", signal)

# kill all hanging previous of instances of this script.


def kill_zombies(proctitle):

    #setproctitle.setproctitle(proctitle)
    me = os.getpid()
    zombie_pids = subprocess.getoutput("pidof %s" % proctitle)
    for pid in (zombie_pids.split()):
        if int(pid) != int(me):
            gentle_kill(pid, proctitle, 9)
    kill_zombie_tcpdump(signal=9)


def init_output_dir():
    output_dir = "%s/OUTPUT" % os.getcwd()

    if os.path.isdir(output_dir):
        try:
            shutil.rmtree(output_dir)
        except PermissionError:
            logging.info("PermissionError to remove %s", output_dir)
            return True

    os.mkdir(output_dir, 0o777)


def sanitize(cmd):
    sanity = subprocess.getoutput(cmd)
    # logging.info ("sanitizer output %s", sanity.replace("\n", " "))
    logging.info("sanitizer output\n %s", sanity)
    return sanity


def split_sanity(sanity):
    if sanity:
        for line in sanity.split("\n")[-1:]:
            try:
                key, name, value = line.split()
                if key == "result":
                    return value
            except:
                pass


def re_write_result(sanity):

    changed = False

    if sanity:
        result = split_sanity(sanity)
    else:
        logging.debug("nothing sane to re-write")
        return

    output_file = "OUTPUT/RESULT"
    if not os.path.exists(output_file):
        logging.debug("cannot rewrite %s missing", output_file)
        return
    logging.debug("read result %s", output_file)

    lines = []
    f = open(output_file, 'r')
    for line in f:
        line = line.strip()
        if not line:
            continue

        x = None
        try:
            x = json.loads(line)
        except:
            continue

        if x and "result" in x and "testname" in x:
            if x["result"] == result:
                logging.debug(
                    "testcase %s result didn't change %s", x["testname"], result)
                continue

            changed = True
            logging.info(
                "testcase %s new result '%s' old '%s'", x["testname"], x["result"],  result)
            x["result"] = result
            x["resanitized"] = time.strftime(
                "%Y-%m-%d %H:%M", time.localtime())
            lines.append(json.dumps(x, ensure_ascii=True))
        else:
            lines.append(line)

    f.close

    if not changed:
        return False

    try:
        with open(output_file, 'w') as f:
            for line in lines:
                f.write(line)
                f.write("\n")
    except:
        logging.error("ERROR could not open file to write %s", output_file)

    return changed


def write_result(args, start, testname, sanity, result='FAILED', e=None, testexpect=''):

    if sanity:
        result = split_sanity(sanity)

    logline = dict()
    output_file = "OUTPUT/RESULT"

    f = open(output_file, 'a')
    logline["epoch"] = int(time.time())
    logline["testname"] = testname
    logline["result"] = result
    logline["time"] = time.strftime("%Y-%m-%d %H:%M", time.localtime())
    logline["runtime"] = round(time.time() - start, 2)
    logline["node"] = args.node
    if testexpect:
        logline["expect"] = testexpect

    if e:
        logline["error"] = e
    f.write(json.dumps(logline, ensure_ascii=True))
    f.write("\n")
    f.close
    logging.debug("wrote result to %s/%s", testname, output_file)

DEFAULTCONFIG = {
    'bootwait': 59,
    'docker': False,
    'dockerimage': "swanbase",
    'kvm': True,
    'prefix': [],
    'prefix': ['t1.', 't2.', 't3.', 't4.', 't5.', 't6.', 't7.'],
    'initshwait': 600,
    'libreswandir': os.path.dirname(os.path.dirname(os.path.abspath(os.path.dirname(__file__)))),
    'newrun': None,
    'node': socket.getfqdn().split('.')[0],
    'npool': 17,
    'regualrhosts': ['nic'],
    'resultsdir': '/home/build/results',
    'retry': 5,
    'rpm': '',
    'sanitizer': "../../utils/sanitize.sh",
    'shutdownwait': 63,
    'stoponerror': None,
    'swanhosts': ['east', 'west', 'road', 'north'],
    'tcpdumpfilter': "-s  0 -n not stp and not port 22",
    'tcpdump': "/usr/sbin/tcpdump",
    'testlist': "TESTLIST",
    'testrun': None
}


def compile_regex_arg(string):
    return re.compile(string)


def cmdline():

    parser = argparse.ArgumentParser(description='swantest arguments.')
    exclusive_grp = parser.add_mutually_exclusive_group()

    # the add_argument lines below are sorted.

    parser.add_argument("--bootwait", default=DEFAULTCONFIG['bootwait'],
                        type=int, help="seconds to reboot guest. Default %s seconds" % DEFAULTCONFIG['bootwait'])

    parser.add_argument('--docker', default=DEFAULTCONFIG[
                        "docker"], action="store_true", help="Test type dockerpluto %s" % DEFAULTCONFIG["docker"])

    parser.add_argument(
        "--dockerimage", default=DEFAULTCONFIG['dockerimage'],
        help="sudo docker image id, default %s" % DEFAULTCONFIG['dockerimage'])

    parser.add_argument("--exclude", default=None,
                        type=compile_regex_arg, help="Exclude tests that match <regex>")

    parser.add_argument("--failed", default=None, action="store_true",
                        help="re-run result=failed and expected == good")

    parser.add_argument("--graphsappend", default='',
                        help="update this graph entry. full path ")

    parser.add_argument("--graphs", default=False, action="store_true",
                        help="generate graphs and JSON tables.")

    parser.add_argument("--include", default=None, type=compile_regex_arg,
                        help="Only include tests that match <regex>")

    parser.add_argument(
        "--initshwait", default=DEFAULTCONFIG['initshwait'],
        type=int, help="seconds to wait for the init.sh, Python Thread, %s seconds" % DEFAULTCONFIG['initshwait'])

    parser.add_argument( "--ipsec-stop", default=False,  action="store_true", dest='ipsecstop',
            help="run ipsec stop after the tests")


    exclusive_grp.add_argument( "--new-prefix", default=[],
        action="append", help="new set of prefix to domain") 

    exclusive_grp.add_argument( "--prefix", default=DEFAULTCONFIG['prefix'],
        action="append", help="prefix to domain %s" % DEFAULTCONFIG['prefix']) 

    exclusive_grp.add_argument( "--with-prefix", default=False, action='store_true', help="with prefixes specified [--prefix <p1> --prefix <p2> ..]  VMs")

    parser.add_argument(
        '--kvm', default=DEFAULTCONFIG["kvm"], action="store_true",
        help="Test type kvmpluto %s" % DEFAULTCONFIG["kvm"])

    parser.add_argument('--leavezombies', default=None, action="store_true",
                        help='leave other instances running. Default kill all other swantest and lingering tcpdump')

    parser.add_argument("--libreswandir", default=DEFAULTCONFIG['libreswandir'],
                        help="libreswan source/repository directory %s" % DEFAULTCONFIG['libreswandir'])

    parser.add_argument("--newrun", default=DEFAULTCONFIG['newrun'],
                        action="store_true", help="overwrite the results in %s/<hostname>/<YYYY-MM-DD>-libreswan-version>. Default %s" % (DEFAULTCONFIG['resultsdir'], DEFAULTCONFIG['newrun']))

    parser.add_argument("--npool", default=DEFAULTCONFIG['npool'],
                        type=int, help="number of parallel docker tests. Default %s" % DEFAULTCONFIG['npool'])

    parser.add_argument("--node", default=DEFAULTCONFIG['node'],
                        help="Default node name %s ." % DEFAULTCONFIG['node'])

    parser.add_argument('--noreboot', default=None, action="store_true",
                        help='Dont reboot vm. Default reboot')

    parser.add_argument("--rerun", default=None,
                        help="<directory> continue previous run")

    parser.add_argument("--resanitize", default=False, action="store_true",
                        help="re-sanitize results, run it from ~/libreswan/testing/pluto directory")

    parser.add_argument("--resultsdir", default=DEFAULTCONFIG['resultsdir'],
                        help="test results directory %s" % DEFAULTCONFIG['resultsdir'])

    parser.add_argument("--retry", default=DEFAULTCONFIG['retry'],
                        type=int, help="retry %d when there is console error." % DEFAULTCONFIG['retry'])
    parser.add_argument("--rpm", default=DEFAULTCONFIG['rpm'],
                        help="binary rpm to install libreswan %s. Default run make install-base" % DEFAULTCONFIG['rpm'])

    parser.add_argument("--sanitizer", default=DEFAULTCONFIG['sanitizer'],
                        help="sanitizer script. %s" % DEFAULTCONFIG['sanitizer'])

    parser.add_argument("--scancwd", default=False, action="store_true",
            help="san/resanitize  results, run in cwd")
    parser.add_argument(
        "--shutdownwait", default=DEFAULTCONFIG['shutdownwait'],
        type=int, help="seconds to wait for guest to shutdown. Default %s seconds" % DEFAULTCONFIG['shutdownwait'])

    parser.add_argument("--stop", default=False, action="store_true",
                        help="stop tests now. call from testing/pluto directory")

    parser.add_argument("--stoponerror", default=DEFAULTCONFIG['stoponerror'],
                        action="store_true", help="Stop on kvm errors. Default continues")

    parser.add_argument("--tcpdump", default=DEFAULTCONFIG['tcpdump'],
                        help="tcpdump . The actual command is made of three parts, this one + -w <file> -i <interface> + tcpdumpfilter. Default %s" % DEFAULTCONFIG['tcpdump'])

    parser.add_argument(
        "--tcpdumpfilter", default=DEFAULTCONFIG['tcpdumpfilter'],
        help="tcpdump_filter used by tcpdump command. Default %s" % DEFAULTCONFIG['tcpdumpfilter'])

    parser.add_argument(
        "--testingdir", default="/testing", help="Change the /testing directory")

    parser.add_argument("--testlist", default=DEFAULTCONFIG['testlist'],
                        help="read kvmpluto tests from  %s" % DEFAULTCONFIG['testlist'])

    parser.add_argument('--testrun', action="store_true", default=DEFAULTCONFIG['testrun'],
                        help="run tests from %s " % DEFAULTCONFIG['testlist'])

    parser.add_argument('--testname', '-t',
                        help='The name of the test to run from directory %s/testing/pluto/' % DEFAULTCONFIG["libreswandir"])

    parser.add_argument("-v", "--verbose", default=1, type=int,
                        help="increase verbosity: 0 = only warnings, 1 = info, 2 = debug. Default info")

    args = parser.parse_args()

    if not args.with_prefix:
        args.prefix = []

    print("prefix %s %d" % (args.prefix, len(args.prefix)))

    if args.verbose == 0:
        logger.setLevel(logging.WARN)
    elif args.verbose == 1:
        logger.setLevel(logging.INFO)
    elif args.verbose == 2:
        logger.setLevel(logging.DEBUG)

    return args


def kvm_error(odir, failed=False):

    r_file = odir + '/' + 'RESULT'
    s_file = odir + '/' + STOP_TESTS_NOW

    if os.path.exists(s_file):
        return "found " + STOP_TESTS_NOW

    if not os.path.exists(r_file):
        return False

    with open(r_file, 'r') as f:
        for line in f:
            x = json.loads(line)

            # re-run the aborted, KVMERROR tests
            if "msg" in x and re.search(r'KVMERROR', x['msg'], re.I):
                logging.info(line)
                return line

            if "msg" in x:
                if re.search(r'aborting', x['msg'], re.I):
                    logging.info(line)
                    f.close
                    return line

            # re-run the failed tests
            if failed and "result" in x and "expect" in x:
                if x['result'] == "failed" and x['expect'] == "good":
                    return line

    return False


def runcmd(cmd):

    logging.info("%s", cmd)
    o = subprocess.getoutput(cmd)
    logging.debug(o)

    return o


def runcmd_check_output(cmd):
    logging.info("%s", cmd)
    try:
        o = subprocess.check_output(cmd, shell=True).decode('ascii')
        logging.debug(o)
        return (o, None)
    except subprocess.CalledProcessError as e:
        logging.error("ERROR %s %s", cmd, e.output.decode('ascii'))
        return (None, e)


def docker_stop_container(dname):
    if not dname:
        return

    prefix = None

    # runcmd("sudo docker exec -ti %s halt" % dname) # this is more gentle. It
    # hangs on F23
    runcmd("sudo docker stop --time=1 %s" % dname)
    runcmd("sudo docker rm -f %s" % dname)


def docker_start_container(dhost, dname, dimage):

    cmd = "sudo docker run -h %s --privileged --net=none --name %s -v /home/build:/home/build -v /sys/fs/cgroup:/sys/fs/cgroup:ro -d %s /usr/sbin/init" % (
        dhost, dname, dimage)
    return (runcmd(cmd))


def docker_flush_eth0(dhost, dname):
    # docker exec -ti dwest  ip address flush dev eth0
    runcmd("sudo docker exec -ti %s ip address flush dev eth0" % dname)


def docker_restart_network(test):
    for host in test["test_hosts"]:
        dname = host + "-" + test["name"]
        runcmd("sudo docker exec -ti %s /testing/guestbin/docker-transmogrify" % dname)

def docker_add_route(test, dnamei, dnamer, dnamen):

    # docker exec -it $dwest ip route add 192.0.2.0/24 via 192.1.2.23

    if test["initiator"] == "west":
        runcmd("sudo docker exec -ti %s ip route add 192.0.2.0/24 via 192.1.2.23" %
               dnamei)
        runcmd("sudo docker exec -ti %s ip route add default via 192.1.2.254" %
               dnamei)
    elif test["initiator"] == "road" or test["initiator"] == "north":
        runcmd("sudo docker exec -ti %s ip route add default via 192.1.3.254" %
               dnamei)
    else:
        pass

    runcmd("sudo docker exec -ti %s ip route add 192.0.1.0/24 via 192.1.2.45" %
           dnamer)
    runcmd(
        "sudo docker exec -ti %s ip route add default via 192.1.2.254" % dnamer)


def docker_swan_build(args, dname):
    # AA comment out make progreams
    # cmd = "docker exec -ti %s /bin/bash -c 'cd /home/build/libreswan; make
    # programs install'"%dname
    if (args.rpm):
        cmd = "sudo docker exec -ti %s /bin/bash -c 'rpm -vhi %s'" % (
            dname, args.rpm)
    else:
        cmd = "sudo docker exec -ti %s /bin/bash -c 'cd /home/build/libreswan; make base install-base'" % dname

    (line, e) = runcmd_check_output(cmd)
    if e:
        return(e.output)

    cmd = "sudo docker exec -ti %s ipsec pluto --version" % dname
    (line, e) = runcmd_check_output(cmd)
    logging.debug("%s", str(line))
    if re.search("not found", str(line)):
        logging.error("ERROR %s" % line)
        return ("no version found")

    cmd = "sudo docker exec -ti %s /bin/bash -c 'cd /home/build/libreswan; make showobjdir'" % dname
    (line, e) = runcmd_check_output(cmd)
    objdir = line.strip()
    if e:
        return(e.output)
    cmd = "sudo chown build.build -R /home/build/libreswan/%s" % objdir
    logging.debug("%s", cmd)
    subprocess.getoutput(cmd)


def docker_containers_start_stop(args, test, st, stop):

    start = time.time()

    logging.info("%s in docker_containers_start_stop ", test["name"])

    if "is_testlist" in st and dqstart and not stop:
        logging.info(
            "%s stop boot q is enabled wait for the turn", test["name"])
        prefix = dqstart.get()
        logging.info("%s got a boot q slot %s %.2f" %
                     (test["name"], prefix, time.time() - start))
    elif "is_testlist" in st and dqstop and stop:
        logging.info(
            "%s stop boot q is enabled wait for the turn", test["name"])
        prefix = dqstop.get()
        logging.info("%s got a boot q slot %s %.2f" %
                     (test["name"], prefix, time.time() - start))
    else:
        logging.info("stop boot q is not enabled")

    for host in test["test_hosts"]:
        dname = host + "-" + test["name"]
        docker_stop_container(dname)

    if stop:
        if "is_testlist" in st:
            logging.debug("%s put boot q slot %s back %.2f" %
                          (test["name"], prefix, time.time() - start))
            dqstop.put(prefix)
        return

    if init_output_dir():
        return True

    for host in test["test_hosts"]:
        dname = host + "-" + test["name"]
        docker_start_container(host, dname, args.dockerimage)

    if "is_testlist" in st and dqstart:
        logging.debug("%s put boot q slot %s back %.2f" %
                      (test["name"], prefix, time.time() - start))
        dqstart.put(prefix)

    try:
        docker_add_net_bridges(args, test)
    except:
        return True

    docker_restart_network(test)

    for host in test["test_hosts"]:
        if host == "nic":
            continue

        dname = host + "-" + test["name"]
        if docker_swan_build(args, dname):
            return True


def docker_net_bridge(dname, brname, iface, prefix, brlist):
    cmd = "sudo /usr/local/bin/pipework %s -i %s %s %s" % (
        brname, iface, dname, prefix)
    output = runcmd(cmd)
    if output:
        logging.error(output)
        raise PipeworkEroor(output)

    cmd = "sudo docker exec -ti %s /bin/bash -c 'ip addr del %s dev %s'" % (
        dname, prefix, iface)
    runcmd_check_output(cmd)
    brlist[brname] = 1


def docker_add_net_bridges(args, test):

    test['bid'] = []

    # shared bridge between east - west, east - nic
    bidc = random.randint(10000001, 19999999)
    test['bid'].append(bidc)

    # for clients behind west
    bidw0 = random.randint(21000001, 21999999)
    bidw2 = random.randint(22000001, 22999999)
    test['bid'].append(bidw0)
    test['bid'].append(bidw2)

    # for clients behind east
    bide0 = random.randint(31000001, 31999999)
    bide2 = random.randint(32000001, 32999999)
    test['bid'].append(bide0)
    test['bid'].append(bide2)

    # shared bridge between nic - road, nic - north
    bidn1 = random.randint(51000001, 51999999)
    bidn2 = random.randint(52000001, 52999999)
    bidn3 = random.randint(53000001, 53999999)
    test['bid'].append(bidn1)
    test['bid'].append(bidn2)
    test['bid'].append(bidn3)

    # shared for clients behind north
    bidno1 = random.randint(61000001, 61999999)
    test['bid'].append(bidno1)

    brname = ''
    pcap_file_ext = '.pcap'
    pcap_file = None

    prefix = "192.0.2.1/24"  # rfc 5737. leave to network restart

    tcpdump_cmds = dict()
    brlist = dict()

    for host in test["test_hosts"]:
        dname = host + "-" + test["name"]
        if host == "west":
            iface = "eth0"
            brname = "br%s" % bidw0
            docker_net_bridge(
                dname=dname, brname=brname, iface=iface, prefix=prefix,
                brlist=brlist)

            iface = "eth1"
            brname = "br%s" % bidc
            docker_net_bridge(
                dname=dname, brname=brname, iface=iface, prefix=prefix,
                brlist=brlist)

            iface = "eth2"
            brname = "br%s" % bidw2
            docker_net_bridge(
                dname=dname, brname=brname, iface=iface, prefix=prefix,
                brlist=brlist)

        elif host == "road":
            iface = "eth0"
            brname = "br%s" % bidn1
            docker_net_bridge(
                dname=dname, brname=brname, iface=iface, prefix=prefix,
                brlist=brlist)

            pcap_file = 'OUTPUT/' + 'swan13' + pcap_file_ext

        elif host == "north":
            iface = "eth0"
            brname = "br%s" % bidno1
            docker_net_bridge(
                dname=dname, brname=brname, iface=iface, prefix=prefix,
                brlist=brlist)
            pcap_file = 'OUTPUT/' + 'swan13' + pcap_file_ext

            iface = "eth1"
            brname = "br%s" % bidn1
            docker_net_bridge(
                dname=dname, brname=brname, iface=iface, prefix=prefix,
                brlist=brlist)

        if host == "nic":
            iface = "eth1"
            brname = "br%s" % bidn1
            docker_net_bridge(
                dname=dname, brname=brname, iface=iface, prefix=prefix,
                brlist=brlist)

            iface = "eth0"
            brname = "br%s" % bidc
            docker_net_bridge(
                dname=dname, brname=brname, iface=iface, prefix=prefix,
                brlist=brlist)

            # adding eth2 and eth3 seems to slowdown network.restart
            # if we need it in the future we could enable those
            iface = "eth2"
            brname = "br%s" % bidn2
            # docker_net_bridge( dname=dname, brname=brname, iface=iface,
            # prefix=prefix, brlist=brlist)

            iface = "eth3"
            brname = "br%s" % bidn3
            # docker_net_bridge(dname=dname, brname=brname, iface=iface,
            # prefix=prefix, brlist=brlist)
            pcap_file = None

        if host == "east":

            iface = "eth0"
            brname = "br%s" % bide0
            docker_net_bridge(
                dname=dname, brname=brname, iface=iface, prefix=prefix,
                brlist=brlist)

            iface = "eth2"
            brname = "br%s" % bide2
            docker_net_bridge(
                dname=dname, brname=brname, iface=iface, prefix=prefix,
                brlist=brlist)

            iface = "eth1"
            brname = "br%s" % bidc
            docker_net_bridge(
                dname=dname, brname=brname, iface=iface, prefix=prefix,
                brlist=brlist)
            pcap_file = 'OUTPUT/' + 'swan12' + pcap_file_ext

            if pcap_file:
                tcpdump_args =  args.tcpdump + \
                    " -w %s -i %s " % (pcap_file, brname) + args.tcpdumpfilter
                tcpdump_cmds[tcpdump_args] = 1

    # tcpdump pid(s) to kill the right tcpdump at the end.
    # killall is not an option, there could be other tests in the root pid
    # name space?

    test["tpids"] = []
    test["brlist"] = brlist
    logging.debug("bridges to remove after the test [%s]" %
                  " ".join(test["brlist"]))

    for cmd in tcpdump_cmds.keys():
        logging.info("%s", cmd)
        tpid = subprocess.Popen(cmd.split()).pid
        test["tpids"].append(tpid)


def docker_run_script(host, testname, script, timer=90):
    dname = "%s-%s" % (host, testname)
    if not os.path.exists(script):
        logging.info(
            "missing file %s for host  %s testname %s", script, host, testname)
        return
    logging.debug("%s execute commands from file %s", host, script)

    logfile = "OUTPUT/%s.console.verbose.txt" % host

    f_log = open(logfile, 'a', encoding='utf-8')
    f_cmds = open(script, "r")
    prompt = "[root@%s %s]# " % (host, testname)
    output = ''
    for line in f_cmds:
        f_log.write(line)

        cmd = "sudo docker exec -ti %s-%s /bin/bash -c 'cd /testing/pluto/%s;%s'" % (
            host, testname, testname, line)
        logging.info("%s", cmd)
        try:
            output = subprocess.check_output(cmd, shell=True, timeout=timer)
        except subprocess.TimeoutExpired:
            f_log.write("%s #timedout %s\n" % (cmd, timer))
        except subprocess.CalledProcessError as e:
            logging.error("ERROR %s %s", cmd, e.output)
            output = e.output

        output = output.decode('ascii')
        output = output.replace('\r', '')
        f_log.writelines(output + prompt)
    f_log.close()
    f_cmds.close()


def docker_run_script_1(host, testname, script):
    output = "OUTPUT/%s.console.verbose.txt" % host
    cmd = "suod docker exec -ti %s-%s /bin/bash -c 'cd /testing/pluto/%s; ./%s' >> %s" % (
        host, testname, testname, script, output)
    logging.info("%s", cmd)
    subprocess.getoutput(cmd)


def docker_run_test(args, test):
    if "nic" in test:
        docker_run_script(test["nic"], test["name"], "%sinit.sh" % test["nic"])
    docker_run_script(test["responder"], test[
                      "name"], "%sinit.sh" % test["responder"])

    if "others" in test:
        for host in test["others"]:
            docker_run_script(host, test["name"], "%sinit.sh" % host)

    docker_run_script(test["initiator"], test[
                      "name"], "%sinit.sh" % test["initiator"])
    docker_run_script(test["initiator"], test[
                      "name"], "%srun.sh" % test["initiator"])
    docker_run_script(test["responder"], test["name"], "final.sh")
    docker_run_script(test["initiator"], test["name"], "final.sh")

    if "others" in test:
        for host in test["others"]:
            docker_run_script(host, test["name"], "final.sh")


def gentle_kill(pid, name, signal=1):
    logging.info("kill -%s %d ; # %s", signal, int(pid), name)
    try:
        os.kill(int(pid), signal)
    except OSError as e:
        logging.error(
            "# kill -%s %d process %s failed %s", signal, int(pid), name, str(e))


def docker_clean(args, test, st, signal=1):
    if "tpids" in test:
        for tpid in test["tpids"]:
            gentle_kill(tpid, "tcpdump", signal)


def do_dockerplutotest(args, start, test, st):
    # chceck for docker image exists
    # only one instance of a test
    # create network bridges
    # run the tests
    # if not a single test
        # delete network bridges
        # shutdown the docker container

    logging.info("***** DOCKER  PLUTO RUNNING test %s *******", test["name"])

    # tcpdump_cmds is only used in kvmplutotest. docker figures that further
    # down
    tcpdump_cmds, test_hosts = orient(args, test)
    if not test_hosts:
        return

    if docker_containers_start_stop(args, test, st, False):
        return True

    docker_run_test(args, test)
    docker_clean(args, test, st)
    if "is_testlist" in st:
        # stop the docker instances
        docker_containers_start_stop(args, test, st, True)


def do_kvmplutotest(args, start, test, st, prefix=None):

    logging.info("***** KVM PLUTO RUNNING test %s *******", test["name"])

    tcpdump_cmds, test_hosts = orient(args, test, prefix=prefix)
    if not test_hosts:
        return "error"

    e = shut_down_hosts(args, test_hosts, prefix=prefix)
    if init_output_dir():
        return True

    if e:
        write_result(args, start, test["name"], None, "abort", e)
        # we can't call exit(1) "make check" will abort then
        return e

    test["tpids"] = []
    for cmd in tcpdump_cmds:
        test["tpids"].append(subprocess.Popen(cmd.split()).pid)

    r_init.clear()
    i_ran.clear()
    n_init.clear()

    # Create new threads
    th_responder = guest(test["responder"], "responder", test[
                         "name"], args, prefix=prefix, ipsecstop=args.ipsecstop)
    if "nic" in test:
        th_nic = guest(test["nic"], "nic", test["name"], args, prefix=prefix,
                ipsecstop=args.ipsecstop)

    th_initiator = guest(test["initiator"], "initiator", test["name"],
            args, prefix=prefix, ipsecstop=args.ipsecstop)

    # Start Threads
    th_responder.start()
    if "nic" in test:
        th_nic.start()
    else:
        n_init.set()

    if "others" in test:
        others = dict()
        for host in test["others"]:
            others[host] = guest(host, "other", test[
                                 "name"], args, prefix=prefix, ipsecstop=args.ipsecstop)
            others[host].start()

    th_initiator.start()

    # now wait for the threads to finish their job
    th_responder.join(timeout=args.initshwait)
    th_initiator.join(timeout=args.initshwait)
    if "nic" in test:
        th_nic.join(timeout=args.initshwait)

    if "others" in test:
        for host in test["others"]:
            others[host].join(timeout=args.initshwait)

    kill_tcpdump(test)

    if "is_testlist" in st:
        e = shut_down_hosts(args, test_hosts, prefix=prefix)
        if (prefix):
            tq.put(prefix)

    return e


def setup_single_test(args, start='', st=None):

    test = dict()

    logging.debug("try to run as single test")
    if args.docker:
        test["type"] = "dockerplutotest"
        if check_host_xfrm_stack():
            return True
    elif args.kvm:
        test["type"] = "kvmplutotest"
    else:
        logging.info("ABORT %s single test unknown type", test["type"])
        return

    if args.testname and args.testrun:
        logging.info("ABORT conflicting options --testname and --testrun")
        return True
    elif args.testname:
        test["name"] = args.testname
        d = "%s/testing/pluto/" % args.libreswandir
        if not os.path.isdir(d):
            logging.error("****** missing directory %s", d)
            return
        t = d + '/' + args.testname
        if not os.path.isdir(t):
            logging.error("****** missing test directory %s", d)
            return
    else:
        test["name"] = os.path.basename(os.getcwd())
        test[
            "expect"] = "XXXX"  # this is a single test don't know the expected outcome
        logging.debug(
            "start test from cwd %s type %s", test["name"], test["type"])

    prefix = None
    if args.prefix:
        prefix = args.prefix[0]

    do_single_test(args=args, start=start, test=test, st=st, prefix=prefix)


def do_single_test(args, start='', test=None, st=None, prefix=None):

    if not start:
        start = time.time()

    if "is_testlist" in st:
        logging.debug("single test from a testlist worker pool")
        if os.path.exists('stop-tests-now'):
            logging.error(
                "****** skip tests found stop-tests-now in dir %s *****", os.getcwd())
            return

        if os.path.exists(test["name"]):
            try:
                logging.debug("chdir %s in dir %s", test["name"], os.getcwd())
                os.chdir(test["name"])  # chdir start
            except:
                logging.exception(
                    "EXCEPTION chdir %s in dir %s", test["name"], os.getcwd())
                return
        else:
            logging.error(
                "******* missing test directory %s in %s", test["name"], os.getcwd())
            return
    else:
        logging.debug("%s is a single test not a testlist", test["name"])

    if os.path.exists('stop-tests-now'):
        logging.error(
            "****** skip test %s found stop-tests-now *****", testdir)
        os.chdir("..")  # chdir end
        return

    e = None

    if test["type"] == 'dockerplutotest':
        if do_dockerplutotest(args=args, start=start, test=test, st=st):
            os.chdir("..")  # chdir end
            return True
    elif test["type"] == 'kvmplutotest':
        if do_kvmplutotest(args=args, start=start, test=test, st=st, prefix=prefix):
            os.chdir("..")  # chdir end
            return True
    else:
        logging.info("ABORT %s single test unknown type", test["type"])
        os.chdir("..")  # chdir end
        return
    grep_4_known_errors(test)
    s = sanitize(args.sanitizer)
    write_result(args, start, test["name"], s, testexpect=test["expect"])

    if "odir" in st:
        logging.info("copying results")
        rsync_file_to_dir(os.getcwd(), st["odir"])
    else:
        logging.debug("not copying results")

    if "is_testlist" in st:
        os.chdir("..")
        # chdir end


def scancwd(args):
    r = args.testlist or none
    stdir = os.getcwd()
    logging.info(
        "re scan results in CWD %s/%s", stdir, r)

    if not os.path.exists(r):
        logging.error(
            "ABORT re sanitize missing %s/%s. Did it start from right place", stdir, r)
        logging.info("run it from libreswan/testing/pluto?")
        return None

    s = scantests(args, stdir=stdir)
    s.scan_test_runs()

    return


def resanitize(args):
    r = args.testlist
    logging.info(
        "re sanitize the existing results in the CWD %s.", r)

    if not os.path.exists(r):
        logging.error(
            "ABORT re sanitize missing %s. Did it start from right place", r)
        logging.info("run it from libreswan/testing/pluto?")
        return None

    s = scantests(args, stdir=os.getcwd())
    s.scan_test_runs()

    return
    output_dir = setup_result_dir(args, True)

    # for h in DEFAULTCONFIG['swanhosts']:
    #       f = "OUTPUT/%s.console.diff"%h
    #       if  os.path.exists(f):
    #               cmd = "grep -E '^[+-]' %s | grep -E -v '^(\+\+\+|---)' | sort -u | cmp -s - ../strongswan-star-diff"%f
    #               (status, output) =  commands.getstatusoutput(cmd);
    #               if not status:
    #                       print "cp %s/OUTPUT/%s.console.txt %s/" % (testdir,h,testdir)
    #                       print "cat %s/%s"%(testdir,f)


def check_host_xfrm_stack():
    # check if there is XFRM stack on the host

    v_line = runcmd("ipsec version")
    if v_line.split()[3] != '(XFRM)':
        logging.debug("no XFRM stack found, try to load it %s", v_line)
        runcmd("sudo /usr/local/sbin/ipsec _stackmanager start --xfrm")
        v_line = runcmd("ipsec version")
        if v_line.split()[3] != '(XFRM)':
            # give up
            logging.error(
                "ABORT no XFRM stack found. cannot run docker test, %s", v_line)
            return True


def run_docker_q(args, tests, st):
    logging.info(
        "%s Docker pluto tests to run. pool size %s, from %s", st[
            "dtorun"], args.npool,
        args.testlist)

    if check_host_xfrm_stack():
        return True

    bootp = [1, 2, 3]
    logging.debug("add %d boot workers to dq" % (len(bootp)))
    for i in bootp:
        logging.debug("add %d to the dq" % (i))
        dqstart.put(i)
        dqstop.put(i)

    for name, test in tests.items():
        logging.debug("queue up %s type %s", test["name"], test["type"])

    with concurrent.futures.ProcessPoolExecutor(max_workers=int(args.npool)) as executor:
        for name, test in tests.items():
            if test["type"] != 'dockerplutotest':
                continue
            logging.debug("queue up %s type %s", test["name"], test["type"])
            executor.submit(do_single_test, args=args, test=test, st=st)


def update_progress_table(args, tests, st):
    return


def tlist_progress_worker(args, st):

    if tp.empty():
        return  # no need to execute

    tp.get()
    progress = scantests(args, stdir=st["odir"], to_append=True)
    progress.scan_test_runs()
    tp.put(1)


def kvm_test_worker(args, test, st):

    logging.debug("work q %s", test["name"])
    start = time.time()
    prefix = tq.get(True)  # Blocking wait to get worker slot.
    logging.info("%s got worker slot %s. Waited for %.2f" %
                 (test["name"], prefix, time.time() - start))
    start = time.time()  # now it is the start of the test
    do_single_test(args, start, test, st, prefix=prefix)


def run_kvm_q_conc(args, tests, st, tried=0):

    maxDomains = len(args.prefix)
    logging.info(
        "%s kvmpluto tests to run concurrently. pool size %s, from %s", st[
            "ktorun"], len(args.prefix), args.testlist)

    for name, test in tests.items():
        logging.debug("queue up %s type %s", test["name"], test["type"])

    if tried and maxDomains > 2:
        maxDomains = 2

    j = 0;
    for i in args.prefix:
        if j <= maxDomains:
            tq.put(i)
        j += 1

    tp.put(1)

    maxWorkers = maxDomains + 1
    with concurrent.futures.ProcessPoolExecutor(max_workers=maxWorkers) as executor:
        progress_period = maxWorkers
        i = 0
        try:
            for name, test in tests.items():
                if test["type"] != 'kvmplutotest':
                    continue
                i += 1
                logging.debug("queue up %s %s", test["type"], test["name"])
                if (i % progress_period == 0):
                    executor.submit(tlist_progress_worker, args=args, st=st)
                    progress_period *= 2
                    progress_period = 50 if progress_period >= 50 else progress_period

                executor.submit(kvm_test_worker, args=args, test=test, st=st)
        except KeyboardInterrupt:
            executor.cancel()
            return


def run_kvm_q(args, tests, st):

    logging.info(
        "%s KVM pluto tests to run. from %s", st["ktorun"], args.testlist)
    for name, test in tests.items():
        if test["type"] != 'kvmplutotest':
            continue
        logging.debug("next up %s type %s", test["name"], test["type"])
        do_single_test(args=args, test=test, st=st)
        st["kran"] = st["kran"] + 1
        st["progress"].scan_single_test_result(tname=name)
        st["progress"].create_test_run_table()


def do_test_list(args, start, tried, output_dir):

    if args.testname and args.testrun:
        logging.info("ABORT conflicting options --testname and --testrun")
        return True
    elif args.testrun:
        tdir = args.libreswandir + '/testing/pluto'
        logging.debug("change working directory to %s", tdir)
        os.chdir(tdir)

    if not os.path.exists(args.testlist):
        logging.debug("no TESTLIST file \"%s\" found" % (args.testlist))
        return ("", False, 0)
    ran = 0
    torun = 0

    st = dict()
    st["is_testlist"] = True

    odir = output_dir

    if not tried:
        odir = setup_result_dir(args, True)

    st["odir"] = odir
    st["ktorun"] = 0
    st["kran"] = 0
    st["dtorun"] = 0
    st["dran"] = 0
    st["n_lines"] = 0
    st["n_lines_skip"] = 0
    st["knottorun"] = 0
    st["dnottorun"] = 0

    s = 'stop-tests-now'
    if os.path.exists(s) and (tried == 0):
        os.unlink(s)
        logging.debug("removing existing %s" % s)

    rsync_file(args.testlist, "%s/TESTLIST" % odir)

    tests = collections.OrderedDict()  # slow but keeps the order
    tests.clear()

    progress = scantests(args, stdir=odir)
    progress.scan_test_runs()
    st["progress"] = progress

    logging.info("** read tests from file %s **", args.testlist)
    for testtype, testdir, testexpect in TestList(args.testlist, args, verbose=False):
        st["n_lines"] = st["n_lines"] + 1
        if os.path.exists(s):
            logging.error("* stop all tests now. Found %s *", s)
            return (odir, True, ran)

        r_file = odir + '/' + testdir + '/OUTPUT/RESULT'
        test_odir = odir + '/' + testdir + '/OUTPUT'

        if os.path.exists(r_file):

            if args.failed or tried:
                if kvm_error(test_odir, failed=args.failed):
                    logging.info(
                        "result [%s] KVMERROR. deleting previous run, retrying %s/%s", r_file, tried, args.retry)
                    o = odir + '/' + testdir
                    # shutil.rmtree(o)
                else:
                    TestList.log_skip(
                        testdir, "%s exists and max retry count %s exceeded" % (r_file, args.retry))
                    continue
            else:
                # output already exists: skip test
                TestList.log_skip(
                    testdir, "%s exists, do not run again" % r_file)
                if testtype == 'kvmplutotest':
                    st["knottorun"] = st["knottorun"] + 1
                if testtype == 'dockerplutotest':
                    st["dnottorun"] = st["dnottorun"] + 1
                continue

        tests[testdir] = {}
        tests[testdir]["name"] = testdir  # do i need it double? may be not
        tests[testdir]["type"] = testtype
        tests[testdir]["expect"] = testexpect

        if testtype == 'kvmplutotest':
            st["ktorun"] = st["ktorun"] + 1

        if testtype == 'dockerplutotest':
            st["dtorun"] = st["dtorun"] + 1

    if not (st["dtorun"] + st["ktorun"]):

        logging.info(
            "red %s lines from %s. to run (kvm %s , docker %s), do not run(kvm %s, docker %s)",
            st["n_lines"], args.testlist, st["ktorun"], st["dtorun"], st["knottorun"], st["dnottorun"])

        logging.info("** no tests to run in file %s. could run %s **",
                     args.testlist, (st["dnottorun"] + st["knottorun"]))

        return (odir, True, 0)

    # in every retry check if directory need to be  created
    if not os.path.isdir(st["odir"]):
        if not makedirs_log(st["odir"], 0o775):
            logging.error(
                "failed to create directory %s. no results will be copied", st["odir"])

    if st["dtorun"]:
        run_docker_q(args, tests, st)

    if st["ktorun"]:
        if len(args.prefix) > 1:
            run_kvm_q_conc(args, tests, st, tried)
        else:
            run_kvm_q(args, tests, st)

    if os.path.isdir(st["odir"]) and ((st["dtorun"] + st["ktorun"]) > 0):
        endscan = scantests(args, stdir=st["odir"], to_append=True)
        endscan.scan_test_runs()
    return (odir, True, (st["dtorun"] + st["ktorun"]))


def makedirs_log(D, mode):  # mode is octal

    try:
        logging.debug("create directory %s mode %s", D, mode)
        os.makedirs(D, mode)
    except:
        logging.error("failed to create directory %s", output_dir)
        return

    return True


def setup_result_dir(args, to_create):
    date_dir = time.strftime("%Y-%m-%d", time.localtime())
    output_dir = args.resultsdir

    if (args.node):
        output_dir = args.resultsdir + '/' + args.node

    if not os.path.isdir(output_dir) and to_create:
        if not makedirs_log(output_dir, 0o775):
            return

    ipsecversion = ''

    if args.rerun:  # this is continuation of previous run
        ipsecversion = args.rerun
        output_dir = output_dir + '/'
    else:
        if args.node:
            output_dir = output_dir + '/' + date_dir + '-' + args.node
        else:
            output_dir = output_dir + '/' + date_dir

        try:
            # inside 'make check' IPSECVERSION is set
            ipsecversion = os.environ["IPSECVERSION"]
        except:
            pass

    if not ipsecversion:
        # Let's assume the path is <LIBRESWANSRC>/testing/utis
        cwd = os.getcwd()
        dn = args.libreswandir
        os.chdir(dn)
        cmd = "make showversion"
        ipsecversion = subprocess.getoutput(cmd)
        ipsecversion = ipsecversion.strip()
        os.chdir(cwd)
        logging.debug("cd %s; %s ; IPSECVERSION %s", dn, cmd, ipsecversion)
    else:
        logging.debug("IPSECVERSION %s was set in environment", ipsecversion)

    if ipsecversion and args.rerun:
        output_dir = output_dir + ipsecversion
    else:
        output_dir = output_dir + '-' + ipsecversion

    if (args.resanitize):
        output_dir = output_dir + "-resanitized"

    if to_create and args.newrun and os.path.exists(output_dir):
        logging.info("newrun, remove results [%s]", output_dir)
        if os.path.islink(output_dir):
            os.unlink(output_dir)
        elif os.path.isdir(output_dir):
            shutil.rmtree(output_dir)
        else:
            os.unlink(output_dir)

    if not os.path.isdir(output_dir) and to_create:
        if not makedirs_log(output_dir, 0o775):
            return

    if to_create and not os.path.isdir(output_dir):
        logging.error("%s is not directory.", output_dir)
        return

    logging.info("results will be in %s", output_dir)

    return output_dir


def append_graphs_tables(args):
    s = scantests(args)
    s.scan_test_runs()
    # testsum.read_dirs(args)


def gen_graphs_tables(args):
    s = scantests(args)
    s.scan_test_runs()
    # testsum.read_dirs(args)


def gen_stop_tests_now(args):
    if not os.path.exists(args.testlist):
        return None

    s = "stop-tests-now"
    f = open(s, 'w')
    f.write("stop\n")
    f.close


def main():
    signal.signal(signal.SIGINT, handler)

    start = time.time()
    args = cmdline()
    tried = 0
    ran = 2  # used for summery generation.

    if args.graphs:
        gen_graphs_tables(args)
        return
    elif args.graphsappend:
        a = scantests(args, stdir=args.graphsappend, to_append=True)
        a.scan_test_runs()
        return
    elif args.stop:
        gen_stop_tests_now(args)
        return
    elif args.resanitize:
        resanitize(args)
        # gen_graphs_tables(args)
        return
    elif args.scancwd:
        scancwd(args)
        return

    if args.prefix:
        args.leavezombies = True

    if not args.leavezombies:
        kill_zombies("swantest")

    output_dir = ''

    # if there is TESTLIST run in batch mode.
    (output_dir, is_testlist, ran_tests) = do_test_list(
        args, start, tried, output_dir)
    if is_testlist:
        tried = 1 + tried
        logging.info("ran %s retry %s %s/%s",
                     ran_tests, args.testlist, tried, args.retry)
        while (tried < args.retry):
            if ran_tests > 0:
                # gen_graphs_tables(args)
                logging.info(
                    "retry %s %s/%s", args.testlist, tried, args.retry)
            (output_dir, testlist, ran_tests) = do_test_list(
                args, start, tried, output_dir)
            logging.info("ran %s retry %s %s/%s",
                         ran_tests, args.testlist, tried, args.retry)
            tried = 1 + tried
            time.sleep(60)

    else:
        # no TESTLIST. Let's try a single test
        st = dict()
        setup_single_test(args=args, start=start, st=st)

if __name__ == "__main__":
    main()