#
# Copyright 2009- ECMWF.
#
# This software is licensed under the terms of the Apache Licence version 2.0
# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
# In applying this licence, ECMWF does not waive the privileges and immunities
# granted to it by virtue of its status as an intergovernmental organisation
# nor does it submit to any jurisdiction.
#

# This test ensures the User API works as expected.

import time
import os
import pwd
import select
import sys
import threading
from datetime import datetime
import shutil  # used to remove directory tree

# ecflow_test_util, see File ecflow_test_util.py
import ecflow_test_util as Test
from ecflow import Defs, Suite, Family, Task, Edit, Meter, Clock, DState, Style, State, RepeatDate, RepeatDateTime, \
    PrintStyle, File, Client, SState, CheckPt, Cron, Late, debug_build, Flag, FlagType

import re
import platform


def disable_on(regex_platform):
    def decorator(func):
        def wrapper(*args, **kwargs):
            if re.match(regex_platform, platform.platform()):
                print(f"Skipping {func.__name__} due to match with {regex_platform}")
            else:
                return func(*args, **kwargs)

        return wrapper

    return decorator


def ecf_includes():  return os.getcwd() + "/test/data/includes"


def debugging(): return False  # Use to enable auto flush and disable log file tests


def create_defs(name, port, protocol):
    defs = Defs()
    suite_name = name
    if len(suite_name) == 0: suite_name = "s1"
    suite = defs.add_suite(suite_name);

    ecfhome = Test.ecf_home(port);
    suite.add_variable("ECF_HOME", ecfhome);
    protocol_option = "--http" if protocol == Test.Protocol.HTTP else ""
    suite.add_variable("ECF_CLIENT_EXE_PATH", f"{File.find_client()} {protocol_option}")
    suite.add_variable("SLEEP", "1");  # not strictly required since default is 1 second
    suite.add_variable("ECF_INCLUDE", ecf_includes());

    family = suite.add_family("f1")
    family.add_task("t1")
    family.add_task("t2")
    return defs


def test_host_port(ci, host, port):
    try:
        ci.set_host_port(host, port)
        return True
    except RuntimeError:
        return False


def test_host_port_(ci, host_port):
    try:
        ci.set_host_port(host_port)
        return True
    except RuntimeError:
        return False


def test_client_host_port(host, port):
    try:
        Client(host, port)
        return True
    except RuntimeError:
        return False


def test_client_host_port_(host_port):
    try:
        Client(host_port)
        return True
    except RuntimeError:
        return False


def sync_local(ci):
    if ci.is_auto_sync_enabled():
        return
    ci.sync_local()


def test_set_host_port():
    print("test_set_host_port")
    ci = Client()
    ci.debug(True)
    ci.enable_http()
    print(" Client.get_host() = " + ci.get_host())
    print(" Client.get_port() = " + ci.get_port())
    assert test_host_port(ci, "host", "3141"), "Expected no errors"
    assert test_host_port(ci, "host", 4444), "Expected no errors"
    assert test_host_port_(ci, "host:4444"), "Expected no errors"
    assert test_host_port_(ci, "host@4444"), "Expected no errors"
    assert test_host_port(ci, "", "") == False, "Expected errors"
    assert test_host_port(ci, "host", "") == False, "Expected errors"
    assert test_host_port(ci, "host", "host") == False, "Expected errors"
    assert test_host_port_(ci, "host:host") == False, "Expected errors"
    assert test_host_port_(ci, "3141:host") == False, "Expected errors"
    assert test_host_port_(ci, "3141@host") == False, "Expected errors"
    assert test_host_port_(ci, "3141@") == False, "Expected errors"

    assert test_client_host_port("host", "3141"), "Expected no errors"
    assert test_client_host_port("host", 4444), "Expected no errors"
    assert test_client_host_port_("host:4444"), "Expected no errors"
    assert test_client_host_port("", "") == False, "Expected errors"
    assert test_client_host_port("host", "") == False, "Expected errors"
    assert test_client_host_port("host", "host") == False, "Expected errors"
    assert test_client_host_port_("host:host") == False, "Expected errors"
    assert test_client_host_port_("3141:host") == False, "Expected errors"


def print_test(ci, test_name):
    print(test_name, flush=True)
    if ci.is_auto_sync_enabled():
        ci.log_msg(test_name + " ============= AUTO SYNC ENABLED ================================ ")
    else:
        ci.log_msg(test_name + " ================================================================ ")


def test_version(ci):
    print_test(ci, "test_version")
    client_version = ci.version();
    server_version = ci.server_version();
    print("  client_version: ", client_version)
    print("  server_version: ", server_version)
    assert client_version == server_version, "Expected client version(" + client_version + ") and server version(" + server_version + ") to match\n";


def test_client_get_server_defs(ci, protocol):
    print_test(ci, "test_client_get_server_defs")
    ci.delete_all()  # start fresh
    ci.load(create_defs("", ci.get_port(), protocol))
    ci.get_server_defs()
    assert ci.get_defs().find_suite("s1") is not None, "Expected to find suite of name s1:\n" + str(ci.get_defs())

    ci.delete_all()  # start fresh
    ci.load(create_defs("", ci.get_port(), protocol))
    sync_local(ci)

    assert ci.get_defs().find_suite("s1") is not None, "Expected to find suite of name s1:\n" + str(ci.get_defs())


def test_client_new_log(ci):
    print_test(ci, "test_client_new_log")
    log_path = Test.log_file_path(ci.get_port())
    if not os.path.exists(log_path):
        print(log_path + " : log does not exist ?")

    new_log_file_name = "./test_client_new_log_" + str(os.getpid()) + ".log"
    try:
        os.remove(new_log_file_name)  # delete file if it exists
    except:
        pass

    ci.new_log(new_log_file_name)
    ci.flush_log()  # close log file and force write to disk
    assert os.path.exists(new_log_file_name), new_log_file_name + " : New log does not exist"
    try:
        os.remove(new_log_file_name)
    except:
        pass

    # reset new log to original
    ci.new_log(log_path)
    ci.ping()
    ci.flush_log()  # close log file and force write to disk
    log_file = open(log_path)
    try:
        log_text = log_file.read();  # assume log file not to big
    finally:
        log_file.close();
    assert log_text.find("--ping") != -1, "Expected to find --ping in log file"

    if not os.path.exists(log_path):
        print(log_path + " : log does not exist ?")


def test_client_clear_log(ci):
    if debugging(): return  # dont run this test when debugging as log file is lost
    print_test(ci, "test_client_clear_log")
    log_path = Test.log_file_path(ci.get_port())
    # populate log
    ci.ping();
    ci.ping();
    ci.flush_log()  # close log file and force write to disk
    log_file = open(log_path)
    try:
        log_text = log_file.read();  # assume log file not to big
    finally:
        log_file.close();
    assert log_text.find("--ping") != -1, "Expected to find --ping in log file"

    ci.clear_log()
    if not os.path.exists(log_path):
        print(log_path + " : log does not exist ?")

    log_file = open(log_path)
    try:
        log_text = log_file.read();  # assume log file not to big
    finally:
        log_file.close();
    assert len(log_text) == 0, "Expected log file to be empty but found " + log_text

    if not os.path.exists(log_path):
        print(log_path + " : log does not exist ?")


def test_client_log_msg(ci):
    if debugging(): return  # dont run this test when debugging

    print_test(ci, "test_client_log_msg")
    log_path = Test.log_file_path(ci.get_port())
    if not os.path.exists(log_path):
        print(log_path + " : log does not exist ?")

    # Send a message to the log file, then make sure it was written
    ci.log_msg("Humpty dumpty sat on a wall!")
    ci.flush_log();  # flush and close log file, so we can open it
    log_file = open(log_path)
    try:
        log_text = log_file.read();  # assume log file not to big
    finally:
        log_file.close();
    assert log_text.find("Humpty dumpty sat on a wall!") != -1, "Expected to find Humpty dumpty in the log file"

    if not os.path.exists(log_path):
        print(log_path + " : log does not exist ?")


def test_client_restart_server(ci):
    print_test(ci, "test_client_restart_server")
    ci.restart_server()
    sync_local(ci)
    assert ci.get_defs().get_server_state() == SState.RUNNING, "Expected server to be running"

    paths = list(ci.changed_node_paths)
    assert len(paths) == 1, "expected changed node to be the root node: " + paths
    assert paths[0] == "/", "Expected root path but found " + str(paths[0])


def test_client_halt_server(ci):
    print_test(ci, "test_client_halt_server")
    ci.halt_server()
    sync_local(ci)
    assert ci.get_defs().get_server_state() == SState.HALTED, "Expected server to be halted"

    paths = list(ci.changed_node_paths)
    assert len(paths) == 1, "expected changed node to be the root node"
    assert paths[0] == "/", "Expected root path but found " + str(paths[0])
    ci.restart_server()


def test_client_shutdown_server(ci):
    print_test(ci, "test_client_shutdown_server")
    ci.shutdown_server()
    sync_local(ci)
    assert ci.get_defs().get_server_state() == SState.SHUTDOWN, "Expected server to be shutdown"

    paths = list(ci.changed_node_paths)
    assert len(paths) == 1, "expected changed node to be the root node"
    assert paths[0] == "/", "Expected root path but found " + str(paths[0])


def test_client_load_in_memory_defs(ci, protocol):
    print_test(ci, "test_client_load_in_memory_defs")
    ci.delete_all()  # start fresh
    ci.load(create_defs("", ci.get_port(), protocol))
    sync_local(ci)
    assert ci.get_defs().find_suite("s1") is not None, "Expected to find suite of name s1:\n" + str(ci.get_defs())


def test_client_load_from_disk(ci, protocol):
    print_test(ci, "test_client_load_from_disk")
    ci.delete_all()  # start fresh
    defs = create_defs("", ci.get_port(), protocol);
    defs_file = "test_client_load_from_disk_" + str(os.getpid()) + ".def"
    defs.save_as_defs(defs_file)
    assert os.path.exists(defs_file), "Expected file " + defs_file + " to exist after defs.save_as_defs()"
    ci.load(defs_file)  # open and parse defs file, and load into server.\n"

    sync_local(ci)
    assert ci.get_defs().find_suite("s1") is not None, "Expected to find suite of name s1:\n" + str(ci.get_defs())
    os.remove(defs_file)


def test_client_checkpt(ci, protocol):
    print_test(ci, "test_client_checkpt")
    # start fresh
    ci.delete_all()
    port = ci.get_port();
    try:
        os.remove(Test.checkpt_file_path(port))
        os.remove(Test.backup_checkpt_file_path(port))
    except:
        pass

    ci.load(create_defs("", ci.get_port(), protocol))
    ci.checkpt()
    assert os.path.exists(Test.checkpt_file_path(port)), "Expected check pt file to exist after ci.checkpt()"
    assert os.path.exists(Test.backup_checkpt_file_path(port)) == False, "Expected back up check pt file to *NOT* exist"

    ci.checkpt()  # second check pt should cause backup check pt to be written
    assert os.path.exists(
        Test.backup_checkpt_file_path(port)), "Expected back up check pt file to exist after second ci.checkpt()"

    ci.checkpt(CheckPt.NEVER)  # switch of check pointing
    ci.checkpt(CheckPt.ALWAYS)  # always check point, at any state change
    ci.checkpt(CheckPt.ON_TIME)  # Check point periodically, by interval set in server
    ci.checkpt(CheckPt.ON_TIME, 200)  # Check point periodically, by interval set in server
    ci.checkpt(CheckPt.UNDEFINED, 0, 35)  # Change check point save time alarm

    os.remove(Test.checkpt_file_path(port))
    os.remove(Test.backup_checkpt_file_path(port))


def test_client_restore_from_checkpt(ci, protocol):
    print_test(ci, "test_client_restore_from_checkpt")
    # start fresh
    ci.delete_all()
    port = ci.get_port()
    try:
        os.remove(Test.checkpt_file_path(port))
        os.remove(Test.backup_checkpt_file_path(port))
    except:
        pass

    ci.load(create_defs("", ci.get_port(), protocol))
    ci.checkpt()
    ci.delete_all()

    sync_local(ci)
    assert ci.get_defs().find_suite("s1") is None, "Expected all suites to be delete:\n"

    ci.halt_server()  # server must be halted, otherwise restore_from_checkpt will throw
    ci.restore_from_checkpt()

    sync_local(ci)
    assert ci.get_defs().find_suite("s1") is not None, "Expected to find suite s1 after restore from checkpt:\n" + str(
        ci.get_defs())

    os.remove(Test.checkpt_file_path(port))
    ci.restart_server()


def get_username(): return pwd.getpwuid(os.getuid())[0]


def test_client_reload_wl_file(ci):
    print_test(ci, "test_client_reload_wl_file")

    expected = False
    try:
        ci.reload_wl_file();
    except:
        expected = True
    assert expected, "Expected reload to fail when no white list specified"

    # create a white list file
    port = ci.get_port()
    wl_file = open(Test.white_list_file_path(port), 'w')
    wl_file.write("#\n")
    wl_file.write("4.4.14   #   comment\n\n")
    wl_file.write("# These user have read and write access to the server\n")
    wl_file.write(
        get_username() + "\n")  # add current user otherwise remaining test's, wont have access from server anymore
    wl_file.write("axel  # admin\n")
    wl_file.write("john  # admin\n\n")
    wl_file.write("# Read only users\n")
    wl_file.write("-fred   # needs read access only\n")
    wl_file.write("-joe90  # needs read access only\n")
    wl_file.close();

    ci.reload_wl_file();
    os.remove(Test.white_list_file_path(port))


def test_client_run(ci, protocol):
    print_test(ci, "test_client_run")
    ci.delete_all()
    port = ci.get_port()
    defs = create_defs("test_client_run", port, protocol)
    suite = defs.find_suite("test_client_run")
    suite.add_defstatus(DState.suspended)

    defs.generate_scripts();
    msg = defs.check_job_creation(verbose=True)
    assert len(msg) == 0, msg

    ci.restart_server()
    ci.load(defs)
    ci.begin_all_suites()
    ci.run("/test_client_run", False)

    count = 0
    while 1:
        count += 1
        ci.sync_local()  # get the changes, synced with local defs
        suite = ci.get_defs().find_suite("test_client_run")
        assert suite is not None, "Expected to find suite test_client_run:\n" + str(ci.get_defs())
        print(f"Suite state: {suite.get_state()}, {count}, at {datetime.now()}")
        if suite.get_state() == State.complete:
            break;
        assert suite.get_state() != State.aborted, "Expected suite to complete, but found it was aborted"
        time.sleep(3)
        if count > 20:
            assert False, "test_client_run aborted after " + str(count) + " loops:\n" + str(ci.get_defs())

    ci.log_msg("Looped " + str(count) + " times")
    dir_to_remove = Test.ecf_home(port) + "/" + "test_client_run"
    shutil.rmtree(dir_to_remove, ignore_errors=True)


def test_client_run_with_multiple_paths(ci, protocol):
    print_test(ci, "test_client_run_with_multiple_paths")
    ci.delete_all()
    port = ci.get_port()
    defs = create_defs("test_client_run_with_multiple_paths", port, protocol)
    suite = defs.find_suite("test_client_run_with_multiple_paths")
    suite.add_defstatus(DState.suspended)

    defs.generate_scripts();
    msg = defs.check_job_creation(verbose=True)
    assert len(msg) == 0, msg

    ci.restart_server()
    ci.load(defs)
    ci.begin_all_suites()
    path_list = ["/test_client_run_with_multiple_paths/f1/t1", "/test_client_run_with_multiple_paths/f1/t2"]
    ci.run(path_list, False)

    count = 0
    while 1:
        count += 1
        ci.sync_local()  # get the changes, synced with local defs
        suite = ci.get_defs().find_suite("test_client_run_with_multiple_paths")
        assert suite is not None, "Expected to find suite test_client_run_with_multiple_paths:\n" + str(ci.get_defs())
        if suite.get_state() == State.complete:
            break;
        time.sleep(3)
        if count > 20:
            assert False, "test_client_run_with_multiple_paths aborted after " + str(count) + " loops:\n" + str(
                ci.get_defs())

    ci.log_msg("Looped " + str(count) + " times")

    dir_to_remove = Test.ecf_home(port) + "/" + "test_client_run_with_multiple_paths"
    shutil.rmtree(dir_to_remove, ignore_errors=True)


def test_client_requeue(ci, protocol):
    print_test(ci, "test_client_requeue")
    ci.delete_all()
    port = ci.get_port()
    defs = create_defs("test_client_requeue", port, protocol)
    suite = defs.find_suite("test_client_requeue")
    suite.add_defstatus(DState.suspended)

    defs.generate_scripts()
    msg = defs.check_job_creation(verbose=True)
    assert len(msg) == 0, msg

    ci.restart_server()
    ci.load(defs)
    ci.begin_all_suites()

    ci.force_state_recursive("/test_client_requeue", State.unknown)
    sync_local(ci)
    suite = ci.get_defs().find_suite("test_client_requeue")
    assert suite.get_state() == State.unknown, "Expected to find suite with state unknown"

    ci.requeue("/test_client_requeue")
    sync_local(ci);
    suite = ci.get_defs().find_suite("test_client_requeue")
    assert suite.get_state() == State.queued, "Expected to find suite with state queued"

    dir_to_remove = Test.ecf_home(port) + "/" + "test_client_requeue"
    shutil.rmtree(dir_to_remove, ignore_errors=True)


def test_client_requeue_with_multiple_paths(ci, protocol):
    print_test(ci, "test_client_requeue_with_multiple_paths")
    ci.delete_all()
    port = ci.get_port()
    defs = create_defs("test_client_requeue_with_multiple_paths", port, protocol)
    suite = defs.find_suite("test_client_requeue_with_multiple_paths")
    suite.add_defstatus(DState.suspended)

    defs.generate_scripts()
    msg = defs.check_job_creation(verbose=True)
    assert len(msg) == 0, msg

    ci.restart_server()
    ci.load(defs)
    ci.begin_all_suites()

    ci.force_state_recursive("/test_client_requeue_with_multiple_paths", State.unknown)
    sync_local(ci);
    task1 = ci.get_defs().find_abs_node("/test_client_requeue_with_multiple_paths/f1/t1")
    task2 = ci.get_defs().find_abs_node("/test_client_requeue_with_multiple_paths/f1/t2")
    assert task1.get_state() == State.unknown, "Expected to find t1 with state unknown"
    assert task2.get_state() == State.unknown, "Expected to find t2 with state unknown"

    path_list = ["/test_client_requeue_with_multiple_paths/f1/t1", "/test_client_requeue_with_multiple_paths/f1/t2"]
    ci.requeue(path_list)
    sync_local(ci);
    task1 = ci.get_defs().find_abs_node("/test_client_requeue_with_multiple_paths/f1/t1")
    task2 = ci.get_defs().find_abs_node("/test_client_requeue_with_multiple_paths/f1/t2")
    assert task1.get_state() == State.queued, "Expected to find task t1 with state queued"
    assert task2.get_state() == State.queued, "Expected to find task t2 with state queued"

    dir_to_remove = Test.ecf_home(port) + "/" + "test_client_requeue_with_multiple_paths"
    shutil.rmtree(dir_to_remove, ignore_errors=True)


def test_client_free_dep(ci, protocol):
    print_test(ci, "test_client_free_dep")
    ci.delete_all()

    # add a real clock, since we are adding date dependencies
    # Note: adding a future time dependency on a task, will cause it to requeue, when complete
    # Hence even when we free these dependency they get requeued.
    # So we use todays date.
    ltime = time.localtime();
    day = ltime.tm_mday
    month = ltime.tm_mon
    year = ltime.tm_year

    defs = Defs()
    suite = defs.add_suite("test_client_free_dep");
    suite.add_clock(Clock(False))  # true means hybrid, False means real
    ecfhome = Test.ecf_home(ci.get_port());
    suite.add_variable("ECF_HOME", ecfhome);
    protocol_option = "--http" if protocol == Test.Protocol.HTTP else ""
    suite.add_variable("ECF_CLIENT_EXE_PATH", f"{File.find_client()} {protocol_option}")
    suite.add_variable("SLEEPTIME", "1");
    suite.add_variable("ECF_INCLUDE", ecf_includes());
    family = suite.add_family("f1")
    family.add_task("t1").add_time("00:01")
    family.add_task("t2").add_date(day, month, year)
    family.add_task("t3").add_trigger("1 == 0")
    t4 = family.add_task("t4")
    t4.add_time("00:01")
    t4.add_date(day, month, year)
    t4.add_trigger("1 == 0")

    defs.generate_scripts();
    msg = defs.check_job_creation(verbose=True)
    assert len(msg) == 0, msg

    ci.restart_server()
    ci.load(defs)
    ci.begin_all_suites()

    t1_path = "/test_client_free_dep/f1/t1"
    t2_path = "/test_client_free_dep/f1/t2"
    t3_path = "/test_client_free_dep/f1/t3"
    t4_path = "/test_client_free_dep/f1/t4"
    while 1:
        ci.sync_local()
        t1 = ci.get_defs().find_abs_node(t1_path)
        t2 = ci.get_defs().find_abs_node(t2_path)
        t3 = ci.get_defs().find_abs_node(t3_path)
        t4 = ci.get_defs().find_abs_node(t4_path)

        if t1.get_state() == State.queued: ci.free_time_dep(t1_path)
        if t2.get_state() == State.queued: ci.free_date_dep(t2_path)
        if t3.get_state() == State.queued: ci.free_trigger_dep(t3_path)
        if t4.get_state() == State.queued: ci.free_all_dep(t4_path)

        suite = ci.get_defs().find_suite("test_client_free_dep")
        if suite.get_state() == State.complete:
            break;
        time.sleep(3)

    dir_to_remove = Test.ecf_home(ci.get_port()) + "/" + "test_client_free_dep"
    shutil.rmtree(dir_to_remove, ignore_errors=True)


class CustomStdOut:

    def __init__(self, stream=sys.stdout):
        self.marker = "<<<OUTPUT CAPTURE END>>>"  # marker to indicate end of capture
        self.captured = ""
        # create a pipe to capture content temporarily
        self.o_pipe, self.i_pipe = os.pipe()
        # store the stream
        self.stream = sys.stdout if stream is None else stream
        self.stream_no = self.stream.fileno()

    def __enter__(self):
        self.captured = ""
        # make a copy of the original stdout
        self.backup_no = os.dup(self.stream_no)
        # redirect stdout into the write pipe
        os.dup2(self.i_pipe, self.stream_no)

        # start a worker thread to capture the output
        self.worker = threading.Thread(target=self._capture_output)
        self.worker.start()

        return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        # write the end marker to indicate the end of capture
        self.stream.write(self.marker)
        self.stream.flush()

        # wait for the worker threadto finish capturing output
        self.worker.join()

        # close the temporary pipe
        os.close(self.i_pipe)
        os.close(self.o_pipe)
        # restore the original output fd
        os.dup2(self.backup_no, self.stream_no)

    def _capture_output(self):
        while True:
            self.captured += os.read(self.o_pipe, 1).decode(self.stream.encoding)
            if self.captured.endswith(self.marker):
                # we reached the end marker, so we stop capturing and remove the marker
                self.captured = self.captured[:-len(self.marker)]
                break

    def value(self):
        return self.captured


@disable_on("macOS-13.*-arm64-.*")
def test_client_stats(ci):
    print_test(ci, "test_client_stats")
    out = CustomStdOut()
    with out:
        stats = ci.stats()
        assert "statistics" in stats, "Expected 'statistics' in the response"
    assert "statistics" in out.value(), "Expected 'statistics' in the captured output"


@disable_on("macOS-13.*-arm64-.*")
def test_client_stats_with_stdout(ci):
    print_test(ci, "test_client_stats_with_stdout")
    out = CustomStdOut()
    with out:
        stats = ci.stats(True)
        assert "statistics" in stats, "Expected 'statistics' in the response"
    assert "statistics" in out.value(), "Expected 'statistics' in the captured output"


@disable_on("macOS-13.*-arm64-.*")
def test_client_stats_without_stdout(ci):
    print_test(ci, "test_client_stats_without_stdout")
    out = CustomStdOut()
    with out:
        stats = ci.stats(False)
        assert "statistics" in stats, "Expected 'statistics' in the response"
    assert not out.value(), "No captured output expected, but found: " + out.value()


def test_client_stats_reset(ci):
    print_test(ci, "test_client_stats_reset")
    ci.stats_reset()
    stats = ci.stats()
    assert "statistics" in stats, "Expected 'statistics' in the response"


def test_client_debug_server_on_off(ci):
    print_test(ci, "test_client_debug_server_on_off")
    ci.debug_server_on()  # writes to standard out
    ci.debug_server_off()


def test_client_check(ci):
    print_test(ci, "test_client_check")
    ci.delete_all()

    defs = Defs()
    defs.add_extern("/a/b/c/d")
    defs.add_extern("/a/b/c/d/e:event")
    defs.add_extern("/a/b/c/d/e:meter")
    defs.add_extern("/made/up/redundant/extren")
    defs.add_extern("/made/up/redundant/extren")
    defs.add_extern("/limits:c1a")
    defs.add_extern("/limits:c1a")
    defs.add_extern("fred")
    defs.add_extern("limits:hpcd")
    defs.add_extern("/suiteName:sg1")
    defs.add_extern("/obs/limits:hpcd")
    suite = defs.add_suite("extern")
    family_f1 = suite.add_family("f1")
    family_f1.add_task("p").add_trigger("/a/b/c/d == complete")  # extern path
    family_f1.add_task("q").add_trigger("/a/b/c/d/e:event == set")  # extern event path
    family_f1.add_task("r").add_trigger("/a/b/c/d/e:meter le 30")  # extern meter path

    suite.add_inlimit("c1a", "/limits")
    suite.add_inlimit("fred")

    family_anon = suite.add_family("anon")
    family_anon.add_inlimit("hpcd", "limits")
    family_anon.add_task("t1").add_inlimit("sg1", "/suiteName")
    family_anon.add_task("t2").add_inlimit("hpcd", "/obs/limits")
    family_anon.add_task("t3").add_inlimit("c1a", "/limits")

    # CLIENT side check
    client_check = defs.check()
    assert len(client_check) == 0, "Expected clean defs check due to externs but found:\n" + client_check + "\n" + str(
        defs)

    # SERVER side check
    ci.load(defs)
    server_check = ci.check("")  # empty string means check the whole defs, otherwise a node path can be specified.
    # print(server_check)
    assert len(server_check) > 0, "Expected defs to fail, since no externs in server "


def test_client_suites(ci, protocol):
    print_test(ci, "test_client_suites")
    ci.delete_all()
    assert len(ci.suites()) == 0, "expected 0 suite "

    defs = create_defs("test_client_suites", ci.get_port(), protocol)
    ci.load(defs)
    assert len(ci.suites()) == 1, "expected 1 suite "

    ci.delete_all()
    defs.add_suite("s2")
    ci.load(defs)
    assert len(ci.suites()) == 2, "expected 2 suite "


def test_client_ch_suites(ci):
    print_test(ci, "test_client_ch_suites")
    try:
        ci.ch_drop_user("")  # drop all handle associated with current user
    except:
        pass  # Drop throws if no handle registered
    ci.delete_all()

    defs = Defs()
    for i in range(1, 7): defs.add_suite("s" + str(i))
    ci.load(defs)
    sync_local(ci)
    assert len(list(ci.get_defs().suites)) == 6, "Expected 6 after load, but found " + str(
        len(list((ci.get_defs().suites))))

    suite_names = ['s1', 's2', 's3']
    ci.ch_register(True, suite_names)  # register interest in suites s1,s2,s3 and any new suites
    print("ch_handle after register : ", ci.ch_handle())
    ci.ch_suites()  # writes to standard out, list of suites and handles
    sync_local(ci)
    assert len(list(ci.get_defs().suites)) == 3, "Expected 3 registered suites but found " + str(
        len(list((ci.get_defs().suites))))

    ci.ch_register(False, ["s1"])  # register interest in suites s1. ci remembers the last client handle
    ci.ch_suites()  # writes to standard out, list of suites and handles
    sync_local(ci)
    ci.ch_suites()  # writes to standard out, list of suites and handles
    assert len(list(ci.get_defs().suites)) == 1, "Expected 1 registered suites but found " + str(
        len(list((ci.get_defs().suites))))


def test_client_ch_register(ci):
    print_test(ci, "test_client_ch_register")
    try:
        ci.ch_drop_user("")  # drop all handle associated with current user
    except:
        pass  # Drop throws if no handle registered
    ci.delete_all()

    defs = Defs()
    for i in range(1, 7): defs.add_suite("s" + str(i))
    ci.load(defs)

    suite_names = ['s1', 's2', 's3']
    ci.ch_register(True, suite_names)  # register interest in suites s1,s2,s3 and any new suites
    ci.ch_register(False, suite_names)  # register interest in suites s1,s2,s3 only

    sync_local(ci)
    assert len(list(ci.get_defs().suites)) == 3, "Expected 3 registered suites but found " + str(
        len(list((ci.get_defs().suites))))


def test_client_ch_drop(ci):
    print_test(ci, "test_client_ch_drop")
    try:
        ci.ch_drop_user("")  # drop all handle associated with current user
    except:
        pass  # Drop throws if no handle registered
    ci.delete_all()

    defs = Defs()
    for i in range(1, 7): defs.add_suite("s" + str(i))
    ci.load(defs)

    try:
        # register interest in suites s1,s2,s3 and any new suites
        suite_names = ['s1', 's2', 's3']
        ci.ch_register(True, suite_names)
    finally:
        ci.ch_drop()  # drop using handle stored in ci., from last register

    sync_local(ci)
    assert len(list(ci.get_defs().suites)) == 6, "Expected 6 suites but found " + str(len(list((ci.get_defs().suites))))


def test_client_ch_drop_user(ci):
    print_test(ci, "test_client_ch_drop_user")
    ci.delete_all()
    try:
        ci.ch_drop_user("")  # drop all handle associated with current user
    except:
        pass  # Drop throws if no handle registered

    defs = Defs()
    for i in range(1, 7): defs.add_suite("s" + str(i))
    ci.load(defs)

    try:
        # register interest in suites s1,s2,s3 and any new suites
        suite_names = ['s1', 's2', 's3']
        ci.ch_register(True, suite_names)
        sync_local(ci)
        assert len(list(ci.get_defs().suites)) == 3, "Expected 3 suites but found " + str(
            len(list((ci.get_defs().suites))))
    except RuntimeError as e:
        print(str(e))

    ci.ch_drop_user("")  # drop all handle associated with current user
    sync_local(ci)
    assert len(list(ci.get_defs().suites)) == 6, "Expected 6 suites but found " + str(len(list((ci.get_defs().suites))))


def test_client_ch_add(ci):
    print_test(ci, "test_client_ch_add")
    ci.delete_all()
    try:
        ci.ch_drop_user("")  # drop all handle associated with current user
    except:
        pass  # Drop throws if no handle registered

    defs = Defs()
    for i in range(1, 7): defs.add_suite("s" + str(i))
    ci.load(defs)

    try:
        suite_names = []
        ci.ch_register(True, suite_names)  # register interest in any new suites
        suite_names = ['s1', 's2']
        ci.ch_add(suite_names)  # add suites s1,s2 to the last added handle
        sync_local(ci)
        assert len(list(ci.get_defs().suites)) == 2, "Expected 2 suites but found " + str(
            len(list((ci.get_defs().suites))))

        suite_names = ['s3', 's4']
        ci.ch_add(ci.ch_handle(), suite_names)  # add suites s3,s4 using last handle
        sync_local(ci)
        assert len(list(ci.get_defs().suites)) == 4, "Expected 4 suites but found " + str(
            len(list((ci.get_defs().suites))))

    except RuntimeError as e:
        print(str(e))

    ci.ch_drop_user("")  # drop all handle associated with current user
    sync_local(ci)
    assert len(list(ci.get_defs().suites)) == 6, "Expected 6 suites but found " + str(len(list((ci.get_defs().suites))))


def test_client_ch_auto_add(ci):
    print_test(ci, "test_client_ch_auto_add")
    ci.delete_all()
    try:
        ci.ch_drop_user("")  # drop all handle associated with current user
    except:
        pass  # Drop throws if no handle registered

    defs = Defs()
    for i in range(1, 7): defs.add_suite("s" + str(i))
    ci.load(defs)

    try:
        suite_names = ['s1', 's2', 's3']
        ci.ch_register(True, suite_names)  # register interest in suites s1,s2,s3 and any new suites
        ci.ch_auto_add(False)  # disable adding newly created suites to last registered handle\n"
        ci.ch_auto_add(True)  # enable adding newly created suites to last registered handle\n"
        ci.ch_auto_add(ci.ch_handle(), False)  # disable adding newly created suites to handle\n"
        sync_local(ci)
        assert len(list(ci.get_defs().suites)) == 3, "Expected 3 suites but found " + str(
            len(list((ci.get_defs().suites))))
    except RuntimeError as e:
        print(str(e))

    ci.ch_drop_user("")  # drop all handle associated with current user
    sync_local(ci)
    assert len(list(ci.get_defs().suites)) == 6, "Expected 6 suites but found " + str(len(list((ci.get_defs().suites))))


def test_client_ch_remove(ci):
    print_test(ci, "test_client_ch_remove")
    ci.delete_all()
    try:
        ci.ch_drop_user("")  # drop all handle associated with current user
    except:
        pass  # Drop throws if no handle registered

    defs = Defs()
    for i in range(1, 7): defs.add_suite("s" + str(i))
    ci.load(defs)

    try:
        suite_names = ['s1', 's2', 's3']
        ci.ch_register(True, suite_names)  # register interest in suites s1,s2,s3 and any new suites
        sync_local(ci)
        assert len(list(ci.get_defs().suites)) == 3, "Expected 3 suites but found " + str(
            len(list((ci.get_defs().suites))))

        suite_names = ['s1']
        ci.ch_remove(suite_names)  # remove suites s1 from the last added handle\n"
        sync_local(ci)
        assert len(list(ci.get_defs().suites)) == 2, "Expected 2 suites but found " + str(
            len(list((ci.get_defs().suites))))

        suite_names = ['s2']
        ci.ch_remove(ci.ch_handle(), suite_names)  # remove suites s2 from the last added handle\n"
        sync_local(ci)
        assert len(list(ci.get_defs().suites)) == 1, "Expected 1 suites but found " + str(
            len(list((ci.get_defs().suites))))

    except RuntimeError as e:
        print(str(e))

    ci.ch_drop_user("")  # drop all handle associated with current user
    sync_local(ci)
    assert len(list(ci.get_defs().suites)) == 6, "Expected 6 suites but found " + str(len(list((ci.get_defs().suites))))


def test_client_get_file(ci):
    print_test(ci, "test_client_get_file")
    ci.delete_all()
    port = ci.get_port()
    defs = create_defs("test_client_get_file", port, protocol)

    # Also test where user has specified his OWN ECF_JOBOUT (rare)
    t2 = defs.find_abs_node("/test_client_get_file/f1/t2")
    t2_jobout = Test.ecf_home(ci.get_port()) + "/test_client_get_file/t2.xx"
    t2.add_variable("ECF_JOBOUT", t2_jobout)

    defs.generate_scripts()
    msg = defs.check_job_creation(verbose=True)
    assert len(msg) == 0, msg

    ci.restart_server()
    ci.load(defs)
    ci.begin_all_suites()

    while 1:
        if ci.news_local():
            ci.sync_local()  # get the changes, synced with local defs
            suite = ci.get_defs().find_suite("test_client_get_file")
            assert suite is not None, "Expected to find suite"
            if suite.get_state() == State.complete:
                break
        time.sleep(1)

    try:
        for task in ['/test_client_get_file/f1/t1', '/test_client_get_file/f1/t2']:

            result_string = ci.get_file(task)
            assert isinstance(result_string, str), \
                f"Expected ci.get_file({task}) to return 'str'"
            assert len(result_string) > 0, \
                f"Expected ci.get_file({task}) to return something"

            result_string = ci.get_file(task, as_bytes=False)
            assert isinstance(result_string, str), \
                f"Expected ci.get_file({task}, as_bytes=False) to return 'str'"
            assert len(result_string) > 0, \
                f"Expected ci.get_file({task}, as_bytes=False) to return something"

            result_bytes = ci.get_file(task, as_bytes=True)
            assert isinstance(result_bytes, bytes), \
                f"Expected ci.get_file({task}, as_bytes=True) to return 'bytes'"
            assert len(result_bytes) > 0, \
                f"Expected ci.get_file({task}, as_bytes=True) to return something"

            assert len(result_string) == len(result_bytes), \
                f"Expected 'str' result and 'bytes' result to have the same length"

            result_string = ci.get_file(task, max_lines='10')
            assert isinstance(result_string, str), \
                f"Expected ci.get_file({task}, max_lines='10') to return 'str'"
            assert len(result_string) > 0, \
                f"Expected ci.get_file({task}, max_lines='10') to return something"

            result_string = ci.get_file(task, max_lines='10', as_bytes=False)
            assert isinstance(result_string, str), \
                f"Expected ci.get_file({task}, max_lines='10', as_bytes=False) to return 'str'"
            assert len(result_string) > 0, \
                f"Expected ci.get_file({task}, max_lines='10', as_bytes=False) to return something"

            result_bytes = ci.get_file(task, max_lines='10', as_bytes=True)
            assert isinstance(result_bytes, bytes), \
                f"Expected ci.get_file({task}, max_lines='10', as_bytes=True) to return 'bytes'"
            assert len(result_bytes) > 0, \
                f"Expected ci.get_file({task}, max_lines='10', as_bytes=True) to return something"

            assert len(result_string) == len(result_bytes), \
                f"Expected 'str' result and 'bytes' result to have the same length"

            for file_type in ['script', 'job', 'jobout', 'manual']:
                result_string = ci.get_file(task, file_type)
                assert isinstance(result_string, str), \
                    f"Expected ci.get_file({task}, {file_type}) to return 'str'"
                assert len(result_string) > 0, \
                    f"Expected ci.get_file({task}, {file_type}) to return something"

                result_string = ci.get_file(task, file_type, as_bytes=False)
                assert isinstance(result_string, str), \
                    f"Expected ci.get_file({task}, {file_type}, as_bytes=False) to return 'str'"
                assert len(result_string) > 0, \
                    f"Expected ci.get_file({task}, {file_type}, as_bytes=False) to return something"

                result_bytes = ci.get_file(task, file_type, as_bytes=True)
                assert isinstance(result_bytes, bytes), \
                    f"Expected ci.get_file({task}, {file_type}, as_bytes=True) to return 'bytes'"
                assert len(result_bytes) > 0, \
                    f"Expected ci.get_file({task}, {file_type}, as_bytes=True) to return something"

                assert len(result_string) == len(result_bytes), \
                    f"Expected 'str' result and 'bytes' result to have the same length"

                result_string = ci.get_file(task, file_type, '10')
                assert isinstance(result_string, str), \
                    f"Expected ci.get_file({task}, {file_type}, '10') to return 'str'"
                assert len(result_string) > 0, \
                    f"Expected ci.get_file({task}, {file_type}, '10') to return something"

                result_string = ci.get_file(task, file_type, '10', as_bytes=False)
                assert isinstance(result_string, str), \
                    f"Expected ci.get_file({task}, {file_type}, '10', as_bytes=False) to return 'str'"
                assert len(result_string) > 0, \
                    f"Expected ci.get_file({task}, {file_type}, '10', as_bytes=False) to return something"

                result_bytes = ci.get_file(task, file_type, '10', as_bytes=True)
                assert isinstance(result_bytes, bytes), \
                    f"Expected ci.get_file({task}, {file_type}, '10', as_bytes=True) to return 'bytes'"
                assert len(result_bytes) > 0, \
                    f"Expected ci.get_file({task}, {file_type}, '10', as_bytes=True) to return something"

                assert len(result_string) == len(result_bytes), \
                    f"Expected 'str' result and 'bytes' result to have the same length"

        assert os.path.exists(t2_jobout), f"User specified ECF_JOBOUT file not created {t2_jobout}"

    except RuntimeError as e:
        print(str(e))

    dir_to_remove = Test.ecf_home(port) + "/" + "test_client_get_file"
    shutil.rmtree(dir_to_remove, ignore_errors=True)


def test_client_plug(ci):
    pass


def test_client_alter_sort(ci, protocol):
    print_test(ci, "test_client_alter_sort")
    ci.delete_all()

    defs = create_defs("test_client_alter_sort", ci.get_port(), protocol)
    t1 = "/test_client_alter_sort/f1/t1"
    task_t1 = defs.find_abs_node(t1)
    task_t1.add_variable("z", "value").add_variable("y", "value").add_variable("x", "value")
    task_t1.add_event("z").add_event("y").add_event("x")
    task_t1.add_meter("z", 0, 100, 100).add_meter("y", 0, 100, 100).add_meter("x", 0, 100, 100)
    task_t1.add_label("z", "name").add_label("y", "name").add_label("x", "name")
    task_t1.add_limit("z", 10).add_limit("y", 10).add_limit("x", 10)

    ci.load(defs)

    ci.sort_attributes(t1, "variable")
    ci.sort_attributes(t1, "event")
    ci.sort_attributes(t1, "meter")
    ci.sort_attributes(t1, "label")
    ci.sort_attributes(t1, "limit")

    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.variables)) == 3, "Expected 3 variable :\n" + str(ci.get_defs())
    assert len(list(task_t1.events)) == 3, "Expected 3 events :\n" + str(ci.get_defs())
    assert len(list(task_t1.meters)) == 3, "Expected 3 meters :\n" + str(ci.get_defs())
    assert len(list(task_t1.labels)) == 3, "Expected 3 labels :\n" + str(ci.get_defs())
    assert len(list(task_t1.limits)) == 3, "Expected 3 limits :\n" + str(ci.get_defs())
    expected = ['x', 'y', 'z'];
    vactual = [];
    eactual = [];
    mactual = [];
    lactual = [];
    liactual = [];
    for v in task_t1.variables: vactual.append(v.name())
    for v in task_t1.events: eactual.append(v.name())
    for v in task_t1.meters: mactual.append(v.name())
    for v in task_t1.labels: lactual.append(v.name())
    for v in task_t1.limits: liactual.append(v.name())
    assert expected == vactual, "variable Attributes not sorted, expected:" + str(expected) + " but found:" + str(
        vactual)
    assert expected == eactual, "event Attributes not sorted, expected:" + str(expected) + " but found:" + str(eactual)
    assert expected == mactual, "meter Attributes not sorted, expected:" + str(expected) + " but found:" + str(mactual)
    assert expected == lactual, "label Attributes not sorted, expected:" + str(expected) + " but found:" + str(lactual)
    assert expected == liactual, "limit Attributes not sorted, expected:" + str(expected) + " but found:" + str(
        liactual)


def test_client_alter_sort_defs(ci, protocol):
    print_test(ci, "test_client_alter_sort_defs")
    ci.delete_all()

    defs = create_defs("test_client_alter_sort_defs", ci.get_port(), protocol)
    t1 = "/test_client_alter_sort_defs/f1/t1"
    task_t1 = defs.find_abs_node(t1)
    task_t1.add_variable("z", "value").add_variable("y", "value").add_variable("x", "value")
    task_t1.add_event("z").add_event("y").add_event("x")
    task_t1.add_meter("z", 0, 100, 100).add_meter("y", 0, 100, 100).add_meter("x", 0, 100, 100)
    task_t1.add_label("z", "name").add_label("y", "name").add_label("x", "name")
    task_t1.add_limit("z", 10).add_limit("y", 10).add_limit("x", 10)

    ci.load(defs)

    ci.sort_attributes("/", "variable", True)
    ci.sort_attributes("/", "event", True)
    ci.sort_attributes("/", "meter", True)
    ci.sort_attributes("/", "label", True)
    ci.sort_attributes("/", "limit", True)

    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.variables)) == 3, "Expected 3 variable :\n" + str(ci.get_defs())
    assert len(list(task_t1.events)) == 3, "Expected 3 events :\n" + str(ci.get_defs())
    assert len(list(task_t1.meters)) == 3, "Expected 3 meters :\n" + str(ci.get_defs())
    assert len(list(task_t1.labels)) == 3, "Expected 3 labels :\n" + str(ci.get_defs())
    assert len(list(task_t1.limits)) == 3, "Expected 3 limits :\n" + str(ci.get_defs())
    expected = ['x', 'y', 'z'];
    vactual = [];
    eactual = [];
    mactual = [];
    lactual = [];
    liactual = [];
    for v in task_t1.variables: vactual.append(v.name())
    for v in task_t1.events: eactual.append(v.name())
    for v in task_t1.meters: mactual.append(v.name())
    for v in task_t1.labels: lactual.append(v.name())
    for v in task_t1.limits: liactual.append(v.name())
    assert expected == vactual, "variable Attributes not sorted, expected:" + str(expected) + " but found:" + str(
        vactual)
    assert expected == eactual, "event Attributes not sorted, expected:" + str(expected) + " but found:" + str(eactual)
    assert expected == mactual, "meter Attributes not sorted, expected:" + str(expected) + " but found:" + str(mactual)
    assert expected == lactual, "label Attributes not sorted, expected:" + str(expected) + " but found:" + str(lactual)
    assert expected == liactual, "limit Attributes not sorted, expected:" + str(expected) + " but found:" + str(
        liactual)


def test_client_alter_add(ci, protocol):
    print_test(ci, "test_client_alter_add")
    ci.delete_all()
    ci.load(create_defs("test_client_alter_add", ci.get_port(), protocol))

    t1 = "/test_client_alter_add/f1/t1"
    ci.alter(t1, "add", "variable", "var", "var_name")
    ci.alter(t1, "add", "variable", "var2", "--")
    ci.alter(t1, "add", "variable", "var3", "--fred")
    ci.alter(t1, "add", "variable", "var4", " --fred ")
    ci.alter(t1, "add", "variable", "var5", "--fred --jake")
    ci.alter(t1, "add", "variable", "var6", " --fred --jake ")
    ci.alter(t1, "add", "variable", "var7", "")
    ci.alter(t1, "add", "time", "+00:30")
    ci.alter(t1, "add", "time", "01:30")
    ci.alter(t1, "add", "time", "01:30 20:00 00:30")
    ci.alter(t1, "add", "today", "+00:30")
    ci.alter(t1, "add", "today", "01:30")
    ci.alter(t1, "add", "today", "01:30 20:00 00:30")
    ci.alter(t1, "add", "date", "01.01.2001")
    ci.alter(t1, "add", "date", "*.01.2001")
    ci.alter(t1, "add", "date", "*.*.2001")
    ci.alter(t1, "add", "date", "*.*.*")
    ci.alter(t1, "add", "day", "sunday")
    ci.alter(t1, "add", "day", "monday")
    ci.alter(t1, "add", "day", "tuesday")
    ci.alter(t1, "add", "day", "wednesday")
    ci.alter(t1, "add", "day", "thursday")
    ci.alter(t1, "add", "day", "friday")
    ci.alter(t1, "add", "day", "saturday")
    ci.alter(t1, "add", "late", "late -s +00:15 -a 20:00 -c +02:00")
    ci.alter(t1, "add", "label", "label_name", "label_value")
    ci.alter(t1, "add", "label", "label_name2", "/a/label/with/path/values")

    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.variables)) == 7, "Expected 7 variable :\n" + str(ci.get_defs())
    assert len(list(task_t1.times)) == 3, "Expected 3 time :\n" + str(ci.get_defs())
    assert len(list(task_t1.todays)) == 3, "Expected 3 today's :\n" + str(ci.get_defs())
    assert len(list(task_t1.dates)) == 4, "Expected 4 dates :\n" + str(ci.get_defs())
    assert len(list(task_t1.days)) == 7, "Expected 7 days :\n" + str(ci.get_defs())
    assert len(list(task_t1.labels)) == 2, "Expected 2 labels :\n" + str(ci.get_defs())
    assert str(
        task_t1.get_late()) == "late -s +00:15 -a 20:00 -c +02:00", "Expected late 'late -s +00:15 -a 20:00 -c +02:00'" + str(
        ci.get_defs())


def test_client_alter_delete(ci, protocol):
    print_test(ci, "test_client_alter_delete")
    ci.delete_all()
    defs = create_defs("test_client_alter_delete", ci.get_port(), protocol)
    suite_with_limits = defs.add_suite("suite_with_limits")
    suite_with_limits.add_limit("limitX", 10)
    suite_with_limits_X = defs.add_suite("suite_with_limits_X")
    suite_with_limits_X.add_limit("limitX", 10)

    t1 = "/test_client_alter_delete/f1/t1"
    task_t1 = defs.find_abs_node(t1)
    task_t1.add_variable("var", "value")
    task_t1.add_variable("var1", "value")
    task_t1.add_time("00:30")
    task_t1.add_time("00:31")
    task_t1.add_today("00:30")
    task_t1.add_today("00:31")
    task_t1.add_date(1, 1, 2001)
    task_t1.add_date(1, 1, 2002)
    task_t1.add_day("sunday")
    task_t1.add_day("monday")
    cron = Cron()
    cron.set_week_days([0, 1, 2, 3, 4, 5, 6])
    cron.set_time_series("+00:30")
    task_t1.add_cron(cron)
    task_t1.add_cron(cron)
    task_t1.add_event("event")
    task_t1.add_event("event1")
    task_t1.add_meter("meter", 0, 100, 100)
    task_t1.add_meter("meter1", 0, 100, 100)
    task_t1.add_label("label", "name")
    task_t1.add_label("label1", "name")
    task_t1.add_limit("limit", 10)
    task_t1.add_limit("limit1", 10)
    task_t1.add_inlimit("limit", t1, 2)
    task_t1.add_inlimit("limit1", t1, 2)
    task_t1.add_inlimit("limitX", "/suite_with_limits", 2)
    task_t1.add_inlimit("limitX", "/suite_with_limits_X", 2)
    task_t1.add_trigger("t2 == active")
    task_t1.add_complete("t2 == complete")

    assert task_t1.get_late() is None, "expected no late"
    late = Late()
    late.submitted(20, 10)
    late.active(20, 10)
    late.complete(20, 10, True)
    task_t1.add_late(late)
    assert task_t1.get_late() is not None, "expected late"

    t2 = "/test_client_alter_delete/f1/t2"
    task_t2 = defs.find_abs_node(t2)
    task_t2.add_repeat(RepeatDate("date", 20100111, 20100115, 2))  # can't add cron and repeat at the same level

    # print(defs)
    ci.load(defs)

    ci.alter(t1, "delete", "variable", "var")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.variables)) == 1, "Expected 1 variable :\n" + str(ci.get_defs())
    ci.alter(t1, "delete", "variable")  # delete all veriables
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.variables)) == 0, "Expected 0 variable :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "time", "00:30")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.times)) == 1, "Expected 1 time :\n" + str(ci.get_defs())
    ci.alter(t1, "delete", "time")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.times)) == 0, "Expected 0 time :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "today", "00:30")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.todays)) == 1, "Expected 1 today :\n" + str(ci.get_defs())
    ci.alter(t1, "delete", "today")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.todays)) == 0, "Expected 0 today :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "date", "01.01.2001")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.dates)) == 1, "Expected 1 date :\n" + str(ci.get_defs())
    ci.alter(t1, "delete", "date")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.dates)) == 0, "Expected 0 date :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "day", "sunday")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.days)) == 1, "Expected 1 day :\n" + str(ci.get_defs())
    ci.alter(t1, "delete", "day")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.days)) == 0, "Expected 0 day :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "event", "event")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.events)) == 1, "Expected 1 event :\n" + str(ci.get_defs())
    ci.alter(t1, "delete", "event")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.events)) == 0, "Expected 0 event :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "meter", "meter")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.meters)) == 1, "Expected 1 meter :\n" + str(ci.get_defs())
    ci.alter(t1, "delete", "meter")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.meters)) == 0, "Expected 0 meter :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "label", "label")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.labels)) == 1, "Expected 1 label :\n" + str(ci.get_defs())
    ci.alter(t1, "delete", "label")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.labels)) == 0, "Expected 0 label :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "inlimit", "limit")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.inlimits)) == 3, "Expected 3 inlimit :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "inlimit", "/suite_with_limits:limitX")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.inlimits)) == 2, "Expected 2 inlimit :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "inlimit", "/suite_with_limits_X:limitX")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.inlimits)) == 1, "Expected 1 inlimit :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "inlimit")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.inlimits)) == 0, "Expected 0 inlimit :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "limit", "limit")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.limits)) == 1, "Expected 1 limit :\n" + str(ci.get_defs())
    ci.alter(t1, "delete", "limit")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.limits)) == 0, "Expected 0 limit :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "cron")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.crons)) == 0, "Expected 0 crons :\n" + str(ci.get_defs())

    ci.alter(t1, "delete", "late")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert task_t1.get_late() is None, "expected no late after delete"

    task_t1 = ci.get_defs().find_abs_node(t1)
    assert task_t1.get_trigger() is not None, "Expected trigger:\n" + str(ci.get_defs())
    ci.alter(t1, "delete", "trigger")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert task_t1.get_trigger() is None, "Expected trigger to be deleted:\n" + str(ci.get_defs())

    task_t1 = ci.get_defs().find_abs_node(t1)
    assert task_t1.get_complete() is not None, "Expected complete:\n" + str(ci.get_defs())
    ci.alter(t1, "delete", "complete")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert task_t1.get_complete() is None, "Expected complete to be deleted:\n" + str(ci.get_defs())

    ci.alter(t2, "delete", "repeat")
    sync_local(ci)
    task_t2 = ci.get_defs().find_abs_node(t2)
    repeat = task_t2.get_repeat()
    assert repeat.empty(), "Expected repeat to be deleted:\n" + str(ci.get_defs())


def test_client_alter_change(ci, protocol):
    print_test(ci, "test_client_alter_change")
    ci.delete_all()
    defs = create_defs("test_client_alter_change", ci.get_port(), protocol)
    t1 = "/test_client_alter_change/f1/t1"
    repeat_date_path = "/test_client_alter_change/f1/repeat_date"
    repeat_datetime_path = "/test_client_alter_change/f1/repeat_datetime"

    task_t1 = defs.find_abs_node(t1)
    task_t1.add_variable("var", "value")
    task_t1.add_variable("var1", "value")
    task_t1.add_event("event")
    task_t1.add_event("event1")
    task_t1.add_meter("meter", 0, 100, 100)
    task_t1.add_meter("meter1", 0, 100, 100)
    task_t1.add_label("label", "name")
    task_t1.add_label("label1", "name1")
    task_t1.add_limit("limit", 10)
    task_t1.add_limit("limit1", 10)
    task_t1.add_inlimit("limit", t1, 2)
    task_t1.add_inlimit("limit1", t1, 2)
    task_t1.add_trigger("t2 == active")
    task_t1.add_complete("t2 == complete")
    task_t1.add_time("00:31")
    task_t1.add_today("00:30")

    late = Late()
    late.submitted(20, 10)
    late.active(20, 10)
    late.complete(20, 10, True)
    task_t1.add_late(late)

    f1 = defs.find_abs_node("/test_client_alter_change/f1")
    # Add a new Repeat (based on date)
    repeat_date = f1.add_task("repeat_date")
    repeat_date.add_repeat(RepeatDate("date", 20100111, 20100115, 2))  # can't add cron and repeat at the same level
    # Add a new Repeat (based on date+time)
    repeat_datetime = f1.add_task("repeat_datetime")
    repeat_datetime.add_repeat(RepeatDateTime("datetime", "20100111T000000", "20100115T000000", "48:00:00"))

    ci.load(defs)

    ci.alter(t1, "change", "time", "00:31", "10:10")
    ci.alter(t1, "change", "today", "00:30", "10:10")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    assert len(list(task_t1.times)) == 1, "Expected 1 time :\n" + str(ci.get_defs())
    assert len(list(task_t1.todays)) == 1, "Expected 1 time :\n" + str(ci.get_defs())
    for t in task_t1.times:
        assert str(t) == "time 10:10", " expected 'time 10:10' but found:\n" + str(t)
    for t in task_t1.todays:
        assert str(t) == "today 10:10", " expected 'today 10:10' but found:\n" + str(t)

    ci.alter(t1, "change", "late", "-s +10:00")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    variable = task_t1.get_late()
    assert str(
        task_t1.get_late()) == "late -s +10:00", "Expected alter of late to be 'late -s +10:00' but found " + str(
        task_t1.get_late())

    ci.alter(t1, "change", "variable", "var", "changed_var")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    variable = task_t1.find_variable("var")
    assert variable.value() == "changed_var", "Expected alter of variable to be 'change_var' but found " + variable.value()
    res = ci.query('variable', task_t1.get_abs_node_path(), "var")
    assert res == 'changed_var', "Expected alter of variable to be 'change_var' but found " + res

    ci.alter(t1, "change", "variable", "var", "--")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    variable = task_t1.find_variable("var")
    assert variable.value() == "--", "Expected alter of variable to be '--' but found " + variable.value()
    res = ci.query('variable', task_t1.get_abs_node_path(), "var")
    assert res == '--', "Expected alter of variable to be '--' but found " + res

    ci.alter(t1, "change", "variable", "var", "--fred")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    variable = task_t1.find_variable("var")
    assert variable.value() == "--fred", "Expected alter of variable to be '--fred' but found " + variable.value()

    ci.alter(t1, "change", "variable", "var", "--fred --bill")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    variable = task_t1.find_variable("var")
    assert variable.value() == "--fred --bill", "Expected alter of variable to be '--fred --bill' but found " + variable.value()

    ci.alter(t1, "change", "meter", "meter", "10")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    meter = task_t1.find_meter("meter")
    assert meter.value() == 10, "Expected alter of meter to be 10 but found " + str(meter.value())
    res = ci.query('meter', task_t1.get_abs_node_path(), "meter")
    assert res == '10', "Expected alter of meter to be 10 but found " + res

    ci.alter(t1, "change", "event", "event", "set")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    event = task_t1.find_event("event")
    assert event.value() == True, "Expected alter of event to be set but found " + str(event.value())
    res = ci.query('event', task_t1.get_abs_node_path(), "event")
    assert res == 'set', "Expected alter of event to be 'set' but found " + res

    ci.alter(t1, "change", "trigger", "t2 == aborted")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    trigger = task_t1.get_trigger()
    assert trigger.get_expression() == "t2 == aborted", "Expected alter of trigger to be 't2 == aborted' but found " + trigger.get_expression()
    res = ci.query('trigger', task_t1.get_abs_node_path(), "t2 == aborted")
    assert res == 'false', "Expected evaluation of trigger to fail, but found: " + res

    ci.alter(t1, "change", "trigger", "/test_client_alter_change/f1/t2 == complete")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    trigger = task_t1.get_trigger()
    assert trigger.get_expression() == "/test_client_alter_change/f1/t2 == complete", "Expected alter of trigger to be '/test_client_alter_change/f1/t2 == complete' but found " + trigger.get_expression()

    ci.alter(t1, "change", "complete", "t2 == aborted")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    complete = task_t1.get_complete()
    assert complete.get_expression() == "t2 == aborted", "Expected alter of complete to be 't2 == aborted' but found " + complete.get_expression()

    ci.alter(t1, "change", "complete", "/test_client_alter_change/f1/t2 == active")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    complete = task_t1.get_complete()
    assert complete.get_expression() == "/test_client_alter_change/f1/t2 == active", "Expected alter of complete to be '/test_client_alter_change/f1/t2 == active' but found " + complete.get_expression()

    ci.alter(t1, "change", "limit_max", "limit", "2")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    limit = task_t1.find_limit("limit")
    assert limit is not None, "Expected to find limit"
    assert limit.limit() == 2, "Expected alter of limit_max to be 2 but found " + str(limit.limit())

    ci.alter(t1, "change", "limit_value", "limit", "2")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    limit = task_t1.find_limit("limit")
    assert limit is not None, "Expected to find limit"
    assert limit.value() == 2, "Expected alter of limit_value to be 2 but found " + str(limit.value())

    ci.alter(t1, "change", "label", "label", "new-value")
    sync_local(ci)
    task_t1 = ci.get_defs().find_abs_node(t1)
    label = task_t1.find_label("label")
    assert label.new_value() == "new-value", "Expected alter of label to be 'new-value' but found " + label.new_value()

    ci.alter(repeat_date_path, "change", "repeat", "20100113")
    sync_local(ci)
    task = ci.get_defs().find_abs_node(repeat_date_path)
    repeat = task.get_repeat()
    assert repeat.value() == 20100113, "Expected alter of repeat to be 20100113 but found " + str(repeat.value())
    res = ci.query('variable', task.get_abs_node_path(), repeat.name())
    assert res == "20100113", "Expected alter of repeat to be 20100113 but found " + res

    ci.alter(repeat_datetime_path, "change", "repeat", "20100113T000000")
    sync_local(ci)
    task = ci.get_defs().find_abs_node(repeat_datetime_path)
    repeat = task.get_repeat()
    assert repeat.value() == 1263340800, "Expected alter of repeat to be 1263340800 (i.e. seconds between 19700101T000000 and 20100113T000000) but found " + str(
        repeat.value())
    res = ci.query('variable', task.get_abs_node_path(), repeat.name())
    assert res == "20100113T000000", "Expected alter of repeat to be 20100113T000000 but found " + res


def test_client_alter_flag(ci, protocol):
    print_test(ci, "test_client_alter_flag")
    ci.delete_all()
    defs = create_defs("test_client_alter_flag", ci.get_port(), protocol)
    t1 = "/test_client_alter_flag/f1/t1"

    task_t1 = defs.find_abs_node(t1)

    ci.load(defs)

    flag = Flag()
    flag_list = flag.list()  # flag_list is of type FlagTypeVec
    for flg in flag_list:
        ci.alter(t1, "set_flag", flag.type_to_string(flg))
        sync_local(ci)
        task_t1 = ci.get_defs().find_abs_node(t1)
        task_flag = task_t1.get_flag()
        assert task_flag.is_set(flg), "expected flag %r to be set" % task_flag.type_to_string(flg)

        # alter itself causes the flag message to be set, and preserved
        if flg == FlagType.message: continue

        ci.alter(t1, "clear_flag", flag.type_to_string(flg))
        sync_local(ci)
        task_t1 = ci.get_defs().find_abs_node(t1)
        task_flag = task_t1.get_flag()
        assert not task_flag.is_set(flg), "expected flag %r NOT to be set" % task_flag.type_to_string(flg)

    # ISSUES:
    # o Currently we can only change clock attr if we have one.
    # o Even when we have a clock attr, it only makes sense to apply clock attr changes
    #   before begin(), i.e how do we apply change in gain after begin ??

    # " change clock-type name        # The name must be one of 'hybrid' or 'real'.\n"
    #       " change clock-gain name        # The gain must be convertible to an integer.\n"
    #        " change label name value       # sets the label\n"
    #        " change repeat value           # If the repeat is a date, then the value must be a valid YMD ( ie. yyyymmdd)\n"
    #        "                               # and be convertible to an integer, additionally the value must be in range\n"
    #        "                               # of the repeat start and end dates. Like wise for repeat integer. For repeat\n"
    #         "                               # string and enum,  the name must either be an integer, that is a valid index or\n"
    #         "                               # if it is a string, it must correspond to one of enum's or strings list\n"


def test_client_force(ci, protocol):
    print_test(ci, "test_client_force")
    ci.delete_all()
    defs = create_defs("test_client_force", ci.get_port(), protocol)

    path_list = ["/test_client_force/f1/t1", "/test_client_force/f1/t2"]
    t1 = path_list[0]
    for path in path_list:
        task = defs.find_abs_node(path)
        assert task is not None, "Expected to find task at path " + path
        task.add_event("event")

    ci.load(defs)

    state_list = [State.unknown, State.active, State.complete, State.queued, State.submitted, State.aborted]
    for state in state_list:
        ci.force_state(t1, state)
        sync_local(ci)
        task = ci.get_defs().find_abs_node(t1)
        assert task.get_state() == state, "Expected state " + state + " but found " + str(task.get_state())
    for state in state_list:
        ci.force_state(path_list, state)
        sync_local(ci)
        for path in path_list:
            task = ci.get_defs().find_abs_node(path)
            assert task.get_state() == state, "Expected state " + state + " but found " + str(task.get_state())

    for state in state_list:
        ci.force_state_recursive("/test_client_force", state)
        sync_local(ci)
        task = ci.get_defs().find_abs_node(t1)
        assert task.get_state() == state, "Expected state " + state + " but found " + str(task.get_state())
    suite_paths = ["/test_client_force"]
    for state in state_list:
        ci.force_state_recursive(suite_paths, state)
        sync_local(ci)
        task = ci.get_defs().find_abs_node(t1)
        assert task.get_state() == state, "Expected state " + state + " but found " + str(task.get_state())

    event_states = ["set", "clear"]
    for ev_state in event_states:
        for path in path_list:
            ci.force_event(path + ":event", ev_state)
            sync_local(ci)
            task = ci.get_defs().find_abs_node(path)
            event_fnd = False
            for event in task.events:
                event_fnd = True
                if ev_state == "set":
                    assert event.value() == True, " Expected event value to be set"
                else:
                    assert event.value() == False, " Expected event value to be clear"
            assert event_fnd == True, " Expected event to be found"

    event_path_list = ["/test_client_force/f1/t1:event", "/test_client_force/f1/t2:event"]
    event_states = ["set", "clear"]
    for ev_state in event_states:
        ci.force_event(event_path_list, ev_state)
        sync_local(ci)
        for path in path_list:
            task = ci.get_defs().find_abs_node(path)
            event_fnd = False
            for event in task.events:
                event_fnd = True
                if ev_state == "set":
                    assert event.value() == True, " Expected event value to be set"
                else:
                    assert event.value() == False, " Expected event value to be clear"
            assert event_fnd == True, " Expected event to be found"


def test_client_replace(ci, on_disk, protocol):
    print_test(ci, "test_client_replace client_defs on disk = " + str(on_disk))
    # Create and load the following defs
    # s1
    #   f1
    #     t1
    #     t2
    ci.delete_all()
    ci.load(create_defs("s1", ci.get_port(), protocol))

    # ===============================================================================
    # Example of using replace to ADD a *NEW* node hierarchy to an existing suite
    # we should end up with:
    # s1
    #   f1
    #     t1
    #     t2
    #   f2
    #     t1
    #     t2
    test_client_replace_def_file = "test_client_replace_" + str(os.getpid()) + ".def"
    client_def = create_defs("s1", ci.get_port(), protocol)
    client_def.find_suite("s1").add_family("f2").add_task("t1")
    if on_disk:
        client_def.save_as_defs(test_client_replace_def_file)
        client_def = test_client_replace_def_file

    ci.replace("/s1/f2", client_def, True,
               False)  # True means create parents as needed, False means don't bypass checks/zombies
    ci.get_server_defs()
    assert len(list(ci.get_defs().suites)) == 1, "Expected 1 suites:\n" + str(ci.get_defs())
    assert ci.get_defs().find_abs_node("/s1/f2/t1") is not None, "Expected to find task /s1/f2/t1\n" + str(
        ci.get_defs())

    # Example of using replace to *REMOVE* node hierarchy to an existing suite, could have used delete
    assert ci.get_defs().find_abs_node("/s1/f1/t1") is not None, "Expected to find task /s1/f1/t1\n" + str(
        ci.get_defs())
    assert ci.get_defs().find_abs_node("/s1/f2/t1") is not None, "Expected to find task /s1/f2/t1\n" + str(
        ci.get_defs())
    client_def = Defs()
    client_def.add_suite("s1")  # should only have the suite
    if on_disk:
        client_def.save_as_defs(test_client_replace_def_file)
        client_def = test_client_replace_def_file

    ci.replace("/s1", client_def)
    ci.get_server_defs()
    assert len(list(ci.get_defs().suites)) == 1, "Expected 1 suites:\n" + str(ci.get_defs())
    assert ci.get_defs().find_abs_node("/s1/f1/t1") is None, "Expected NOT to find task /s1/f1/t1\n" + str(
        ci.get_defs())
    assert ci.get_defs().find_abs_node("/s1/f2/t1") is None, "Expected NOT to find task /s1/f2/t1\n" + str(
        ci.get_defs())

    # ===============================================================================
    # Example of using replace to add a *NEW* suite
    client_def = Defs();
    client_def.add_suite("s2")
    if on_disk:
        client_def.save_as_defs(test_client_replace_def_file)
        client_def = test_client_replace_def_file

    ci.replace("/s2", client_def, True,
               False)  # True means create parents as needed, False means don't bypass checks/zombies
    ci.get_server_defs()
    assert len(list(ci.get_defs().suites)) == 2, " Expected two suites:\n" + str(ci.get_defs())

    # replace added suite s2 with a new s2 which has a task, 
    # s2 must exist on the client defs
    client_def = Defs();
    client_def.add_suite("s2").add_task("t1")
    if on_disk:
        client_def.save_as_defs(test_client_replace_def_file)
        client_def = test_client_replace_def_file

    ci.replace("/s2", client_def)

    ci.get_server_defs()
    assert len(list(ci.get_defs().suites)) == 2, " Expected two suites:\n" + str(ci.get_defs())
    assert ci.get_defs().find_abs_node("/s2/t1") is not None, "Expected to find task /s2/t1\n" + str(ci.get_defs())
    if on_disk:
        os.remove(test_client_replace_def_file)


def test_node_replace(ci):
    print_test(ci, "test_node_replace")
    PrintStyle.set_style(Style.MIGRATE)  # show node state
    ci.delete_all()
    defs = Defs() + (Suite("s1") + Family('f1').add(Task('t1'), Task('t2')))
    ci.load(defs)

    # We should have 4 nodes
    sync_local(ci)
    ci_defs = ci.get_defs()
    node_vec = ci_defs.get_all_nodes()
    assert len(list(node_vec)) == 4, "Expected two 4 nodes: \n" + str(ci.get_defs())

    # replace each node, add variable first, then check, it was added
    for node in node_vec:
        node += Edit(var="XX", var2="xx")
        node.replace_on_server(ci)  # default is to suspend server node first, so replaced node is also suspended

        sync_local(ci)
        replace_node = ci.get_defs().find_abs_node(node.get_abs_node_path())
        assert len(list(replace_node.variables)) == 2, "Expected two 2 variable: \n" + str(replace_node) + "\n" + str(
            ci.get_defs())
        assert replace_node.get_dstate() == DState.suspended, "Expected node to be suspended:\n" + str(
            replace_node) + "\n" + str(ci.get_defs())

    # resume nodes, test that when False passed in we do not suspend the replaced node
    for node in node_vec:
        ci.resume(node.get_abs_node_path())
        sync_local(ci)
        replace_node = ci.get_defs().find_abs_node(node.get_abs_node_path())
        assert replace_node.get_dstate() != DState.suspended, "Expected node not to suspended:\n" + str(replace_node)

        replace_node += Meter("meter", 0, 100)

        # cant use node i.e this is only client side, and is still suspended
        # When we replace is server the client defs is sent to server. hence node would be suspended
        replace_node.replace_on_server(ci, suspend_node_first=False)
        sync_local(ci)

        replace_node = ci.get_defs().find_abs_node(node.get_abs_node_path())
        assert len(list(replace_node.meters)) == 1, "Expected 1 meter: \n" + str(replace_node)
        assert len(list(replace_node.variables)) == 2, "Expected two 2 variable: \n" + str(replace_node)
        assert replace_node.get_dstate() != DState.suspended, "Expected node not to suspended:\n" + str(replace_node)

    # replace the suite
    defs = Defs() + Suite("s1")
    host_port = ci.get_host() + ':' + ci.get_port()
    defs.s1.replace_on_server(ci)

    ci.get_server_defs()
    assert len(list(ci.get_defs().s1)) == 0, "Expected 0 family: \n" + str(ci.get_defs())
    assert ci.get_defs().s1.get_dstate() == DState.suspended, "Expected node to be suspended:\n" + str(ci.get_defs())


def test_client_kill(ci):
    pass


def test_client_status(ci):
    pass


def test_client_order(ci):
    pass


def test_client_group(ci):
    pass


def test_client_suspend(ci, protocol):
    print_test(ci, "test_client_suspend")
    ci.delete_all()
    defs = create_defs("test_client_suspend", ci.get_port(), protocol)
    suite = defs.find_suite("test_client_suspend")
    suite.add_variable("ECF_DUMMY_TASK", "")

    ci.load(defs)
    ci.begin_all_suites()

    ci.suspend("/test_client_suspend")

    sync_local(ci);
    suite = ci.get_defs().find_suite("test_client_suspend")
    assert suite.is_suspended(), "Expected to find suite suspended"

    assert ci.query('dstate',
                    suite.get_abs_node_path()) == "suspended", "Expected to find suite dstate suspended but found: " + res
    assert ci.query('state',
                    suite.get_abs_node_path()) == "queued", "Expected to find suite state queued but found: " + res


def test_client_suspend_multiple_paths(ci, protocol):
    print_test(ci, "test_client_suspend_multiple_paths")
    ci.delete_all()
    defs = create_defs("test_client_suspend_multiple_paths", ci.get_port(), protocol)
    suite = defs.find_suite("test_client_suspend_multiple_paths")
    suite.add_variable("ECF_DUMMY_TASK", "")

    ci.load(defs)
    ci.begin_all_suites()

    path_list = ["/test_client_suspend_multiple_paths/f1/t1", "/test_client_suspend_multiple_paths/f1/t2"]
    ci.suspend(path_list)

    sync_local(ci);
    task_t1 = ci.get_defs().find_abs_node("/test_client_suspend_multiple_paths/f1/t1")
    task_t2 = ci.get_defs().find_abs_node("/test_client_suspend_multiple_paths/f1/t2")
    assert task_t1.is_suspended(), "Expected to find task t1 to be suspended"
    assert task_t2.is_suspended(), "Expected to find task t2 to be suspended"


def test_client_resume(ci, protocol):
    print_test(ci, "test_client_resume")
    ci.delete_all()
    defs = create_defs("test_client_resume", ci.get_port(), protocol)
    suite = defs.find_suite("test_client_resume")
    suite.add_variable("ECF_DUMMY_TASK", "")

    ci.load(defs)
    ci.begin_all_suites()

    ci.suspend("/test_client_resume")
    sync_local(ci);
    suite = ci.get_defs().find_suite("test_client_resume")
    assert suite.is_suspended(), "Expected to find suite suspended"

    ci.resume("/test_client_resume")
    sync_local(ci);
    suite = ci.get_defs().find_suite("test_client_resume")
    assert suite.is_suspended() == False, "Expected to find suite resumed"


def test_client_resume_multiple_paths(ci, protocol):
    print_test(ci, "test_client_resume_multiple_paths")
    ci.delete_all()
    defs = create_defs("test_client_resume_multiple_paths", ci.get_port(), protocol)
    suite = defs.find_suite("test_client_resume_multiple_paths")
    suite.add_variable("ECF_DUMMY_TASK", "")

    ci.load(defs)
    ci.begin_all_suites()

    path_list = ["/test_client_resume_multiple_paths/f1/t1", "/test_client_resume_multiple_paths/f1/t2"]
    ci.suspend(path_list)

    sync_local(ci);
    task_t1 = ci.get_defs().find_abs_node("/test_client_resume_multiple_paths/f1/t1")
    task_t2 = ci.get_defs().find_abs_node("/test_client_resume_multiple_paths/f1/t2")
    assert task_t1.is_suspended(), "Expected to find task t1 to be suspended"
    assert task_t2.is_suspended(), "Expected to find task t2 to be suspended"

    ci.resume(path_list)
    sync_local(ci);
    task_t1 = ci.get_defs().find_abs_node("/test_client_resume_multiple_paths/f1/t1")
    task_t2 = ci.get_defs().find_abs_node("/test_client_resume_multiple_paths/f1/t2")
    assert task_t1.is_suspended() == False, "Expected to find task t1 to be resumed"
    assert task_t2.is_suspended() == False, "Expected to find task t2 to be resumed"


def test_client_delete_node(ci, protocol):
    print_test(ci, "test_client_delete_node")
    ci.delete_all()
    defs = create_defs("test_client_delete_node", ci.get_port(), protocol)

    task_vec = defs.get_all_tasks();
    assert len(task_vec) > 0, "Expected some tasks but found none:\n" + str(defs)

    ci.load(defs)
    sync_local(ci);
    for task in task_vec:
        node = ci.get_defs().find_abs_node(task.get_abs_node_path())
        assert node is not None, "Expected to find task " + task.get_abs_node_path() + ":\n" + str(ci.get_defs())

    for task in task_vec:
        ci.delete(task.get_abs_node_path())

    sync_local(ci);
    for task in task_vec:
        node = ci.get_defs().find_abs_node(task.get_abs_node_path())
        assert node is None, "Expected not to find task " + task.get_abs_node_path() + " as it should have been deleted:\n" + str(
            ci.get_defs())


def test_client_delete_node_multiple_paths(ci, protocol):
    print_test(ci, "test_client_delete_node_multiple_paths")
    ci.delete_all()
    defs = create_defs("test_client_delete_node_multiple_paths", ci.get_port(), protocol)

    task_vec = defs.get_all_tasks();
    assert len(task_vec) > 0, "Expected some tasks but found none:\n" + str(defs)

    paths = []
    for task in task_vec:
        paths.append(task.get_abs_node_path())

    ci.load(defs)

    sync_local(ci);
    for task in task_vec:
        node = ci.get_defs().find_abs_node(task.get_abs_node_path())
        assert node is not None, "Expected to find task " + task.get_abs_node_path() + ":\n" + str(ci.get_defs())

    ci.delete(paths)

    sync_local(ci);
    for task in task_vec:
        node = ci.get_defs().find_abs_node(task.get_abs_node_path())
        assert node is None, "Expected not to find task " + task.get_abs_node_path() + " as it should have been deleted:\n" + str(
            ci.get_defs())


def test_client_archive_and_restore(ci, protocol):
    suite_name = "test_client_archive_and_restore"
    print_test(ci, suite_name)
    ci.delete_all()
    defs = create_defs(suite_name, ci.get_port(), protocol)
    suite = defs.find_suite(suite_name)
    suite_path = suite.get_abs_node_path();

    ci.restart_server()
    ci.load(defs)

    sync_local(ci)  # get the changes, synced with local defs
    node_vec = ci.get_defs().get_all_nodes()
    assert len(node_vec) == 4, "Expected 4 nodes, but found " + str(len(node_vec))

    ci.archive(suite_path)
    sync_local(ci)  # get the changes, synced with local defs
    node_vec = ci.get_defs().get_all_nodes()
    assert len(node_vec) == 1, "Expected 1 nodes, but found " + str(len(node_vec))
    the_suite = ci.get_defs().find_suite(suite_name)
    assert the_suite is not None, "Expected to find suite"
    assert the_suite.get_flag().is_set(FlagType.archived), " expected archive flag to be set"

    ci.restore(suite_path)
    sync_local(ci)  # get the changes, synced with local defs
    node_vec = ci.get_defs().get_all_nodes()
    assert len(node_vec) == 4, "Expected 4 nodes, but found " + str(len(node_vec))
    the_restored_suite = ci.get_defs().find_suite(suite_name)
    assert the_restored_suite is not None, "Expected to find suite"
    assert the_restored_suite.get_flag().is_set(FlagType.restored), " expected restored flag to be set"
    assert not the_restored_suite.get_flag().is_set(FlagType.archived), "expected archive flag to be cleared"


def test_client_check_defstatus(ci, protocol):
    print_test(ci, "test_client_check_defstatus")
    ci.delete_all()
    port = ci.get_port()
    defs = create_defs("test_client_check_defstatus", port, protocol)

    # stop defs form running when begin is called.
    suite = defs.find_suite("test_client_check_defstatus")
    suite.add_defstatus(DState.suspended)

    t1 = "/test_client_check_defstatus/f1/t1"
    t2 = "/test_client_check_defstatus/f1/t2"
    task_t1 = defs.find_abs_node(t1)
    task_t1.add_defstatus(DState.suspended)

    defs.generate_scripts();
    msg = defs.check_job_creation(verbose=True)
    assert len(msg) == 0, msg

    ci.restart_server()
    ci.load(defs)
    ci.begin_all_suites()

    sync_local(ci)  # get the changes, synced with local defs
    # print(ci.get_defs())
    task_t1 = ci.get_defs().find_abs_node(t1)
    task_t2 = ci.get_defs().find_abs_node(t2)
    assert task_t1 is not None, "Could not find t1"
    assert task_t2 is not None, "Could not find t2"

    assert task_t1.get_state() == State.queued, "Expected state queued but found " + str(task_t1.get_state())
    assert task_t2.get_state() == State.queued, "Expected state queued " + str(task_t2.get_state())

    assert task_t1.get_dstate() == DState.suspended, "Expected state suspended but found " + str(task_t1.get_state())
    assert task_t2.get_dstate() == DState.queued, "Expected state queued but found " + str(task_t2.get_state())

    dir_to_remove = Test.ecf_home(port) + "/" + "test_client_check_defstatus"
    shutil.rmtree(dir_to_remove, ignore_errors=True)


def test_ECFLOW_189(ci, protocol):
    # Bug, when a node is resumed it ignored holding dependencies higher up the tree.
    # i.e Previously when we resumed a node, it ignored trigger/time/node state, dependencies higher up the tree
    print_test(ci, "test_ECFLOW_189")
    ci.delete_all()
    port = ci.get_port()
    defs = create_defs("test_ECFLOW_189", port, protocol)
    defs.generate_scripts();
    msg = defs.check_job_creation(verbose=True)
    assert len(msg) == 0, msg

    ci.restart_server()
    ci.load(defs)

    ci.suspend("/test_ECFLOW_189")
    ci.suspend("/test_ECFLOW_189/f1/t1")
    ci.suspend("/test_ECFLOW_189/f1/t2")

    ci.begin_all_suites()

    sync_local(ci)  # get the changes, synced with local defs
    # print(ci.get_defs())
    task_t1 = ci.get_defs().find_abs_node("/test_ECFLOW_189/f1/t1")
    task_t2 = ci.get_defs().find_abs_node("/test_ECFLOW_189/f1/t2")
    assert task_t1 is not None, "Could not find /test_ECFLOW_189/f1/t1"
    assert task_t2 is not None, "Could not find /test_ECFLOW_189/f1/t2"

    assert task_t1.get_state() == State.queued, "Expected state queued but found " + str(task_t1.get_state())
    assert task_t2.get_state() == State.queued, "Expected state queued but found " + str(task_t2.get_state())
    assert task_t1.get_dstate() == DState.suspended, "Expected state suspended but found " + str(task_t1.get_dstate())
    assert task_t2.get_dstate() == DState.suspended, "Expected state suspended but found " + str(task_t2.get_dstate())

    # ok now resume t1/t2, they should remain queued, since the Suite is still suspended
    ci.resume("/test_ECFLOW_189/f1/t1")
    ci.resume("/test_ECFLOW_189/f1/t2")

    time.sleep(3)
    sync_local(ci)  # get the changes, synced with local defs
    # print(ci.get_defs())
    task_t1 = ci.get_defs().find_abs_node("/test_ECFLOW_189/f1/t1")
    task_t2 = ci.get_defs().find_abs_node("/test_ECFLOW_189/f1/t2")
    assert task_t1.get_state() == State.queued, "Expected state queued but found " + str(task_t1.get_state())
    assert task_t2.get_state() == State.queued, "Expected state queued but found " + str(task_t2.get_state())
    assert task_t1.get_dstate() == DState.queued, "Expected state queued but found " + str(task_t1.get_dstate())
    assert task_t2.get_dstate() == DState.queued, "Expected state queued but found " + str(task_t2.get_dstate())

    dir_to_remove = Test.ecf_home(port) + "/" + "test_ECFLOW_189"
    shutil.rmtree(dir_to_remove, ignore_errors=True)


def test_ECFLOW_1761(ci):
    print_test(ci, "test_ECFLOW_1761")
    print("# get log: last 10 lines")
    ci.get_log(10)

    ci_defs = ci.get_defs()
    task = ci_defs.get_all_nodes()[0]  # take first task

    # print(ci.edit_script_edit(task.get_abs_node_path())

    # print(ci.edit_script_preprocess(task.get_abs_node_path()))

    # ci.edit_script_submit(
    #    task.get_abs_node_path(),
    #        [],  # name-value variables list
    #        [],  # lines (script)
    #        False, # True,  $ alias
    #        False)  # run


def test_ECFLOW_199(ci, protocol):
    # Test ClientInvoker::changed_node_paths
    print_test(ci, "test_ECFLOW_199")
    ci.delete_all()
    port = ci.get_port()
    defs = create_defs("test_ECFLOW_199", port, protocol)
    defs.generate_scripts();
    msg = defs.check_job_creation(verbose=True)
    assert len(msg) == 0, msg

    ci.restart_server()
    ci.load(defs)

    ci.suspend("/test_ECFLOW_199")
    ci.suspend("/test_ECFLOW_199/f1/t1")
    ci.suspend("/test_ECFLOW_199/f1/t2")

    ci.begin_all_suites()

    sync_local(ci)  # get the changes, synced with local defs
    # print(ci.get_defs())
    assert len(list(
        ci.changed_node_paths)) == 0, "Expected first call to sync_local, to have no changed paths but found " + str(
        len(list(ci.changed_node_paths)))

    # ok now resume t1/t2, they should remain queued, since the Suite is still suspended
    # Note: ECFLOW-1512, we may get additional paths.( i.e. suite path) due to changes in suite calendar
    ci.resume("/test_ECFLOW_199/f1/t1")
    sync_local(ci)
    found_path = False
    for path in ci.changed_node_paths:
        print("   changed node path " + path);
        if path == "/test_ECFLOW_199/f1/t1":
            found_path = True
    assert found_path, "Expected '/test_ECFLOW_199/f1/t1' in list of changed paths"

    ci.resume("/test_ECFLOW_199/f1/t2")
    sync_local(ci)
    found_path = False
    for path in ci.changed_node_paths:
        print("   changed node path " + path);
        if path == "/test_ECFLOW_199/f1/t2":
            found_path = True
    assert found_path, "Expected '/test_ECFLOW_199/f1/t2' in list of changed paths"

    dir_to_remove = Test.ecf_home(port) + "/" + "test_ECFLOW_199"
    shutil.rmtree(dir_to_remove, ignore_errors=True)


def test_client_ch_with_drops_handles(ci, protocol):
    print_test(ci, "test_client_ch_with_drops_handles")
    try:
        port = ci.get_port()
        print("  test using with but without register, handle should be zero, ie do nothing")
        with Client("localhost", port) as local_ci:
            local_ci.debug(True)
            if protocol == Test.Protocol.HTTP:
                local_ci.enable_http()
            assert local_ci.ch_handle() == 0, "Expected handle to be zero"
            local_ci.ch_suites()
        print("  Test Client with register, should drop handle ")
        with Client("localhost", port) as local_ci:
            local_ci.debug(True)
            if protocol == Test.Protocol.HTTP:
                local_ci.enable_http()
            local_ci.ch_register(True, ['s1', 's2'])
            # print("Handle:",local_ci .ch_handle())
            local_ci.ch_suites()
            # raise RuntimeError("xxx") # check exeption is still caught
    except RuntimeError as e:
        print("Exception", e)
    print("after with:")
    # really need a way to get hold of the suites, via python api.
    ci.ch_suites()  # should be empty


def do_tests(ci, protocol):
    test_version(ci)
    PrintStyle.set_style(Style.STATE)  # show node state

    test_client_get_server_defs(ci, protocol)
    test_client_new_log(ci)
    test_client_clear_log(ci)
    test_client_log_msg(ci)

    test_client_restart_server(ci)
    test_client_halt_server(ci)
    test_client_shutdown_server(ci)

    test_client_load_in_memory_defs(ci, protocol)
    test_client_load_from_disk(ci, protocol)
    test_client_checkpt(ci, protocol)
    test_client_restore_from_checkpt(ci, protocol)

    test_client_reload_wl_file(ci)

    test_client_run(ci, protocol)
    test_client_run_with_multiple_paths(ci, protocol)
    test_client_requeue(ci, protocol)
    test_client_requeue_with_multiple_paths(ci, protocol)
    test_client_free_dep(ci, protocol)

    test_client_suites(ci, protocol)
    test_client_ch_with_drops_handles(ci, protocol)
    test_client_ch_suites(ci)
    test_client_ch_register(ci)
    test_client_ch_drop(ci)
    test_client_ch_drop_user(ci)
    test_client_ch_add(ci)
    test_client_ch_auto_add(ci)
    test_client_ch_remove(ci)

    test_client_get_file(ci)
    # test_client_plug(ci)
    test_client_alter_sort(ci, protocol)
    test_client_alter_sort_defs(ci, protocol)
    test_client_alter_add(ci, protocol)
    test_client_alter_delete(ci, protocol)
    test_client_alter_change(ci, protocol)
    test_client_alter_flag(ci, protocol)

    test_client_force(ci, protocol)
    test_client_replace(ci, False, protocol)
    test_client_replace(ci, True, protocol)
    test_node_replace(ci)

    # test_client_kill(ci)
    # test_client_status(ci)
    # test_client_order(ci)
    # test_client_group(ci)
    test_client_suspend(ci, protocol)
    test_client_suspend_multiple_paths(ci, protocol)
    test_client_resume(ci, protocol)
    test_client_resume_multiple_paths(ci, protocol)
    test_client_delete_node(ci, protocol)
    test_client_delete_node_multiple_paths(ci, protocol)
    test_client_archive_and_restore(ci, protocol)

    test_client_check(ci)
    test_client_check_defstatus(ci, protocol)

    test_client_stats(ci)
    test_client_stats_with_stdout(ci)
    test_client_stats_without_stdout(ci)
    test_client_stats_reset(ci)
    test_client_debug_server_on_off(ci)

    test_ECFLOW_189(ci, protocol)
    test_ECFLOW_199(ci, protocol)
    test_ECFLOW_1761(ci)


def launch_tests(ci, protocol):
    server_version = ci.server_version()
    print("Running ecflow server version " + server_version)
    print("Running ecflow client version " + ci.version())
    assert ci.version() == server_version, " Client version not same as server version"

    # test with sync_local
    do_tests(ci, protocol)

    # test with auto sync
    ci.set_auto_sync(True)
    do_tests(ci, protocol)

    print("All Tests pass ======================================================================")


if __name__ == "__main__":
    Test.print_test_start(os.path.basename(__file__))

    # server independent tests
    test_set_host_port();

    # Run tests using ecFlow server (using custom TCP/IP protocol)
    with Test.Server(Test.Protocol.CUSTOM) as ctx:
        ci = ctx[0]
        protocol = ctx[1]
        launch_tests(ci, protocol)

    # Run tests using ecFlow server (using HTTP protocol)
    with Test.Server(Test.Protocol.HTTP) as ctx:
        ci = ctx[0]
        protocol = ctx[1]
        launch_tests(ci, protocol)