import sys import os print(os.environ['CI_SITE_CONFIG']) sys.path.append(os.environ['CI_SITE_CONFIG']) import subprocess import re import ci_site_config import common import shlex # A Jenkins env variable for job name is composed of the name of the jenkins job and the branch name # it is building for. for e.g. in our case jobname = 'ofi_libfabric/master' class Test: def __init__ (self, jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, mpitype=None, util_prov=None): self.jobname = jobname self.buildno = buildno self.testname = testname self.core_prov = core_prov self.util_prov = f'ofi_{util_prov}' if util_prov != None else '' self.fabric = fabric self.hosts = hosts self.run_test = run_test self.mpi_type = mpitype self.ofi_build_mode = ofi_build_mode if (len(hosts) == 2): self.server = hosts[0] self.client = hosts[1] self.nw_interface = ci_site_config.interface_map[self.fabric] self.libfab_installpath = f'{ci_site_config.install_dir}/'\ f'{self.jobname}/{self.buildno}/'\ f'{self.ofi_build_mode}' self.ci_middlewares_path = f'{ci_site_config.install_dir}/'\ f'{self.jobname}/{self.buildno}/'\ 'ci_middlewares' self.ci_logdir_path = f'{ci_site_config.install_dir}/'\ f'{self.jobname}/{self.buildno}/'\ 'log_dir' self.env = eval(user_env) self.mpi = '' if (self.mpi_type == 'impi'): self.mpi = IMPI(self.core_prov, self.hosts, self.libfab_installpath, self.nw_interface, self.server, self.client, self.env, self.util_prov) elif (self.mpi_type == 'ompi'): self.mpi = OMPI(self.core_prov, self.hosts, self.libfab_installpath, self.nw_interface, self.server, self.client, self.env, self.ci_middlewares_path, self.util_prov) elif (self.mpi_type == 'mpich'): self.mpi = MPICH(self.core_prov, self.hosts, self.libfab_installpath, self.nw_interface, self.server, self.client, self.env, self.ci_middlewares_path, self.util_prov) class FiInfoTest(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, None, util_prov) self.fi_info_testpath = f'{self.libfab_installpath}/bin' @property def cmd(self): return f"{self.fi_info_testpath}/fi_info " @property def options(self): if (self.util_prov): opts = f"-f {self.fabric} -p {self.core_prov};{self.util_prov}" elif (self.core_prov == 'psm3'): opts = f"-p {self.core_prov}" else: opts = f"-f {self.fabric} -p {self.core_prov}" return opts def execute_cmd(self): command = self.cmd + self.options outputcmd = shlex.split(command) common.run_command(outputcmd, self.ci_logdir_path, self.run_test, self.ofi_build_mode) class Fabtest(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, None, util_prov) self.fabtestpath = f'{self.libfab_installpath}/bin' self.fabtestconfigpath = f'{self.libfab_installpath}/share/fabtests' def get_exclude_file(self): path = self.libfab_installpath efile_path = f'{path}/share/fabtests/test_configs' prov = self.util_prov if self.util_prov else self.core_prov efile_old = f'{efile_path}/{prov}/{prov}.exclude' if self.util_prov: efile = f'{efile_path}/{self.util_prov}/{self.core_prov}/exclude' else: efile = f'{efile_path}/{self.core_prov}/exclude' if os.path.isfile(efile): return efile elif os.path.isfile(efile_old): return efile_old else: print(f"Exclude file: {efile} not found!") return None @property def cmd(self): return f"{self.fabtestpath}/runfabtests.sh " @property def options(self): opts = f"-T 300 -vvv -p {self.fabtestpath} -S " if (self.core_prov != 'shm' and self.nw_interface): opts += f"-s {common.get_node_name(self.server, self.nw_interface)} " opts += f"-c {common.get_node_name(self.client, self.nw_interface)} " if (self.core_prov == 'shm'): opts += f"-s {self.server} " opts += f"-c {self.client} " opts += "-N " if (self.ofi_build_mode == 'dl'): opts += "-t short " else: opts += "-t all " if (self.core_prov == 'sockets' and self.ofi_build_mode == 'reg'): complex_test_file = f'{self.libfab_installpath}/share/fabtests/'\ f'test_configs/{self.core_prov}/quick.test' if (os.path.isfile(complex_test_file)): opts += "-u {complex_test_file} " else: print(f"{self.core_prov} Complex test file not found") if (self.ofi_build_mode != 'reg' or self.core_prov == 'udp'): opts += "-e \'ubertest,multinode\' " efile = self.get_exclude_file() if efile: opts += "-R " opts += f"-f {efile} " for key in self.env: opts += f"-E {key}={self.env[key]} " if self.util_prov: opts += f"{self.core_prov};{self.util_prov} " else: opts += f"{self.core_prov} " if (self.core_prov == 'shm'): opts += f"{self.server} {self.server} " else: opts += f"{self.server} {self.client} " return opts @property def execute_condn(self): return True def execute_cmd(self): curdir = os.getcwd() os.chdir(self.fabtestconfigpath) command = self.cmd + self.options outputcmd = shlex.split(command) common.run_command(outputcmd, self.ci_logdir_path, self.run_test, self.ofi_build_mode) os.chdir(curdir) class ShmemTest(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, None, util_prov) #self.n - number of hosts * number of processes per host self.n = 4 # self.ppn - number of processes per node. self.ppn = 2 self.shmem_dir = f'{self.ci_middlewares_path}/shmem' @property def cmd(self): return f"{ci_site_config.testpath}/run_shmem.sh " def options(self, shmem_testname): if self.util_prov: prov = f"{self.core_prov};{self.util_prov} " else: prov = self.core_prov opts = f"-n {self.n} " opts += f"-hosts {self.server},{self.client} " opts += f"-shmem_dir={self.shmem_dir} " opts += f"-libfabric_path={self.libfab_installpath}/lib " opts += f"-prov {prov} " opts += f"-test {shmem_testname} " opts += f"-server {self.server} " opts += f"-inf {ci_site_config.interface_map[self.fabric]}" return opts @property def execute_condn(self): #make always true when verbs and sockets are passing return True if (self.core_prov == 'tcp') \ else False def execute_cmd(self, shmem_testname): command = self.cmd + self.options(shmem_testname) outputcmd = shlex.split(command) common.run_command(outputcmd, self.ci_logdir_path, f'{shmem_testname}_{self.run_test}', self.ofi_build_mode) class MultinodeTests(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, None, util_prov) self.fabtestpath = f'{self.libfab_installpath}/bin' self.fabtestconfigpath = f'{self.libfab_installpath}/share/fabtests' self.n = 2 self.ppn = 64 self.iterations = 1 self.method = 'msg' self.pattern = "full_mesh" @property def cmd(self): return f"{self.fabtestpath}/runmultinode.sh " @property def options(self): opts = f"-h {common.get_node_name(self.server, self.nw_interface)}" opts += f",{common.get_node_name(self.client, self.nw_interface)}" opts += f" -n {self.ppn}" opts += f" -I {self.iterations}" opts += f" -z {self.pattern}" opts += f" -C {self.method}" if self.util_prov: opts += f" -p {self.core_prov};{self.util_prov}" else: opts += f" -p {self.core_prov}" opts += f" --ci {self.fabtestpath}/" #enable ci mode to disable tput return opts @property def execute_condn(self): return True def execute_cmd(self): if self.util_prov: prov = f"{self.core_prov}-{self.util_prov} " else: prov = self.core_prov curdir = os.getcwd() os.chdir(self.fabtestconfigpath) command = self.cmd + self.options outputcmd = shlex.split(command) common.run_command(outputcmd, self.ci_logdir_path, prov, self.ofi_build_mode) os.chdir(curdir) class ZeFabtests(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, None, util_prov) self.fabtestpath = f'{self.libfab_installpath}/bin' self.zefabtest_script_path = f'{ci_site_config.ze_testpath}' self.fabtestconfigpath = f'{self.libfab_installpath}/share/fabtests' @property def cmd(self): return f'{self.zefabtest_script_path}/runfabtests_ze.sh ' def options(self, test_name): opts = f"-p {self.fabtestpath} " opts += f"-B {self.fabtestpath} " opts += f"-t {test_name} " opts += f"{self.server} {self.client} " return opts @property def execute_condn(self): return True if (self.core_prov == 'shm') else False def execute_cmd(self, test_name): curdir = os.getcwd() os.chdir(self.fabtestconfigpath) command = self.cmd + self.options(test_name) outputcmd = shlex.split(command) common.run_command(outputcmd, self.ci_logdir_path, f'{test_name}', self.ofi_build_mode) os.chdir(curdir) class OMPI: def __init__(self, core_prov, hosts, libfab_installpath, nw_interface, server, client, environ, ci_middlewares_path, util_prov=None): self.ompi_src = f'{ci_middlewares_path}/ompi' self.core_prov = core_prov self.hosts = hosts self.util_prov = util_prov self.libfab_installpath = libfab_installpath self.nw_interface = nw_interface self.server = server self.client = client self.environ = environ self.n = 4 self.ppn = 2 @property def env(self): cmd = "bash -c \'" if (self.util_prov): cmd += f"export FI_PROVIDER={self.core_prov}\\;{self.util_prov}; " else: cmd += f"export FI_PROVIDER={self.core_prov}; " cmd += "export I_MPI_FABRICS=ofi; " cmd += f"export LD_LIBRARY_PATH={self.ompi_src}/lib:$LD_LIBRARY_PATH; " cmd += f"export LD_LIBRARY_PATH={self.libfab_installpath}/lib/:"\ "$LD_LIBRARY_PATH; " cmd += f"export PATH={self.ompi_src}/bin:$PATH; " cmd += f"export PATH={self.libfab_installpath}/bin:$PATH; " return cmd @property def options(self): opts = f"-np {self.n} " hosts = '\',\''.join([':'.join([common.get_node_name(host, \ self.nw_interface), str(self.ppn)]) \ for host in self.hosts]) opts += f"--host \'{hosts}\' " if self.util_prov: opts += f"--mca mtl_ofi_provider_include {self.core_prov}\\;"\ f"{self.util_prov} " opts += f"--mca btl_ofi_provider_include {self.core_prov}\\;"\ f"{self.util_prov} " else: opts += f"--mca mtl_ofi_provider_include {self.core_prov} " opts += f"--mca btl_ofi_provider_include {self.core_prov} " opts += "--mca orte_base_help_aggregate 0 " # This is necessary to prevent verbs from printing warning messages # The test still uses libfabric verbs even when enabled. # if (self.core_prov == 'verbs'): # opts += "--mca btl_openib_allow_ib 1 " opts += "--mca mtl ofi " opts += "--mca pml cm -tag-output " for key in self.environ: opts += f"-x {key}={self.environ[key]} " return opts @property def cmd(self): return f"{self.ompi_src}/bin/mpirun {self.options}" class MPICH: def __init__(self, core_prov, hosts, libfab_installpath, nw_interface, server, client, environ, ci_middlewares_path, util_prov=None): self.mpich_src = f'{ci_middlewares_path}/mpich' self.core_prov = core_prov self.hosts = hosts self.util_prov = util_prov self.libfab_installpath = libfab_installpath self.nw_interface = nw_interface self.server = server self.client = client self.environ = environ self.n = 4 self.ppn = 1 @property def env(self): cmd = "bash -c \'" if (self.util_prov): cmd += f"export FI_PROVIDER={self.core_prov}\\;{self.util_prov}; " else: cmd += f"export FI_PROVIDER={self.core_prov}; " cmd += "export I_MPI_FABRICS=ofi; " cmd += "export MPIR_CVAR_CH4_OFI_ENABLE_ATOMICS=0; " cmd += "export MPIR_CVAR_CH4_OFI_CAPABILITY_SETS_DEBUG=1; " cmd += f"export LD_LIBRARY_PATH={self.mpich_src}/lib:$LD_LIBRARY_PATH; " cmd += f"export LD_LIBRARY_PATH={self.libfab_installpath}/lib/:"\ "$LD_LIBRARY_PATH; " cmd += f"export PATH={self.mpich_src}/bin:$PATH; " cmd += f"export PATH={self.libfab_installpath}/bin:$PATH; " return cmd @property def options(self): opts = f"-n {self.n} " opts += f"-ppn {self.ppn} " opts += f"-hosts {common.get_node_name(self.server, self.nw_interface)},"\ f"{common.get_node_name(self.client, self.nw_interface)} " for key in self.environ: opts += f"-genv {key} {self.environ[key]} " return opts @property def cmd(self): return f"{self.mpich_src}/bin/mpirun {self.options}" class IMPI: def __init__(self, core_prov, hosts, libfab_installpath, nw_interface, server, client, environ, util_prov=None): self.impi_src = ci_site_config.impi_root self.core_prov = core_prov self.hosts = hosts self.util_prov = util_prov self.libfab_installpath = libfab_installpath self.nw_interface = nw_interface self.server = server self.client = client self.environ = environ self.n = 4 self.ppn = 1 @property def env(self): cmd = f"bash -c \'source {self.impi_src}/env/vars.sh "\ "-i_mpi_ofi_internal=0; " if (self.util_prov): cmd += f"export FI_PROVIDER={self.core_prov}\\;{self.util_prov}; " else: cmd += f"export FI_PROVIDER={self.core_prov}; " cmd += "export I_MPI_FABRICS=ofi; " cmd += f"export LD_LIBRARY_PATH={self.impi_src}/lib:$LD_LIBRARY_PATH; " cmd += f"export LD_LIBRARY_PATH={self.impi_src}/lib/release:"\ "$LD_LIBRARY_PATH; " cmd += f"export LD_LIBRARY_PATH={self.libfab_installpath}/lib/:"\ "$LD_LIBRARY_PATH; " cmd += f"export PATH={self.libfab_installpath}/bin:$PATH; " return cmd @property def options(self): opts = f"-n {self.n} " opts += f"-ppn {self.ppn} " opts += f"-hosts {common.get_node_name(self.server, self.nw_interface)},"\ f"{common.get_node_name(self.client, self.nw_interface)} " for key in self.environ: opts += f"-genv {key} {self.environ[key]} " return opts @property def cmd(self): return f"{self.impi_src}/bin/mpiexec {self.options}" class IMBtests(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, mpitype, ofi_build_mode, user_env, run_test, test_group, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, mpitype, util_prov) self.test_group = test_group self.mpi_type = mpitype self.imb_src = '' self.imb_tests = { '1' :[ 'MPI1', 'P2P' ], '2' :[ 'EXT', 'IO' ], '3' :[ 'NBC', 'RMA', 'MT' ] } self.iter = 100 self.include = { 'MPI1':[ 'Biband', 'Uniband', 'PingPongAnySource', 'PingPingAnySource', 'PingPongSpecificSource', 'PingPingSpecificSource' ], 'P2P':[], 'EXT':[], 'IO':[], 'NBC':[], 'RMA':[], 'MT':[] } self.exclude = { 'MPI1':[], 'P2P':[], 'EXT':[], 'IO':[], 'NBC':[], 'RMA':[ 'Accumulate', 'Get_accumulate', 'Fetch_and_op', 'Compare_and_swap' ], 'MT':[] } if (self.mpi_type == 'impi'): self.imb_src = ci_site_config.impi_root elif (self.mpi_type == 'ompi' or self.mpi_type == 'mpich'): self.imb_src = f'{self.ci_middlewares_path}/{self.mpi_type}/imb' @property def execute_condn(self): # Mpich and ompi are excluded to save time. Run manually if needed return (self.mpi_type == 'impi') def imb_cmd(self, imb_test): print(f"Running IMB-{imb_test}") cmd = f"{self.imb_src}/bin/IMB-{imb_test} " if (imb_test != 'MT'): cmd += f"-iter {self.iter} " if (len(self.include[imb_test]) > 0): cmd += f"-include {','.join(self.include[imb_test])}" if (len(self.exclude[imb_test]) > 0): cmd += f"-exclude {','.join(self.exclude[imb_test])}" return cmd def execute_cmd(self): for test_type in self.imb_tests[self.test_group]: outputcmd = shlex.split(self.mpi.env + self.mpi.cmd + \ self.imb_cmd(test_type) + '\'') common.run_command(outputcmd, self.ci_logdir_path, f'{self.mpi_type}_{self.run_test}', self.ofi_build_mode) class OSUtests(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, mpitype, ofi_build_mode, user_env, run_test, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, mpitype, util_prov) self.n_ppn = { 'pt2pt': (2, 1), 'collective': (4, 2), 'one-sided': (2, 1), 'startup': (2, 1) } self.osu_src = f'{self.ci_middlewares_path}/{mpitype}/osu/libexec/'\ 'osu-micro-benchmarks/mpi/' self.mpi_type = mpitype @property def execute_condn(self): # mpich-tcp, ompi are the only osu test combinations failing return False if ((self.mpi_type == 'mpich' and self.core_prov == 'tcp') or \ self.mpi_type == 'ompi') \ else True def osu_cmd(self, test_type, test): print(f"Running OSU-{test_type}-{test}") cmd = f'{self.osu_src}/{test_type}/{test} ' return cmd def execute_cmd(self): assert(self.osu_src) p = re.compile('osu_put*') for root, dirs, tests in os.walk(self.osu_src): for test in tests: self.mpi.n = self.n_ppn[os.path.basename(root)][0] self.mpi.ppn = self.n_ppn[os.path.basename(root)][1] if (test == 'osu_latency_mp' and self.core_prov == 'verbs'): self.env['IBV_FORK_SAFE'] = '1' if(p.search(test) == None): osu_command = self.osu_cmd(os.path.basename(root), test) outputcmd = shlex.split(self.mpi.env + self.mpi.cmd + \ osu_command + '\'') common.run_command(outputcmd, self.ci_logdir_path, f'{self.mpi_type}_{self.run_test}', self.ofi_build_mode) if (test == 'osu_latency_mp' and self.core_prov == 'verbs'): self.env.pop('IBV_FORK_SAFE') class MpichTestSuite(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, mpitype, ofi_build_mode, user_env, run_test, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, mpitype, util_prov) self.mpichsuitepath = f'{self.ci_middlewares_path}/{mpitype}/'\ 'mpichsuite/test/mpi/' self.pwd = os.getcwd() self.mpi_type = mpitype def testgroup(self, testgroupname): testpath = f'{self.mpichsuitepath}/{testgroupname}' tests = [] with open(f'{testpath}/testlist') as file: for line in file: if(line[0] != '#' and line[0] != '\n'): tests.append((line.rstrip('\n')).split(' ')) return tests def set_options(self, nprocs, timeout=None): self.mpi.n = nprocs if (timeout != None): os.environ['MPIEXEC_TIMEOUT']=timeout @property def execute_condn(self): return (self.mpi_type == 'impi' or \ (self.mpi_type == 'mpich' and self.core_prov == 'verbs')) def execute_cmd(self, testgroupname): print("Running Tests: " + testgroupname) tests = [] time = None os.chdir(f'{self.mpichsuitepath}/{testgroupname}') tests = self.testgroup(testgroupname) for test in tests: testname = test[0] nprocs = test[1] args = test[2:] for item in args: itemlist = item.split('=') if (itemlist[0] == 'timelimit'): time = itemlist[1] self.set_options(nprocs, timeout=time) testcmd = f'./{testname}' outputcmd = shlex.split(self.mpi.env + self.mpi.cmd + testcmd + '\'') if self.util_prov: util_prov = self.util_prov.strip('ofi_') log_file_name = f'{self.core_prov}-{util_prov}_' \ f'{self.mpi_type}_{self.run_test}' else: log_file_name = f'{self.core_prov}_{self.mpi_type}_{self.run_test}' common.run_command(outputcmd, self.ci_logdir_path, log_file_name, self.ofi_build_mode) os.chdir(self.pwd) class OneCCLTests(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, None, util_prov) self.n = 2 self.ppn = 1 self.oneccl_path = f'{self.ci_middlewares_path}/oneccl/build' self.examples_tests = { 'allgatherv', 'allreduce', 'alltoallv', 'broadcast', 'communicator', 'cpu_allgatherv_test', 'cpu_allreduce_bf16_test', 'cpu_allreduce_test', 'custom_allreduce', 'datatype', 'external_kvs', 'priority_allreduce', 'reduce', 'reduce_scatter', 'unordered_allreduce' } self.functional_tests = { 'allgatherv_test', 'allreduce_test', 'alltoall_test', 'alltoallv_test', 'bcast_test', 'reduce_scatter_test', 'reduce_test' } @property def cmd(self): return f"{ci_site_config.testpath}/run_oneccl.sh " def options(self, oneccl_test): opts = f"-n {self.n} " opts += f"-ppn {self.ppn} " opts += f"-hosts {self.server},{self.client} " opts += f"-prov '{self.core_prov}' " opts += f"-test {oneccl_test} " opts += f"-libfabric_path={self.libfab_installpath}/lib " opts += f'-oneccl_root={self.oneccl_path}' return opts @property def execute_condn(self): return True def execute_cmd(self, oneccl_test): if oneccl_test == 'examples': for test in self.examples_tests: command = self.cmd + self.options(oneccl_test) + \ f" {test}" outputcmd = shlex.split(command) common.run_command(outputcmd, self.ci_logdir_path, self.run_test, self.ofi_build_mode) elif oneccl_test == 'functional': for test in self.functional_tests: command = self.cmd + self.options(oneccl_test) + \ f" {test}" outputcmd = shlex.split(command) common.run_command(outputcmd, self.ci_logdir_path, self.run_test, self.ofi_build_mode) class OneCCLTestsGPU(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, None, util_prov) self.n = 2 self.ppn = 4 self.oneccl_path = f'{self.ci_middlewares_path}/oneccl_gpu/build' self.examples_tests = { 'sycl_allgatherv_custom_usm_test', 'sycl_allgatherv_inplace_test', 'sycl_allgatherv_inplace_usm_test', 'sycl_allgatherv_test', 'sycl_allgatherv_usm_test', 'sycl_allreduce_inplace_usm_test', 'sycl_allreduce_test', 'sycl_allreduce_usm_test', 'sycl_alltoall_test', 'sycl_alltoall_usm_test', 'sycl_alltoallv_test', 'sycl_alltoallv_usm_test', 'sycl_broadcast_test', 'sycl_broadcast_usm_test', 'sycl_reduce_inplace_usm_test', 'sycl_reduce_scatter_test', 'sycl_reduce_scatter_usm_test', 'sycl_reduce_test', 'sycl_reduce_usm_test' } self.functional_tests = { 'allgatherv_test', 'alltoall_test', 'alltoallv_test', 'bcast_test', 'reduce_scatter_test', 'reduce_test' } @property def cmd(self): return f"{ci_site_config.testpath}/run_oneccl_gpu.sh " def options(self, oneccl_test_gpu): opts = f"-n {self.n} " opts += f"-ppn {self.ppn} " opts += f"-hosts {self.server},{self.client} " opts += f"-prov '{self.core_prov}' " opts += f"-test {oneccl_test_gpu} " opts += f"-libfabric_path={self.libfab_installpath}/lib " opts += f'-oneccl_root={self.oneccl_path}' return opts @property def execute_condn(self): return True def execute_cmd(self, oneccl_test_gpu): if oneccl_test_gpu == 'examples': for test in self.examples_tests: command = self.cmd + self.options(oneccl_test_gpu) + \ f" {test}" outputcmd = shlex.split(command) common.run_command(outputcmd, self.ci_logdir_path, self.run_test, self.ofi_build_mode) elif oneccl_test_gpu == 'functional': for test in self.functional_tests: command = self.cmd + self.options(oneccl_test_gpu) + \ f" {test}" outputcmd = shlex.split(command) common.run_command(outputcmd, self.ci_logdir_path, self.run_test, self.ofi_build_mode) class DaosCartTest(Test): def __init__(self, jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, util_prov=None): super().__init__(jobname, buildno, testname, core_prov, fabric, hosts, ofi_build_mode, user_env, run_test, None, util_prov) self.set_paths() self.set_environment(core_prov,util_prov) print(core_prov) self.daos_nodes = ci_site_config.prov_node_map[core_prov] print(self.daos_nodes) self.cart_tests = { 'corpc_one_node' : {'tags' :'cart,corpc,one_node', 'numservers':1, 'numclients':0}, 'corpc_two_node' : {'tags' :'cart,corpc,two_node', 'numservers':2, 'numclients':0}, 'ctl_one_node' : {'tags' :'cart,ctl,one_node', 'numservers':1, 'numclients':1}, # 'ghost_rank_rpc_one_node' : {'tags' :'cart,ghost_rank_rpc,one_node', 'numservers':1, 'numclients':0}, 'group_test' : {'tags' :'cart,group_test,one_node', 'numservers':1, 'numclients':0}, 'iv_one_node' : {'tags' :'cart,iv,one_node', 'numservers':1, 'numclients':1}, 'iv_two_node' : {'tags' :'cart,iv,two_node', 'numservers':2, 'numclients':1}, 'launcher_one_node' : {'tags' :'cart,no_pmix_launcher,one_node','numservers':1, 'numclients':1}, # 'multictx_one_node' : {'tags' :'cart,no_pmix,one_node', 'numservers':1, 'numclients':0}, 'rpc_one_node' : {'tags' :'cart,rpc,one_node', 'numservers':1, 'numclients':1}, 'rpc_two_node' : {'tags' :'cart,rpc,two_node','numservers':2, 'numclients':1}, 'swim_notification' : {'tags' :'cart,rpc,swim_rank_eviction,one_node', 'numservers':1, 'numclients':1} } def set_paths(self): self.ci_middlewares_path = f'{ci_site_config.ci_middlewares}' self.daos_install_root = f'{self.ci_middlewares_path}/daos/install' self.cart_test_scripts = f'{self.daos_install_root}/lib/daos/TESTING/ftest' self.mpipath = f'{ci_site_config.daos_mpi}/bin' self.pathlist = [f'{self.daos_install_root}/bin/', self.cart_test_scripts, self.mpipath, \ f'{self.daos_install_root}/lib/daos/TESTING/tests'] self.daos_prereq = f'{self.daos_install_root}/prereq' common.run_command(['rm','-rf', f'{self.daos_prereq}/debug/ofi']) common.run_command(['ln', '-sfn', self.libfab_installpath, f'{self.daos_prereq}/debug/ofi']) def set_environment(self, core_prov, util_prov): prov_name = f'ofi+{core_prov}' if util_prov: prov_name = f'{prov_name};ofi_{util_prov}' if (core_prov == 'verbs'): os.environ["OFI_DOMAIN"] = 'mlx5_0' else: os.environ["OFI_DOMAIN"] = 'ib0' os.environ["OFI_INTERFACE"] = 'ib0' os.environ["CRT_PHY_ADDR_STR"] = prov_name os.environ["PATH"] += os.pathsep + os.pathsep.join(self.pathlist) os.environ["DAOS_TEST_SHARED_DIR"] = ci_site_config.daos_share os.environ["DAOS_TEST_LOG_DIR"] = ci_site_config.daos_logs os.environ["LD_LIBRARY_PATH"] = f'{self.ci_middlewares_path}/daos/install/lib64:{self.mpipath}' @property def cmd(self): return "./launch.py " def options(self, testname): opts = "-s " opts += f"{self.cart_tests[testname]['tags']} " if (self.cart_tests[testname]['numservers'] != 0): servers = ",".join(self.daos_nodes[:self.cart_tests[testname]['numservers']]) opts += f"--test_servers={servers} " if (self.cart_tests[testname]['numclients'] != 0): clients = ",".join(self.daos_nodes[:self.cart_tests[testname]['numclients']]) opts += f"--test_clients={clients}" return opts @property def execute_condn(self): return True def execute_cmd(self): sys.path.append(f'{self.daos_install_root}/lib64/python3.6/site-packages') os.environ['PYTHONPATH']=f'{self.daos_install_root}/lib64/python3.6/site-packages' print("PATH:" + os.environ["PATH"]) print("LD_LIBRARY_PATH:" + os.environ["LD_LIBRARY_PATH"]) print("MODULEPATH:" + os.environ["MODULEPATH"]) test_dir=self.cart_test_scripts curdir=os.getcwd() os.chdir(test_dir) for test in self.cart_tests: print(test) command = self.cmd + self.options(test) outputcmd = shlex.split(command) common.run_command(outputcmd, self.ci_logdir_path, self.run_test, self.ofi_build_mode) print("--------------------TEST COMPLETED----------------------") os.chdir(curdir)