############################################################################## # # Copyright (c) 2003-2020 by The University of Queensland # http://www.uq.edu.au # # Primary Business: Queensland, Australia # Licensed under the Apache License, version 2.0 # http://www.apache.org/licenses/LICENSE-2.0 # # Development until 2012 by Earth Systems Science Computational Center (ESSCC) # Development 2012-2013 by School of Earth Sciences # Development from 2014 by Centre for Geoscience Computing (GeoComp) # Development from 2019 by School of Earth and Environmental Sciences # ############################################################################## from __future__ import print_function, division __copyright__="""Copyright (c) 2003-2020 by The University of Queensland http://www.uq.edu.au Primary Business: Queensland, Australia""" __license__="""Licensed under the Apache License, version 2.0 http://www.apache.org/licenses/LICENSE-2.0""" __url__="https://launchpad.net/escript-finley" filter# $Id:$ """ A simple framework to run benchmarks under OpenMP and to summarize the results in tables for instance in HTML :var __author__: name of author :var __license__: licence agreement :var __copyright__: copyrights :var __url__: url entry point on documentation :var __version__: version :var __date__: date of the version """ __author__="Lutz Gross, l.gross@uq.edu.au" import os, socket, sys, time, traceback from . import escriptcpp as esc class BenchmarkSuite(object): """ Framework to run a bunch of `Benchmark` s using the object and creating a table of statistics. :cvar MAX_LEVEL: maximum number of level in headers for output """ MAX_LEVEL=5 def __init__(self,name=None): """ Sets up a suite of benchmarks. :param name: name of the benchmark suite. If no name is given the class name is used. :type name: ``str`` """ super(BenchmarkSuite,self).__init__() self.__benchmarks=[] self.__scale=1 if name is None: self.__name=self.__class__.__name__ else: self.__name=name def __str__(self): """ Returns the name of the benchmark suite. :return: the name :rtype: ``str`` """ return self.__name def addBenchmark(self,benchmark): """ Adds a new `Benchmark` to the suite. :param benchmark: the benchmark to add :type benchmark: `Benchmark` """ self.__benchmarks.append(benchmark) def __len__(self): """ Returns the number of benchmarks in the suite. :return: number of benchmarks :rtype: ``int`` """ return len(self.__benchmarks) def __getitem__(self,i): """ Returns the i-th benchmark in the suite through self[i]. :param i: index of the requested benchmark :type i: ``int`` :return: i-th benchmark :rtype: `Benchmark` """ return self.__benchmarks[i] def run(self,scale=1): """ Runs all benchmarks. :param scale: defines the number of (OpenMP) threads to be used. If ``scale`` is a scalar all benchmarks are run with ``scale`` number of threads. If ``scale`` is a ``list``, the p-th problem in each of the benchmarks in the suite is run with ``scale[p]`` threads. If ``scale[p]`` <1 the p-th problem is omitted. :type scale: ``int`` or ``list`` of ``int`` """ self.__scale=scale for i in range(len(self)): self[i].run(scale=scale) def getHTML(self,filter,level=1): """ Returns the results of the last benchmark run in HTML format. :param filter: filter to be applied to the results :type filter: `BenchmarkFilter` :param level: level used in header tags :type level: ``int`` :return: HTML document :rtype: ``str`` """ out="" if level==1: out+="Benchmark: %s\n"%str(self) out+="%s\n"%(level,str(self),level) if level==1: m="" if isinstance(self.__scale,int): if self.__scale>1: m=" (%s threads)"%self.__scale out+="

platform: %s%s

\n"%(socket.gethostname(),m) for i in range(len(self)): out+="

\n" out+=self[i].getHTML(filter=filter,level=min(level+1,self.MAX_LEVEL)) out+="

\n" if level==1: try: name=os.getlogin() out+="

by %s at %s

\n"%(name,time.strftime('%X %x %Z')) except OSError: out+="

%s

\n"%(time.strftime('%X %x %Z')) out+="\n" return out class Benchmark(object): """ Runs a bunch of similar `BenchmarkProblem` s with a bunch of `Options`. """ def __init__(self,name=None,description=None): """ Sets up a benchmark. :param name: name of the benchmark. If no name is given the class name is used. :type name: ``str`` :param description: description of the benchmark :type description: ``str`` or ``None`` """ super(Benchmark,self).__init__() self.__options=[] self.__problems=[] self.__results=[] self.__scale=1 if name is None: self.__name=self.__class__.__name__ else: self.__name=name self.__description=description def __str__(self): """ Returns the name of the benchmark suite. :return: the name :rtype: ``str`` """ return self.__name def addProblem(self,problem): """ Adds a problem to the benchmark. :param problem: the problem to be added :type problem: `BenchmarkProblem` """ self.__problems.append(problem) def addOptions(self,options): """ Adds options to the benchmark. :param options: the options to be added to the benchmark. If options is None the options are left unchanged. :type options: `Options` """ if options!=None: self.__options.append(options) def run(self,scale=1): """ Runs all problems with all options. :param scale: defines the number of (OpenMP) threads to be used. If ``scale`` is a scalar all benchmarks are run with ``scale`` number of threads. If ``scale`` is a ``list`` , the p-th problem in each of the benchmarks in the suite is run with ``scale[p]`` threads. If ``scale[p]`` <1 the p-th problem is omitted. :type scale: ``int`` or ``list`` of ``int`` s """ if isinstance(scale,list): c_max=min(len(scale),len(self.__problems)) else: c_max=len(self.__problems) self.__filter=filter self.__scale=scale self.__results=[] for c in range(c_max): r=self.__problems[c] if isinstance(scale,list): s=scale[c] else: s=scale row=[] if s>0: t0=time.time() print(("%s with %s threads started."%(r.__class__,s))) for p in self.__options: esc.setNumberOfThreads(s) try: row.append(r.run(p)) except: traceback.print_exc(file=sys.stdout) row.append(None) t0=time.time()-t0 print(("%s with %s threads finished (walltime=%s sec)."%(r.__class__,s,t0))) self.__results.append(row) def getHTML(self,filter,level=1): """ Returns the results of the last benchmark run in HTML format. :param filter: filter to be applied to the results :type filter: `BenchmarkFilter` :param level: level used in header tags :type level: ``int`` :return: HTML document :rtype: ``str`` """ out="" if level==1: out+="Benchmark: %s\n"%str(self) out+="%s\n"%(level,str(self),level) if level==1: m="" if isinstance(self.__scale,int): if self.__scale>1: m=" (%s threads)"%self.__scale out+="

platform: %s%s

\n"%(socket.gethostname(),m) if self.__description: out+="

%s

\n"%str(self.__description) if len(self.__problems)>0: out+="\n" h1_seg="" rn=filter.getResultNames() if len(rn)==0: h1_seg+="" else: for n in rn: h1_seg+=""%n h0="" h1="" if isinstance(self.__scale,list): h0+="" for o in self.__options: if len(rn)==0: h0+=""%str(o) colspan=1 elif len(rn)==1: h0+=""%str(o) colspan=1 empty_h1=False else: colspan=len(rn) h0+=""%(colspan,str(o)) h1+=h1_seg out+=h0+"\n"+h1+"\n" c=0 for r in range(len(self.__results)): out+=""%str(self.__problems[r]) if isinstance(self.__scale,list): out+=""%self.__scale[c] for col in self.__results[r]: if col is None: out+=""%colspan else: for e in filter(col): out+=""%e out+="\n" c+=1 out+="
%s
Case
Threads%s%s%s
%s%sfailed.%s
" if level==1: out+="

by %s at %s

\n"%(os.getlogin(),time.strftime('%X %x %Z')) out+="\n" return out class BenchmarkProblem(object): """ Represents a benchmark problem that can be run and which returns a list of characteristics such as timing, MFlops, error, etc. """ def __init__(self,name=None): """ Sets up a benchmark problem. :param name: name of the problem. If no name is given the class name is used. :type name: ``str`` """ super(BenchmarkProblem,self).__init__() if name is None: self.__name=self.__class__.__name__ else: self.__name=name def __str__(self): """ Returns the name of the benchmark suite. :return: the name :rtype: ``str`` """ return self.__name def run(self,options=None): """ Runs the problem and returns a list of run characteristics. :param options: the options that are used for the run. Note that the number of OpenMP threads is controlled by the `Benchmark` the problem is run in. :type options: `Options` :return: run characteristics :rtype: any type that can be read by the `BenchmarkFilter` applied to it :note: this function has to be overwritten by a particular problem """ raise NotImplementedError return [] class BenchmarkFilter(object): """ Object to filter the characteristics returned by Benchmark runs. """ def __init__(self): """ Sets up a filter. """ pass def getResultNames(self): """ Returns the names of the results produced when ``run()`` is called. :return: the list of the names to be used when the results of the ``run()`` call are printed :rtype: ``list`` of ``str`` :note: this function has to overwritten by a particular problem """ raise NotImplementedError return [] def __call__(self,result): """ Filters out results returned as characteristics of a problem run. :param result: values to be filtered :type result: any type that is produced by the `BenchmarkProblem` it is applied to :return: a list of strings selected from result :rtype: ``list`` of ``str`` :note: this function has to be overwritten by a particular problem """ raise NotImplementedError return [] class Options(object): """ Defines a set of options to be used to run a `BenchmarkProblem`. """ def __init__(self,name=None): """ Sets up the options. :param name: name of the option. If no name is given the class name is used. :type name: ``str`` """ super(Options,self).__init__() if name is None: self.__name=self.__class__.__name__ else: self.__name=name def __str__(self): """ Returns the name of this options object. :return: the name :rtype: ``str`` """ return self.__name if __name__=="__main__": class OptionsTest1(Options): pass class OptionsTest2(Options): pass class BenchmarkProblemTest1(BenchmarkProblem): def __init__(self): super(BenchmarkProblemTest1,self).__init__(name="TEST1") def run(self,options=None): import time return time.time(),"A" class BenchmarkProblemTest2(BenchmarkProblem): def __init__(self): super(BenchmarkProblemTest2,self).__init__(name="TEST2") def run(self,options=None): import time return -time.time(),"B" class SimpleFilter(BenchmarkFilter): def getResultNames(self): return ["r0","r1"] def __call__(self,result): return [str(result[0]),str(result[1])] bm=Benchmark("Example") bm.addProblem(BenchmarkProblemTest1()) bm.addProblem(BenchmarkProblemTest2()) bm.addOptions(OptionsTest1()) bm.addOptions(OptionsTest2()) bms=BenchmarkSuite("A Test") bms.addBenchmark(bm) bms.run() print((bms.getHTML(filter=SimpleFilter()))) bms.run(scale=4) print((bms.getHTML(filter=SimpleFilter()))) bms.run(scale=[1,2]) print((bms.getHTML(filter=SimpleFilter())))