1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
|
#!/usr/bin/env python
#
# Author: Mike McKerns (mmckerns @caltech and @uqfoundation)
# Copyright (c) 1997-2016 California Institute of Technology.
# Copyright (c) 2016-2025 The Uncertainty Quantification Foundation.
# License: 3-clause BSD. The full license text is available at:
# - https://github.com/uqfoundation/pyina/blob/master/LICENSE
"""
helper script for ``pyina.mpi`` maps using the *'worker pool'* strategy
Notes:
this uses the same code as ``ezscatter``, but with ``pyina.mpi_pool``.
Warning:
this is a helper script for ``pyina.mpi.Mapper`` -- don't use it directly.
"""
import logging
log = logging.getLogger("ezpool")
log.addHandler(logging.StreamHandler())
def _debug(boolean):
"""print debug statements"""
if boolean: log.setLevel(logging.DEBUG)
else: log.setLevel(logging.WARN)
return
if __name__ == '__main__':
from pyina.mpi_pool import parallel_map
import dill as pickle
import sys
import os
from pyina import mpi
world = mpi.world
funcname = sys.argv[1]
argfilename = sys.argv[2]
outfilename = sys.argv[3]
if funcname.endswith('.pik'): # used pickled func
workdir = None
func = pickle.load(open(funcname,'rb'))
else: # used tempfile for func
workdir = sys.argv[4]
sys.path = [workdir] + sys.path
modname = os.path.splitext(os.path.basename(funcname))[0]
module = __import__(modname)
sys.path.pop(0)
func = module.FUNC
args,kwds = pickle.load(open(argfilename,'rb'))
if world.rank == 0:
log.info('funcname: %s' % funcname) # sys.argv[1]
log.info('argfilename: %s' % argfilename) # sys.argv[2]
log.info('outfilename: %s' % outfilename) # sys.argv[3]
log.info('workdir: %s' % workdir) # sys.argv[4]
log.info('func: %s' % func)
log.info('args: %s' % str(args))
log.info('kwds: %s' % str(kwds))
res = parallel_map(func, *args, **kwds) #XXX: called on ALL nodes ?
if world.rank == 0:
log.info('res: %s' % str(res))
pickle.dump(res, open(outfilename,'wb'))
# end of file
|
