1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
|
#!/usr/bin/env python3
# Input on stdin; output on stdout.
# WARNING: This script assumes throughout that dictionaries retain their
# order, which is guaranteed as of Python 3.7.
import copy
import itertools
import os
import string
import sys
import yaml
import yaml.representer
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)) + "/../../lib")
import charliecloud as ch
def main():
ch.INFO("starting")
yin = yaml.safe_load(sys.stdin)
yout = dict()
assert(isinstance(yin, dict))
ch.INFO("loaded top-level dictionary with %d keys" % len(yin))
ch.INFO("expanding")
for (name, job) in yin.items():
ch.INFO("job: %s ... " % name, end="")
if ("parallel" in job):
ch.INFO(" expanding")
yout |= expand_matrices(name, job)
else:
ch.INFO(" not matrixed")
yout[name] = job
ch.INFO("updating needs")
for (name, job) in yout.items():
ch.INFO("job: %s ..." % name, end="")
if ("needs" in job):
ch.INFO(" updating")
needs_update(job)
else:
ch.INFO(" no needs")
ch.INFO("outputting")
yaml.dump(yout, sys.stdout, indent=2,
width=80,
allow_unicode=True)
ch.INFO("done")
def expand_matrices(name, job):
jobs = dict()
for (i, g) in enumerate(job["parallel"]["matrix"], 1):
ch.INFO(" group %d:" % i)
group_ct = 0
for vs in list(itertools.product(*g.values())):
(name_sub, job_sub) = expand_one(name, job, dict(zip(g.keys(), vs)))
assert (name_sub not in jobs)
ch.INFO(" %s" % name_sub)
jobs[name_sub] = job_sub
group_ct += 1
ch.INFO(" expanded to %d jobs" % group_ct)
ch.INFO(" total jobs: %d" % len(jobs))
return jobs
def expand_one(name, job, row):
job = copy.deepcopy(job)
del job["parallel"]
job["variables"] = job.get("variables", dict()) | row
return (name_expand(name, row), job)
def name_expand(name, row):
assert (name[-1] == "@")
return name + "/".join(row.values())
def needs_update(job):
# this lets us use shell-style variable substitution
job["needs"] = [string.Template(n).substitute(job["variables"])
for n in job["needs"]]
# Use block style for multi-line strings, default for other strings.
def multiline_block(dumper, data):
if "\n" in data:
return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
else:
return yaml.representer.SafeRepresenter.represent_str(dumper, data)
yaml.add_representer(str, multiline_block)
if (__name__ == "__main__"):
main()
|
