1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
|
#!/bin/sh
set -e
KUROBAKO=${KUROBAKO:-kurobako}
DIR=$(cd $(dirname $0); pwd)
REPEATS=${REPEATS:-5}
BUDGET=${BUDGET:-300}
SEED=${SEED:-1}
DIM=${DIM:-2}
SURROGATE_ROOT=${SURROGATE_ROOT:-$(dirname $DIR)/tmp/surrogate-models}
WARM_START=${WARM_START:-0}
usage() {
cat <<EOF
$(basename ${0}) is an entrypoint to run benchmarkers.
Usage:
$ $(basename ${0}) <problem> <json-path>
Problem:
rosenbrock : https://www.sfu.ca/~ssurjano/rosen.html
six-hump-camel : https://www.sfu.ca/~ssurjano/camel6.html
himmelblau : https://en.wikipedia.org/wiki/Himmelblau%27s_function
ackley : https://www.sfu.ca/~ssurjano/ackley.html
rastrigin : https://www.sfu.ca/~ssurjano/rastr.html
sphere : https://www.sfu.ca/~ssurjano/spheref.html
toxic-lightgbm : https://github.com/c-bata/benchmark-warm-starting-cmaes
Options:
--help, -h print this
Example:
$ $(basename ${0}) rosenbrock ./tmp/kurobako.json
$ cat ./tmp/kurobako.json | kurobako plot curve --errorbar -o ./tmp
EOF
}
case "$1" in
himmelblau)
PROBLEM=$($KUROBAKO problem command python $DIR/problem_himmelblau.py)
;;
rosenbrock)
PROBLEM=$($KUROBAKO problem command python $DIR/problem_rosenbrock.py)
;;
six-hump-camel)
PROBLEM=$($KUROBAKO problem command python $DIR/problem_six_hump_camel.py)
;;
ackley)
PROBLEM=$($KUROBAKO problem sigopt --dim $DIM ackley)
;;
rastrigin)
# "kurobako problem sigopt --dim 8 rastrigin" only accepts 8-dim.
PROBLEM=$($KUROBAKO problem command python $DIR/problem_rastrigin.py $DIM)
;;
sphere)
# "kurobako problem sigopt --dim 8 rastrigin" only accepts 8-dim.
PROBLEM=$($KUROBAKO problem command python $DIR/problem_sphere.py $DIM)
;;
toxic-lightgbm)
PROBLEM=$($KUROBAKO problem warm-starting \
$($KUROBAKO problem surrogate $SURROGATE_ROOT/wscmaes-toxic-source/) \
$($KUROBAKO problem surrogate $SURROGATE_ROOT/wscmaes-toxic-target/))
;;
help|--help|-h)
usage
exit 0
;;
*)
echo "[Error] Invalid problem '${1}'"
usage
exit 1
;;
esac
RANDOM_SOLVER=$($KUROBAKO solver random)
CMAES_SOLVER=$($KUROBAKO solver --name 'cmaes' command -- python $DIR/optuna_solver.py cmaes)
SEP_CMAES_SOLVER=$($KUROBAKO solver --name 'sep-cmaes' command -- python $DIR/optuna_solver.py sep-cmaes)
IPOP_CMAES_SOLVER=$($KUROBAKO solver --name 'ipop-cmaes' command -- python $DIR/optuna_solver.py ipop-cmaes)
IPOP_SEP_CMAES_SOLVER=$($KUROBAKO solver --name 'ipop-sep-cmaes' command -- python $DIR/optuna_solver.py ipop-sep-cmaes)
PYCMA_SOLVER=$($KUROBAKO solver --name 'pycma' command -- python $DIR/optuna_solver.py pycma)
WS_CMAES_SOLVER=$($KUROBAKO solver --name 'ws-cmaes' command -- python $DIR/optuna_solver.py ws-cmaes --warm-starting-trials $WARM_START)
$KUROBAKO studies \
--solvers $RANDOM_SOLVER $CMAES_SOLVER $PYCMA_SOLVER \
--problems $PROBLEM \
--seed $SEED --repeats $REPEATS --budget $BUDGET \
| $KUROBAKO run --parallelism 6 > $2
|
