1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
#!/bin/bash
SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
echo $SCRIPTPATH
PARSL_ROOT=$(dirname $(dirname $(dirname $SCRIPTPATH)))
PARSL_GITHASH=$(git rev-parse --short HEAD)
CONDA_TARGET=parsl_$PARSL_GITHASH.py3.7
export CONDA_TARGET
if [[ "$CONDA_TARGET" == "$CONDA_DEFAULT_ENV" ]]
then
echo "Conda target env $CONDA_TARGET loaded"
exit 0
fi
create_conda() {
pushd .
cd $PARSL_ROOT
if [[ "$(hostname)" =~ .*thetalogin.* ]]
then
echo "On theta"
module load miniconda-3/latest
conda create -p $CONDA_TARGET --clone $CONDA_PREFIX --yes --force
conda activate $CONDA_TARGET
# Theta is weird, we do explicit install
pip install -r test-requirements.txt
conda install pip psutil --yes
python3 setup.py install
echo "module load miniconda-3/latest;" > ~/setup_parsl_test_env.sh
echo "conda activate $PWD/$CONDA_TARGET" >> ~/setup_parsl_test_env.sh
return
elif [[ "$(hostname)" =~ .*frontera.* ]]
then
echo "On Frontera"
if [[ -d ~/anaconda3 ]]
then
echo "Loading anaconda3 from ~/anaconda3"
source ~/anaconda3/bin/activate
else
echo "Please install conda to your home dir at ~/anaconda3"
fi
conda create -p $CONDA_TARGET python=3.7 --yes --force
conda activate $CONDA_TARGET
echo "source ~/anaconda3/bin/activate;" > ~/setup_parsl_test_env.sh
echo "conda activate $PWD/$CONDA_TARGET" >> ~/setup_parsl_test_env.sh
elif [[ "$(hostname)" =~ .*nscc.* ]]
then
echo "On NSCC Singapore"
if [[ -d ~/anaconda3 ]]
then
echo "Loading anaconda3 from ~/anaconda3"
source ~/anaconda3/bin/activate
else
echo "Please install conda to your home dir at ~/anaconda3"
fi
conda create -p $CONDA_TARGET python=3.7 --yes --force
conda activate $CONDA_TARGET/
echo "source ~/anaconda3/bin/activate;" > ~/setup_parsl_test_env.sh
echo "conda activate $PWD/$CONDA_TARGET/" >> ~/setup_parsl_test_env.sh
echo "module load openmpi/intel/1.10.2;" >> ~/setup_parsl_test_env.sh
elif [[ "$(hostname -f)" =~ .*summit.* ]]
then
echo "On Summit"
module load ibm-wml-ce
conda create -p $CONDA_TARGET --yes --force
conda activate $PWD/$CONDA_TARGET
conda install paramiko>=2.7.1 pip numpy psutil pandas --yes
# conda install --file requirements.txt --yes
echo "module load ibm-wml-ce" > ~/setup_parsl_test_env.sh
echo "conda activate $PWD/$CONDA_TARGET" >> ~/setup_parsl_test_env.sh
elif [[ "$(hostname)" =~ .*stampede2.* ]]
then
echo "On Stampede2"
if [[ -d ~/anaconda3 ]]
then
echo "Loading anaconda3 from ~/anaconda3"
source ~/anaconda3/bin/activate
else
echo "Please install conda to your home dir at ~/anaconda3"
fi
conda create -p $CONDA_TARGET python=3.7 --yes --force
conda activate $PWD/$CONDA_TARGET
echo "source ~/anaconda3/bin/activate;" > ~/setup_parsl_test_env.sh
echo "conda activate $PWD/$CONDA_TARGET" >> ~/setup_parsl_test_env.sh
elif [[ "$(hostname)" =~ .*comet.* ]]
then
echo "On Comet"
if [[ -d ~/anaconda3 ]]
then
echo "Loading anaconda3 from ~/anaconda3"
source ~/anaconda3/bin/activate
else
echo "Please install conda to your home dir at ~/anaconda3"
fi
conda create -p $CONDA_TARGET python=3.7 --yes --force
conda activate $PWD/$CONDA_TARGET
echo "source ~/anaconda3/bin/activate;" > ~/setup_parsl_test_env.sh
echo "conda activate $PWD/$CONDA_TARGET" >> ~/setup_parsl_test_env.sh
elif [[ "$(hostname)" =~ .*midway.* ]]
then
echo "On Midway"
module unload python
module load Anaconda3/5.3.0
conda create -p $CONDA_TARGET python=3.7 --yes --force
source /software/Anaconda3-5.3.0-el7-x86_64/bin/activate
conda activate $PWD/$CONDA_TARGET
echo "module unload python" > ~/setup_parsl_test_env.sh
echo "module load Anaconda3/5.3.0" >> ~/setup_parsl_test_env.sh
echo "source /software/Anaconda3-5.3.0-el7-x86_64/bin/activate" >> ~/setup_parsl_test_env.sh
echo "conda activate $PWD/$CONDA_TARGET" >> ~/setup_parsl_test_env.sh
elif [[ "$(hostname -f)" =~ .*in2p3.* ]]
then
echo "On CC-IN2P3"
export PATH=/pbs/throng/lsst/software/anaconda/anaconda3-5.0.1/bin:$PATH
conda create -p $CONDA_TARGET python=3.7 --yes --force
source activate $PWD/$CONDA_TARGET
conda install pip --yes
cat <<'EOF' > ~/setup_parsl_test_env.sh
export PATH=/pbs/throng/lsst/software/anaconda/anaconda3-5.0.1/bin:$PATH
export SHARED_FS_OPTIONS='-k "not issue363"'
EOF
echo "source activate $PWD/$CONDA_TARGET" >> ~/setup_parsl_test_env.sh
elif [[ "$(hostname)" =~ .*cooley.* ]]
then
echo "On Cooley, testing Kubernetes on PetrelKube"
if [[ -d ~/anaconda3 ]]
then
echo "Loading anaconda3 from ~/anaconda3"
source ~/anaconda3/bin/activate
else
echo "Please install conda to your home dir at ~/anaconda3"
fi
conda create -p $CONDA_TARGET python=3.7 --yes --force
conda activate $PWD/$CONDA_TARGET
pip install kubernetes
echo "pip install git+https://github.com/Parsl/parsl@$PARSL_GITHASH" > ~/setup_parsl_test_env.sh
echo 'pip install pytest==4.6' >> ~/setup_parsl_test_env.sh
elif [[ "$(hostname -f)" =~ .*h2ologin.* ]]
then
echo "On Blue Waters"
module load bwpy
python_version=3.6
export EPYTHON="python$python_version"
mkdir -p parsl_$PARSL_GITHASH.py$python_version
virtualenv --system-site-packages $PWD/parsl_$PARSL_GITHASH.py$python_version
source $PWD/parsl_$PARSL_GITHASH.py$python_version/bin/activate
pip install --upgrade pip
echo "module load bwpy;" > ~/setup_parsl_test_env.sh
echo "cd $PWD;" >> ~/setup_parsl_test_env.sh
echo "source $PWD/parsl_$PARSL_GITHASH.py$python_version/bin/activate;" >> ~/setup_parsl_test_env.sh
elif [[ "$(hostname -f)" =~ .*bridges.* ]]
then
echo "On bridges"
module load anaconda3
conda create -p $CONDA_TARGET python=3.7 --yes --force
source activate $PWD/$CONDA_TARGET
echo "module load anaconda3" > ~/setup_parsl_test_env.sh
echo "source activate $PWD/$CONDA_TARGET" >> ~/setup_parsl_test_env.sh
else
echo "Unknown site"
exit -1
fi
echo "Installing parsl from $PARSL_ROOT"
python3 -m pip install .
python3 -m pip install -r test-requirements.txt
popd
}
create_conda
|
