1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
|
#!/usr/bin/env bash
set -eou pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source "${DIR}/common_utils.sh"
# Allow for users to pass PACKAGE_NAME
# For use with other packages, i.e. torchvision, etc.
PACKAGE_NAME=${PACKAGE_NAME:-pytorch}
PYTORCH_CONDA_FROM=${PYTORCH_CONDA_FROM:-pytorch-test}
PYTORCH_CONDA_TO=${PYTORCH_CONDA_TO:-pytorch}
CONDA_PLATFORMS="linux-64 osx-64 win-64 noarch"
pytorch_version="$(get_pytorch_version)"
tmp_dir="$(mktemp -d)"
pushd "${tmp_dir}"
trap 'rm -rf ${tmp_dir}' EXIT
conda_search() {
conda search -q "${PYTORCH_CONDA_FROM}::${PACKAGE_NAME}==${pytorch_version}" -c "${PYTORCH_CONDA_FROM}" --platform "${platform}" \
| grep -e "^${PACKAGE_NAME}" \
| awk -F ' *' '{print $3}' \
| xargs -I % echo "https://anaconda.org/${PYTORCH_CONDA_FROM}/${PACKAGE_NAME}/${pytorch_version}/download/${platform}/${PACKAGE_NAME}-${pytorch_version}-%.tar.bz2"
}
pkgs_to_download=()
for platform in ${CONDA_PLATFORMS}; do
pkgs_to_download+=($(\
conda_search 2>/dev/null || true
))
# Create directory where packages will eventually be downloaded
mkdir -p "${platform}"
done
my_curl() {
local dl_url=$1
local start=$(date +%s)
# downloads should be distinguished by platform which should be the second
# to last field in the url, this is to avoid clobbering same named files
# for different platforms
dl_dir=$(echo "${dl_url}" | rev | cut -d'/' -f 2 | rev)
dl_name=$(echo "${dl_url}" | rev | cut -d'/' -f 1 | rev)
curl -fsSL -o "${dl_dir}/${dl_name}" "${dl_url}"
local end=$(date +%s)
local diff=$(( end - start ))
echo "+ ${dl_url} took ${diff}s"
}
export -f my_curl
# Download all packages in parallel
printf '%s\n' "${pkgs_to_download[@]}" \
| xargs -P 10 -I % bash -c '(declare -t my_curl); my_curl %'
# dry run by default
DRY_RUN=${DRY_RUN:-enabled}
ANACONDA="true anaconda"
if [[ $DRY_RUN = "disabled" ]]; then
ANACONDA="anaconda"
fi
(
# We use --skip here to avoid re-uploading files we've already uploaded
set -x
${ANACONDA} upload --skip -u ${PYTORCH_CONDA_TO} $(find . -name '*.bz2')
)
popd
|
