1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
|
#!/bin/bash
set -ex
UNKNOWN=()
# defaults
PARALLEL=1
export TORCH_ONNX_EXPERIMENTAL_RUNTIME_TYPE_CHECK=ERRORS
while [[ $# -gt 0 ]]
do
arg="$1"
case $arg in
-p|--parallel)
PARALLEL=1
shift # past argument
;;
*) # unknown option
UNKNOWN+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
set -- "${UNKNOWN[@]}" # leave UNKNOWN
# allows coverage to run w/o failing due to a missing plug-in
pip install -e tools/coverage_plugins_package
# realpath might not be available on MacOS
script_path=$(python -c "import os; import sys; print(os.path.realpath(sys.argv[1]))" "${BASH_SOURCE[0]}")
top_dir=$(dirname $(dirname $(dirname "$script_path")))
test_paths=(
"$top_dir/test/onnx"
)
args=()
args+=("-v")
args+=("--cov")
args+=("--cov-report")
args+=("xml:test/coverage.xml")
args+=("--cov-append")
time python "${top_dir}/test/run_test.py" --onnx --shard "$SHARD_NUMBER" 2 --verbose
if [[ "$SHARD_NUMBER" == "2" ]]; then
# xdoctests on onnx
xdoctest torch.onnx --style=google --options="+IGNORE_WHITESPACE"
fi
if [[ "$SHARD_NUMBER" == "2" ]]; then
# Sanity check on torchbench w/ onnx
pip install pandas
log_folder="test/.torchbench_logs"
device="cpu"
modes=("accuracy" "performance")
compilers=("dynamo-onnx" "torchscript-onnx")
suites=("huggingface" "timm_models")
mkdir -p "${log_folder}"
for mode in "${modes[@]}"; do
for compiler in "${compilers[@]}"; do
for suite in "${suites[@]}"; do
output_file="${log_folder}/${compiler}_${suite}_float32_inference_${device}_${mode}.csv"
bench_file="benchmarks/dynamo/${suite}.py"
bench_args=("--${mode}" --float32 "-d${device}" "--output=${output_file}" "--output-directory=${top_dir}" --inference -n5 "--${compiler}" --no-skip --dashboard --batch-size 1)
# Run only selected model for each suite to quickly validate the benchmark suite works as expected.
case "$suite" in
"torchbench")
bench_args+=(-k resnet18)
;;
"huggingface")
bench_args+=(-k ElectraForQuestionAnswering)
;;
"timm_models")
bench_args+=(-k lcnet_050)
;;
*)
echo "Unknown suite: ${suite}"
exit 1
;;
esac
python "${top_dir}/${bench_file}" "${bench_args[@]}"
done
done
done
fi
# Our CI expects both coverage.xml and .coverage to be within test/
if [ -d .coverage ]; then
mv .coverage test/.coverage
fi
|
