1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
#!/usr/bin/env bash
# spell-checker:ignore (env/flags) Ccodegen Cinstrument Coverflow Cpanic Zpanic
# spell-checker:ignore PROFDATA PROFRAW coreutil librairies nextest profdata profraw rustlib
# This script will build, run and generate coverage reports for the whole
# testsuite.
# The biggest challenge of this process is managing the overwhelming generation
# of trace files that are generated after EACH SINGLE invocation of a coreutil
# in the testsuite. Moreover, because we run the testsuite against the multicall
# binary, each trace file contains coverage information about the WHOLE
# multicall binary, dependencies included, which results in a 5-6 MB file.
# Running the testsuite easily creates +80 GB of trace files, which is
# unmanageable in a CI environment.
#
# A workaround is to run the testsuite util per util, generate a report per
# util, and remove the trace files. Therefore, we end up with several reports
# that will get uploaded to codecov afterwards. The issue with this
# approach is that the `grcov` call, which is responsible for transforming
# `.profraw` trace files into a `lcov` file, takes a lot of time (~20s), mainly
# because it has to browse all the sources. So calling it for each of the 100
# utils (with --all-features) results in an absurdly long execution time
# (almost an hour).
# TODO: Do not instrument 3rd party librairies to save space and performance
# Exit the script if an unexpected error arise
set -e
# Treat unset variables as errors
set -u
# Ensure pipeline failures are caught (not just the last command's exit code)
set -o pipefail
# Print expanded commands to stdout before running them
set -x
ME="${0}"
ME_dir="$(dirname -- "$(readlink -fm -- "${ME}")")"
REPO_main_dir="$(dirname -- "${ME_dir}")"
# Features to enable for the `coreutils` package
FEATURES_OPTION=${FEATURES_OPTION:-"--features=feat_os_unix"}
COVERAGE_DIR=${COVERAGE_DIR:-"${REPO_main_dir}/coverage"}
# Find llvm-profdata (which is used for coverage builds)
LLVM_PROFDATA="$(find "$(rustc --print sysroot)" -name llvm-profdata)"
if [ -z "${LLVM_PROFDATA}" ]; then
echo "Error: llvm-profdata not found. Install it with: rustup component add llvm-tools"
exit 1
fi
PROFRAW_DIR="${COVERAGE_DIR}/traces"
PROFDATA_DIR="${COVERAGE_DIR}/data"
REPORT_DIR="${COVERAGE_DIR}/report"
REPORT_PATH="${REPORT_DIR}/total.lcov.info"
rm -rf "${PROFRAW_DIR}" && mkdir -p "${PROFRAW_DIR}"
rm -rf "${PROFDATA_DIR}" && mkdir -p "${PROFDATA_DIR}"
rm -rf "${REPORT_DIR}" && mkdir -p "${REPORT_DIR}"
#shellcheck disable=SC2086
UTIL_LIST=$("${ME_dir}"/show-utils.sh ${FEATURES_OPTION})
export RUSTC_BOOTSTRAP=1
export CARGO_INCREMENTAL=0
export RUSTFLAGS="-Cinstrument-coverage -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort"
export RUSTDOCFLAGS="-Cpanic=abort"
export LLVM_PROFILE_FILE="${PROFRAW_DIR}/coverage-%4m.profraw"
# Disable expanded command printing for the rest of the program
set +x
run_test_and_aggregate() {
echo "# Running coverage tests for ${1}"
# Build and run tests for the UTIL
cargo nextest run \
--profile coverage \
--no-fail-fast \
--color=always \
2>&1 \
${2} \
| grep -v 'SKIP'
# Note: Do not print the skipped tests on the output as there will be many.
echo "## Tests for (${1}) generated $(du -h -d1 ${PROFRAW_DIR} | cut -f 1) of profraw files"
# Aggregate all the trace files into a profdata file
PROFDATA_FILE="${PROFDATA_DIR}/${1}.profdata"
echo "## Aggregating coverage files under ${PROFDATA_FILE}"
"${LLVM_PROFDATA}" merge \
-sparse \
-o ${PROFDATA_FILE} \
${PROFRAW_DIR}/*.profraw \
|| true
# We don't want an error in `llvm-profdata` to abort the whole program
}
for UTIL in ${UTIL_LIST}; do
if [ "${UTIL}" = "stty" ]; then
run_test_and_aggregate \
"${UTIL}" \
"-p coreutils -p uu_${UTIL} -E test(/^test_${UTIL}::/) ${FEATURES_OPTION}"
else
run_test_and_aggregate \
"${UTIL}" \
"-p coreutils -E test(/^test_${UTIL}::/) ${FEATURES_OPTION}"
fi
echo "## Clear the trace directory to free up space"
rm -rf "${PROFRAW_DIR}" && mkdir -p "${PROFRAW_DIR}"
done;
echo "Running coverage tests over uucore"
run_test_and_aggregate "uucore" "-p uucore --all-features"
echo "# Aggregating all the profraw files under ${REPORT_PATH}"
grcov \
"${PROFDATA_DIR}" \
--binary-path "${REPO_main_dir}/target/debug/" \
--output-types lcov \
--output-path ${REPORT_PATH} \
--llvm \
--excl-start "^mod test.*\{" \
--excl-stop "^\}" \
--keep-only "${REPO_main_dir}"'/src/*'
# Notify the report file to github
echo "report=${REPORT_PATH}" >> $GITHUB_OUTPUT
|
