1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
|
#!/bin/sh
set -euC
cd "$(dirname "$0")"
VERBOSE=${VERBOSE:-}
realpath () (
cd "$(dirname "$1")"
printf '%s' "$PWD"
[ "$PWD" != '/' ] && printf '/'
printf '%s\n' "$(basename "$1")"
)
printf "* Testsuite configuration\n"
# cleanup
find . \( -name '*.sjson' -o -name '*.morbigerror' \) -delete
[ "${1:-}" = 'clean' ] && exit 0
## ========= Find morbig; either in this directory, or in the $PATH ========= ##
localmorbig=../bin/morbig
if [ -e "$localmorbig" ]; then
morbig=$(realpath "$localmorbig")
printf 'Using local %s.\n' "$morbig"
elif command -v morbig >/dev/null 2>&1; then
morbig=$(command -v morbig)
printf '[WARN] Using from $PATH: %s.\n' "$morbig"
else
printf '[FAIL] Could not find morbig. Did you compile it?\n'
exit 1
fi
# Define counters
good_total=0
good_accepted=0
good_rejected=0
good_unexpected=0
good_open_total=0
good_open_accepted=0
good_open_rejected=0
good_open_unexpected=0
bad_total=0
bad_accepted=0
bad_rejected=0
bad_open_total=0
bad_open_accepted=0
bad_open_rejected=0
incr () {
eval "$1=\$(($1 + 1))"
}
last_test_printed=
printf_for_test () {
if [ "$test" != "$last_test_printed" ]; then
last_test_printed=$test
printf '* %s\n' "$test"
fi
printf "$@"
}
## For all test categories. We only handle `good` and `bad` tests, because
## there is not much to do about `unspecified` and `unknown` ones.
for category in good bad; do
## For all test files in the test category in question.
for test in $(find "$category" -type f -name '*.sh' | sort -n); do
test=${test%.sh}
## If the test file does not exist, then something is wrong, and it is
## probably due to the fact that one test name contains spaces and was
## therefore cut in two by the `$(find ...)` above.
if ! [ -f "$test.sh" ]; then
printf_for_test '[WARN] File `%s` does not exist or is not a file.
Did you create a test whose name contains a space?\n' "$test"
continue
fi
## The variable `$open` is either empty or `_open`. That way, we can
## write `$category$open` and it will be either `good`, `good_open` or
## `bad`. The underscore therefore matters.
open=
[ -e "$test.sh.open" ] && open=_open
incr ${category}${open}_total
## If Morbig succeeds, a file "$test.sh.sjson" gets created.
if "$morbig" --as simple "$test.sh" 2>/dev/null; then
case $category$open in
bad)
printf_for_test '[FAIL] Wrongly accepted: %s\n' "$test"
incr bad_accepted
;;
bad_open)
printf_for_test '[WARN] Open wrongly accepted: %s\n' "$test"
incr bad_open_accepted
;;
good*)
## Normalise the output using `jq`. This ensures that our
## tests do not fail just because Yojson changed its
## printing behaviour.
cat "$test.sh.sjson" | jq . > "$test.sh.sjson.clean"
mv "$test.sh.sjson.clean" "$test.sh.sjson"
## If there is no `.expected` file, then we complain, and we
## count this case as “unexpected”.
if ! [ -f "$test.sh.expected" ]; then
printf_for_test '[WARN] Missing `.expected` file: %s\n' "$test"
incr good${open}_unexpected
## Otherwise, if there is a difference between the produced
## file and the expected one, then we complain.
elif ! diff "$test.sh.sjson" "$test.sh.expected" 2>&1 >/dev/null; then
if [ -n "$open" ]; then
printf_for_test '[WARN] Open with wrong output: %s\n' "$test"
else
printf_for_test '[FAIL] Wrong output: %s\n' "$test"
fi
if [ -n "$VERBOSE" ]; then
echo '** .sh'
cat "$test.sh"
echo '** .sh.sjson'
cat "$test.sh.sjson"
echo '** .sh.expected'
cat "$test.sh.expected"
fi
incr good${open}_unexpected
## Otherwise all is good.
else
incr good${open}_accepted
fi
esac
## If Morbig failed.
else
case "$category" in
bad)
incr bad${open}_rejected
;;
good)
incr good${open}_rejected
if [ -n "$open" ]; then
printf_for_test '[WARN] Open wrongly rejected: %s\n' "$test"
else
printf_for_test '[FAIL] Wrongly rejected: %s\n' "$test"
fi
if [ -n "$VERBOSE" ]; then
echo "** $test.sh"
cat "$test.sh"
fi
esac
fi
done
done
passed=$((good_accepted + good_open_accepted + bad_rejected + bad_open_rejected))
unexpected=$((good_unexpected + good_open_unexpected))
unexpected_open=$good_open_unexpected
failed=$((good_rejected + good_open_rejected + bad_accepted + bad_open_accepted))
failed_open=$((good_open_rejected + bad_open_accepted))
total=$((good_total + good_open_total + bad_total + bad_open_total))
if [ "$((passed + unexpected + failed))" -ne "$total" ]; then
printf '* *Erk... there must be a problem in this test script*.\n'
fi
printf '* Summary:
------------------------------------
| Passed | Unexp. | Failed | Total |
|--------|--------|--------|--------|-------|
| good | %3d | %3d | %3d | %3d |
| (open) | %3d | %3d | %3d | %3d |
|--------|--------|--------|--------|-------|
| bad | %3d | n/a | %3d | %3d |
| (open) | %3d | n/a | %3d | %3d |
|--------|--------|--------|--------|-------|
| all | %3d | %3d | %3d | %3d |
---------------------------------------------\n' \
"$good_accepted" "$good_unexpected" "$good_rejected" "$good_total" \
"$good_open_accepted" "$good_open_unexpected" "$good_open_rejected" "$good_open_total" \
"$bad_rejected" "$bad_accepted" "$bad_total" \
"$bad_open_rejected" "$bad_open_accepted" "$bad_open_total" \
"$passed" "$unexpected" "$failed" "$total"
if [ "$failed" -gt 0 ] || [ "$unexpected" -gt 0 ]; then
if [ "$failed" -eq "$failed_open" ] && [ "$unexpected" -eq "$unexpected_open" ]; then
printf '[INFO] All failures are due to open bugs. This script will succeed.\n'
exit 0
else
exit 2
fi
else
printf "\n ----------------\n"
printf " Congratulations!\n"
printf " ----------------\n"
exit 0
fi
|
