File: run_wml_tests

package info (click to toggle)
wesnoth-1.14 1:1.14.7-1
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, sid
  • size: 817,348 kB
  • sloc: cpp: 228,458; java: 29,664; python: 22,615; lisp: 7,550; sh: 3,325; xml: 1,767; perl: 1,501; ansic: 1,083; makefile: 264; php: 98; sed: 44
file content (384 lines) | stat: -rwxr-xr-x 9,723 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
#!/bin/bash
#This script runs a sequence of wml unit test scenarios.
#Use -h to get help with usage.

usage()
{
  echo "Usage:" $0 "[OPTIONS] [EXTRA-ARGS]"
  echo "Executes a series of wml unit test scenarios found in a file."
  echo
  echo -e "Options:"
  echo -e "\t-h\tShows this help."
  echo -e "\t-v\tVerbose mode."
  echo -e "\t-w\tVery verbose mode. (Debug script.)"
  echo -e "\t-c\tClean mode. (Don't load any add-ons. Used for mainline tests.)"
  echo -e "\t-a arg\tAdditional arguments to go to wesnoth."
  echo -e "\t-t arg\tNew timer value to use, instead of 10s as default."
  echo -e "\t  \t0s means no timer, and also skips tests that expect timeout."
  echo -e "\t-s\tDisable strict mode. By default, we run wesnoth with option"
  echo -e "\t  \t'--log-strict=warning' to ensure errors result in a failed test."
  echo -e "\t-d\tRun wesnoth-debug binary instead of wesnoth."
  echo -e "\t-g\tIf we encounter a crash, generate a backtrace using gdb. Must have gdb installed for this option."
  echo -e "\t-p arg\tPath to wesnoth binary. By default assume it is with this script."
  if [ $(uname) = "Darwin" ]; then
    echo -e "\t  \tThe special value xcode searches in XCode's DerivedProducts directory."
  fi
  echo -e "\t-l arg\tLoads list of tests from the given file."
  echo -e "\t  \tBy default, the file is wml_test_schedule."
  echo
  echo "Each line in the list of tests should be formatted:"
  echo -e "\n\t<expected return code> <name of unit test scenario>\n"
  echo "Lines beginning # are treated as comments."
  echo "Expected return codes:"
  for i in `seq 0 4`;
  do
    get_code_string $i
    echo -e "\t" $i "-" $CodeString
  done
  echo
  echo "Extra arguments besides these options are saved and passed on to wesnoth."
}

get_code_string()
{
  case ${1} in
  0)
    CodeString="PASS"
    ;;
  1)
    CodeString="FAIL"
    ;;
  2)
    CodeString="FAIL (TIMEOUT)"
    ;;
  3)
    CodeString="FAIL (INVALID REPLAY)"
    ;;
  4)
    CodeString="FAIL (ERRORED REPLAY)"
    ;;
  124)
    CodeString="FAIL (TIMEOUT, by TERM signal)"
    ;;
  137)
    CodeString="FAIL (TIMEOUT, by KILL signal)"
    ;;
  134)
    CodeString="FAIL (ASSERTION FAILURE ? ? ?)"
    ;;
  139)
    CodeString="FAIL (SEGFAULT ? ? ?)"
    ;;
  *)
    CodeString="FAIL (? ? ?)"
    ;;
  esac
}

check_errs()
{
  # Argument 1 is the name of the test.
  # Argument 2 is the wesnoth error code for the test.
  # Argument 3 is the expected error code.
  if [ "${2}" -eq 124 -a "${3}" -eq 2 ]; then
    if [ "$Verbose" -ge 2 ]; then
      echo "Caught return code 124 from timeout"
      echo "This signal means that the unix timeout utility killed wesnoth with TERM."
      echo "Since we expected timeout, the test passes."
    fi
    return 0
  elif [ "${2}" -eq 137 -a "${3}" -eq 2 ]; then
    if [ "$Verbose" -ge 2 ]; then
      echo "Caught return code 137 from timeout"
      echo "This signal means that the unix timeout utility killed wesnoth with KILL."
      echo "Since we expected timeout, the test passes."
    fi
    return 0
  elif [ "${2}" -ne "${3}" ]; then
    echo "${1}" ":"
    get_code_string ${2}
    printf '%-55s %3d - %s\n' "  Observed result :" "${2}" "$CodeString"
    get_code_string ${3}
    printf '%-55s %3d - %s\n' "  Expected result :" "${3}" "$CodeString"
    if [ "$Verbose" -ge 2 -a -f "error.log" ]; then
      echo ""
      echo "Found error.log:"
      cat error.log
    fi
    return 1
  fi
  return 0
}

handle_error_log()
{
  if [ -f "error.log" ]; then
    if [ "${1}" -ne 0 ]; then
      if [ -f "errors.log" ]; then
        echo -e "\n--- next unit test ---\n" >> errors.log
        cat error.log >> errors.log
      else
        cp error.log errors.log
      fi
    fi

    rm error.log
  fi
}

run_test()
{
  # Argument 1 is the expected error code
  # Argument 2 is the name of the test scenario
  # Argument 3 (optional) overrides strict mode (0 to disable, 1 to enable)

  preopts=""
  binary="$BinPath"
  opts="-u$2 "

  timer=$basetimer

  if [ "$DebugMode" -eq 1 ]; then
    binary+="wesnoth-debug "
  else
    binary+="wesnoth "
  fi

  # Use validcache on tests that aren't the first test.
  if [ "$FirstTest" -eq 1 ]; then
    ((timer *= 2))
  else
    opts+="--validcache "
  fi

  # Add a timeout using unix timeout utility
  if [ $timer -gt 0 ]; then
    # Some versions might not support the --kill-after option
    timeout --kill-after=5 1 true
    # Exit code of timeout is 125 if the command itself fails, like if we passed a bad argument
    if [ $? -ne 125 ]; then
      timer1=$((timer+1))
      preopts+="timeout --kill-after=$timer1 $timer "
    else
      preopts+="timeout $timer "
    fi
  elif [ $1 -eq 2 ]; then
    # If timeout is disabled, skip tests that expect it
    return 100
  fi

  strict_mode=$3
  if [ -z "$strict_mode" ]; then
    strict_mode=$StrictMode
  fi

  # If running strict, then set strict level to "warning"
  if [ "$strict_mode" -eq 1 ]; then
    opts+="--log-strict=warning "
  fi

  # If running clean mode, then pass "--noaddons"
  if [ "$CleanMode" -eq 1 ]; then
    opts+="--noaddons "
  fi

  # Assemble command
  command="$preopts"
  command+="$binary"
  command+="$opts"
  command+="$extra_opts"
  if [ "$Verbose" -eq 1 ]; then
    echo "$command"
  elif [ "$Verbose" -eq 2 ]; then
    echo "$command" "2> error.log"
  fi
  $command 2> error.log
  error_code="$?"
  if check_errs $2 $error_code $1; then
    FirstTest=0 #Only start using validcache flag when at least one test has passed without error
    handle_error_log 0
    return 0
  else
    # If we got a code of value at least 128, and it wasn't KILL timeout, it means we segfaulted / failed assertion etc. most likely, so run gdb to get a backtrace
    if [ "$GdbBacktraceMode" -eq 1 -a "$error_code" -ge 128 -a "$error_code" -ne 137 ]; then
        echo -e "\n* Launching gdb for a backtrace...\n" >>error.log
        gdb -q -batch -ex start -ex continue -ex bt -ex quit --args $binary $opts $extra_opts >>error.log
    fi

    handle_error_log 1
    return 1
  fi
}


### Main Script Starts Here ###

Verbose=0
UnixTimeout=0
CleanMode=0
LoadFile="wml_test_schedule"
BinPath="./"
StrictMode=1
DebugMode=0
GdbBacktraceMode=0
extra_opts=""
basetimer=10
export OMP_WAIT_POLICY=PASSIVE

while getopts ":hvwcusdgp:l:a:t:" Option
do
  case $Option in
    h )
      usage
      exit 0;
      ;;
    v )
      if [ "$Verbose" -lt 1 ]; then
        Verbose=1
      fi
      ;;
    w )
      if [ "$Verbose" -lt 2 ]; then
        Verbose=2
      fi
      ;;
    c )
      CleanMode=1
      ;;
    u )
      UnixTimeout=1
      ;;
    s )
      StrictMode=0
      ;;
    d )
      DebugMode=1
      ;;
    g )
      GdbBacktraceMode=1
      ;;
    p )
      if [ "$OPTARG" = "XCode" -o "$OPTARG" = "xcode" -o "$OPTARG" = "Xcode" ]; then
        # Find it in XCode's build dir
        path_list=( ~/Library/Developer/XCode/DerivedData/Wesnoth*/Build/Products/{Debug,Release}/Wesnoth.app/Contents/MacOS/ )
        BinPath="${path_list[0]}"
      else
        BinPath="$OPTARG"
      fi
      ;;
    l )
      LoadFile="$OPTARG"
      ;;
    a )
      extra_opts+=" $OPTARG"
      ;;
    t )
      echo "Replacing default timer of 10 with" "$OPTARG" "seconds."
      basetimer="$OPTARG"
      ;;
  esac
done
shift $(($OPTIND - 1))

extra_opts+="$*"

# Make sure the binary exists
if [ "$DebugMode" -eq 1 ]; then
  if [ ! -x "$BinPath/wesnoth-debug" ]; then
    echo "Wesnoth binary not found at $BinPath/wesnoth-debug"
    exit 1
  fi
else
  if [ ! -x "$BinPath/wesnoth" ]; then
    echo "Wesnoth binary not found at $BinPath/wesnoth"
    exit 1
  fi
fi

if [ "$Verbose" -ge 2 ]; then
  if [ "${#extra_opts}" -ge 0 ]; then
    echo "Found additional arguments to wesnoth: " "$extra_opts"
  fi
  if [ "$UnixTimeout" -eq 1 ]; then
    echo "Wesnoth built-in timeout was disabled. This script was updated, and -u is now unnecessary and has no effect."
  fi
fi

# Disable timeouts if the timeout utility is missing
if [ ! $(which timeout) ]; then
  echo 'timeout not found; skipping timeout tests'
  basetimer=0
fi

echo "Getting tests from" "$LoadFile" "..."

old_IFS=$IFS
IFS=$'\n'
schedule=($(cat $LoadFile)) # array
IFS=$old_IFS

NumTests=0
NumComments=0

for line in "${schedule[@]}"
do
    if [[ "$line" =~ \#.* ]]; then
      NumComments=$((NumComments+1))
    else
      NumTests=$((NumTests+1))
    fi
done

echo "Running" $NumTests "test scenarios."

if [ -f "errors.log" ]; then
  rm errors.log
fi

AllPassed=1
FirstTest=1
TotalPassed=0

for line in "${schedule[@]}"
do
    if [[ "$line" =~ \#.* ]]; then
      if [ "$Verbose" -ge 2 ]; then
        echo "comment:" $line
      fi
    else
      if [ "$Verbose" -eq 0 ]; then
        echo -n "."
      fi
      if run_test $line; then #note: don't put run_test inside a pipe implicitly by using ! or something, this will cause the FirstTest variable not to work properly
        if [ "$Verbose" -ge 2 ]; then
          echo "good"
        elif [ "$Verbose" -eq 0 ]; then
          echo -ne '\b:'
        fi
        TotalPassed=$((TotalPassed+1))
      elif [ $? -ne 100 ]; then
        if [ "$Verbose" -eq 0 ]; then
          echo -ne '\b!'
        fi
        AllPassed=0
      fi
    fi
done

if [ "$Verbose" -eq 0 ]; then
  echo ''
fi

if [ "$AllPassed" -eq 0 ]; then
  if [ "$StrictMode" -eq 1 ]; then
    echo "$TotalPassed" "out of" "$NumTests" "tests were correct."
  else
    echo "$TotalPassed" "out of" "$NumTests" "tests were correct. (However, some tests may expect to be running in strict mode, and not fail as expected otherwise.)"
  fi
  echo "Not all tests gave the correct result."
  echo "Check errors.log for error reports."
  exit 2
else
  echo "All tests gave the correct result."
  exit 0
fi