1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
|
#!/usr/bin/env perl
# run-test-suites.pl
#
# This file is part of mbed TLS (https://tls.mbed.org)
#
# Copyright (c) 2015-2018, ARM Limited, All Rights Reserved
=head1 SYNOPSIS
Execute all the test suites and print a summary of the results.
run-test-suites.pl [[-v|--verbose] [VERBOSITY]] [--skip=SUITE[...]]
Options:
-v|--verbose Print detailed failure information.
-v 2|--verbose=2 Print detailed failure information and summary messages.
-v 3|--verbose=3 Print detailed information about every test case.
--skip=SUITE[,SUITE...]
Skip the specified SUITE(s). This option can be used
multiple times.
=cut
use warnings;
use strict;
use utf8;
use open qw(:std utf8);
use Getopt::Long qw(:config auto_help gnu_compat);
use Pod::Usage;
my $verbose = 0;
my @skip_patterns = ();
GetOptions(
'skip=s' => \@skip_patterns,
'verbose|v:1' => \$verbose,
) or die;
# All test suites = executable files, excluding source files, debug
# and profiling information, etc. We can't just grep {! /\./} because
# some of our test cases' base names contain a dot.
my @suites = grep { -x $_ || /\.exe$/ } glob 'test_suite_*';
@suites = grep { !/\.c$/ && !/\.data$/ && -f } @suites;
die "$0: no test suite found\n" unless @suites;
# "foo" as a skip pattern skips "test_suite_foo" and "test_suite_foo.bar"
# but not "test_suite_foobar".
my $skip_re =
( '\Atest_suite_(' .
join('|', map {
s/[ ,;]/|/g; # allow any of " ,;|" as separators
s/\./\./g; # "." in the input means ".", not "any character"
$_
} @skip_patterns) .
')(\z|\.)' );
# in case test suites are linked dynamically
$ENV{'LD_LIBRARY_PATH'} = '../library';
$ENV{'DYLD_LIBRARY_PATH'} = '../library';
my $prefix = $^O eq "MSWin32" ? '' : './';
my ($failed_suites, $total_tests_run, $failed, $suite_cases_passed,
$suite_cases_failed, $suite_cases_skipped, $total_cases_passed,
$total_cases_failed, $total_cases_skipped );
my $suites_skipped = 0;
sub pad_print_center {
my( $width, $padchar, $string ) = @_;
my $padlen = ( $width - length( $string ) - 2 ) / 2;
print $padchar x( $padlen ), " $string ", $padchar x( $padlen ), "\n";
}
for my $suite (@suites)
{
print "$suite ", "." x ( 72 - length($suite) - 2 - 4 ), " ";
if( $suite =~ /$skip_re/o ) {
print "SKIP\n";
++$suites_skipped;
next;
}
my $command = "$prefix$suite";
if( $verbose ) {
$command .= ' -v';
}
my $result = `$command`;
$suite_cases_passed = () = $result =~ /.. PASS/g;
$suite_cases_failed = () = $result =~ /.. FAILED/g;
$suite_cases_skipped = () = $result =~ /.. ----/g;
if( $result =~ /PASSED/ ) {
print "PASS\n";
if( $verbose > 2 ) {
pad_print_center( 72, '-', "Begin $suite" );
print $result;
pad_print_center( 72, '-', "End $suite" );
}
} else {
$failed_suites++;
print "FAIL\n";
if( $verbose ) {
pad_print_center( 72, '-', "Begin $suite" );
print $result;
pad_print_center( 72, '-', "End $suite" );
}
}
my ($passed, $tests, $skipped) = $result =~ /([0-9]*) \/ ([0-9]*) tests.*?([0-9]*) skipped/;
$total_tests_run += $tests - $skipped;
if( $verbose > 1 ) {
print "(test cases passed:", $suite_cases_passed,
" failed:", $suite_cases_failed,
" skipped:", $suite_cases_skipped,
" of total:", ($suite_cases_passed + $suite_cases_failed +
$suite_cases_skipped),
")\n"
}
$total_cases_passed += $suite_cases_passed;
$total_cases_failed += $suite_cases_failed;
$total_cases_skipped += $suite_cases_skipped;
}
print "-" x 72, "\n";
print $failed_suites ? "FAILED" : "PASSED";
printf( " (%d suites, %d tests run%s)\n",
scalar(@suites) - $suites_skipped,
$total_tests_run,
$suites_skipped ? ", $suites_skipped suites skipped" : "" );
if( $verbose > 1 ) {
print " test cases passed :", $total_cases_passed, "\n";
print " failed :", $total_cases_failed, "\n";
print " skipped :", $total_cases_skipped, "\n";
print " of tests executed :", ( $total_cases_passed + $total_cases_failed ),
"\n";
print " of available tests :",
( $total_cases_passed + $total_cases_failed + $total_cases_skipped ),
"\n";
if( $suites_skipped != 0 ) {
print "Note: $suites_skipped suites were skipped.\n";
}
}
exit( $failed_suites ? 1 : 0 );
|
