File: regression_tests.py

package info (click to toggle)
o2 1.0~repack-1
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, buster, sid
  • size: 1,156 kB
  • sloc: ansic: 9,132; python: 137; sh: 111; makefile: 10
file content (182 lines) | stat: -rw-r--r-- 6,410 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
# regression_tests.py -- a rewrite of regression_tests.sh for Windows
#    using Python 3 (and hopefully compatible with Python 2.7 on OS X)
#
# Roger Dannenberg   Jan 2017
#
# Run this in the o2/tests directory where it is found
#

# get print to be compatible even if using python 2.x:
from __future__ import print_function

import os
import platform
import subprocess
import shlex
import threading

allOK = True

# Is this Windows?
EXE = ""
if os.name == 'nt':
    EXE = ".exe"


# Find the binaries
if os.path.isdir(os.path.join(os.getcwd(), '../Debug')):
   BIN="../Debug"
else:
   BIN=".."
# In linux, there is likely to be a debug version of the
# library copied to ../Debug, but tests are built in ..
if platform.system() == 'Linux':
    BIN=".."

def findLineInString(line, aString):
    return ('\n' + line + '\n') in aString


class RunInBackground(threading.Thread):
    def __init__(self, command):
        self.command = command
        self.output = ""
        self.errout = ""
        threading.Thread.__init__(self)

    def run(self):
        output1 = ""
        output2 = ""
        args = shlex.split(self.command)
        args[0] = BIN + '/' + args[0] + EXE
        process = subprocess.Popen(args,
                                   shell=False, stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
        (self.output, self.errout) = process.communicate()
        if os.name == 'nt':
            self.output = self.output.decode("utf-8").replace('\r\n', '\n')
            self.errout = self.errout.decode("utf-8").replace('\r\n', '\n')
        

# runTest testname - runs testname, saving output in output.txt,
#    searches output.txt for single full line containing "DONE",
#    returns status=0 if DONE was found (indicating success), or
#    else status=-1 (indicating failure).
def runTest(command):
    global allOK
    print(command.rjust(30) + ": ", end='')
    args = shlex.split(command)
    args[0] = BIN + '/' + args[0] + EXE
    process = subprocess.Popen(args,
                               shell=False, stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
    (stdout, stderr) = process.communicate()
    if os.name == 'nt':
        stdout = stdout.decode("utf-8").replace('\r\n', '\n')
        stderr = stderr.decode("utf-8").replace('\r\n', '\n')
    if findLineInString("DONE", stdout):
        print("PASS")
        return True
    else:
        print("FAIL")
        print("**** Failing output:")
        print(stdout)
        print("**** Failing error output:")
        print(stderr)
        allOK = False
        return False


def runDouble(prog1, out1, prog2, out2):
    global allOK
    print((prog1 + '+' + prog2).rjust(30) + ": ", end='')
    p1 = RunInBackground(prog1)
    p1.start()
    p2 = RunInBackground(prog2)
    p2.start()
    p1.join()
    p2.join()
    if findLineInString(out1, p1.output):
        if findLineInString(out2, p2.output):
            print("PASS")
            return True
    print("FAIL")
    print("**** Failing output from " + prog1)
    print(p1.output)
    print("**** Failing error output from " + prog1)
    print(p1.errout)

    print("**** Failing output from " + prog2)
    print(p2.output)
    print("**** Failing error output from " + prog2)
    print(p2.errout)

    allOK = False
    return False


def runAllTests():
    print("Running regression tests for O2 ...")
    if not runTest("dispatchtest"): return
    if not runTest("typestest"): return
    if not runTest("coercetest"): return
    if not runTest("longtest"): return
    if not runTest("arraytest"): return
    if not runTest("bundletest"): return

    if not runDouble("clockmaster", "CLOCKMASTER DONE",
                     "clockslave", "CLOCKSLAVE DONE"): return
    if not runDouble("o2client", "CLIENT DONE",
                     "o2server", "SERVER DONE"): return
    if not runDouble("oscsendtest u", "OSCSEND DONE",
                     "oscrecvtest u", "OSCRECV DONE"): return
    if not runDouble("oscsendtest u", "OSCSEND DONE",
                     "oscanytest u", "OSCANY DONE"): return
    if not runDouble("oscsendtest", "OSCSEND DONE",
                     "oscrecvtest", "OSCRECV DONE"): return
    if not runDouble("tcpclient", "CLIENT DONE",
                     "tcpserver", "SERVER DONE"): return
    if not runDouble("oscbndlsend u", "OSCSEND DONE",
                     "oscbndlrecv u", "OSCRECV DONE"): return
    if not runDouble("oscbndlsend", "OSCSEND DONE",
                     "oscbndlrecv", "OSCRECV DONE"): return

# tests for compatibility with liblo are run only if the binaries were built
# In CMake, set BUILD_TESTS_WITH_LIBLO to create the binaries

    if os.path.isfile(BIN + '/' + "lo_oscrecv" + EXE):
        if not runDouble("oscsendtest Mu", "OSCSEND DONE",
                         "lo_oscrecv u", "OSCRECV DONE"): return
        if not runDouble("oscsendtest M", "OSCSEND DONE",
                         "lo_oscrecv", "OSCRECV DONE"): return

    if os.path.isfile(BIN + '/' + "lo_oscsend" + EXE):
        if not runDouble("lo_oscsend u", "OSCSEND DONE",
                         "oscrecvtest u", "OSCRECV DONE"): return
        if not runDouble("lo_oscsend", "OSCSEND DONE",
                         "oscrecvtest", "OSCRECV DONE"): return

    if os.path.isfile(BIN + '/' + "lo_bndlsend" + EXE):
        if not runDouble("lo_bndlsend u", "OSCSEND DONE",
                         "oscbndlrecv u", "OSCRECV DONE"): return
        if not runDouble("lo_bndlsend", "OSCSEND DONE",
                         "oscbndlrecv", "OSCRECV DONE"): return

    if os.path.isfile(BIN + '/' + "lo_bndlrecv" + EXE):
        if not runDouble ("oscbndlsend Mu", "OSCSEND DONE",
                          "lo_bndlrecv u", "OSCRECV DONE"): return
        if not runDouble ("oscbndlsend M", "OSCSEND DONE",
                          "lo_bndlrecv", "OSCRECV DONE"): return


runAllTests()
if allOK:
    print("****    All O2 regression tests PASSED.")
else:
    print("ERROR: Exiting regression tests because a test failed.")
    print("       See above for output from the failing test(s).")
    print("\nNOTE:  If firewall pop-ups requested access to the network,")
    print("       that *might* affect timing and cause a test to fail.")
    print("       If you granted access, permission should be granted")
    print("       without delay if you run the test again.")