File: utils.py

package info (click to toggle)
tarantool 2.6.0-1.4
  • links: PTS, VCS
  • area: main
  • in suites: sid, trixie
  • size: 85,412 kB
  • sloc: ansic: 513,775; cpp: 69,493; sh: 25,650; python: 19,190; perl: 14,973; makefile: 4,178; yacc: 1,329; sql: 1,074; pascal: 620; ruby: 190; awk: 18; lisp: 7
file content (260 lines) | stat: -rw-r--r-- 8,115 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
import os
import sys
import six
import collections
import signal
import random
import fcntl
import difflib
import time
from gevent import socket
from lib.colorer import color_stdout
try:
    # Python3.5 or above
    from signal import Signals
except ImportError:
    # Python2
    Signals = None


UNIX_SOCKET_LEN_LIMIT = 107


def check_libs():
    deps = [
        ('msgpack', 'msgpack-python'),
        ('tarantool', 'tarantool-python')
    ]
    base_path = os.path.dirname(os.path.abspath(__file__))

    for (mod_name, mod_dir) in deps:
        mod_path = os.path.join(base_path, mod_dir)
        if mod_path not in sys.path:
            sys.path = [mod_path] + sys.path

    for (mod_name, _mod_dir) in deps:
        try:
            __import__(mod_name)
        except ImportError as e:
            color_stdout("\n\nNo %s library found\n" % mod_name,
                         schema='error')
            print(e)
            sys.exit(1)


def non_empty_valgrind_logs(paths_to_log):
    """ Check that there were no warnings in the log."""
    non_empty_logs = []
    for path_to_log in paths_to_log:
        if os.path.exists(path_to_log) and os.path.getsize(path_to_log) != 0:
            non_empty_logs.append(path_to_log)
    return non_empty_logs


def print_tail_n(filename, num_lines=None):
    """ Print N last lines of a file. If num_lines is not set,
    prints the whole file.
    """
    with open(filename, "r") as logfile:
        tail_n = collections.deque(logfile, num_lines)
        for line in tail_n:
            color_stdout(line, schema='tail')


def check_port(port, rais=True, ipv4=True, ipv6=True):
    """ True -- it's possible to listen on this port for TCP/IPv4 or TCP/IPv6
    connections (UNIX Sockets in case of file path). False -- otherwise.
    """
    try:
        if isinstance(port, (int, long)):
            if ipv4:
                sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                sock.bind(('127.0.0.1', port))
                sock.listen(5)
                sock.close()
            if ipv6:
                sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
                sock.bind(('::1', port))
                sock.listen(5)
                sock.close()
        else:
            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
            sock.connect(port)
    except socket.error:
        if rais:
            raise RuntimeError(
                "The server is already running on port {0}".format(port))
        return False
    return True


# A list of ports used so far. Avoid reusing ports
# to reduce race conditions between starting and stopping servers.
# We're using tarantoolctl for instance control, and it reports
# a successful stop of the server before it really closes its
# network sockets
ports = {}


is_ipv6_supported = check_port(port=0, rais=False, ipv4=False, ipv6=True)


def find_port():
    global ports
    start_port = int(os.environ.get('TEST_RUN_TCP_PORT_START', '3000'))
    end_port = int(os.environ.get('TEST_RUN_TCP_PORT_END', '65535'))
    port = random.randrange(start_port, end_port + 1)

    while port <= end_port:
        is_free = check_port(port, False, ipv4=True, ipv6=is_ipv6_supported)
        if port not in ports and is_free:
            ports[port] = True
            return port
        port += 1

    # We've made a full circle, clear the list of used ports and start
    # from scratch
    ports = {}
    return find_port()


def find_in_path(name):
    path = os.curdir + os.pathsep + os.environ["PATH"]
    for _dir in path.split(os.pathsep):
        exe = os.path.join(_dir, name)
        if os.access(exe, os.X_OK):
            return exe
    return ''


# http://stackoverflow.com/a/2549950
SIGNAMES = dict((int(v), k) for k, v in reversed(sorted(
    signal.__dict__.items())) if k.startswith('SIG') and
    not k.startswith('SIG_'))
SIGNUMS = dict((k, int(v)) for k, v in reversed(sorted(
    signal.__dict__.items())) if k.startswith('SIG') and
    not k.startswith('SIG_'))


def signame(signal):
    if isinstance(signal, six.integer_types):
        return SIGNAMES[signal]
    if Signals and isinstance(signal, Signals):
        return SIGNAMES[int(signal)]
    if isinstance(signal, six.string_types):
        return signal
    raise TypeError('signame(): signal argument of unexpected type: {}'.format(
                    str(type(signal))))


def signum(signal):
    if isinstance(signal, six.integer_types):
        return signal
    if Signals and isinstance(signal, Signals):
        return int(signal)
    if isinstance(signal, six.string_types):
        if not signal.startswith('SIG'):
            signal = 'SIG' + signal
        return SIGNUMS[signal]
    raise TypeError('signum(): signal argument of unexpected type: {}'.format(
                    str(type(signal))))


def warn_unix_sockets_at_start(vardir):
    max_unix_socket_rel = '???_replication/autobootstrap_guest3.control'
    real_vardir = os.path.realpath(vardir)
    max_unix_socket_abs = os.path.join(real_vardir, max_unix_socket_rel)
    max_unix_socket_real = os.path.realpath(max_unix_socket_abs)
    if len(max_unix_socket_real) > UNIX_SOCKET_LEN_LIMIT:
        color_stdout(
            'WARGING: unix sockets can become longer than %d symbols:\n'
            % UNIX_SOCKET_LEN_LIMIT,
            schema='error')
        color_stdout('WARNING: for example: "%s" has length %d\n' %
                     (max_unix_socket_real, len(max_unix_socket_real)),
                     schema='error')


def warn_unix_socket(path):
    real_path = os.path.realpath(path)
    if len(real_path) <= UNIX_SOCKET_LEN_LIMIT or \
            real_path in warn_unix_socket.warned:
        return
    color_stdout(
        '\nWARGING: unix socket\'s "%s" path has length %d symbols that is '
        'longer than %d. That likely will cause failing of tests.\n' %
        (real_path, len(real_path), UNIX_SOCKET_LEN_LIMIT), schema='error')
    warn_unix_socket.warned.add(real_path)


warn_unix_socket.warned = set()


def safe_makedirs(directory):
    if os.path.isdir(directory):
        return
    # try-except to prevent races btw processes
    try:
        os.makedirs(directory)
    except OSError:
        pass


def format_process(pid):
    cmdline = 'unknown'
    try:
        with open('/proc/%d/cmdline' % pid, 'r') as f:
            cmdline = ' '.join(f.read().split('\0')).strip() or cmdline
    except (OSError, IOError):
        pass
    status = 'unknown'
    try:
        with open('/proc/%d/status' % pid, 'r') as f:
            for line in f:
                if ':' not in line:
                    continue
                key, value = line.split(':', 1)
                if key == 'State':
                    status = value.strip()
    except (OSError, IOError):
        pass
    return 'process %d [%s; %s]' % (pid, status, cmdline)


def set_fd_cloexec(socket):
    flags = fcntl.fcntl(socket, fcntl.F_GETFD)
    fcntl.fcntl(socket, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)


def print_unidiff(filepath_a, filepath_b):
    def process_file(filepath):
        fh = None
        try:
            fh = open(filepath, 'r')
            lines = fh.readlines()
            ctime = time.ctime(os.stat(filepath).st_mtime)
        except Exception:
            if not os.path.exists(filepath):
                color_stdout('[File does not exists: {}]'.format(filepath),
                             schema='error')
            lines = []
            ctime = time.ctime()
        if fh:
            fh.close()
        return lines, ctime

    lines_a, time_a = process_file(filepath_a)
    lines_b, time_b = process_file(filepath_b)
    diff = difflib.unified_diff(lines_a,
                                lines_b,
                                filepath_a,
                                filepath_b,
                                time_a,
                                time_b)
    color_stdout.writeout_unidiff(diff)


def prefix_each_line(prefix, data):
    data = data.rstrip('\n')
    lines = [(line + '\n') for line in data.split('\n')]
    return prefix + prefix.join(lines)