1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
#!/usr/bin/env python
#===----------------------------------------------------------------------===##
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===----------------------------------------------------------------------===##
"""
Runs an executable on a remote host.
This is meant to be used as an executor when running the LLVM and the Libraries
tests on a target.
"""
import argparse
import os
import posixpath
import shlex
import subprocess
import sys
import tarfile
import tempfile
import re
def ssh(args, command):
cmd = ['ssh', '-oBatchMode=yes']
if args.extra_ssh_args is not None:
cmd.extend(shlex.split(args.extra_ssh_args))
return cmd + [args.host, command]
def scp(args, src, dst):
cmd = ['scp', '-q', '-oBatchMode=yes']
if args.extra_scp_args is not None:
cmd.extend(shlex.split(args.extra_scp_args))
return cmd + [src, '{}:{}'.format(args.host, dst)]
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--host', type=str, required=True)
parser.add_argument('--execdir', type=str, required=False)
parser.add_argument('--extra-ssh-args', type=str, required=False)
parser.add_argument('--extra-scp-args', type=str, required=False)
parser.add_argument('--codesign_identity', type=str, required=False, default=None)
parser.add_argument('--env', type=str, nargs='*', required=False, default=dict())
# Note: The default value is for the backward compatibility with a hack in
# libcxx test suite.
# If an argument is a file that ends in `.tmp.exe`, assume it is the name
# of an executable generated by a test file. We call these test-executables
# below. This allows us to do custom processing like codesigning test-executables
# and changing their path when running on the remote host. It's also possible
# for there to be no such executable, for example in the case of a .sh.cpp
# test.
parser.add_argument('--exec-pattern', type=str, required=False, default='.*',
help='The name regex pattern of the executables generated by \
a test file. Specifying it allows us to do custom \
processing like codesigning test-executables \
and changing their path when running on \
the remote host. It\'s also possible for there \
to be no such executable, for example in \
the case of a .sh.cpp test.')
parser.add_argument("command", nargs=argparse.ONE_OR_MORE)
args = parser.parse_args()
commandLine = args.command
execdir = args.execdir
if execdir == '.':
# Retrieve the exec directory from the command line.
execdir, _ = os.path.split(commandLine[0])
if execdir == '':
# Get the current directory in that case.
execdir = os.getcwd()
arcname = os.path.basename(execdir) if execdir else None
# Create a temporary directory where the test will be run.
# That is effectively the value of %T on the remote host.
tmp = subprocess.check_output(
ssh(args, 'mktemp -d /tmp/llvm.XXXXXXXXXX'),
universal_newlines=True
).strip()
isExecutable = lambda exe: re.match(args.exec_pattern, exe) and os.path.exists(exe)
pathOnRemote = lambda file: posixpath.join(tmp, os.path.basename(file))
remoteCommands = []
try:
# Do any necessary codesigning of test-executables found in the command line.
if args.codesign_identity:
for exe in filter(isExecutable, commandLine):
subprocess.check_call(
['xcrun', 'codesign', '-f', '-s', args.codesign_identity, exe],
env={})
# tar up the execution directory (which contains everything that's needed
# to run the test), and copy the tarball over to the remote host.
if execdir:
try:
tmpTar = tempfile.NamedTemporaryFile(suffix='.tar', delete=False)
with tarfile.open(fileobj=tmpTar, mode='w') as tarball:
tarball.add(execdir, arcname=arcname)
# Make sure we close the file before we scp it, because accessing
# the temporary file while still open doesn't work on Windows.
tmpTar.close()
remoteTarball = pathOnRemote(tmpTar.name)
subprocess.check_call(scp(args, tmpTar.name, remoteTarball))
finally:
# Make sure we close the file in case an exception happens before
# we've closed it above -- otherwise close() is idempotent.
tmpTar.close()
os.remove(tmpTar.name)
# Untar the dependencies in the temporary directory and remove the tarball.
remoteCommands.extend([
'tar -xf {} -C {} --strip-components 1'.format(remoteTarball, tmp),
'rm {}'.format(remoteTarball)
])
else:
# Copy only the files, which are specified in the command line.
# Copy them to remote host one by one.
for x in commandLine:
_, f = os.path.split(x)
subprocess.check_call(scp(args, x, pathOnRemote(f)))
# Make sure all executables in the remote command line have 'execute'
# permissions on the remote host. The host that compiled the test-executable
# might not have a notion of 'executable' permissions.
for exe in filter(isExecutable, commandLine):
remoteCommands.append('chmod +x {}'.format(pathOnRemote(exe)))
# Execute the command through SSH in the temporary directory, with the
# correct environment. We tweak the command line to run it on the remote
# host by transforming the path of test-executables to their path in the
# temporary directory on the remote host.
for i, x in enumerate(commandLine):
if isExecutable(x):
commandLine[i] = pathOnRemote(x)
remoteCommands.append('cd {}'.format(tmp))
if args.env:
remoteCommands.append('export {}'.format(' '.join(args.env)))
remoteCommands.append(subprocess.list2cmdline(commandLine))
# Finally, SSH to the remote host and execute all the commands.
rc = subprocess.call(ssh(args, ' && '.join(remoteCommands)))
return rc
finally:
# Make sure the temporary directory is removed when we're done.
subprocess.check_call(ssh(args, 'rm -r {}'.format(tmp)))
if __name__ == '__main__':
exit(main())
|
