File: linker_driver.py

package info (click to toggle)
chromium 139.0.7258.127-1
  • links: PTS, VCS
  • area: main
  • in suites:
  • size: 6,122,068 kB
  • sloc: cpp: 35,100,771; ansic: 7,163,530; javascript: 4,103,002; python: 1,436,920; asm: 946,517; xml: 746,709; pascal: 187,653; perl: 88,691; sh: 88,436; objc: 79,953; sql: 51,488; cs: 44,583; fortran: 24,137; makefile: 22,147; tcl: 15,277; php: 13,980; yacc: 8,984; ruby: 7,485; awk: 3,720; lisp: 3,096; lex: 1,327; ada: 727; jsp: 228; sed: 36
file content (546 lines) | stat: -rwxr-xr-x 21,065 bytes parent folder | download | duplicates (5)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
#!/usr/bin/env python3

# Copyright 2016 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import os
import os.path
import re
import shutil
import subprocess
import sys
import tempfile

# The path to `whole_archive`.
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))

import whole_archive

# Prefix for all custom linker driver arguments.
LINKER_DRIVER_ARG_PREFIX = '-Wcrl,'
LINKER_DRIVER_COMPILER_ARG_PREFIX = '-Wcrl,driver,'

# The linker_driver.py is responsible for forwarding a linker invocation to
# the compiler driver, while processing special arguments itself.
#
# Usage: linker_driver.py -Wcrl,driver,clang++ main.o -L. -llib -o prog \
#            -Wcrl,dsym,out
#
# On Mac, the logical step of linking is handled by three discrete tools to
# perform the image link, debug info link, and strip. The linker_driver.py
# combines these three steps into a single tool.
#
# The compiler driver invocation for the linker is specified by the following
# required argument.
#
# -Wcrl,driver,<path_to_compiler_driver>
#    Specifies the path to the compiler driver.
#
# After running the compiler driver, the script performs additional actions,
# based on these arguments:
#
# -Wcrl,installnametoolpath,<install_name_tool_path>
#    Sets the path to the `install_name_tool` to run with
#    -Wcrl,installnametool, in which case `xcrun` is not used to invoke it.
#
# -Wcrl,installnametool,<arguments,...>
#    After invoking the linker, this will run install_name_tool on the linker's
#    output. |arguments| are comma-separated arguments to be passed to the
#    install_name_tool command.
#
# -Wcrl,dsym,<dsym_path_prefix>
#    After invoking the linker, this will run `dsymutil` on the linker's
#    output, producing a dSYM bundle, stored at dsym_path_prefix. As an
#    example, if the linker driver were invoked with:
#        "... -o out/gn/obj/foo/libbar.dylib ... -Wcrl,dsym,out/gn ..."
#    The resulting dSYM would be out/gn/libbar.dylib.dSYM/.
#
# -Wcrl,dsymutilpath,<dsymutil_path>
#    Sets the path to the dsymutil to run with -Wcrl,dsym, in which case
#    `xcrun` is not used to invoke it.
#
# -Wcrl,unstripped,<unstripped_path_prefix>
#    After invoking the linker, and before strip, this will save a copy of
#    the unstripped linker output in the directory unstripped_path_prefix.
#
# -Wcrl,strip,<strip_arguments>
#    After invoking the linker, and optionally dsymutil, this will run
#    the strip command on the linker's output. strip_arguments are
#    comma-separated arguments to be passed to the strip command.
#
# -Wcrl,strippath,<strip_path>
#    Sets the path to the strip to run with -Wcrl,strip, in which case
#    `xcrun` is not used to invoke it.
# -Wcrl,object_path_lto
#    Creates temporary directory for LTO object files.
#
# -Wcrl,otoolpath,<otool path>
#    Sets the path to the otool for solink process.
# -Wcrl,nmpath,<nm path>
#    Sets the path to the nm for solink process.
#
# -Wcrl.tocname,<tocname>
#    Output TOC for solink.
#    It would be processed both before the linker (to check reexport
#    in old module) and after the linker (to produce TOC if needed).

class LinkerDriver(object):
    def __init__(self, args):
        """Creates a new linker driver.

        Args:
            args: list of string, Arguments to the script.
        """
        self._args = args

        # List of linker driver pre-actions that need to run before the link.
        # **The sort order of this list affects the order in which
        # the actions are invoked.**
        # The first item in the tuple is the argument's -Wcrl,<sub_argument>
        # and the second is the function to invoke.
        self._pre_actions = [
            ('object_path_lto', self.prepare_object_path_lto),
            ('installnametoolpath,', self.set_install_name_tool_path),
            ('dsymutilpath,', self.set_dsymutil_path),
            ('strippath,', self.set_strip_path),
            ('otoolpath,', self.set_otool_path),
            ('nmpath,', self.set_nm_path),
            ('tocname,', self.check_reexport_in_old_module),
        ]

        # List of linker driver actions. **The sort order of this list affects
        # the order in which the actions are invoked.**
        # The first item in the tuple is the argument's -Wcrl,<sub_argument>
        # and the second is the function to invoke.
        self._actions = [
            ('installnametool,', self.run_install_name_tool),
            ('dsym,', self.run_dsymutil),
            ('unstripped,', self.run_save_unstripped),
            ('strip,', self.run_strip),
            ('tocname,', self.output_toc),
        ]

        # Linker driver actions can modify the these values.
        self._driver_path = None  # Must be specified on the command line.
        self._otool_cmd = ['xcrun', 'otool']
        self._nm_cmd = ['xcrun', 'nm']
        self._install_name_tool_cmd = ['xcrun', 'install_name_tool']
        self._dsymutil_cmd = ['xcrun', 'dsymutil']
        self._strip_cmd = ['xcrun', 'strip']

        # The linker output file, lazily computed in self._get_linker_output().
        self._linker_output = None

        # may not need to reexport unless LC_REEXPORT_DYLIB is used.
        self._reexport_in_old_module = False

        # temp directory for lto cache.
        self._object_path_lto = None

    def run(self):
        """Runs the linker driver, separating out the main compiler driver's
        arguments from the ones handled by this class. It then invokes the
        required tools, starting with the compiler driver to produce the linker
        output.
        """
        # Collect arguments to the linker driver (this script) and remove them
        # from the arguments being passed to the compiler driver.
        self._linker_driver_actions = {}
        self._linker_driver_pre_actions = {}
        self._compiler_driver_args = []
        for index, arg in enumerate(self._args[1:]):
            if arg.startswith(LINKER_DRIVER_COMPILER_ARG_PREFIX):
                assert not self._driver_path
                self._driver_path = arg[len(LINKER_DRIVER_COMPILER_ARG_PREFIX
                                            ):]
            elif arg.startswith(LINKER_DRIVER_ARG_PREFIX):
                # Convert driver actions into a map of name => lambda to invoke.
                self._process_driver_arg(arg)
            else:
                # TODO(crbug.com/40268754): On Apple, the linker command line
                # produced by rustc for LTO includes these arguments, but the
                # Apple linker doesn't accept them.
                # Upstream bug: https://github.com/rust-lang/rust/issues/60059
                BAD_RUSTC_ARGS = '-Wl,-plugin-opt=O[0-9],-plugin-opt=mcpu=.*'
                if not re.match(BAD_RUSTC_ARGS, arg):
                    self._compiler_driver_args.append(arg)

        if not self._driver_path:
            raise RuntimeError(
                "Usage: linker_driver.py -Wcrl,driver,<compiler-driver> "
                "[linker-args]...")

        if self._get_linker_output() is None:
            raise ValueError(
                'Could not find path to linker output (-o or --output)')

        # We want to link rlibs as --whole-archive if they are part of a unit
        # test target. This is determined by switch
        # `-LinkWrapper,add-whole-archive`.
        self._compiler_driver_args = whole_archive.wrap_with_whole_archive(
            self._compiler_driver_args, is_apple=True)

        linker_driver_outputs = [self._get_linker_output()]

        try:
            # Zero the mtime in OSO fields for deterministic builds.
            # https://crbug.com/330262.
            env = os.environ.copy()
            env['ZERO_AR_DATE'] = '1'

            # Run the driver pre-actions, in the order specified by the
            # actions list.
            for action in self._pre_actions:
                name = action[0]
                if name in self._linker_driver_pre_actions:
                    self._linker_driver_pre_actions[name]()

            # Run the linker by invoking the compiler driver.
            subprocess.check_call([self._driver_path] +
                                  self._compiler_driver_args,
                                  env=env)

            # Run the linker driver actions, in the order specified by the
            # actions list.
            for action in self._actions:
                name = action[0]
                if name in self._linker_driver_actions:
                    linker_driver_outputs += self._linker_driver_actions[name](
                    )
        except:
            # If a linker driver action failed, remove all the outputs to make
            # the build step atomic.
            map(_remove_path, linker_driver_outputs)

            # Re-report the original failure.
            raise

    def _get_linker_output(self):
        """Returns the value of the output argument to the linker."""
        if not self._linker_output:
            for index, arg in enumerate(self._args):
                if arg in ('-o', '-output', '--output'):
                    self._linker_output = self._args[index + 1]
                    break
        return self._linker_output

    def _process_driver_arg(self, arg):
        """Processes a linker driver argument and returns a tuple containing the
        name and unary lambda to invoke for that linker driver action.

        Args:
            arg: string, The linker driver argument.

        Returns:
            A 2-tuple:
                0: The driver action name, as in |self._actions|.
                1: A lambda that calls the linker driver action with its direct
                   argument and returns a list of outputs from the action.
        """
        if not arg.startswith(LINKER_DRIVER_ARG_PREFIX):
            raise ValueError('%s is not a linker driver argument' % (arg, ))

        sub_arg = arg[len(LINKER_DRIVER_ARG_PREFIX):]

        found = False
        for driver_action in self._pre_actions:
            (pre_name, pre_action) = driver_action
            if sub_arg.startswith(pre_name):
                assert pre_name not in self._linker_driver_pre_actions, \
                    f"Name '{pre_name}' found in linker driver pre actions"
                self._linker_driver_pre_actions[pre_name] = \
                    lambda: pre_action(sub_arg[len(pre_name):])
                # same sub_arg may be used in actions.
                found = True
                break

        for driver_action in self._actions:
            (name, action) = driver_action
            if sub_arg.startswith(name):
                assert name not in self._linker_driver_actions, \
                    f"Name '{name}' found in linker driver actions"
                self._linker_driver_actions[name] = \
                        lambda: action(sub_arg[len(name):])
                return

        if not found:
            raise ValueError('Unknown linker driver argument: %s' % (arg, ))

    def prepare_object_path_lto(self, arg):
        """Linker driver pre-action for -Wcrl,object_path_lto.

        Prepare object_path_lto path in temp directory.
        """
        # TODO(lgrey): Remove if/when we start running `dsymutil`
        # through the clang driver. See https://crbug.com/1324104
        # The temporary directory for intermediate LTO object files. If it
        # exists, it will clean itself up on script exit.
        self._object_path_lto = tempfile.TemporaryDirectory(dir=os.getcwd())
        self._compiler_driver_args.append('-Wl,-object_path_lto,{}'.format(
            os.path.relpath(self._object_path_lto.name)))

    def check_reexport_in_old_module(self, tocname):
        """Linker driver pre-action for -Wcrl,tocname,<path>.

        Check whether it contains LC_REEXPORT_DYLIB in old module, so that
        needs to ouptupt TOC file for solink even if the same TOC.

        Returns:
           True if old module have LC_REEXPORT_DYLIB
        """
        if not os.path.exists(tocname):
            return
        dylib = self._get_linker_output()
        if not os.path.exists(dylib):
            return
        p = subprocess.run(self._otool_cmd + ['-l', dylib],
                           capture_output=True)
        if p.returncode != 0:
            return
        if re.match(rb'\s+cmd LC_REEXPORT_DYLIB$', p.stdout, re.MULTILINE):
            self._reexport_in_old_module = True

    def set_install_name_tool_path(self, install_name_tool_path):
        """Linker driver pre-action for -Wcrl,installnametoolpath,<path>.

        Sets the invocation command for install_name_tool, which allows the
        caller to specify an alternate path. This action is always
        processed before the run_install_name_tool action.

        Args:
            install_name_tool_path: string, The path to the install_name_tool
                binary to run
        """
        self._install_name_tool_cmd = [install_name_tool_path]

    def run_install_name_tool(self, args_string):
        """Linker driver action for -Wcrl,installnametool,<args>. Invokes
        install_name_tool on the linker's output.

        Args:
            args_string: string, Comma-separated arguments for
                `install_name_tool`.

        Returns:
            No output - this step is run purely for its side-effect.
        """
        command = list(self._install_name_tool_cmd)
        command.extend(args_string.split(','))
        command.append(self._get_linker_output())
        subprocess.check_call(command)
        return []

    def run_dsymutil(self, dsym_path_prefix):
        """Linker driver action for -Wcrl,dsym,<dsym-path-prefix>. Invokes
        dsymutil on the linker's output and produces a dsym file at |dsym_file|
        path.

        Args:
            dsym_path_prefix: string, The path at which the dsymutil output
                should be located.

        Returns:
            list of string, Build step outputs.
        """
        if not len(dsym_path_prefix):
            raise ValueError('Unspecified dSYM output file')

        linker_output = self._get_linker_output()
        base = os.path.basename(linker_output)
        dsym_out = os.path.join(dsym_path_prefix, base + '.dSYM')

        # Remove old dSYMs before invoking dsymutil.
        _remove_path(dsym_out)

        tools_paths = _find_tools_paths(self._args)
        if os.environ.get('PATH'):
            tools_paths.append(os.environ['PATH'])
        dsymutil_env = os.environ.copy()
        dsymutil_env['PATH'] = ':'.join(tools_paths)
        subprocess.check_call(self._dsymutil_cmd +
                              ['-o', dsym_out, linker_output],
                              env=dsymutil_env)
        return [dsym_out]

    def set_dsymutil_path(self, dsymutil_path):
        """Linker driver pre-action for -Wcrl,dsymutilpath,<dsymutil_path>.

        Sets the invocation command for dsymutil, which allows the caller to
        specify an alternate dsymutil. This action is always processed before
        the RunDsymUtil action.

        Args:
            dsymutil_path: string, The path to the dsymutil binary to run
        """
        self._dsymutil_cmd = [dsymutil_path]

    def run_save_unstripped(self, unstripped_path_prefix):
        """Linker driver action for -Wcrl,unstripped,<unstripped_path_prefix>.
        Copies the linker output to |unstripped_path_prefix| before stripping.

        Args:
            unstripped_path_prefix: string, The path at which the unstripped
                output should be located.

        Returns:
            list of string, Build step outputs.
        """
        if not len(unstripped_path_prefix):
            raise ValueError('Unspecified unstripped output file')

        base = os.path.basename(self._get_linker_output())
        unstripped_out = os.path.join(unstripped_path_prefix,
                                      base + '.unstripped')

        shutil.copyfile(self._get_linker_output(), unstripped_out)
        return [unstripped_out]

    def run_strip(self, strip_args_string):
        """Linker driver action for -Wcrl,strip,<strip_arguments>.

        Args:
            strip_args_string: string, Comma-separated arguments for `strip`.
        """
        strip_command = list(self._strip_cmd)
        if len(strip_args_string) > 0:
            strip_command += strip_args_string.split(',')
        strip_command.append(self._get_linker_output())
        subprocess.check_call(strip_command)
        return []

    def set_strip_path(self, strip_path):
        """Linker driver pre-action for -Wcrl,strippath,<strip_path>.

        Sets the invocation command for strip, which allows the caller to
        specify an alternate strip. This action is always processed before the
        RunStrip action.

        Args:
            strip_path: string, The path to the strip binary to run
        """
        self._strip_cmd = [strip_path]

    def set_otool_path(self, otool_path):
        """Linker driver pre-action for -Wcrl,otoolpath,<otool_path>.

        Sets the invocation command for otool.

        Args:
           otool_path: string. The path to the otool binary to run

        """
        self._otool_cmd = [otool_path]

    def set_nm_path(self, nm_path):
        """Linker driver pre-action for -Wcrl,nmpath,<nm_path>.

        Sets the invocation command for nm.

        Args:
           nm_path: string. The path to the nm binary to run

        Returns:
           No output - this step is run purely for its side-effect.
        """
        self._nm_cmd = [nm_path]

    def output_toc(self, tocname):
        """Linker driver action for -Wcrl,tocname,<path>.

        Produce *.TOC from linker output.

        TODO(ukai): recursively collect symbols from all 'LC_REEXPORT_DYLIB'-
        exported modules and present them all in the TOC, and
        drop self._reexport_in_old_module.

        Args:
           tocname: string, The path to *.TOC file.
        Returns:
           list of string, TOC file as output.
        """
        new_toc = self._extract_toc()
        old_toc = None
        if not self._reexport_in_old_module:
            try:
                with open(tocname, 'rb') as f:
                    old_toc = f.read()
            except OSError:
                pass

        if self._reexport_in_old_module or new_toc != old_toc:
            # TODO: use delete_on_close in python 3.12 or later.
            with tempfile.NamedTemporaryFile(prefix=tocname + '.',
                                             dir='.',
                                             delete=False) as f:
                f.write(new_toc)
                f.close()
                os.rename(f.name, tocname)
        return [tocname]

    def _extract_toc(self):
        """Extract TOC from linker output.

        Returns:
           output contents in bytes.
        """
        toc = b''
        dylib = self._get_linker_output()
        out = subprocess.check_output(self._otool_cmd + ['-l', dylib])
        lines = out.split(b'\n')
        found_id = False
        for i, line in enumerate(lines):
            # Too many LC_ID_DYLIBs? We didn’t understand something about
            # the otool output. Raise an exception and die, rather than
            # proceeding.

            # Not any LC_ID_DYLIBs? Probably not an MH_DYLIB. Probably fine, we
            # can proceed with ID-less TOC generation.
            if line == b'      cmd LC_ID_DYLIB':
                if found_id:
                    raise ValueError('Too many LC_ID_DYLIBs in %s' % dylib)
                toc += line + b'\n'
                for j in range(5):
                    toc += lines[i + 1 + j] + b'\n'
                found_id = True

        # -U ignores undefined symbols
        # -g display only global (external) symbols
        # -p unsorted https://crrev.com/c/2173969
        out = subprocess.check_output(self._nm_cmd + ['-Ugp', dylib])
        lines = out.split(b'\n')
        for line in lines:
            fields = line.split(b' ', 2)
            if len(fields) < 3:
                continue
            # fields = (value, type, name)
            # emit [type, name]
            toc += b' '.join(fields[1:3]) + b'\n'
        return toc


def _find_tools_paths(full_args):
    """Finds all paths where the script should look for additional tools."""
    paths = []
    for idx, arg in enumerate(full_args):
        if arg in ['-B', '--prefix']:
            paths.append(full_args[idx + 1])
        elif arg.startswith('-B'):
            paths.append(arg[2:])
        elif arg.startswith('--prefix='):
            paths.append(arg[9:])
    return paths


def _remove_path(path):
    """Removes the file or directory at |path| if it exists."""
    if os.path.exists(path):
        if os.path.isdir(path):
            shutil.rmtree(path)
        else:
            os.unlink(path)


if __name__ == '__main__':
    LinkerDriver(sys.argv).run()
    sys.exit(0)