File: pump.in

package info (click to toggle)
distcc 3.3.2-10
  • links: PTS
  • area: main
  • in suites: buster
  • size: 4,772 kB
  • sloc: ansic: 27,818; python: 6,425; sh: 4,435; makefile: 919; perl: 52
file content (550 lines) | stat: -rwxr-xr-x 19,759 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
#!/bin/sh
#
# Copyright 2007 Google Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
# USA.
#
# Authors: Nils Klarlund, Fergus Henderson
#
# 'pump': a script for using distcc-pump with build commands.

# This file is processed by configure, which substitutes in the right
# value for @PYTHON@, @prefix@, etc.
# NOTE: do we need to do the expansion of these variables by using sed
# in Makefile.in rather than by having configure process this file?
# The autoconf manual suggests that we do.  Right now it doesn't seem
# to be needed in practice, though.

PYTHON=@PYTHON@
prefix=@prefix@
exec_prefix=@exec_prefix@
sysconfdir=@sysconfdir@
srcdir=@srcdir@

# This is where include_server.py lives.  It's set to the empty string
# before pump is installed into its final location.  At install-time,
# the 'make install' command will rewrite this line of pump to point
# to the installed include_server.py.
# NOTE: DO NOT CHANGE THE LINE BELOW WITHOUT CHANGING THE SED IN
#       Makefile.in:install-include-server.
include_server=''


CheckUsage() {
  if [ "$1" = "" -o "$1" = '-h' -o "$1" = '--help' ]; then
    cat <<'EOF'
Usage:
    pump COMMAND [ARG...]
or
    pump --startup
    pump --shutdown

Description:
  Pump, also known as distcc-pump, accelerates remote compilation with
  distcc by also distributing preprocessing to the servers.

  The simplest usage is the form "pump COMMAND [ARG...]".
  This will start an include server for distcc-pump; set some environment
  variables; change PATH to use the distcc-pump "distcc" client; execute
  COMMAND with the specified ARG(s); and then shutdown the include server.
  The COMMAND is typically a parallel build command, such as
  "make -j80", that will do many concurrent invocations of distcc.

  An alternative way of invoking pump is to explicitly invoke "pump --startup"
  to start the include server and "pump --shutdown" to stop the include server.
  The "pump --startup" command will start up the include server, and will print
  out some environment variable settings.  These environment variables are used
  to communicate between the distcc-pump "distcc" client and the include
  server, and to communicate between "pump --startup" and "pump --shutdown".
  The caller of "pump --startup" is responsible for setting those environment
  variables before invoking "distcc" or "pump --shutdown".  For example:

      eval `pump --startup`
      make -j80
      pump --shutdown

  Note that distcc-pump assumes that sources files will not be modified during
  the lifetime of the include server, so modifying source files during a build
  may cause inconsistent results.

Environment variables (all optional):

  DISTCC_LOCATION        Either the distcc 'bin' install directory (when the
                         pump script in the installation executes) or the build
                         directory (when the pump script is run during 
                         development).

                         This location is normally inferred from the dirname
                         of argv[0].  It is resolved with respect to links.

                         If this location is the bin directory inside an
                         installation, then Python executables and distcc
                         itself will be retrieved from the parent directory of
                         the location.

                         If this location is the build directory, then Python 
                         sources are retrived from the source directory.

  DISTCC_POTENTIAL_HOSTS The distcc servers that will be queried by lsdistcc
                         in order to produce a value for DISTCC_HOSTS.
                         This value may be unset or null. In such
                         cases, use DISTCC_HOSTS.

  DISTCC_HOSTS           This variable is passed through to distcc but only if 
                         DISTCC_POTENTIAL_HOSTS is not set.

  LSDISTCC_ARGS          Extra arguments to lsdistcc.

  INCLUDE_SERVER_ARGS    Extra arguments to the include server.

  PYTHONOPTIMIZE         If set to "", then Python optimization is disabled.

Example: 
  pump make
EOF
    exit 0
  fi
}

# Get the directory of a command. The argument is $0 as received from
# argv[0].  If $0 is of the form /path/symlink and symlink points to
# another directory than /path, then `basename $0` does not return
# the directory where the executable lives. We fix this here.
GetScriptDir() {
  if [ -h "$0" ]; then
    dir=`dirname "$0"`
    link=`readlink "$0"`
    linkdir=`dirname "$link"`
    case $linkdir in
         /*) echo "$linkdir" ;;
         *)  echo "$dir/$linkdir" ;;
    esac
  else
    dirname "$0"
  fi
}

# Constants used by this process.

# The default value must be the same as that in src/compile.c. See
# comment there.
max_discrepancies_before_demotion=1
if [ -n "$DISTCC_MAX_DISCREPANCY" ] ; then
  if [ "$DISTCC_MAX_DISCREPANCY" -le 0 ] ; then
    echo "$0:" \
            "Bad DISTCC_MAX_DISCREPANCY value: \"$DISTCC_MAX_DISCREPANCY\"." \
            "Needs to be 1 or greater." \
            1>&2
    exit 1
  fi
  max_discrepancies_before_demotion="$DISTCC_MAX_DISCREPANCY"
fi


# Variables inherited from the environment of the caller.
if [ -z "$DISTCC_LOCATION" ]; then
   # Set the default for location of 'pump' script'.
   DISTCC_LOCATION=`GetScriptDir "$0"`
fi


# Variables used by this process.
program_name="$0"
socket_dir=''           # Temporary directory created by this process.
tmp_pid_file=''         # Temporary file created by this process.
available_hosts_file='' # Temporary file for newline separated list of hosts.
socket=''               # Temporary socket file, inside $socket_dir.
include_server_stdout=''  # Temporary file, inside $socket_dir.
include_server_stderr=''  # Temporary file, inside $socket_dir.
include_server_pid=''
include_server_relative=''  # Set by Initialize(). This path is relative to
                            # the 'bin' directory of a distcc-pump installation.
distcc_location='' # Set by Initialize().  The directory of the distcc binary.
DISTCC='distcc'    # the name of the distcc binary
verbose=1          # Print progress messages to stdout? (1 means yes.)
redirect_io=0      # Redirect include server's I/O streams? (1 means yes.)

# Make temp file using distinguishing prefix $1. Use optional argument
# $2="-d" to make a directory. The name of the created temp file or
# directory is written to stdout.
MakeTmpFile() {
  if mktemp $2 "${TMPDIR-/tmp}/$1.XXXXXX"; then
    :   # mktemp prints the output we want; no need to do more
  else
    echo "$program_name: Could not make temp \"$1\"" 1>&2
    exit 1
  fi
}

Initialize() {
  # We know the value of $PYTHON when this function is executed. The value is
  # set when the 'pump' script is made from 'pump.in'. Use this value in the
  # formation of the relative path that takes us from the 'bin' directory of an
  # installation to the 'include_server' directory.
  include_server_relative="../$PYTHON_RELATIVE_LIB/include_server"

  distcc_location="$DISTCC_LOCATION"

  # Check that we're getting exactly the distcc client we want: the
  # one that is part of this build.
  if [ ! -x "$distcc_location/$DISTCC" ]; then
    echo "$0: error: can't find distcc (looked in $distcc_location)" 1>&2
    exit 1
  fi
}

PrintIncludeServerStatusMessage() {
  include_server_status=$1
  if [ "$verbose" = 1 ]; then
    if [ "$include_server_status" = 0 ]; then
      echo '__________Started distcc-pump include server'
    else
      echo '__________Could not start distcc-pump include server' 1>&2
    fi
  fi
}

Announce() {
  echo "__________Using distcc-pump from $DISTCC_LOCATION"
}

# Starts up the include server.  Sets $socket, $socket_dir, and
# $include_server_pid.  If successful (with exit status 0), sets exported
# variable $INCLUDE_SERVER_PORT to the socket file ($socket), to tell the distcc
# clients where to find the include server.
StartIncludeServer() {
  # If include_server already exists, that means we're a
  # installed pump (in /usr/local/bin somewhere or something), and
  # include_server points to the installed include_server.py.
  if [ -n "$include_server" ]; then
    pythonpath=`dirname "$include_server"`
  else
    # We assume this script is run from the build directory.  We pick up .py
    # files from the include_server directory in the source tree, and we pick up
    # the .so file from the include_server/build/libXXX/include_server
    # directory. 
    #
    # Calculate the location of the source directory.
    distcc_srcdir=`cd "$DISTCC_LOCATION"; cd "$srcdir"; pwd`
    # The source tree location must be passed.
    include_server="$distcc_srcdir/include_server/include_server.py"
    # Now locate the single directory containing the .so file from the build
    # directory. Possibly there may be more than one such file; first identify
    # them all.
    so_dir=`"$distcc_srcdir/find_c_extension.sh" "$DISTCC_LOCATION"`
    pythonpath="$so_dir"
  fi

  # Create a temporary directory $socket_dir.
  socket_dir=`MakeTmpFile "distcc-pump" -d` || exit 1

  # The socket file on which the include server accepts connections.
  socket="$socket_dir/socket"

  # Files for the include server's stdout/stderr.
  # When a build tool invokes 'pump --startup', stdout/stderr may be
  # pipes, in which case the invocation may hang unless the include
  # server process closes them.  So to avoid this, we need to redirect
  # the include server's output to temporary files.
  # We print these files during shutdown; better late than never!
  include_server_stdout="$socket_dir/stdout"
  include_server_stderr="$socket_dir/stderr"

  # File for the include server process id.
  tmp_pid_file=`MakeTmpFile "distcc-pump-pid"` || exit 1

  # Start include server in optimized mode (no assertions) and with
  # debug level 1 for tracing warnings.
  # The include server will fork off
  # a background process to handle the requests;
  # the main process will exit only when the background
  # process is ready to accept connections.

  (
    # Optionally redirect the I/O streams for the include server.
    case $redirect_io in
      1) exec < /dev/null            \
           > $include_server_stdout  \
           2> $include_server_stderr
         ;;
      *)
          rm -f $include_server_stdout $include_server_stderr
          ;;
    esac

    if [ -z "$PYTHONOPTIMIZE" ]; then PYTHONOPTIMIZE=1; fi
    # We use 'eval' so that INCLUDE_SERVER_ARGS can use shell quoting such 
    # as in:
    #
    #   INCLUDE_SERVER_ARGS='--stat_reset_triggers="*"'
    #
    # which will pass '*' to the include server (that is, the string consisting
    # of one asterisk) without filename expansion.
    eval \
      "PYTHONOPTIMIZE='$PYTHONOPTIMIZE' " \
      "PYTHONPATH='$pythonpath${PYTHONPATH:+:$PYTHONPATH}' " \
      "'$PYTHON'"                       \
         "'$include_server'"            \
         --port "'$socket'"             \
         --pid_file "'$tmp_pid_file'"   \
         -d1                            \
         $INCLUDE_SERVER_ARGS
  )

  # solaris sh's built-in test does not support -S, so we need to use
  # the binary.
  if [ -x /bin/test ]; then
    TEST=/bin/test
  elif [ -x /usr/bin/test ]; then
    TEST=/usr/bin/test
  else
    TEST=test
  fi
  if $TEST ! -S "$socket"; then
    echo "__________Expected a socket at '$socket'" 1>&2
    PrintIncludeServerStatusMessage 1
    return 1
  fi

  include_server_pid=`cat $tmp_pid_file`
  rm "$tmp_pid_file"

  # Make sure $include_server_pid looks like a pid
  if echo "$include_server_pid" | grep '^[0-9][0-9]*$' >/dev/null; then
    # We got a number.  Tell the distcc clients where to find it.
    INCLUDE_SERVER_PORT="$socket"
    export INCLUDE_SERVER_PORT
  else
    # We got something that's not a number.  This indicates the
    # socket is not working.
    include_server_pid=''
    PrintIncludeServerStatusMessage 1
    return 1
  fi
}

ReportDiscrepancies() {
  # The discrepancy_counter file is created when a pump-mode distcc invocation
  # failed remotely but succeeded locally.
  num_discrepancies=`(cat $socket_dir/discrepancy_counter 2>/dev/null | wc -c \
                      || echo 0)`
  if [ $num_discrepancies -eq 0 ]; then
    return 0
  fi
  echo "__________Warning: $num_discrepancies" \
       'pump-mode compilation(s) failed on server,' \
       'but succeeded locally.' 1>&2
  if [ $num_discrepancies -ge $max_discrepancies_before_demotion ]; then
    echo '__________Distcc-pump was demoted to plain mode.' 1>&2
  fi
  echo 'See the Distcc Discrepancy Symptoms section in the' \
       'include_server(1) man page.' 1>&2
}

ShutDown() {
  ReportDiscrepancies
  # Always -- at exit -- shut down include_server and remove $socket_dir
  if [ -n "$include_server_pid" ] && \
    ps -p "$include_server_pid" > /dev/null; then
    echo '__________Shutting down distcc-pump include server'
    kill $include_server_pid
    # Wait until it's really dead.  We need to do this because the
    # include server may produce output after receiving SIGTERM.
    # Note that while 'sleep 0.01' is relying on a feature of GNU sleep,
    # that's OK; on systems that don't support it, it's effectively the
    # same as 'sleep 0', i.e. we'll just busy-wait rather than sleeping.
    while kill -0 $include_server_pid; do sleep 0.01; done >/dev/null 2>&1
  fi

  if [ -f "$include_server_stdout" ]; then
    cat "$include_server_stdout"
  fi
  if [ -f "$include_server_stderr" ]; then
    cat "$include_server_stderr" >&2
  fi

  if [ -n "$socket_dir" ];  then
    rm -rf "$socket_dir"
  fi
  if [ -n "$tmp_pid_file" ]; then
    rm -f "$tmp_pid_file"
  fi
  if [ -n "$available_hosts_file" ]; then
    rm -f "$available_hosts_file"
  fi
}

# Invokes lsdistcc to find the available servers. This list is
# \n-separated and written to the filepath provided as $1.
AvailableHosts() {
  available_hosts="$1"
  lsdistcc="$DISTCC_LOCATION/lsdistcc"
  if [ ! -x "$lsdistcc" ]; then
     echo "$0: error: can't find lsdistcc (looked in $DISTCC_LOCATION)" 1>&2
     exit 1
  fi
  # Call lsdistcc and let it wait no more than 150ms unless overridden in
  # LSDISTCC_ARGS.
  "$lsdistcc" -c150 $LSDISTCC_ARGS $DISTCC_POTENTIAL_HOSTS > "$available_hosts"
}

# Exports DISTCC_HOSTS as a function of the single argument and the global
# variable $available_hosts_file.
ExportDISTCC_HOSTS() {
  include_server_status="$1"
  opts=''
  if [ "$include_server_status" = 0 ]; then
    # The include server is up.
    # Make 'cpp' mode (with lzo) the default.
    opts=",cpp,lzo"
  else
    # The include server is not ready.
    # Make 'lzo' mode the default.
    if [ "$?" = 1 ]; then
      echo "__________Warning: distcc-pump include server failed;" \
        "running unpumped" 1>&2
    fi
    opts=",lzo"
  fi
  # When calculating final value of DISTCC_HOSTS, we get rid of
  # newlines (thanks to "`sed ...`").
  export DISTCC_HOSTS="--randomize `sed s/'$'/$opts/ $available_hosts_file`"
  if [ "$verbose" = 1 ]; then
    echo "__________Found" \
         "`wc -l < $available_hosts_file` available distcc servers"
  fi
}

StartIncludeServerAndDetermineHosts() {
  include_server_status=1 # ie, not running (yet)
  if [ -n "$DISTCC_POTENTIAL_HOSTS" ]; then
    # Probe the distcc servers. It may take up to a second. But
    # starting the include server takes about 150ms. Make these
    # activities happen in parallel. 

    # First, the lsdistcc command goes in the background.
    available_hosts_file=`MakeTmpFile "distcc-pump-hosts"` || exit 1
    AvailableHosts "$available_hosts_file" &

    # Second, the include server goes in the foreground so variables can be set.
    StartIncludeServer && include_server_status=0

    # Await for AvailableHosts to finish.
    wait 
    ExportDISTCC_HOSTS "$include_server_status"
    rm -f "$available_hosts_file"
  else
    if [ -z "$DISTCC_HOSTS" ] &&
       [ -z "$DISTCC_DIR" -o ! -f "$DISTCC_DIR/hosts" ] &&
       [ -z "$HOME" -o ! -f "$HOME/.distcc/hosts" ] &&
       [ ! -f "$sysconfdir/distcc/hosts" ]
    then
      echo "$program_name:" \
              "expected environment variables \"DISTCC_HOSTS\" or" \
              "\"DISTCC_POTENTIAL_HOSTS\" to be set, or to find a" \
              "distcc hosts file in \"\$DISTCC_DIR/hosts\"," \
              "\"\$HOME/.distcc/hosts\", or \"$sysconfdir/distcc/hosts\"." \
              1>&2
      exit 1
    else
      hosts=`$distcc_location/distcc --show-hosts`
      num_hosts=`echo "$hosts" | wc -l`
      num_pump_hosts=`echo "$hosts" | grep ',cpp' | wc -l`
      if [ $num_hosts -eq 0 ]; then
        echo "$program_name: error: distcc hosts list is empty!" 1>&2
        exit 1
      elif [ $num_pump_hosts -eq 0 ]; then
        echo "$program_name: error: pump mode requested, but distcc" \
          "hosts list does not contain any hosts with ',cpp' option" 1>&2
        exit 1
      else
        if [ "$verbose" = 1 ]; then
          server_word="servers"
          if [ $num_hosts = 1 ]; then
            server_word="server"
          fi
          if [ $num_hosts -eq $num_pump_hosts ]; then
            echo "__________Using $num_hosts distcc $server_word in pump mode"
          else
            echo "__________Using $num_hosts distcc $server_word, of which" \
                 "only $num_pump_hosts support(s) pump mode"
          fi
        fi
      fi
    fi
    StartIncludeServer && include_server_status=0
  fi
  return "$include_server_status"
}

# Prints out environment variable settings, for the --startup option.
DumpEnvironmentVariables() {
  # Variables used by "pump --shutdown"
  echo export INCLUDE_SERVER_PID=\'$include_server_pid\'
  echo export INCLUDE_SERVER_DIR=\'$socket_dir\'
  # Variables used by the distcc client
  echo export INCLUDE_SERVER_PORT=\'$INCLUDE_SERVER_PORT\'
  if [ -n "$DISTCC_HOSTS" ]; then
    echo export DISTCC_HOSTS=\'$DISTCC_HOSTS\'
  fi
  echo export PATH=\'$distcc_location:$PATH\'
}

Main() {
  CheckUsage "$@"

  Initialize

  case "$*" in
    --startup)
      # Don't put ordinary progress messages on stdout,
      # because they interfere with the environment variable
      # settings that we print out.
      verbose=0
      # Redirect the include server's stdin/stdout/stderr
      redirect_io=1
      trap 'ShutDown' EXIT  # In case we get interrupted.
      StartIncludeServerAndDetermineHosts
      include_server_status=$?
      trap '' EXIT
      if [ "$include_server_status" = 0 ]; then
        DumpEnvironmentVariables
        exit 0
      else
        exit 1
      fi
      ;;
    --shutdown)
      include_server_pid="$INCLUDE_SERVER_PID"
      socket_dir="$INCLUDE_SERVER_DIR"
      include_server_stdout="$socket_dir/stdout"
      include_server_stderr="$socket_dir/stderr"
      ShutDown
      exit 0
      ;;
    *)
      trap 'ShutDown' EXIT
      Announce
      StartIncludeServerAndDetermineHosts || exit 1
      # Now execute the command that is the argument of 'pump'.
      PATH="$distcc_location:$PATH" \
        "$@"
      # When we exit, the ShutDown function will be called.
      ;;
  esac
}

Main "$@"