File: logbt

package info (click to toggle)
postgis 3.1.1%2Bdfsg-1%2Bdeb11u2
  • links: PTS, VCS
  • area: main
  • in suites: bullseye
  • size: 83,792 kB
  • sloc: ansic: 151,982; sql: 146,734; xml: 51,051; sh: 6,186; cpp: 6,110; perl: 4,852; makefile: 3,002; python: 1,205; yacc: 447; lex: 133; javascript: 6
file content (429 lines) | stat: -rw-r--r-- 13,443 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
#!/usr/bin/env bash

set -eu
set -o pipefail
# This setting enables the desired behavior in the "for corefile in ${core_directory}/${SEARCH_PATTERN_NON_TRACKED}; do" loop
# more details at https://gist.github.com/springmeyer/6dd234ff89ba306a73608a6f45cb5506
shopt -s nullglob

PLATFORM_UNAME=$(uname -s)
REQUIRED_FILENAME="core"
LOGBT_VERSION="v2.0.3"
BASE_CORE_DIRECTORY=/tmp/logbt-coredumps

if [[ ${PLATFORM_UNAME} == "Linux" ]]; then
  REQUIRED_PATTERN="${REQUIRED_FILENAME}.%p.%E"
  DEBUGGER="gdb"
elif [[ ${PLATFORM_UNAME} == "Darwin" ]]; then
  # Recommend running with the following setting to only show crashes
  # in the notification center
  # defaults write com.apple.CrashReporter UseUNC 1
  REQUIRED_PATTERN="${REQUIRED_FILENAME}.%P"
  DEBUGGER="lldb"
else
  error "Unsupported platform: ${PLATFORM_UNAME}"
fi

function error() {
  >&2 echo "[logbt] $@"
  exit 1
}

function process_core() {
  local program=${1}
  local corefile=${2}
  local debugger=${3}
  if [[ ${debugger} =~ "lldb" ]]; then
    lldb --core ${corefile} --batch -o "thread backtrace all" -o "quit"
  else
    gdb ${program} --core ${corefile} -ex "set pagination 0" -ex "thread apply all bt full" --batch
  fi
  # note: on OS X the -f avoids a hang on prompt "remove write-protected regular file?"
  rm -f ${corefile}
}

function find_core_by_pid() {
  local program=${1}
  local core_directory=${2}
  local debugger=${3}
  local child_pid=${4}
  if [[ ${PLATFORM_UNAME} == "Darwin" ]]; then
    local single_corefile="${core_directory}/${REQUIRED_FILENAME}.${child_pid}"
    if [ -e ${single_corefile} ]; then
      echo "[logbt] Found corefile at ${single_corefile}"
      process_core ${program} ${single_corefile} ${debugger}
    fi
  else
    local SEARCH_PATTERN_BY_PID="${REQUIRED_FILENAME}.${child_pid}.*"
    # note: this for loop depends on the `shopt -s nullglob` above
    for corefile in ${core_directory}/${SEARCH_PATTERN_BY_PID}; do
      echo "[logbt] Found corefile at ${corefile}"
      # extract program name from corefile
      filename=$(basename "${corefile}")
      binary_program=/$(echo ${filename##*.\!} | tr "!" "/")
      process_core ${binary_program} ${corefile} ${debugger}
    done
  fi  
}

function find_remaining_cores() {
  local program=${1}
  local core_directory=${2}
  local debugger=${3}
  local SEARCH_PATTERN_NON_TRACKED="${REQUIRED_FILENAME}.*"
  local hit=false
  for corefile in ${core_directory}/${SEARCH_PATTERN_NON_TRACKED}; do
    echo "[logbt] Found corefile (non-tracked) at ${corefile}"
    hit=true
  done
  if [[ ${hit} == true ]]; then
    echo "[logbt] Processing cores..."
  fi
  for corefile in ${core_directory}/${SEARCH_PATTERN_NON_TRACKED}; do
    # below two lines are linux specific, but harmless to run on osx
    filename=$(basename "${corefile}")
    binary_program=/$(echo ${filename##*.\!} | tr "!" "/")
    process_core ${binary_program} ${corefile} ${debugger}
  done
}

function snapshot {
  local program=${1}
  local debugger=${2}
  local child_pid=${3}
  echo "[logbt] snapshotting ${program} (${child_pid})"
  if [[ ${debugger} =~ "lldb" ]]; then
    lldb -p ${child_pid} --batch -o "thread backtrace all" -o "quit"
  else
    gdb --pid ${child_pid} -ex "set pagination 0" -ex "thread apply all bt full" --batch
  fi
}

function backtrace {
  local program=${1}
  local core_directory=${2}
  local debugger=${3}
  local child_pid=${4}
  local child_return=${5}
  find_core_by_pid ${program} ${core_directory} ${debugger} ${child_pid}
  find_remaining_cores ${program} ${core_directory} ${debugger}
}

function warn_on_existing_cores() {
  local core_directory=${1}
  local SEARCH_PATTERN_NON_TRACKED="${REQUIRED_FILENAME}.*"
  for corefile in ${core_directory}/${SEARCH_PATTERN_NON_TRACKED}; do
    echo "[logbt] WARNING: Found corefile (existing) at ${corefile}"
  done
}

function error_on_existing_cores() {
  local core_directory=${1}
  local SEARCH_PATTERN_NON_TRACKED="${REQUIRED_FILENAME}.*"
  for corefile in ${core_directory}/${SEARCH_PATTERN_NON_TRACKED}; do
    error "Error: Found corefile (unexpected) at ${corefile}"
  done  
}

function ensure_directory_is_writeable() {
  # ensure we can write to the directory, otherwise
  # core files might not be able to be written
  WRITE_RETURN=0
  touch ${core_directory}/test.txt || WRITE_RETURN=$?
  if [[ ${WRITE_RETURN} != 0 ]]; then
    error "Permissions problem: unable to write to ${core_directory} (exited with ${WRITE_RETURN})"
  else
    # cleanup from test
    rm ${core_directory}/test.txt
  fi  
}

function get_target_core_pattern() {
  echo ${BASE_CORE_DIRECTORY}/${REQUIRED_PATTERN}
}

function get_core_pattern() {
  if [[ ${PLATFORM_UNAME} == "Linux" ]]; then
    local core_pattern=$(cat /proc/sys/kernel/core_pattern)
  elif [[ ${PLATFORM_UNAME} == "Darwin" ]]; then
    # Recommend running with the following setting to only show crashes
    # in the notification center
    # defaults write com.apple.CrashReporter UseUNC 1
    local core_pattern=$(sysctl -n kern.corefile)
  fi
  echo ${core_pattern}
}

function validate_core_pattern() {
  local core_pattern=${1}
  if [[ ! ${core_pattern} =~ ${REQUIRED_PATTERN} ]]; then
    error "unexpected core_pattern: ${core_pattern}"
  fi  
}

function generic_signal_handler() {
  local code=$?
  local program=${1}
  local child_pid=${2}
  local sig=${3}
  # Bug note: On darwin ${code} will be incorrectly 0 after snapshot here
  # so we ignore ${code} and instead get it from the signal
  code=$(($(kill -l ${sig})+128))
  echo "[logbt] received signal:${code} (${sig})"
  echo "[logbt] sending SIGTERM to ${program} (${child_pid})"
  # sleep here to help the stdout show in the right
  # order (accounts for an intermittant case where the above lines print after
  # the child outputs stdout during shutdown)
  sleep 1
  KILL_RETURN=0
  kill -TERM ${child_pid} || KILL_RETURN=$?
  if [[ ${KILL_RETURN} != 0 ]]; then
    echo "[logbt] could not terminate child process (kill returned ${KILL_RETURN})"
  else
    CHILD_EXIT=0
    wait ${child_pid} || CHILD_EXIT=$?
    if [[ ${CHILD_EXIT} != 143 ]]; then
      error "child process exited abnormally: ${CHILD_EXIT}"
    fi
  fi
  echo "[logbt] exiting with ${code}"
  exit ${code}
}


: '
NOTE: SIGINT (aka ctrl-c) is special. First of all SIGINT is sent to the whole process group
automatically in bash (http://stackoverflow.com/a/6804155). This means we do not need to reap
the child process manually. And if logbt is a "foreground" process then SIGINT is ignored if
sent directly with ./bin/logbt <args> & kill -INT $! (http://stackoverflow.com/a/14697034).
So the only way to send SIGINT is with ctrl-c or via another terminal that has a different
process group.
'

function sigint_handler() {
  local code=$?
  local program=${1}
  local child_pid=${2}
  local sig=${3}
  echo "[logbt] received signal:${code} (${sig})"
  CHILD_EXIT=0
  wait ${child_pid} || CHILD_EXIT=$?
  if [[ ${CHILD_EXIT} != 130 ]]; then
    echo "[logbt] child process exited with:${CHILD_EXIT}"
  fi
  exit ${code}
}

function launch_and_wait() {

  local program=${1}
  local core_pattern=$(get_core_pattern)
  validate_core_pattern ${core_pattern}

  local core_directory=$(dirname ${core_pattern})
  echo "[logbt] using corefile location: ${core_directory}"
  echo "[logbt] using core_pattern: $(basename ${core_pattern})"

  # ensure we have a debugger installed
  if ! which ${DEBUGGER} > /dev/null; then
    error "Could not find required command '${DEBUGGER}'"
  fi

  if [[ ! -d ${core_directory} ]]; then
    echo "[logbt] creating directory for core files at '${core_directory}'"
    mkdir -p -m a+w ${core_directory}
  fi

  ensure_directory_is_writeable ${core_directory}

  warn_on_existing_cores ${core_directory}

  # Enable corefile generation
  ulimit -c unlimited

  # Run the child process in a background process
  # in order to get the PID
  if [[ ${LD_PRELOAD:-} ]] && [[ ${PLATFORM_UNAME} == 'Darwin' ]]; then
      # on os x DYLD_INSERT_LIBRARIES is blocked from being inherited
      # so we accept LD_PRELOAD and foreward along
      DYLD_INSERT_LIBRARIES=${LD_PRELOAD} LOGBT_PID=$$ "$@" & CHILD_PID=$!
  else
      LOGBT_PID=$$ "$@" & CHILD_PID=$!
  fi

  # Hook up function to run when logbt received signal
  trap "snapshot ${program} ${DEBUGGER} ${CHILD_PID}" USR1
  trap "generic_signal_handler ${program} ${CHILD_PID} TERM" TERM
  trap "generic_signal_handler ${program} ${CHILD_PID} HUP" HUP
  trap "sigint_handler ${program} ${CHILD_PID} INT" INT

  # Wait for child and attempt to generate a backtrace if child exits in non-zero way
  wait_for_child ${program} ${core_directory} ${DEBUGGER} ${CHILD_PID}
}

function wait_for_child() {
  local program=${1}
  local core_directory=${2}
  local debugger=${3}
  local child_pid=${4}
  CHILD_RETURN=0
  wait ${child_pid} || CHILD_RETURN=$?
  # Bug note: on linux USR1 will trigger an exit which makes it looks like the child
  # has returned when it has not. So the below code ensures we stay alive and watching
  # the child if USR1 is hit
  if [[ $(uname -s) == 'Linux' ]] && [[ $(kill -l ${CHILD_RETURN}) == USR1 ]]; then
    wait_for_child ${program} ${core_directory} ${debugger} ${child_pid}
  fi
  if [[ ${CHILD_RETURN} == 127 ]]; then
    # command not found : http://www.tldp.org/LDP/abs/html/exitcodes.html
    echo "[logbt] command not found: ${program}"
  elif [[ ${CHILD_RETURN} != 0 ]]; then
    local exit_msg="saw '${program}' exit with code:${CHILD_RETURN}"
    exit_msg="${exit_msg} ($(kill -l ${CHILD_RETURN}))"
    echo "[logbt] ${exit_msg}"
    backtrace ${program} ${core_directory} ${DEBUGGER} ${child_pid} ${CHILD_RETURN}
  fi
  # exit logbt with the same code as the child
  exit ${CHILD_RETURN}
}

function setup_logbt() {
  local settable_core_pattern=$(get_target_core_pattern)
  if [[ ${PLATFORM_UNAME} == "Linux" ]]; then
    echo "[logbt] setting $(cat /proc/sys/kernel/core_pattern) -> ${settable_core_pattern}"
    # write new value to /proc/sys/kernel/core_pattern
    echo "${settable_core_pattern}" > /proc/sys/kernel/core_pattern
  elif [[ ${PLATFORM_UNAME} == "Darwin" ]]; then
    echo "[logbt] setting $(sysctl -n kern.corefile) -> ${settable_core_pattern}"
    sysctl kern.corefile=${settable_core_pattern}
  fi

  local core_pattern=$(get_core_pattern)
  validate_core_pattern ${core_pattern}
  local core_directory=$(dirname ${core_pattern})

  if [[ ! -d ${core_directory} ]]; then
    echo "[logbt] creating directory for core files at '${core_directory}'"
    mkdir -p -m a+w ${core_directory}
  fi

  error_on_existing_cores ${core_directory}

}

function test_logbt() {
  ulimit -c unlimited
  # First we create a program that crashes itself
  # We use bash to avoid needing an external dep on some runtime

  # Due to https://github.com/mapbox/logbt/issues/29 we need to copy the bash
  # exe on OS X to a new location since coredumps are disabled for /usr/bin/env bash for
  # reasons I don't understand
  if [[ ${PLATFORM_UNAME} == "Darwin" ]]; then
      # first touch file to create it with writable permissions for this user
      # such that we can cleanup after
      # then copy the system bash there
      cp --no-preserve=all $(which bash) /tmp/tmp-bash
      chmod +x /tmp/tmp-bash
      echo '#!/tmp/tmp-bash' > /tmp/crasher.sh
  else
    # on linux the default bash is okay
      echo '#!/usr/bin/env bash' > /tmp/crasher.sh
  fi
  echo 'kill -SIGSEGV $$' >> /tmp/crasher.sh
  chmod +x /tmp/crasher.sh
  # run it in logbt
  RETURN=0
  ${BASH_SOURCE} -- /tmp/crasher.sh >/tmp/logbt-stdout 2>/tmp/logbt-stderr || RETURN=$?
  local err_message
  if [[ ${RETURN} != 139 ]] || [[ ! $(cat /tmp/logbt-stdout) =~ "Found corefile at" ]]; then
    cat /tmp/logbt-stdout
    cat /tmp/logbt-stderr
    err_message="Expected return code of 139 and a corefile to be generated"
  fi
  if [[ ! $(cat /tmp/logbt-stdout) =~ "Found corefile at" ]]; then
    cat /tmp/logbt-stdout
    cat /tmp/logbt-stderr
    err_message="Expected a corefile to be generated"
  fi
  # cleanup
  rm -f /tmp/logbt-stderr
  rm -f /tmp/logbt-stdout
  rm -f /tmp/crasher.sh
  rm -f /tmp/tmp-bash
  if [[ ${err_message:-} ]]; then
    error ${err_message}
  else
    echo "[logbt] test success (coredumps are working with core pattern: '$(get_core_pattern)')"
  fi
}

function usage() {
  >&2 echo "Usage for logbt:"
  >&2 echo ""
  >&2 echo "Setup logbt (requires root privileges):"
  >&2 echo ""
  >&2 echo "$ sudo logbt --setup"
  >&2 echo ""
  >&2 echo "Test logbt is setup correctly"
  >&2 echo ""
  >&2 echo "$ logbt --test"
  >&2 echo ""
  >&2 echo "Launch a program with logbt:"
  >&2 echo ""
  >&2 echo "$ logbt -- ./program"
  >&2 echo ""
  >&2 echo "Other commands are:"
  >&2 echo ""
  >&2 echo " --current-pattern"
  >&2 echo " --target-pattern"
  >&2 echo " --version"
  exit 1
}

function get_version() {
  echo ${LOGBT_VERSION}
}

if [[ ! ${1:-} ]]; then
  usage
fi

# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash
for i in "$@"
do
case $i in
    --)
    if [[ ! ${2:-} ]]; then
      usage
    fi
    shift
    launch_and_wait "$@"
    ;;
    --setup)
    setup_logbt
    shift
    ;;
    --test)
    test_logbt
    shift
    ;;
    --current-pattern)
    get_core_pattern
    ;;
    --target-pattern)
    get_target_core_pattern
    ;;
    -v | --version)
    get_version
    shift
    ;;
    -h | --help)
    usage
    shift
    ;;
    *)
    usage
    ;;
esac
done