#
# common preliminary check procedures for QA scripts
#
# Copyright (c) 2014-2017 Red Hat.
# Copyright (c) 1997-2002 Silicon Graphics, Inc.  All Rights Reserved.
#

if [ ! -f localconfig ]
then
    ./mk.localconfig
fi

# need to know what version we're running
. ./localconfig

# if systemctl is present and functional, then use this to start
# and stop services
#
PCPQA_SYSTEMD=no
if which systemctl >/dev/null 2>&1
then
    # we have a systemctl executable, but it might be disabled,
    # e.g. on MX Linux
    #
    if systemctl -q is-active local-fs.target >/dev/null 2>&1
    then
	# OK, good to go with systemd and systemctl
	#
	PCPQA_SYSTEMD=yes
    fi
fi
export PCPQA_SYSTEMD

# set $__whatami (if possible) for later use
#
if [ -x $here/admin/whatami ]
then
    __whatami=`$here/admin/whatami`
elif [ -x $HOME/src/pcp/qa/admin/whatami ]
then
    __whatami=`$HOME/src/pcp/qa/admin/whatami`
elif [ -x $HOME/admin/whatami ]
then
    __whatami=`$HOME/admin/whatami`
else
    __whatami="unknown"
fi

# common routine for preventing a test from running on some platforms
# use: _notrun "Reason for not running this test is ..."
# 
_notrun()
{
    echo "$@" >$here/$seq.notrun
    echo "$seq: [not run] `cat $here/$seq.notrun`"
    rm -f $here/$seq.pre-avc
    _exit 0
}

# common routine for failing a test in a standard way
# use: _fail "Reason for failing this test is ..."
#
_fail()
{
    echo FAIL: $@ 1>&2
    _exit 1
}

# calculate the size of a given fail, echo it on standard output
# 
_filesize()
{
    __filename=$1

    if [ $PCP_PLATFORM = darwin -o $PCP_PLATFORM = openbsd ]
    then
	# macOS format (same for OpenBSD)
	# stat(1) format
	# 234881026 5304024 -rwxr-xr-x 1 kenj kenj 0 2016 "May  4 14:00:42 2011" "Apr 27 20:14:16 2011" "Apr 27 20:14:16 2011" "Apr 27 20:14:16 2011" 4096 8 0 441
	stat "$__filename" 2>&1 | $PCP_AWK_PROG '{ print $8 }'
    else
	# stat(1) format
	#   File: `441'
	#   Size: 2016      	Blocks: 8          IO Block: 4096   regular file
	# Device: 816h/2070d	Inode: 758237      Links: 1
	# Access: (0755/-rwxr-xr-x)  Uid: ( 1000/    kenj)   Gid: ( 1000/    kenj)
	# Access: 2011-05-09 06:52:58.000000000 +1000
	# Modify: 2011-02-27 14:42:30.000000000 +1100
	# Change: 2011-02-28 19:21:27.000000000 +1100
	stat "$__filename" 2>&1 | $PCP_AWK_PROG '$1 == "Size:" { print $2 }'
    fi
}

# determine whether sufficient free space exists to run a test;
# passed in parameter is the size needed, in megabytes.
# 
_check_freespace()
{
    __needspace=$1

    # Filesystem           1M-blocks Used Available Use% Mounted on
    # /dev/sda5                57349     24838     29621  46% /
    case "$PCP_PLATFORM"
    in
	darwin)
	    __free=`df -m . | $PCP_AWK_PROG 'NR == 2 { print $4 }'`
	    ;;
	openbsd)	# only Kbyte units available
	    __free=`df -P . | $PCP_AWK_PROG 'NR == 2 { print int($4/1024) }'`
	    ;;
	*)
	    __free=`df -mP . | $PCP_AWK_PROG 'NR == 2 { print $4 }'`
	    ;;
    esac

    if [ -z "$__free" ]
    then
	echo "Cannot determine free space (df -mP fails)"
    elif [ "$__free" -lt "$__needspace" ]
    then
	echo "Insufficient free space ($__free MB, need $__needspace MB)"
    fi
}

# save and restore configuration files in a way that labels which test
# was responsible in the event of a test abort, but also preserves the
# mode and ownership of the configuration file
#
# pmlogger control files are special and need to be saved *outside*
# $PCP_PMLOGGERCONTROL_PATH.d in the peer save_config.d directory, else
# _parse_config() in utilproc.sh will find "duplicate" direcotries
#
_save_config()
{
    case "$1"
    in
	$PCP_PMLOGGERCONTROL_PATH.d/*)
	    _src="$1"
	    _dst=`echo "$1" | sed -e 's@/control.d/@/save_control.d/@'`.$seq
	    if [ ! -d `dirname $_dst` ]
	    then
		if $sudo mkdir `dirname $_dst`
		then
		    :
		else
		    echo "Arrgh, failed to make backup pmlogger control.d"
		    exit
		fi
	    fi
	    ;;
	*)
	    _src="$1"
	    _dst="$1.$seq"
	    ;;
    esac

    if [ -f "$_src" ]
    then
	$sudo rm -f "$_dst"
	$sudo mv "$_src" "$_dst"
	$sudo cp "$_dst" "$_src"
    elif [ -d "$_src" ]
    then
	$sudo rm -rf "_dst"
	$sudo mv "$_src" "$_dst"
	$sudo cp -r "$_dst" "$_src"
    else
	echo "Botch: _save_config: $1 does not exist"
    fi
}

# reverse the changes from _save_config()
#
# pmlogger control files are special and need to be saved *outside*
# $PCP_PMLOGGERCONTROL_PATH.d in the peer save_config.d directory, else
# _parse_config() in utilproc.sh will find "duplicate" direcotries
#
_restore_config()
{
    case "$1"
    in
	$PCP_PMLOGGERCONTROL_PATH.d/*)
	    _src="$1"
	    _dst=`echo "$1" | sed -e 's@/control.d/@/save_control.d/@'`.$seq
	    ;;
	*)
	    _src="$1"
	    _dst="$1.$seq"
	    ;;
    esac
    if [ -f "$_dst" ]
    then
	$sudo rm -f "$_src"
	$sudo mv "$_dst" "$_src"
    elif [ -d "$_dst" ]
    then
	$sudo rm -rf "$_src"
	$sudo mv "$_dst" "$_src"
    else
	echo "Warning: _restore_config: $_dst does not exist"
    fi
}

# systemd config changes
#
_stop_auto_restart()
{
    if [ -n "$PCP_SYSTEMDUNIT_DIR" -a -f $PCP_SYSTEMDUNIT_DIR/$1.service ]
    then
	if grep '^Restart=always' $PCP_SYSTEMDUNIT_DIR/$1.service >/dev/null
	then
	    _save_config $PCP_SYSTEMDUNIT_DIR/$1.service
	    sed -e 's/Restart=always/Restart=no/' <$PCP_SYSTEMDUNIT_DIR/$1.service >$tmp.tmp
	    $sudo cp $tmp.tmp $PCP_SYSTEMDUNIT_DIR/$1.service
	    if [ "$PCPQA_SYSTEMD" = yes ]
	    then
		$sudo systemctl daemon-reload
	    else
		$sudo kill -HUP 1
	    fi
	fi
    fi
}

_restore_auto_restart()
{
    # Note this is idempotent, so repeated calls for the same service
    # are safe, e.g. in the body of a test and in _cleanup() from a
    # trap
    #
    if [ -n "$PCP_SYSTEMDUNIT_DIR" -a -f $PCP_SYSTEMDUNIT_DIR/$1.service.$seq ]
    then
	_restore_config $PCP_SYSTEMDUNIT_DIR/$1.service
	if [ "$PCPQA_SYSTEMD" = yes ]
	then
	    $sudo systemctl daemon-reload
	else
	    $sudo kill -HUP 1
	fi
    fi
}

# Wrapper for PCP daemon init scripts ... use systemctl if available,
# else run the PCP scripts directly
#
# Based on similar logic in the $PCP_RC_DIR/pcp script (retired), but
# note this one has to do the $sudo and handle "pcp" as the special
# non-service case ...
#
# Usage: _service [-v] <service> [on|off|restart|...]
#
_service()
{
    if [ -n "$1" -a "$1" = "-v" ]
    then
	__verbose=true
	__vflag='-v'
	shift
    else
	__verbose=false
	__vflag=''
    fi
    __do_systemctl=false
    if [ "$PCPQA_SYSTEMD" = no ]
    then
	# don't ever use systemctl
	$__verbose && echo "_service: no systemctl, PCPQA_SYSTEMD=$PCPQA_SYSTEMD"
    else
	[ "$PCPQA_SYSTEMD" = yes ] && __do_systemctl=true
	if $__do_systemctl
	then
	    case "$1"
	    in
		pcp)
		    echo >&2 "Botch: _service pcp ... retired"
		    ;;
		*)
		    if [ -n "$PCP_SYSTEMDUNIT_DIR" -a -f $PCP_SYSTEMDUNIT_DIR/$1.service ]
		    then
			:
		    else
			$__verbose && echo "_service: no systemctl, need $1 service"
			__do_systemctl=false
		    fi
		    ;;
	    esac
	else
	    $__verbose && echo "_service: no systemctl executable"
	fi
    fi

    if $__do_systemctl
    then
	# smells like systemctl is the go ...
	#
	case "$1"
	in
	    pcp)
		echo >&2 "Botch: _service pcp ... retired"
		;;

	    *)
		_extra_svc=''
		case "$1"
		in
		    pmlogger)
			    _extra_svc='pmlogger_daily pmlogger_check pmlogger_farm'
			    ;;
		esac
		# if service (or related service) is failed, need
		# reset-failed before trying anything else
		#
		for _svc in $1 $_extra_svc
		do
		    if systemctl show --property=ActiveState $_svc.service 2>&1 \
		      | grep '=failed$' >/dev/null
		    then
			$__verbose && echo "_service: using systemctl: reset-failed $_svc"
			$sudo systemctl reset-failed $_svc.service 2>>$seq_full
		    fi
		done

		$__verbose && echo "_service: using systemctl $2 $1.service"
		$sudo systemctl $2 $1.service
		if [ "$1" = pmlogger -a "$2" = stop ]
		then
		    # ensure pmlogger_check, pmlogger_daily and
		    # pmlogger_farm and pmlogger_farm_check are stopped
		    # ... see note above
		    #
		    for __svc in pmlogger_check pmlogger_daily pmlogger_farm pmlogger_farm_check
		    do
			if systemctl show --property=ActiveState $__svc.timer 2>&1 \
			  | grep '=active$' >/dev/null
			then
			    $__verbose && echo "_service: also stop $__svc.timer and $__svc.service"
			    $sudo systemctl stop $__svc.timer $__svc.service
			fi
		    done
		fi
		eval `systemctl show --property=ActiveState $1.service`
		if [ "$2" = start -o "$2" = restart ]
		then
		    if [ "$ActiveState" != active -a "$ActiveState" != activating ]
		    then
			echo "_service $1 $2: expecting state \"active\", found state \"$ActiveState\"" >>$seq_full
			_systemctl_status $1 >>$seq_full
		    fi
		elif [ "$2" = stop ]
		then
		    if [ "$ActiveState" != inactive ]
		    then
			echo "_service $1 $2: expecting state \"inactive\", found state \"$ActiveState\"" >>$seq_full
			_systemctl_status $1 >>$seq_full
		    fi
		else
		    echo "_service $1 $2: Botch: don't know what state to expect, found state \"$ActiveState\"" >>$seq_full
		fi
		;;
	esac
    elif [ -f $PCP_RC_DIR/$1 ]
    then
	$__verbose && echo "_service: using $PCP_RC_DIR/$1 $2"
	$sudo $PCP_RC_DIR/$1 $__vflag $2
    else
	echo "_service: $1 is not a systemctl service nor a $PCP_RC_DIR script!"
	return 1
    fi
    return 0
}

# check that a given agent (argument 1) is up and running;
# indicated by return status and agent warnings to stdout.
#
_check_agent() 
{
    __agent=$1
    __quiet=$2
    __sts=0

    [ X"$__quiet" = "X" ] && __quiet=false

    pminfo -f $__agent 2>&1 | \
        $PCP_AWK_PROG '
BEGIN           { value = 0; metric = 0; warn = 0 }
NF==0           { next }
/ value /       { value++; next }
/^'$__agent'/         { metric++
                  if ($0 !~ /:/) next
		}
		{ warn++ }
END             { if (warn) printf "%d warnings, ",warn
		  printf "%d metrics and %d values\n",metric,value
		}' > $tmp.fetch 2>&1

    pminfo -v $__agent 2>&1 | LC_COLLATE=POSIX sort >$tmp.verify
    $__quiet || cat $tmp.fetch
    if grep warnings $tmp.fetch > /dev/null 2>&1
    then
	# X warnings, Y metrics and Z values
	num_warn=`sed -n -e '/warnings/s/ .*//p' < $tmp.fetch`
	if [ "$num_warn" -gt 0 ]
	then
	    echo "Warnings when fetching $__agent metrics:"
	    cat $tmp.verify
	    __sts=1
	fi
    else
    	num_warn=0
    fi

    return $__sts
}

#
# prepare and cleanup pmda routines work together to ensure
# pmcd+pmda state is left as it was at the start of a test.
#
_prepare_pmda()
{
    __agent=$1
    __names=$2

    iam=$__agent
    [ "X$__names" = "X" ] && __names=$iam
    __done_clean=false
    __install_on_cleanup=false
    pminfo $__names >/dev/null 2>&1 && __install_on_cleanup=true
    echo "_prepare_pmda(agent=$__agent, names=$__names) __install_on_cleanup=$__install_on_cleanup" >>$seq_full
    # copy the pmcd config file to restore state later.
    _save_config $PCP_PMCDCONF_PATH
    unset ROOT MAKEFLAGS
}

_cleanup_pmda()
{
    __agent=$1
    __iopts=$2

    if $__done_clean
    then
	echo "_cleanup_pmda($__agent,$__iopts) called twice?" >>$seq_full
    else
	_restore_config $PCP_PMCDCONF_PATH
	_service pmcd restart 2>&1 | _filter_pcp_restart
	_wait_for_pmcd
	_restore_auto_restart pmcd
	_service pmlogger restart 2>&1 | _filter_pcp_restart
	_wait_for_pmlogger
	if $__install_on_cleanup
	then
	    [ "X$__iopts" = "X" ] && __iopts=/dev/null
	    ( cd $PCP_PMDAS_DIR/$__agent; $sudo ./Install <$__iopts >/dev/null 2>&1 )
	    echo "_cleanup_pmda($__agent,$__iopts) reinstall PMDA" >>$seq_full
	else
	    ( cd $PCP_PMDAS_DIR/$__agent; $sudo ./Remove >/dev/null 2>&1 )
	    echo "_cleanup_pmda($__agent,$__iopts) remove PMDA" >>$seq_full
	fi
	_wait_for_pmcd
	_wait_for_pmlogger
	__done_clean=true
    fi
    $sudo rm -rf $tmp $tmp.*
}

#
# create a Linux /proc/stat-alike file for a given number of CPUs
#
_make_proc_stat()
{
    __path=$1
    __ncpu=$2

    echo cpu 0 0 0 0 0 0 0 0 0 > $__path
    __cpu=0
    while [ $__cpu -lt $__ncpu ]
    do
	echo cpu$__cpu 0 0 0 0 0 0 0 0 0 >> $__path
	__cpu=`expr $__cpu + 1`
    done
}

_make_helptext()
{
    __pmda=$1
    __home=`pwd`

    if [ -f $PCP_PMDAS_DIR/$__pmda/help.dir ]
    then
	# ./Install already run, or QA stumbled past to run newhelp
	if [ -f $PCP_PMDAS_DIR/$__pmda/help -a $PCP_PMDAS_DIR/$__pmda/help -nt $PCP_PMDAS_DIR/$__pmda/help.dir ]
	then
	    :
	else
	    return 0
	fi
    fi

    cd $PCP_PMDAS_DIR/$__pmda
    $sudo $PCP_BINADM_DIR/newhelp -n root help >$tmp.out 2>&1
    if [ -f $PCP_PMDAS_DIR/$__pmda/help.dir ]
    then
	:
    else
	cat $tmp.out
	echo "Arrgh ... failed to create help.dir for $__pmda PMDA"
	return 1
    fi
    cd $__home
    return 0
}

#
# Get all IPv6 connection strings for a hosts interfaces, excluding loopback
#
_host_to_ipv6addrs()
{
    # extract string from lines like:
    # inst [2 or "br0"] value "fe80::d217:c2ff:fea7:4ae9/64"
    #
    pminfo -f -h "$1" network.interface.ipv6_addr \
    | sed \
	-e '/^$/d' \
	-e '/^network/d' \
	-e '/"lo"/d' \
	-e 's/.* or "//' \
	-e 's/"] value "/ /' \
	-e 's/\/[0-9][0-9]*"//g' \
    | while read __iface __addr
    do
	case "$__addr"
	in
	    fe80::*)
		echo "$__addr%$__iface"
		;;
	    *)
		echo $__addr
		;;
	esac
    done
}

_host_to_ipaddr()
{
    perl -MSocket -e '
	my $__packed_ip = gethostbyname("'$1'");
	if (defined $__packed_ip) {
	    my $__ipaddr = inet_ntoa($__packed_ip);
	    print $__ipaddr;
	}'
}

_ipaddr_to_host()
{
    perl -MSocket -e '
	my $__ipaddr = inet_aton("'$1'");
	if (defined $__ipaddr) {
	    my $__name  = gethostbyaddr($__ipaddr, AF_INET);
	    print $__name;
	}'
}

_host_to_fqdn()
{
    __ans=''
    if which nslookup >/dev/null 2>&1
    then
	__ans=`nslookup $1 2>&1 | sed -n -e '/^Name:[ 	][ 	]*/{
s///p
q
}'`
    elif which host >/dev/null 2>&1
    then
	__ans=`host $1 2>&1 | sed -n -e '/ has address /{
s/ .*//p
q
}'`
    fi

    if [ -z "$__ans" ]
    then
	# no working DNS ... just go with what we were given
	#
	__ans=$1
    fi
    echo "$__ans"
}

_ipv6_localhost()
{
    if [ ! -f /etc/hosts ]
    then
	echo >&2 "Arrgh ... cannot find /etc/hosts to determine IPv6 localhost name"
	return
    fi
    for __sniff in localhost6 ip6-localhost ipv6-localhost
    do
	if sed -e '/^#/d' -e 's/$/ /' /etc/hosts | grep -q "[ 	]$__sniff "
	then
	    echo $__sniff
	    return
	fi
    done
    echo >&2 "Arrgh ... cannot determine IPv6 localhost name from /etc/hosts"
    return
}

_domain_name()
{
    if which domainname >/dev/null 2>&1
    then
	__domainname=`domainname`
    else
	__domainname=`hostname -d`
    fi
    [ -z "$__domainname" ] && __domainname=localdomain
    [ "$__domainname" = "(none)" ] && __domainname=localdomain
    [ "$__domainname" = "(null)" ] && __domainname=localdomain
    # OpenIndiana sickness => domainname returns " "
    #
    [ "$__domainname" = " " ] && __domainname=localdomain
    echo "$__domainname"
}

_machine_id()
{
    cat /etc/machine-id 2> /dev/null || echo localmachine
}

# Check metric ($1) availability from pmcd on host ($2, defaults to
# a local: pminfo connection) ... return 1 if no values available,
# else return 0
#
_check_metric()
{
    if pminfo -h ${2-local:} -f $1 2>&1 | grep " value " >/dev/null
    then
	return 0
    else
	echo "Check failed for metric \"$1\" at host \"${2-local:}\" ... is PMDA installed?"
	return 1
    fi
}

# Check for metric availability from local pmcd and _notrun if not
# available.
# Use this check for metrics that are requireed by a qa test but are
# not universally available, e.g. kernel metrics across different
# platforms.
#
_need_metric()
{
    __numval=`pmprobe $1 | cut -f 2 -d ' '`
    case "$__numval"
    in
	-*)
	    _notrun "metric \"$1\" not available"
	    ;;
	[0-9]*)
	    ;;
	*)
	    echo "_need_metric: unexpected output: `pmprobe $1`"
	    ;;
    esac
}

# Check for metric availability from local pmcd.
# Like _need_metric() above, but return status is 0 for
# available and 1 for not available (rather than _notrun).
#
_avail_metric()
{
    __numval=`pmprobe $1 | cut -f 2 -d ' '`
    case "$__numval"
    in
	-*)
	    return 1
	    ;;
	[0-9]*)
	    return 0
	    ;;
	*)
	    echo "_avail_metric: unexpected output: `pmprobe $1`"
	    return 1
	    ;;
    esac
}

# if systemctl is in play, see if there is any additional info
# from systemctl for the $1 service
#
_systemctl_status()
{
    if [ $# -ne 1 ]
    then
	echo "_systemctl_status: botch: usage #args=$# not 1 as expected"
	return
    fi
    
    if [ "$PCPQA_SYSTEMD" = yes ]
    then
	echo "--- start systemctl status for $1.service ---"
	LC_CTYPE=C SYSTEMD_COLORS=false systemctl --no-pager -l status "$1.service"
	echo "--- end systemctl status for $1.service ---"
	echo "--- start journalctl entries in last minute for -u $1 ---"
	journalctl --since -1m -u $1 -l --no-page
	echo "--- end recent journalctl entries for -u $1 ---"
    fi
}

# who's listening on a port ($1)
#
_ps_tcp_port()
{
    if which fstat >/dev/null 2>&1
    then
	# for OpenBSD (at least) fstat produces lines like ...
	# USER     CMD          PID   FD MOUNT        INUM  MODE         R/W    SZ|DV
	# pcp      pmcd        7731    0* internet stream tcp 0x0 *:44321
	# pcp      pmcd        7731    3* internet6 stream tcp 0x0 *:44321
	#
	echo "--- IPv4 port $1 user(s) ---"
	fstat | sed -n -e '1s/MOUNT.*/.../p' -e "/internet .*:$1\$/p"
	echo "--- IPv6 port $1 user(s) ---"
	fstat | sed -n -e '1s/MOUNT.*/.../p' -e "/internet6 .*:$1\$/p"
    elif which lsof >/dev/null 2>&1
    then
	echo "--- IP port $1 user(s) ---"
	sudo lsof -i :$1
    elif which fuser >/dev/null 2>&1
    then
	echo "--- IPv4 port $1 user(s) ---"
	sudo fuser -4 $1/tcp
	echo "--- IPv6 port $1 user(s) ---"
	sudo fuser -6 $1/tcp
    else
	echo "_ps_tcp_port: no lsof(1) or fuser(1)?"
    fi
}

# local pmcd not running or not responding, see what we can find
# out
#
_triage_pmcd()
{
    echo "=== triage for pmcd ==="
    for log in pmcd.log.prev pmcd.log
    do
	if [ -f $PCP_LOG_DIR/pmcd/$log ]
	then
	    echo "--- start $log ---"
	    cat $PCP_LOG_DIR/pmcd/$log
	    echo "--- end $log ---"
	else
	    echo "--- $log is missing"
	fi
    done

    $PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID]|/[p]mcd( |$)' >$tmp.triage 2>&1
    if [ -s $tmp.triage ]
    then
	echo "--- likely looking processes ..."
	cat $tmp.triage
    else
	echo "--- there are no likely looking processes!"
    fi

    if [ -f $PCP_RUN_DIR/pmcd.pid ]
    then
	echo "--- $PCP_RUN_DIR/pmcd.pid => `cat $PCP_RUN_DIR/pmcd.pid`"
    else
	echo "--- $PCP_RUN_DIR/pmcd.pid is missing"
    fi

    if [ -S $PCP_RUN_DIR/pmcd.socket ]
    then
	echo "--- UNIX doman socket $PCP_RUN_DIR/pmcd.socket exists"
    elif [ -e $PCP_RUN_DIR/pmcd.socket ]
    then
	echo "--- $PCP_RUN_DIR/pmcd.socket is not a UNIX domain socket"
	ls -l $PCP_RUN_DIR/pmcd.socket
    else
	echo "--- UNIX doman socket is missing"
    fi

    _ps_tcp_port 44321

    _systemctl_status pmcd
}

# wait_for_pmcd [maxdelay [host [port]]]
#
_wait_for_pmcd()
{
    # 20 seconds default seems like a reasonable max time to get going
    __can_wait=${1-20}
    if [ $# -ge 2 -a -n "$2" ]
    then
	__host="$2"
	__host_opt="$2"
    else
	__host=localhost
	__host_opt='localhost'
    fi
    if [ $# -ge 3 -a -n "$3" ]
    then
	__host_opt="$__host_opt:$3"
    fi
    __i=0
    __dead=true
    #debug# ping -c 2 $__host
    #debug# pcp -h $__host_opt
    #debug# pcp -h `hostname`
    rm -f $tmp._wait_for_pmcd.err $tmp._wait_for_pmcd.out
    while [ $__i -lt $__can_wait ]
    do
	echo "-- $__i --" >>$tmp._wait_for_pmcd.err
	echo "-- $__i --" >>$tmp._wait_for_pmcd.out
	#debug# pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd -h $__host_opt pmcd.numclients
	__sts=`pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd -h $__host_opt pmcd.numclients 2>>$tmp._wait_for_pmcd.err | tee -a $tmp._wait_for_pmcd.out | $PCP_AWK_PROG '{print $2}'`
	if [ "${__sts:-0}" -gt 0 ]
	then
	    # numval really > 0, we're done
	    #
	    __dead=false
	    break
	fi
	sleep 1
	__i=`expr $__i + 1`
    done
    if $__dead
    then
	date
	echo "Arrgh ... pmcd at $__host_opt failed to start after $__can_wait seconds"
	echo "=== failing pmprobes stdout ==="
	cat $tmp._wait_for_pmcd.out
	echo "=== failing pmprobes stderr ==="
	cat $tmp._wait_for_pmcd.err
	echo "... and now"
	echo "+ pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd -h $__host_opt pmcd.numclients"
	pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd -h $__host_opt pmcd.numclients
	case $__host
	in
	    localhost|unix:|local:|`hostname`)
		# these are all local PMCD's
		#
		_triage_pmcd
		;;
	    *)
		# assume a remote PMCD
		#
		ssh pcpqa@$__host "sh -c '. \$PCP_DIR/etc/pcp.env; echo; echo "=== pmcd.log ==="; [ -f \$PCP_LOG_DIR/pmcd/pmcd.log ] && cat \$PCP_LOG_DIR/pmcd/pmcd.log; [ -f \$PCP_LOG_DIR/pmcd.log ] && cat \$PCP_LOG_DIR/pmcd.log; echo; echo likely looking processes ...; ( \$PCP_PS_PROG \$PCP_PS_ALL_FLAGS | grep -E \"[P]PID|/[p]mcd( |\\\$)\" )'" </dev/null
		;;
	esac
	rm -f $tmp._wait_for_pmcd.err $tmp._wait_for_pmcd.out
	return 1
    fi
    return 0
}

# wait_for_pmcd_stop [maxdelay]
#
_wait_for_pmcd_stop()
{
    # 20 seconds default seems like a reasonble max time to get shutdown
    __can_wait=${1-20}
    __i=0
    __stopped=false
    rm -f $tmp._wait_for_pmcd_stop.err $tmp._wait_for_pmcd_stop.out
    while [ $__i -lt $__can_wait ]
    do
	echo "-- $__i --" >>$tmp._wait_for_pmcd_stop.out.all
	echo "-- $__i --" >>$tmp._wait_for_pmcd_stop.err.all
	#debug# pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd pmcd.numclients
	pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd pmcd.numclients >$tmp._wait_for_pmcd_stop.out 2>$tmp._wait_for_pmcd_stop.err
	if grep 'pmprobe: Cannot connect to PMCD' $tmp._wait_for_pmcd_stop.err >/dev/null
	then
	    # we're done
	    #
	    __stopped=true
	    break
	fi
	cat $tmp._wait_for_pmcd_stop.out >>$tmp._wait_for_pmcd_stop.out.all
	cat $tmp._wait_for_pmcd_stop.err >>$tmp._wait_for_pmcd_stop.err.all
	sleep 1
	__i=`expr $__i + 1`
    done
    if $__stopped
    then
	:
    else
	date
	echo "Arrgh ... pmcd failed to stop after $__can_wait seconds"
	$PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID|/[p]mcd( |$)'
	echo "=== successful pmprobes stdout ==="
	cat $tmp._wait_for_pmcd_stop.out.all
	echo "=== successful pmprobes stderr ==="
	cat $tmp._wait_for_pmcd_stop.err.all
	echo "... and now"
	echo "+ pmprobe -n $PCP_VAR_DIR/pmns/root_pmcd pmcd.numclients"
	pmprobe -Dpdu,context -n $PCP_VAR_DIR/pmns/root_pmcd pmcd.numclients
	echo "=== pmcd.log ==="
	cat $PCP_LOG_DIR/pmcd/pmcd.log
	_systemctl_status pmcd
	rm -f $tmp._wait_for_pmcd_stop.err $tmp._wait_for_pmcd_stop.out
	return 1
    fi
    return 0
}

__triage_systemd()
{
    if [ "$PCPQA_SYSTEMD" = yes ]
    then
	if which systemctl >/dev/null 2>&1
	then
	    echo
	    echo "++ systemctl status for $1.service ..."
	    systemctl status $1.service | cat
	else
	    echo "No systemctl for $1.service"
	fi

	if which journalctl >/dev/null
	then
	    last=4
	    echo
	    echo "++ last $last journal entries for $1.service"
	    $sudo journalctl -l -n $last -xeu $1.service | cat
	else
	    echo "No journalctl for $1.service"
	fi
    else
	echo
	echo "++ no systemctl/journalctl status for $1.service (\$PCPQA_SYSTEMD=$PCPQA_SYSTEMD)"
    fi
}

__filter_pmlogger_log()
{
    $PCP_AWK_PROG '
BEGIN					{ ingroup = -1 }
$1 == "Group" && $3 == "metrics]"	{ ingroup = 0 }
ingroup >= 0 && $1 == "}"		{ ingroup = -1 }
ingroup >= 0				{ ingroup++
					  if (ingroup < 4) print
					  else if (ingroup == 4) print "        ..."
					  next
					}
					{ print }'

}

# wait_for_pmlogger [pid logfile [maxdelay]]
#
_wait_for_pmlogger()
{
    # 20 seconds default seems like a reasonable max time to get going
    __maxdelay=20

    case $#
    in
	0)
	    __pid="-P"
	    __sudo="$sudo -u $PCP_USER"
	    __dir_hostname=`hostname || echo localhost`
	    __logfile="$PCP_ARCHIVE_DIR/$__dir_hostname/pmlogger.log" 
            __ident="primary"
	    ;;
	2)
	    __pid=$1
	    __sudo=''
	    __logfile=$2
            __ident="pid=$__pid"
	    ;;
	3)
	    __pid=$1
	    __sudo=''
	    __logfile=$2
	    __maxdelay=$3
            __ident="pid=$__pid"
	    ;;
	*)
	    echo "_wait_for_pmlogger(): wrong number of arguments"
	    return 1
	    ;;
    esac

    __i=0
    __dead=true
    rm -f $tmp._wait_for_pmlogger.pmlc
    while [ $__i -lt $__maxdelay ]
    do
	$PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID|/[p]mlogger( |$)' >>$tmp._wait_for_pmlogger.pmlc
	echo "pmlc $__pid" >>$tmp._wait_for_pmlogger.pmlc
	# May need sudo -u $PCP_USER here in case we're doing credentials checks
	# in pmlogger and this is the primary pmlogger (started by user
	# pcp) ... $__sudo contains the right secret sauce.
	#
	# For hard debugging, add -Dpdu,context,desperate to pmlc invocation
	# below.
	#
	echo status | pmlc $__pid >$tmp._wait_for_pmlogger.tmp 2>&1
	if grep "^Connected to .*pmlogger" $tmp._wait_for_pmlogger.tmp >/dev/null
	then
	    # pmlogger socket has been set up ...
	    __dead=false
	    echo "_wait_for_pmlogger() success at iter $__i ..." >>$seq_full
	    cat $tmp._wait_for_pmlogger.tmp >>$seq_full
	    # give pmlogger a chance to detect that pmlc has gone away
	    # so the port is free
	    pmsleep 0.25
	    break
        else
	    cat $tmp._wait_for_pmlogger.tmp >>$tmp._wait_for_pmlogger.pmlc
	    sleep 1
	    __i=`expr $__i + 1`
	fi
    done
    if $__dead
    then
	date
	echo "Arrgh ... pmlogger ($__ident) failed to start after $__maxdelay seconds"
	echo "at `date`."
	echo "++ pmlogger log ($__logfile) ..."
	ls -l $__logfile
	__filter_pmlogger_log <$__logfile
	echo
	if [ -f "$__logfile.prev" ]
	then
	    echo "++ previous pmlogger log ($__logfile.prev) ..."
	    ls -l $__logfile.prev
	    __filter_pmlogger_log <$__logfile.prev
	fi
	for file in pmlogger_check.log pmlogger_check.log.prev \
	    pmlogger_janitor.log pmlogger_janitor.log.prev
	do
	    if [ -f "$PCP_ARCHIVE_DIR/$file" ]
	    then
		echo "++ $PCP_ARCHIVE_DIR/$file"
		ls -l $PCP_ARCHIVE_DIR/$file
		cat $PCP_ARCHIVE_DIR/$file
	    fi
	done
	__triage_systemd pmcd
	__triage_systemd pmlogger
	__triage_systemd pmlogger_farm

	#  If pmlc could not connect to pmlogger, report details
	if [ -s $tmp._wait_for_pmlogger.pmlc ]
	then
	    echo "++ pmlc output ..."
	    cat $tmp._wait_for_pmlogger.pmlc
	    echo
	fi

	#  If pmlogger could not connect to PMCD, find which host it was
	#  connecting to, and get the pmcd.log file from that host.
	cat $__logfile | $PCP_AWK_PROG '/pmlogger: Cannot connect to PMCD on host/' \
	  | sed -e '	s/pmlogger: Cannot connect to PMCD on host "//g' \
	  -e '	s/": .*//g' >$tmp._wait_for_pmlogger.host
	if [ -s $tmp._wait_for_pmlogger.host ]
	then
	    __pmcdhost=`cat $tmp._wait_for_pmlogger.host`
	    echo "++ pmcd log ($__pmcdhost:$PCP_LOG_DIR/pmcd/pmcd.log) ..."
	    if [ -r /hosts/$__pmcdhost$PCP_LOG_DIR/pmcd/pmcd.log ]
	    then
	    	cat /hosts/$__pmcdhost$PCP_LOG_DIR/pmcd/pmcd.log
	    elif [ -r /hosts/$__pmcdhost$PCP_LOG_DIR/pmcd.log ]
	    then
	    	cat /hosts/$__pmcdhost$PCP_LOG_DIR/pmcd.log
	    else
	    	if [ "`hostname | sed -e 's/\..*//'`" != $__pmcdhost ]
	    	then
	    	    if scp -q $__pmcdhost:$PCP_LOG_DIR/pmcd/pmcd.log \
	    	      $tmp._wait_for_pmlogger.pmcdlog
	    	    then
	    	    	cat $tmp._wait_for_pmlogger.pmcdlog
	    	    elif scp -q $__pmcdhost:$PCP_LOG_DIR/pmcd.log \
	    	      $tmp._wait_for_pmlogger.pmcdlog
	    	    then
	    	    	cat $tmp._wait_for_pmlogger.pmcdlog
	    	    fi
	    	else
	    	    cat $PCP_LOG_DIR/pmcd/pmcd.log
	    	fi
	    fi
	fi
	__base=`cat $__logfile | sed -n '/^Archive basename:[ 	]*/s///p'`
	if [ -n "$__base" -a -f $__base.0 ]
	then
	    echo "++ archive created ..."
	    pmdumplog -l $__base.0
	    __nres=`pmdumplog $__base.0 | grep '^[0-9]' | wc -l | sed -e 's/ //g'`
	    echo "++ archive contains $__nres records"
	else
	    echo "++ archive not created"
	fi
	echo
	echo "++ local pmlogger map ..."
	for __map in $PCP_TMP_DIR/pmlogger/*
	do
	    if [ "`echo $__map`" = "$PCP_TMP_DIR"'/pmlogger/*' ]
	    then
		echo "++ No files in $PCP_TMP_DIR/pmlogger !?"
	    else
		ls -l $__map
		cat $__map
	    fi
	done
	echo
	echo "++ Likely looking processes ..."
	# Note: no ( |$) for egrep, want to find pmlogger_* as well
	$PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID|/[p]mlogger'
	rm -f $tmp._wait_for_pmlogger.*
	return 1
    fi
    rm -f $tmp._wait_for_pmlogger.*
    return 0
}

# Start a pmlogger with appropriate privs that it can create
# portmap files in PCP_TMP_DIR (requires pcp or root account)
#
# Couple of side-effects to be aware of:
# Returns with $pid containing the backgrounded pmlogger PID;
# Creates the pmlogger archive, logfile, etc as root/pcp, not
# as the user running QA (so, tmp cleanup needs to use sudo).
#
_start_up_pmlogger()
{
    cat >$tmp.cmd <<End-of-File
#!/bin/sh
pmlogger $@ &
echo pid=\$!
End-of-File

    __user=root
    id pcp >/dev/null 2>&1 && __user=pcp

    $sudo -u $__user sh $tmp.cmd $@ >$tmp.pid
    eval `cat $tmp.pid`
}

# wait for a pmlogger process to end that is not our child
# (else we could just use shell wait command).  Instead we
# must keep tabs on the PID file and bail when its gone.
#
_wait_pmlogger_end()
{
    if [ $# = 0 ]
    then
	# default is to wait for the primary pmlogger
	__pid=''
        if [ -f "$PCP_TMP_DIR/pmlogger/primary" ]
	then
	    __tmp=`ls -l $PCP_TMP_DIR/pmlogger/primary 2>/dev/null`
	    case "_$tmp"
	    in
		*' -> '*)
		    __pid=`echo "$__tmp" | sed -e 's@.*/@@'`
		    ;;
		*)
		    __pid=''
		    ;;
	    esac
	fi
	if [ -z "$__pid" ]
	then
	    # either not a symlink or does not exist, nothing
	    # to wait for
	    #
	    return
	fi
    else
	__pid="$1"
    fi

    if $sudo kill -s 0 "$__pid" 2>/dev/null
    then
	# pmlogger process exists, so wait for PID file to go away
	#
	echo "pmlogger process $__pid present ..." >>$seq_full
	$PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E "[P]PID|/[p]mlogger( |$)|$__pid" >>$seq_full
    else
	# pmlogger process is not here, warn if PID file is here,
	# and we're done
        if [ -f "$PCP_TMP_DIR/pmlogger/$__pid" ]
	then
	    echo "_wait_pmlogger_end: warning process $__pid gone but $PCP_TMP_DIR/pmlogger/$__pid still present"
	fi
	return 0
    fi

    __i=0
    while true
    do
        if [ -f "$PCP_TMP_DIR/pmlogger/$__pid" ]
	then
	    # potential race between [ ... ] and ls ... syphon of stderr
	    #
	    [ $__i -eq 0 ] && ls -l "$PCP_TMP_DIR/pmlogger/$__pid" >>$seq_full 2>&1
	    __i=`expr $__i + 1`
	    if [ $__i -ge 100 ]
	    then
		echo "_wait_pmlogger_end: failed to see $PCP_TMP_DIR/pmlogger/$__pid removed after 100 iterations"
		cat "$PCP_TMP_DIR/pmlogger/$__pid"
		$PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E "[P]PID|/[p]mlogger( |$)|$__pid"
		_systemctl_status pmlogger
		return 1
	    fi
	    pmsleep 0.1
	else
	    # now there is an ugly race condition at the end of pmlogger
	    # ... once the control file has been removed (as per the test
	    # above) the pmlogger log file is not complete until the atexit()
	    # code is run ... so hang around waiting for the process to
	    # go away
	    #
	    if _wait_process_end "_wait_pmlogger_end" "$__pid"
	    then
		return 0
	    else
		_systemctl_status pmlogger
		return 1
	    fi
	fi
    done
}

# Wait for pmcd to end
#
_wait_pmcd_end()
{
    __pid=`cat $PCP_RUN_DIR/pmcd.pid 2>/dev/null`
    if [ -z "$__pid" ]
    then
	# no $PCP_RUN_DIR/pmcd.pid file, try ps(1) the hard way
	#
	__pid=`$PCP_PS_PROG $PCP_PS_ALL_FLAGS \
	| sed -n -e 's/$/ /' -e '/\[p]mcd /{
s/ $//
s/^[^ 	]*//
s/^[ 	]*//
s/[ 	].*//
p
}'`
    fi
    if [ -z "$__pid" ]
    then
	return 0
    fi

    if _wait_process_end "_wait_pmcd_end" "$__pid"
    then
	return 0
    else
	_systemctl_status pmcd
	return 1
    fi
}

# Wait for at primary pmie to start ...
#
_wait_for_pmie()
{
    __i=0
    while true
    do
	if [ ! -s $PCP_RUN_DIR/pmie.pid ]
	then
	    # "run" file does not exist or is empty =>
	    # pmie process not started yet ...
	    __i=`expr $__i + 1`
	    if [ "$__i" -ge 100 ]
	    then
		echo "_wait_for_pmie: failed to see pmie start after 100 iterations"
		_systemctl_status pmie
		return 1
	    fi
	    pmsleep 0.1
	else
	    # process has really started ...
	    return 0
	fi
    done
}

# Wait for primary pmie to end
#
_wait_pmie_end()
{
    __pidlist=`cat $PCP_RUN_DIR/pmie.pid 2>/dev/null`
    if [ -z "$__pidlist" ]
    then
	# no $PCP_RUN_DIR/pmie.pid file, try ps(1) the hard way ...
	# Note, this will wait for ALL pmie's running on the local machine.
	#
	__pidlist=`$PCP_PS_PROG $PCP_PS_ALL_FLAGS \
	| sed -n -e 's/$/ /' -e '/\[p]mie /{
s/ $//
s/^[^ 	]*//
s/^[ 	]*//
s/[ 	].*//
p
}'`
    fi
    if [ -z "$__pidlist" ]
    then
	return 0
    fi

    for __pid in $__pidlist
    do
	if _wait_process_end "_wait_pmie_end" "$__pid"
	then
	    continue
	else
	    $PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID|/[p]mie( |$)'
	    _systemctl_status pmie
	    return 1
	fi
    done
    return 0
}

# Wait for pmproxy to start up on given port
# Usage: _wait_for_pmproxy [port [logfile]]
#
_wait_for_pmproxy()
{
    case $#
    in
	0)
	    __port=44322
	    __logfile="$PCP_LOG_DIR/pmproxy/pmproxy.log"
	    ;;
	1)
	    __port="$1"
	    __logfile="$PCP_LOG_DIR/pmproxy/pmproxy.log"
	    ;;
	2)
	    __port="$1"
	    __logfile="$2"
	    ;;
	*)
	    echo "_wait_for_pmproxy(): wrong number of arguments"
	    return 1
	    ;;
    esac

    # 20 seconds default seems like a reasonable max time to get going
    __maxdelay=20

    __count=0
    while ! $PCP_BINADM_DIR/telnet-probe -v -c localhost $__port >>$seq_full 2>&1
    do
	__count=`expr $__count + 1`
	if [ $__count -ge $__maxdelay ]
	then
	    echo "pmproxy failed to start on port $__port after $__maxdelay seconds"
	    echo "likely looking processes ..."
	    $PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID|/[p]mproxy( |$)'
	    if [ -f "$__logfile" ]
	    then
		echo "pmproxy logfile ($__logfile) ..."
		cat "$__logfile"
	    else
		echo "pmproxy logfile ($__logfile) not created"
	    fi
	    _systemctl_status pmproxy
	    return 1
	fi
	sleep 1
    done
    return 0
}

# Wait for pmproxy server metrics. This should always be called after
# _wait_for_pmproxy. Note: some platforms might not have pmproxy metrics.
#
_wait_for_pmproxy_metrics()
{
    __n=0
    while true; do
	pminfo -f pmproxy.pid pmproxy.cpu pmproxy.map.instance.size >/dev/null 2>&1 && return 0
	sleep 0.25
	__n=`expr $__n + 1`
	[ $__n -lt 20 ] && continue
    done
    echo _wait_for_pmproxy_metrics failed after 20 iterations
    return 1
}

# Wait for pmproxy logfile creation. This should always be called after
# _wait_for_pmproxy.
#
_wait_for_pmproxy_logfile()
{
    __n=0
    __log="$1"
    while true; do
	test -f "$__log" && return 0
	sleep 0.25
	__n=`expr $__n + 1`
	[ $__n -lt 20 ] && continue
    done
    echo _wait_for_pmproxy_logfile failed after 20 iterations
    return 1
}

# Wait for pmproxy to end
#
_wait_pmproxy_end()
{
    __pid=`cat $PCP_RUN_DIR/pmproxy.pid 2>/dev/null`
    if [ -z "$__pid" ]
    then
	# no $PCP_RUN_DIR/proxy.pid file, try ps(1) the hard way
	#
	__pid=`$PCP_PS_PROG $PCP_PS_ALL_FLAGS \
	| sed -n -e 's/$/ /' -e '/\[p]mproxy /{
s/ $//
s/^[^ 	]*//
s/^[ 	]*//
s/[ 	].*//
p
}'`
    fi
    if [ -z "$__pid" ]
    then
	return 0
    fi

    if _wait_process_end "_wait_proxy_end" "$__pid"
    then
	return 0
    else
	_systemctl_status proxy
	return 1
    fi
}

# ensure primary logger allows pmlc state changes
#
_writable_primary_logger()
{
    __configfile="$PCP_VAR_DIR/config/pmlogger/config.default"

    echo "_open_primary_logger checking config file: $__configfile" >>$seq_full

    # first, move aside the default configuration file
    _save_config "$__configfile"

    # next, create a new default configuration file
    PMLOGCONF="$PCP_BINADM_DIR/pmlogconf"
    test -f "$__configfile" || \
	$sudo $PMLOGCONF -q -h localhost "$__configfile" >>$seq_full 2>&1

    # finally, open it up to local access from pmlc
    sed <"$__configfile" >$tmp.sed-i \
	-e 's/^allow localhost.*$/allow localhost : all;/g' \
	-e 's/^allow local:.*$/allow local:* : all;/g'
    $sudo cp $tmp.sed-i "$__configfile"
    rm -f $tmp.sed-i
}

# restore primary logger pmlc access restrictions
#
_restore_primary_logger()
{
    __configfile="$PCP_VAR_DIR/config/pmlogger/config.default"

    # if there was an initial configuration, put it back
    if test -f "$__configfile.$seq"
    then
	_restore_config "$__configfile"
    else
	# otherwise, close out access from local pmlc once more
	if test -f "$__configfile"
	then
	    sed <"$__configfile" >$tmp.sed-i \
		-e 's/^allow localhost.*$/allow localhost : enquire;/g' \
		-e 's/^allow local:.*$/allow local:* : enquire;/g'
	    $sudo cp $tmp.sed-i "$__configfile"
	    rm -f $tmp.sed-i
	fi
    fi
}

# restore pmlogger control file(s) to ensure only primary logger
# would be started (as in a default install) - i.e. remove farm.
#
_restore_pmlogger_control()
{
    for __file in `find $PCP_SYSCONF_DIR/pmlogger/control $PCP_SYSCONF_DIR/pmlogger/control.d -type f`
    do
	$PCP_AWK_PROG '/^\$/ || /^#/ || (/PCP_ARCHIVE_DIR/ && $2 == "y") {print}' \
		$__file >$tmp.new
	$sudo mv $tmp.new $__file
    done
}

# purify support
#
# typical usage:
#
# At the top before outputting $seq.out but after setting $seq ...
#   _check_purify prog
#
# Main code...
#   _setup_purify prog
#   _run_purify [arg1] [arg2]
#

# initial setup for running purify
# creates purified binary in $__purify_dir
#
_setup_purify()
{
    # Need to change these to match Purify setup locally, if you
    # have Purify!
    #
    LM_LICENSE_FILE=27000@snort.melbourne.sgi.com
    RSU_LICENSE_MAP=/usr/Rational/config/LICENSE_MAP
    export LM_LICENSE_FILE RSU_LICENSE_MAP

    __path=$1
    __prog=`echo $__path | sed -e 's#.*/##'`
    __pure_prog="$__prog.pure"
    __purify_dir=$tmp.purify

    rm -rf $__purify_dir
    mkdir $__purify_dir
    cp $__path $__purify_dir
    __here=`pwd`
    cd $__purify_dir

    cat >.purify<<EOF
suppress umr _write
suppress miu
EOF
    unset PURIFYOPTIONS PUREOPTIONS
    purify -log-file=stderr $__prog >out 2>&1
    if [ ! -x $__prog.pure ]
    then
	cat out
	echo "Hmm ... purify failed to create $__prog.pure"
	return 1
    fi
    cd $__here
    return 0
}

_run_purify()
{
    __here=`pwd`
    cd $__purify_dir
    $__purify_dir/$__pure_prog "$@" > $tmp._purify.out 2>&1
    cat $tmp._purify.out >>$__here/$seq_full
    if grep -i expired $tmp._purify.out >/dev/null; then
	cat $tmp._purify.out
    else
	_filter_purify < $tmp._purify.out
    fi
    cd $__here
}

_filter_purify()
{
    $PCP_AWK_PROG '
state == 0 && /License successfully checked out/	{ state = 1; next }
state == 0 && /Purify checking enabled/			{ state = 1; next }
state == 1						{ print }' \
    | sed \
	-e 's/pid [0-9][0-9]*/pid PID/g' \
	-e "s;$__purify_dir;TMP;g" \
	-e '/reserved for Purify internal use/d' \
	-e 's/suppressed chunks/suppressed blocks/g' \
    | $PCP_AWK_PROG -v __extra="$PURIFY_FILTER_EXTRA" '
/Purify instrumented/		{ skip = 0 }
/bytes leaked\./		{ print "..."; skip = 1; next }
skip == 1			{ next }
				{ print }
/Purify Heap Analysis/		{ print "..."; skip = 1 }
/Basic memory usage \(including Purify/		{ print "..."; skip = 1 }
__extra != "" && /Current file descriptors/	{ print "..."; skip = 1 }' \
    | (if [ "$PURIFY_FILTER_EXTRA" ]
    then sed -e 's/in use: [0-9][0-9]*/in use: N/'
    else cat -
    fi)
}

_check_purify()
{
    [ $# -eq 1 ] || _notrun "_check_purify needs executable as argument"
    __path=$1
    which purify >/dev/null 2>&1
    [ $? -eq 0 ] || _notrun "No purify binary found"
}

# check that test is not running on a 32 bit environment
_check_64bit_platform()
{
    __message="$@"
    [ -z "$__message" ] && __message="Test requires 64-bit platform"
    #
    # use src/sizeof not hinv.machine (or uname -m) to avoid confusion
    # when the environment is a 32-bit container running on a 64-bit
    # host, like test-ubuntu1804i386-container in CI
    #
    eval __`$here/src/sizeof ptr`
    echo "__ptr=$__ptr" >>$seq_full
    case "$__ptr"
    in
	4)
	    _notrun "$__message"
	    ;;
    esac
}

# check we have pmseries and a key server -cli and -server binary installed.
# Note: use _check_key_server to test if a key server is running locally.
#
_check_series()
{
    which pmseries >/dev/null 2>&1 || \
	_notrun "pmseries command line utility not installed"

    if which valkey-cli >/dev/null 2>&1
    then
	key_server=valkey-server
    elif which redis-cli >/dev/null 2>&1
    then
	key_server=redis-server
    else
	_notrun "No key server command line interface found"
    fi

    which $key_server >/dev/null 2>&1 || _notrun "No $key_server binary found"
}

# check key server version
# ... assumes _check_series already done and a key server is running
#
_check_key_server_version()
{
    if [ $# -ne 1 ]
    then
	echo "_check_key_server_version: botch: needs port# as only argument"
	return
    fi

    
    if which valkey-cli >/dev/null 2>&1 && test -n `_get_pids_by_name valkey-server`
    then
	keys_cli=valkey-cli
	keys_ver=valkey_version
    elif which redis-cli >/dev/null 2>&1 && test -n _get_pids_by_name redis-server
    then
	keys_cli=redis-cli
	keys_ver=redis_version
    else
	_notrun "No key server command line interface found or the key server is not running"
    fi
    eval __key_server_version=`$keys_cli -p "$1" info server |
	grep "$keys_ver:" | sed -e 's/^.*://g' -e 's/\..*//g'`
    [ -z "$__key_server_version" ] && _notrun "No key server version extracted"
    [ $__key_server_version -ge 5 ] || _notrun "Key server @ port $1: version $__key_server_version too old"
}

# check key server version without a running key server
_check_key_server_version_offline()
{
    if which valkey-server >/dev/null 2>&1
    then
	key_server=valkey-server
    elif which redis-server >/dev/null 2>&1
    then
	key_server=redis-server
    else
	_notrun "No key server binary found"
    fi
    __key_server_version=`$key_server --version | sed -E 's/.*v=([0-9]).*/\1/g'`
    [ $__key_server_version -ge 5 ] || _notrun "Key server version $__key_server_version too old"
}

# Check a key server is installed and running locally and the version
# is not too old. Optional port parameter defaults to 6379
#
_check_key_server()
{
    _check_series

    if which valkey-cli >/dev/null 2>&1
    then
	keys_cli=valkey-cli
    elif which redis-cli >/dev/null 2>&1
    then
	keys_cli=redis-cli
    else
	_notrun "No key server command line interface found"
    fi

    __key_server_port=6379
    [ -n "$1" ] && __key_server_port="$1"
    $keys_cli -p $__key_server_port ping >/dev/null
    [ $? -eq 0 ] || _notrun "Key server not running locally"
    _check_key_server_version $__key_server_port
}

# be prepared to "ping" the key server a few times until we see
# the expected response
#
_check_key_server_ping()
{
    if [ $# -ne 1 ]
    then
	echo "_check_key_server_ping: botch: needs port# as only argument"
	return
    fi

    if which valkey-cli >/dev/null 2>&1
    then
	keys_cli=valkey-cli
    elif which redis-cli >/dev/null 2>&1
    then
	keys_cli=redis-cli
    else
	_notrun "No key server command line interface found"
    fi

    echo "PING"
    __i=0
    while [ $__i -lt 10 ]
    do
	$keys_cli -p $1 ping >$tmp.keys.tmp 2>&1
	if grep PONG $tmp.keys.tmp
	then
	    rm -f $tmp.keys.tmp
	    break
	fi
	pmsleep 0.125
	__i=`expr $__i + 1`
    done
    [ -f $tmp.keys.tmp ] && cat $tmp.keys.tmp
}

# Check if the key server search module is installed
_check_search()
{
    which pmsearch >/dev/null 2>&1 || \
	_notrun "pmsearch command line utility not installed"

    __key_search=`_find_key_server_search`
    __key_server_path=`_find_key_server_modules`
    __key_server_search="$__key_server_path/$__key_search.$DSO_SUFFIX"
    # module path is not accessibly by the pcpqa user on Fedora/RHEL
    sudo test -f $__key_server_search || \
	_notrun "$__key_search module is not installed"
}

_find_key_server_search()
{
    if which valkey-server >/dev/null 2>&1
    then
	echo valkeysearch
    elif which redis-server >/dev/null 2>&1
    then
	echo redisearch
    else
	echo
    fi
}

_find_key_server_name()
{
    if which valkey-server >/dev/null 2>&1
    then
	echo valkey
    elif which redis-server >/dev/null 2>&1
    then
	echo redis
    else
	echo
    fi
}

_find_key_server_modules()
{
    __key_server_name=`_find_key_server_name`
    # scan through a list of known locations for modules
    if sudo test -d "$PCP_LIB_DIR/$__key_server_name/modules"
    then
	echo "$PCP_LIB_DIR/$__key_server_name/modules"
    else
	echo "$PCP_LIB32_DIR/$__key_server_name/modules"
    fi
}

_check_display()
{
    # note: non-X systems (macOS, Windows) must pass here unchallenged
    if [ -z "$PCPQA_CLOSE_X_SERVER" ]
    then
	_notrun "\$PCPQA_CLOSE_X_SERVER not set in ./common.config"
	# NOTREACHED
    fi
    DISPLAY=$PCPQA_CLOSE_X_SERVER; export DISPLAY
    if which xdpyinfo >/dev/null 2>&1
    then
	xdpyinfo >/dev/null 2>&1 || \
	    _notrun "Failed sanity check on DISPLAY $PCPQA_CLOSE_X_SERVER"
    fi

    if [ "$PCPQA_DESKTOP_HACK" = true ]
    then
	# hackery for warning:
	# QStandardPaths: XDG_RUNTIME_DIR not set, defaulting to '/tmp/...'
	mkdir -p "$tmp/xdg-runtime"
	chmod 0700 "$tmp/xdg-runtime"
	XDG_RUNTIME_DIR="$tmp/xdg-runtime"; export XDG_RUNTIME_DIR
    fi
}

_clean_display()
{
    # on some systems, we see these processes left running
    # kenj      6612     1  0 08:10 ?        00:00:00 /usr/lib/gvfs/gvfsd-fuse /var/tmp/1003-6479/xdg-runtime/gvfs -f
    # ... need to kill 'em before the xdg-runtime dir can be culled
    #
    if [ "$PCPQA_DESKTOP_HACK" = true ]
    then
	$PCP_PS_PROG $PCP_PS_ALL_FLAGS \
	| grep " $tmp/[x]dg-runtime" \
	| $PCP_AWK_PROG '{print $2}' \
	| while read pid
	do
	    kill -HUP $pid >/dev/null 2>&1
	    sleep 1
	    kill -TERM $pid >/dev/null 2>&1
	    sleep 1
	done
	test -d "$tmp/xdg-runtime" && rm -fr "$tmp/xdg-runtime"
    fi
}

# valgrind support
#
# typical usage:
#
# At the top before outputting $seq.out but after setting $seq ...
#   _check_valgrind
#
# Main code...
#   _run_valgrind [--sudo] [--save-output] app [arg1 [arg2] ... ]
# or
#   _run_helgrind [--sudo] app [arg1 [arg2] ... ]
#

_check_valgrind()
{
    which valgrind >/dev/null 2>&1
    [ $? -eq 0 ] || _notrun "No valgrind binary found"
    if [ "$PCP_PLATFORM" = freebsd ]
    then
	_notrun "valgrind and PCP QA broken on FreeBSD"
    fi
    __extra=''
}

# if both valgrind and non-valgrind tests are available, then prefer
# to run only one of the variants ... should we run the valgrind one
# only?
#
_prefer_valgrind()
{
    # QA hook to run _both_ valgrind and non-valgrind variants
    #
    [ "$PCPQA_PREFER_VALGRIND" = no ] && return 1

    # no valgrind executable => don't prefer valgrind
    #
    if which valgrind >/dev/null 2>&1
    then
	:
    else
	return 1
    fi

    # and on some QA platforms, even when valgrind is present it
    # is broken or produces radically different output/results,
    # so don't use it in these cases
    #
    if [ "$PCP_PLATFORM" = freebsd ]
    then
	return 1
    fi

    return 0
}

# helgrind is just broken in some places ... assume we've already
# run _check_valgrind, so only issue here is do we trust the helgrind
# module of valgrind
#
_check_helgrind()
{
    case "$__whatami"
    in
	*Fedora\ 33*)
	    _notrun "helgrind broken on Fedora 33"
	    ;;
	*Fedora\ 34*)
	    _notrun "helgrind broken on Fedora 34"
	    ;;
	*Arch\ Linux*)
	    _notrun "helgrind broken on Arch Linux"
	    ;;
    esac
}

# Note: because we may be avoiding leaks in system libraries
# via the suppressions, there is potential indeterminism for
# the following case:
# -All heap blocks were freed -- no leaks are possible
# +LEAK SUMMARY:
# +definitely lost: 0 bytes in 0 blocks
# +indirectly lost: 0 bytes in 0 blocks
#
# Note
#	- last 6 sed expressions are for broken valgrind(1) on
#	  Debian stretch circa 2016
#	- one before that is for broken valgrind(1) on Arch Linux
#	  circa Nov 2019
#       - awk at the end is to handle this observed on Arch Linux
#	  circa May 2023
#	   WARNING: unhandled amd64-linux syscall: 441
#	   You may be able to write your own handler.
#	   Read the file README_MISSING_SYSCALL_OR_IOCTL.
#	   Nevertheless we consider this a bug.  Please report
#	   it at http://valgrind.org/support/bug_reports.html.
# 
_filter_valgrind()
{
    __noleak="LEAK SUMMARY:\ndefinitely lost: 0 bytes in 0 blocks\nindirectly lost: 0 bytes in 0 blocks"
    sed \
	-e 's/^==*[1-9][0-9]*==* *//' \
	-e '/^$/d' \
	-e '/^Copyright (/d' \
	-e '/^Using Valgrind-/d' \
	-e '/^Parent PID:/d' \
	-e '/^HEAP SUMMARY:/d' \
	-e '/^in use at exit:/d' \
	-e '/^total heap usage:/d' \
	-e '/^possibly lost:/d' \
	-e '/^still reachable:/d' \
	-e '/^of which reachable via heuristic:/d' \
	-e '/^newarray .*:/d' \
	-e '/^suppressed:/d' \
	-e '/^Reachable blocks (those to which a pointer was found)/d' \
	-e '/^To see them, rerun with:/d' \
	-e '/^For lists of detected and suppressed errors,/d' \
	-e '/^For counts of detected and suppressed errors,/d' \
	-e '/^ERROR SUMMARY:/s/ (suppressed: [^)]*)/ .../' \
	-e "s/^All heap blocks were freed -- .*/$__noleak/g" \
	-e '/warning: addVar: in range .* outside all rx mapped areas/d' \
	-e '/WARNING: Serious error when reading debug info/d' \
	-e '/When reading debug info from /d' \
	-e '/Ignoring non-Dwarf2\/3\/4 block in .debug_info/d' \
	-e '/Last block truncated in .debug_info; ignoring/d' \
	-e '/parse_CU_Header: is neither DWARF2 nor DWARF3 nor DWARF4/d' \
	-e '/DW_FORM_GNU_strp_alt used, but no alternate .debug_str/d' \
	-e '/^parse DIE/,/confused by the above DIE/d' \
    | $PCP_AWK_PROG '
BEGIN					{ skip = 0 }
/WARNING: unhandled .* syscall/		{ skip = 1 }
skip == 0				{ print }
/at http:\/\/valgrind.org\/support/	{ skip = 0 }'
}

_filter_helgrind()
{
    sed \
	-e 's/^==*[1-9][0-9]*==* *//' \
	-e '/^$/d' \
	-e '/^Parent PID:/d' \
	-e '/^Copyright (/d' \
	-e '/^Using Valgrind-/d' \
	-e '/^Command: /d' \
	-e '/^For lists of detected and suppressed errors,/d' \
	-e '/^For counts of detected and suppressed errors,/d' \
	-e '/^Use --history-level=approx or =none/d' \
	-e '/^the cost of reduced accuracy of conflicting-access/d' \
	-e '/^ERROR SUMMARY:/s/ (suppressed: [^)]*)/ .../' \
    # end
}

# options:
# --sudo
# 	run $sudo valgrind ...
# --save-output
# 	don't cat output files, save 'em to $tmp.out and $tmp.err
#
_run_valgrind()
{
    if [ x"$1" = "x--sudo" ]
    then
	__valgrind_prefix=$sudo
	shift
    else
	__valgrind_prefix=''
    fi
    if [ x"$1" = "x--save-output" ]
    then
	__valgrind_save_output=true
	shift
    else
	__valgrind_save_output=false
    fi

    # extract version number I.J.K ... ignore anything after that,
    # e.g. .SVN or .SVN-Debian for Debian-based distros
    #
    __version=`valgrind --version | sed -e 's/valgrind-//' -e 's/\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*/\1/'`
    __extra=''
    if [ -f $here/valgrind-suppress-$__version ]
    then
	__extra="--suppressions=$here/valgrind-suppress-$__version"
    else
	echo "Warning: no extra suppressions found for valgrind version $__version" >>$seq_full
    fi
    valgrind --help >$tmp.valgrind.help 2>&1
    if grep -q .--read-inline-info= <$tmp.valgrind.help
    then
	# --read-inline-info added in valgrind 3.13, --read-var-info appears
	# in much earlier releases, but was fixed around valgrind 3.8 or 3.9
	#
	# *Note* - for more detailed diagnostics, at the expense of runtime,
	# use
	# __extra="$__extra --track-origins=yes --read-var-info=yes --read-inline-info=yes"
	__extra="$__extra --read-var-info=no --read-inline-info=yes"
    fi
    # $grind_extra may be used in the caller to pass additional command
    # line options to valgrind
    #
    [ -n "$grind_extra" ] && __extra="$__extra $grind_extra"
    $sudo rm -f $tmp.valgrind.help
    if [ -n "$__extra" ]
    then
	echo "Warning: using extra $__extra" >>$seq_full
    fi
    case "$__whatami"
    in
	*Arch\ Linux*)
	    # need $DEBUGINFOD_URLS set or valgrind dies on Arch Linux
	    # -- seen on vm34 circa May 2023
	    #
	    ;;
	*)
	    unset DEBUGINFOD_URLS
	    ;;
    esac
    $__valgrind_prefix valgrind \
	--leak-check=full \
	--gen-suppressions=all --suppressions=$here/valgrind-suppress \
	$__extra \
	--log-file=$tmp._valgrind \
	    "$@" 2>$tmp._valgrind.err >$tmp._valgrind.out
    if $__valgrind_save_output
    then
	touch $tmp.out
	$sudo cp $tmp._valgrind.out $tmp.out
	touch $tmp.err
	$sudo cp $tmp._valgrind.err $tmp.err
    else
	echo "=== std out ==="
	$sudo cat $tmp._valgrind.out
	echo "=== std err ==="
	$sudo cat $tmp._valgrind.err
    fi
    echo "=== valgrind report ===" >>$seq_full
    $sudo cat $tmp._valgrind >>$seq_full
    echo "=== filtered valgrind report ==="
    $sudo cat $tmp._valgrind | _filter_valgrind
}

# Alternate invocation, using valgrind-silence as an assertion.
# Produces no stdout/stderr from valgrind - except in case of errors!
# Uses same suppression logic as _run_valgrind.
#
# Usage:                    $valgrind_clean_assert CMD ARGS ...
# just as you'd have used   CMD ARGS ...
# alone, or                 |filtered
# or                        & backgrounded
#
# It works even if valgrind is not available (expanding to nothing).
#
if which valgrind >/dev/null 2>&1; then
    # Note:
    # 	$here and $seq may not be set here, and this is not a shell
    # 	procedure, so skip any echoing
    #
    __version=`valgrind --version | sed -e 's/valgrind-//'`
    __extra=''
    if [ -f $here/valgrind-suppress-$__version ]
    then
	__extra="--suppressions=$here/valgrind-suppress-$__version"
    fi
    valgrind --help >$tmp.valgrind.help 2>&1
    grep -q .--vgdb= <$tmp.valgrind.help && __extra="$__extra --vgdb=no"
    if grep -q .--show-leak-kinds= <$tmp.valgrind.help
    then
	__extra="$__extra --show-leak-kinds=definite"
    else
	if grep -q .--show-possibly-lost= <$tmp.valgrind.help
	then
	    __extra="$__extra --show-reachable=no --show-possibly-lost=no"
	else
	    __extra="$__extra --show-reachable=no"
	fi
    fi
    if grep -q .--read-inline-info= <$tmp.valgrind.help
    then
	# --read-inline-info added in valgrind 3.13, --read-var-info appears
	# in much earlier releases, but was fixed around valgrind 3.8 or 3.9
	#
	# *Note* - for more detailed diagnostics, at the expense of runtime,
	# use
	# $__extra="$__extra --track-origins=yes --read-var-info=yes --read-inline-info=yes"
	__extra="$__extra --read-inline-info=no --read-var-info=no"
    fi
    $sudo rm -f $tmp.valgrind.help
    case "$__whatami"
    in
	*Arch\ Linux*)
	    # need $DEBUGINFOD_URLS set or valgrind dies on Arch Linux
	    # -- seen on vm34 circa May 2023
	    #
	    ;;
	*)
	    unset DEBUGINFOD_URLS
	    ;;
    esac
    # $grind_extra may be used in the caller to pass additional command
    # line options to valgrind
    #
    [ -n "$grind_extra" ] && __extra="$__extra $grind_extra"
    valgrind_clean_assert="valgrind -q \
	     --leak-check=full \
	     --gen-suppressions=all --suppressions=$here/valgrind-suppress \
	     $__extra \
	     --log-fd=1"
else
    valgrind_clean_assert=
fi

_run_helgrind()
{
    $sudo rm -f $tmp.valgrind.err $tmp.valgrind.out $tmp.valgrind.log
    touch $tmp.valgrind.err $tmp.valgrind.out
    if [ x"$1" = "x--sudo" ]
    then
	$sudo touch $tmp.valgrind.log
	__helgrind_prefix=$sudo
	shift
    else
	touch $tmp.valgrind.log
	__helgrind_prefix=''
    fi
    __version=`valgrind --version | sed -e 's/valgrind-//'`
    __extra=''
    if [ -f $here/helgrind-suppress-$__version ]
    then
	__extra="--suppressions=$here/helgrind-suppress-$__version"
    else
	echo "Warning: no extra suppressions found for helgrind version $__version" >>$seq_full
    fi
    # $grind_extra may be used in the caller to pass additional command
    # line options to valgrind
    #
    [ -n "$grind_extra" ] && __extra="$__extra $grind_extra"
    if [ -n "$__extra" ]
    then
	echo "Warning: using extra $__extra" >>$seq_full
    fi

    $__helgrind_prefix valgrind \
	--tool=helgrind --gen-suppressions=all \
	--suppressions=$here/helgrind-suppress $__extra \
	--log-file=$tmp.valgrind.log \
	    "$@" 2>$tmp.valgrind.err >$tmp.valgrind.out
    echo "=== valgrind report ===" >>$seq_full
    $sudo cat $tmp.valgrind.log >>$seq_full
    echo "=== filtered valgrind report ==="
    $sudo cat $tmp.valgrind.log | _filter_helgrind
}

#
# Checks that given_value is in range of correct_value +/- tolerance.
# Tolerance can be an absolute value or a percentage of the correct value
# (see examples with tolerances below).
# Outputs suitable message to stdout if it's not in range.
#
# A verbose option, -v, may be used as the LAST argument
# 
# e.g. 
# foo: 0.0298 = 0.03 +/- 5%
# _within_tolerance "foo" 0.0298 0.03 5%  
# 
# foo: 0.0298 = 0.03 +/- 0.01
# _within_tolerance "foo" 0.0298 0.03 0.01
#
# foo: 0.0298 = 0.03 -0.01 +0.002
# _within_tolerance "foo" 0.0298 0.03 0.01 0.002
#
# foo: verbose output of 0.0298 = 0.03 +/- 5% 
# _within_tolerance "foo" 0.0298 0.03 5% -v 

_within_tolerance()
{
  __name=$1
  __given_val=$2
  __correct_val=$3
  __mintol=$4
  __maxtol=$__mintol
  __verbose=0
  __debug=false

  # maxtol arg is optional
  # verbose arg is optional
  if [ $# -ge 5 ]
  then 
     if [ "$5" = "-v" ]
     then
	__verbose=1
     else
        __maxtol=$5
     fi
  fi
  if [ $# -ge 6 ]
  then
     [ "$6" = "-v" ] && __verbose=1
  fi

  # find min with or without %
  __mintolerance=`echo $__mintol | sed -e 's/%//'` 
  if [ $__mintol = $__mintolerance ]
  then 
      __min=`echo "scale=5; $__correct_val-$__mintolerance" | bc`
  else
      __min=`echo "scale=5; $__correct_val-$__mintolerance*0.01*$__correct_val" | bc`
  fi

  # find max with or without %
  __maxtolerance=`echo $__maxtol | sed -e 's/%//'` 
  if [ $__maxtol = $__maxtolerance ]
  then 
      __max=`echo "scale=5; $__correct_val+$__maxtolerance" | bc`
  else
      __max=`echo "scale=5; $__correct_val+$__maxtolerance*0.01*$__correct_val" | bc`
  fi

  $__debug && echo "min = $__min"
  $__debug && echo "max = $__max"

  cat <<EOF > $tmp._bc.1
scale=5;
if ($__min <= $__given_val) 1;
if ($__min > $__given_val) 0; 
EOF

  cat <<EOF > $tmp._bc.2
scale=5;
if ($__given_val <= $__max) 1;
if ($__given_val > $__max) 0;
EOF

  __above_min=`bc < $tmp._bc.1`
  __below_max=`bc < $tmp._bc.2`

  __in_range=`expr $__above_min \& $__below_max` 

  # fix up min, max precision for output
  # can vary for 5.3, 6.2
  __min=`echo $__min | sed -e 's/\(.\)0*$/\1/'` # get rid of trailing zeroes
  __max=`echo $__max | sed -e 's/\(.\)0*$/\1/'` # get rid of trailing zeroes

  if [ $__in_range -eq 1 ] 
  then
	[ $__verbose -eq 1 ] && echo $__name is in range
	return 0
  else
	[ $__verbose -eq 1 ] && echo $__name has value of $__given_val
	[ $__verbose -eq 1 ] && echo $__name is NOT in range $__min .. $__max	
	return 1
  fi
}

# disable systemd timers and / or comment pmlogger_check and pmsnap entries in
# the crontab file (also cron.pmcheck and cron.pmsnap entries for backwards
# compatibility)
# Usage: _remove_job_scheduler cron_backup systemd_state sudo
#
# cron_backup - where to keep the old crontab file
# systemd_state - where to keep systemd timer state
# sudo - location of sudo
#
_remove_job_scheduler()
{
    __rc_cron_backup=${1:-crontab}
    __rc_systemd_state=${2:-systemd.state}
    __rc_sudo=${3:-sudo}

    $__rc_sudo rm -f $__rc_cron_backup $__rc_systemd_state

    if systemctl cat pmie_daily.timer >/dev/null 2>&1; then
	for __i in pmie.service pmie_daily.timer \
		pmie_check.timer pmie_farm_check.timer \
		pmlogger_daily.timer \
		pmlogger_check.timer pmlogger_farm_check.timer ; do
	    $__rc_sudo systemctl is-active $__i > /dev/null || continue
	    $__rc_sudo systemctl stop $__i >/dev/null
	    echo $__i >> $__rc_systemd_state
	done
    fi

    which crontab >/dev/null 2>&1 || return	# systemd timers only

    if $__rc_sudo crontab -l 2>/dev/null >$__rc_cron_backup
    then
	# Need to cull lines like this, so we end up whith just the
	# real crontab entries (if any)
	#
	# DO NOT EDIT THIS FILE - edit the primary and reinstall.
	# (/dev/null installed on Sun Feb  2 07:30:33 2020)
	# (Cronie version 4.2)
	sed <$__rc_cron_backup >$tmp.__cron \
	    -e '/^# DO NOT EDIT THIS FILE/d' \
	    -e '/^# (.* installed on /d' \
	    -e '/^# (Cronie/d' \
	# end
	cp $tmp.__cron $__rc_cron_backup
    else
	# error, remove the backup file so no changes are made
	rm -f $__rc_cron_backup
    fi

    if [ -s $__rc_cron_backup ]
    then
	$__rc_sudo cat $__rc_cron_backup \
	| sed \
	    -e 's/^[^#].*pmlogger_check/#&/' \
	    -e 's/^[^#].*pmsnap/#&/' \
	    -e 's/^[^#].*cron.pmcheck/#&/' \
	    -e 's/^[^#].*cron.pmsnap/#&/' \
	| $__rc_sudo crontab > /dev/null 2>&1
    fi
}

# restore systemd timers and / or crontab back to original state
# Usage: _restore_job_scheduler cron_backup systemd_state sudo
#
# cron_backup - where to keep the old crontab file
# systemd_state - where to keep systemd timer state
# sudo - location of sudo
#
_restore_job_scheduler()
{
    __rc_cron_backup=${1:-crontab}
    __rc_systemd_state=${2:-systemd.state}
    __rc_sudo=${3:-sudo}

    if [ -s $__rc_systemd_state ]; then
	for __i in `cat $__rc_systemd_state`; do
	    $__rc_sudo systemctl start $__i >/dev/null
	done
	rm -f $__rc_systemd_state
    fi

    if [ -s $__rc_cron_backup ]
    then
	$__rc_sudo rm -f rc_cron_out rc_cron_check rc_cron_diff
	if $__rc_sudo crontab $__rc_cron_backup >rc_cron_out 2>&1
	then
	    # check everything is OK
	    #
	    $__rc_sudo crontab -l >rc_cron_check
	    sed -e '/^#/d' $__rc_cron_backup >$__rc_cron_backup.clean
	    sed -e '/^#/d' rc_cron_check >rc_cron_check.clean
	    if diff -u $__rc_cron_backup.clean rc_cron_check.clean >rc_cron_diff 2>&1
	    then
		:
	    else
		echo "_restore_cron: Warning: could not restore crontab to original state"
		echo "               Unexpected differences ..."
		diff -u $__rc_cron_backup rc_cron_check
	    fi
	    $__rc_sudo rm -f rc_cron_check rc_cron_check.clean rc_cron_diff
	else
	    echo "_restore_cron: Warning: could not restore crontab to original state"
	    echo "               crontab(1) failed ..."
	    cat rc_cron_out
	fi
	$__rc_sudo rm -f rc_cron_out rc_cron_check rc_cron_diff
    fi
}

# running QA within a container can mean no crontab setup;
# _check_job_scheduler() is a convenience routine checking the external
# dependencies of _remove_job_scheduler() and _restore_job_scheduler()
#
_check_job_scheduler()
{
    systemctl cat pmie_daily.timer >/dev/null 2>&1 && return
    which crontab >/dev/null 2>&1 || _notrun "No crontab binary found"
}

# get offset into an archive relative to the first pmResult
# past the preamble
#
# Usage: _arch_start archive [offset]
#
_arch_start()
{
    pmdumplog -z $1 \
    | $PCP_AWK_PROG '
/^[0-9][0-9]:[0-9][0-9]:/	{ if ($3 ~ /pmcd.pmlogger.host/) next
				  split($1, t, ":")
				  t[3] += '"${2-0}"'
				  while (t[3] < 0) {
				    t[3] += 60
				    t[2]--
				  }
				  while (t[3] > 60) {
				    t[3] -= 60
				    t[2]++
				  }
				  while (t[2] < 0) {
				    t[2] += 60
				    t[1]--
				  }
				  while (t[2] > 60) {
				    t[2] -= 60
				    t[1]++
				  }
				  while (t[1] < 0)
				    t[1] += 24
				  while (t[1] > 23)
				    t[1] -= 24
				  printf "@%02d:%02d:%06.3f",t[1],t[2],t[3]
				  exit
				}'
}

# get an unused ipc port ... returned on std out, empty for failure
# Usage: _get_port tcp|udp low_port high_port
#
_get_port()
{
    [ $# -ne 3 ] && return
    __proto=$1
    __port=$2
    while [ $__port -le "$3" ]
    do
	if $sudo fuser $__port/$__proto >/dev/null 2>&1
	then
	    :
	else
	    echo $__port
	    return
	fi
	__port=`expr $__port + 1`
    done
}

# get host endianness ("le" or "be")
_get_endian()
{
    __check=`echo a | od -x 2>&1 | sed -e 's/^0[^ ]* *//' -e 's/ //g' -e '/^$/d'`
    case "$__check"
    in
    0a61) echo le ;;
    610a) echo be ;;
    *)
	echo >&2 "Arrgh ... od -x returned something odd ($__check)"
	echo >&2 a | od -x
	echo "??"
	return 1
	;;
    esac
    return 0
}

# get host word size ("32" or "64")
_get_word_size()
{
    __machine=`uname -m`
    case "$__machine"
    in
    i?86|athlon|ppc)
	__size=32
	# but, apps are 64-bit on modern macOS
	[ $PCP_PLATFORM = darwin ] && __size=64
	;;
    x86_64|ia64|ppc64|ppc64le|s390x|aarch64)
	__size=64
	;;
    *)
	echo >&2 "uname gave machine type with unknown word size ($__machine)"
	echo 0
	return 1
	;;
    esac
    echo $__size
    return 0
}

# _all_hostnames host - generate all hostnames (or IP addresses) for this host,
#			that map to some network interface, excluding loopback
#
_all_hostnames()
{
    touch $tmp._addr
    ssh pcpqa@$1 </dev/null netstat -in 2>/dev/null >$tmp._tmp
    if grep 'Network.*Address' $tmp._tmp >/dev/null
    then
	# This is the IRIX version of netstat -in, get IP addr from the
	# Address field
	# 
	# Name   Mtu   Network         Address            Ipkts Ierrs ...
	# ef0    1500  134.14.55.128   134.14.55.149  712168207    10 ...
	#                              134.14.55.159      
	#                              134.14.55.147      
	# ef2*   1500  none            none                   0     0 ...
	# lo0    32992 127             127.0.0.1       23628402     0 ...
	#
	$PCP_AWK_PROG <$tmp._tmp >$tmp._addr '
/^lo/									{ next }
NF >= 4 && $4 ~ /^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$/ 	{ print $4 }
NF == 1 && $1 ~ /^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$/	{ print $1 }
END									{ print "End-of-List" }'
    else
	ssh pcpqa@$1 </dev/null /sbin/ifconfig 2>/dev/null >$tmp._tmp
	if grep 'UP.*RUNNING' $tmp._tmp >/dev/null
	then
	    # This is the Linux version of ifconfig, get IP addr from the
	    # inet addr: line
	    #
	    # eth0      Link encap:Ethernet  HWaddr 00:90:27:98:EE:A8  
	    #	        inet addr:134.14.55.176  Bcast:134.14.55.255  ...
	    #	        UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
	    #	        ...
	    #
	    # lo        Link encap:Local Loopback  
	    #	        inet addr:127.0.0.1  Mask:255.0.0.0
	    #	        UP LOOPBACK RUNNING  MTU:16436  Metric:1
	    #	        ...
	    #
	    # Note addr: tag is not present in some ifconfig output
	    #
	    $PCP_AWK_PROG <$tmp._tmp '
/^lo/						{ skip = 1; next }
skip == 1 && NF > 0				{ next }
skip == 1 					{ skip = 0 }
$1 == "inet" && $2 ~ /addr:/			{ print $2 }
$1 == "inet" && $2 ~ /^[0-9]/			{ print $2 }
END						{ print "End-of-List" }' \
	    | sed -e 's/addr://' >$tmp._addr
	else
	    # Nothing we can do really, as there is no way of passing
	    # an error back from here, other than returning an empty
	    # list
	    return
	fi
    fi
    cat $tmp._addr \
    | while read __ip
    do
	if [ "$__ip" = "End-of-List" ]
	then
	    echo
	    break
	fi
	# check that ip addr is reachable
	if ping -c 1 $__ip >/dev/null 2>&1
	then
	    __host=`_ipaddr_to_host $__ip`
	    if [ ! -z "$__host" ]
	    then
		$PCP_ECHO_PROG $PCP_ECHO_N ",$__host""$PCP_ECHO_C"
	    else
		$PCP_ECHO_PROG $PCP_ECHO_N ",$__ip""$PCP_ECHO_C"
	    fi
	fi
    done \
    | sed -e 's/^,//'
}

# _all_ipaddrs - generate all IP addresses for this host,
#			that map to some network interface, excluding
#			loopback, slip, ppp
#
# See _all_hostnames() above for comments on the method used.
#
_all_ipaddrs()
{
    touch $tmp._addr
    if [ "$1" = "localhost" ]
    then 
	netstat -in 2>/dev/null >$tmp._tmp
    else
	ssh pcpqa@$1 </dev/null netstat -in 2>/dev/null >$tmp._tmp
    fi
    if grep 'Network.*Address' $tmp._tmp >/dev/null
    then
	# this is the IRIX version of netstat -in, get IP addr from the
	# Address field
	#
	$PCP_AWK_PROG <$tmp._tmp >$tmp._addr '
/^lo/ || /^sl/ || /^pp/							{ next }
NF >= 4 && $4 ~ /^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$/ 	{ print $4 }
NF == 1 && $1 ~ /^[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*$/	{ print $1 }
END									{ print "End-of-List" }'
    else
        if [ "$1" = "localhost" ]
	then 
	    /sbin/ifconfig 2>/dev/null >$tmp._tmp
	else
	    ssh pcpqa@$1 </dev/null /sbin/ifconfig 2>/dev/null >$tmp._tmp
	fi
	if grep 'UP.*RUNNING' $tmp._tmp >/dev/null
	then
	    # This is the Linux version of ifconfig, get IP addr from the
	    # inet addr: line
	    #
# ppp0      Link encap:Point-to-Point Protocol
#           inet addr:134.14.52.219  P-t-P:134.14.52.189  Mask:255.255.255.255
#           UP POINTOPOINT RUNNING NOARP MULTICAST  MTU:1522  Metric:1
#           RX packets:50119 errors:0 dropped:0 overruns:0 frame:0
#           TX packets:47474 errors:0 dropped:0 overruns:0 carrier:0
#           collisions:0 txqueuelen:3
#           RX bytes:7017171 (6.6 Mb)  TX bytes:3952015 (3.7 Mb)
	    #
	    # Note addr: tag is not present in some ifconfig output,
	    #      AND UP comes first!
	    #
	    $PCP_AWK_PROG <$tmp._tmp '
/^[a-z]/					{ loopback = 0 }
/^lo/						{ loopback = 1; next }
$1 == "inet" && $2 ~ /addr:/			{ save = $2; next }
$1 == "inet" && $2 ~ /^[0-9]/			{ save = $2; next }
$1 == "TX"					{ if (loopback == 0 && save != 0)
						    print save
						  save = 0
						}
END						{ print "End-of-List" }' \
	    | sed -e 's/addr://' >$tmp._addr
	else
	    # Nothing we can do really, as there is no way of passing
	    # an error back from here, other than returning an empty
	    # list
	    return
	fi
    fi
    cat $tmp._addr \
    | while read __ip
    do
	if [ "$__ip" = "End-of-List" ]
	then
	    echo
	    break
	fi
	$PCP_ECHO_PROG $PCP_ECHO_N ",$__ip""$PCP_ECHO_C"
    done \
    | sed -e 's/^,//'
}

# fqdn for localhost
# 
_get_fqdn()
{
    _host_to_fqdn `hostname`
}

# Distro-specific filtering for init, rc scripts, chkconfig, et al
#
_filter_init_distro()
{
    if [ -f /etc/mandriva-release ]
    then
	# looks like this is a Mandriva bug ... see
	# http://mandriva.598463.n5.nabble.com/Bug-24409-initscripts-New-netfs-provides-local-fs-scripts-can-t-be-turned-off-td869820.html
	#
	sed \
	    -e '/Warning: netfs is needed by pcp in runlevel/d'
    else
	cat
    fi
}

# deal with chkconfig et al
# assumes $sudo is set correctly
# try very hard to _not_ emit messages unless serious errors encountered
#
_change_config()
{
    if [ $PCP_PLATFORM = linux ]
    then
	__pat=$1
	[ "$1" = "verbose" ] && __pat=""

	__have_systemctl=false
	[ "$PCPQA_SYSTEMD" = yes ] && __have_systemctl=true
	if [ -z "$__pat" ]
	then
	    __have_service=true
	elif [ -n "$PCP_SYSTEMDUNIT_DIR" -a -f $PCP_SYSTEMDUNIT_DIR/$__pat.service ]
	then
	    __have_service=true
	else
	    __have_service=false
	fi
	if $__have_systemctl && $__have_service
	then
	    # Run with systemd whenever it is available now
	    #
	    case "$2"
	    in
		on)	__act=enable ;;
		off)	__act=disable ;;
		*)
		    echo "_change_config: Error: \$2=$2 not \"on\" or \"off\" as expected"
		    return 1
		    ;;
	    esac
	    if [ -n "$__pat" ]
	    then
		$sudo systemctl $__act $__pat.service >$tmp._tmp 2>$tmp._err
		[ $? -eq 0 ] || cat $tmp._tmp
	    fi
	elif which chkconfig >/dev/null 2>&1
	then
	    # Try the older RedHat and SuSE way ..
	    #
	    [ -n "$__pat" ] && $sudo chkconfig $__pat $2 2>&1 \
	    | sed \
		-e '/^insserv: warning: current start runlevel(s) .* of script .pcp./d' \
		-e '/^insserv: Service .* is missed in the runlevels /d' \
	    # end
	elif [ -x /usr/sbin/sysv-rc-conf ]
	then
	    # Try the Debian and Ubuntu way ..
	    #
	    [ -n "$__pat" ] && $sudo /usr/sbin/sysv-rc-conf $__pat $2
	elif which rc-update >/dev/null 2>&1
	then
	    # Try the Gentoo way ..
	    #
	    if [ -n "$__pat" ]
	    then
		case $2
		in
		    on)		__act=add ;;
		    off)	__act=delete ;;
		    *)
			echo "_change_config: Error: \$2=$2 not \"on\" or \"off\" as expected"
			return 1
			;;
		esac
		$sudo rc-update $__act $__pat default >$tmp._tmp 2>&1
		[ $? -eq 0 ] || cat $tmp._tmp
	    fi
	elif which update-rc.d >/dev/null 2>&1
	then
	    # Try the Linux MX way ...
	    #
	    if [ -n "$__pat" ]
	    then
		case $2
		in
		    on)		__act=enable ;;
		    off)	__act=disable ;;
		    *)
			echo "_change_config: Error: \$2=$2 not \"on\" or \"off\" as expected"
			return 1
			;;
		esac
		$sudo update-rc.d $__pat $__act >$tmp._tmp 2>&1
		[ $? -eq 0 ] || cat $tmp._tmp
	    fi
	elif [ -f /etc/slackware-version ]
	then
	    # Slackware uses BSD-style init, so no control
	    :
	else
	    # I have no clue!
	    #
	    echo "_change_config: Error: cannot change config \"$1 $2\""
	    return 1
	fi
    elif [ $PCP_PLATFORM = solaris ]
    then
	# Try the Solaris way ..
	#
	if which svcadm >/dev/null 2>&1
	then
	    case $1 
	    in
		pmcd)       __pat=pmcd ;;
		pmlogger)   __pat=pmlogger ;;
		verbose)    __pat="" ;;
		*)	    __pat=$1 ;;
	    esac
	    if [ -n "$__pat" ]
	    then
		if [ "$2" = on ]
		then
		    __state=`svcs -l svc:/application/pcp/$__pat | sed -n '/^state[ 	].*/s///p'`
		    [ -n "$__state" -a "$__state" != "online" ] \
			&& $sudo svcadm clear svc:/application/pcp/$__pat
		    $sudo svcadm enable svc:/application/pcp/$__pat
		elif [ "$2" = off ]
		then
		    $sudo svcadm disable svc:/application/pcp/$__pat
		else
		    echo "_change_config: Error: \$2=$2 not \"on\" or \"off\" as expected"
		    return 1
		fi
	    fi
	else
	    echo "_change_config: Error: cannot find svcs for Solaris"
	    return 1
	fi
    elif [ $PCP_PLATFORM = darwin ]
    then
	case $1 
	in
	    pmcd)	__pat=PMCD ;;
	    pmlogger)	__pat=PMLOGGER ;;
	    pmie)	__pat=PMIE ;;
	    pmproxy)	__pat=PMPROXY ;;
	    verbose)	__pat="" ;;
	    *)		__pat=$1 ;;
	esac
	if [ -n "$__pat" ]
	then
	    __state=`sed -n -e "/^$__pat=/{"'
s/.*=//
s/-//g
p
}' /etc/hostconfig`
	    if [ -z "$__state" ]
	    then
		echo "_change_config: Error: No $__pat control line in /etc/hostconfig"
		echo "You need to add a $__pat=-YES- line to this file"
		return 1
	    fi
	    if [ "$2" = "on" ]
	    then
		__req_state=YES
	    elif [ "$2" = "off" ]
	    then
		__req_state=NO
	    else
		echo "_change_config: Error: bad state ($2) should be on or off"
		return 1
	    fi
	    if [ "$__state" != "$__req_state" ]
	    then
		sed </etc/hostconfig >$tmp._state \
		    -e "/^$__pat=/s/-.*/-$__req_state-/"
		$sudo cp $tmp._state /etc/hostconfig
	    fi
	fi
    elif [ $PCP_PLATFORM = freebsd ]
    then
	# no control for FreeBSD
	:
    elif [ $PCP_PLATFORM = netbsd ]
    then
	# no control for NetBSD
	:
    elif [ $PCP_PLATFORM = openbsd ]
    then
	# no control for OpenBSD
	:
    else
	# I have no idea what to do for this platform!
	#
	echo "_change_config: Error: cannot \"$1 $2\" for $PCP_PLATFORM"
	return 1
    fi 2>&1 \
    | _filter_init_distro
    return 0
}

_get_config()
{
    if [ "$PCP_PLATFORM" = linux ]
    then
	case $1 
	in
	    pmlogger|pmcd) __pat=$1 ;;
	    verbose) __pat="" ;;
	    *)	__pat=$1 ;;
	esac

	__have_systemctl=false
	[ "$PCPQA_SYSTEMD" = yes ] && __have_systemctl=true
	if [ -z "$__pat" ]
	then
	    __have_service=true
	elif [ -n "$PCP_SYSTEMDUNIT_DIR" -a -f $PCP_SYSTEMDUNIT_DIR/$__pat.service ]
	then
	    __have_service=true
	else
	    __have_service=false
	fi
	if $__have_systemctl && $__have_service
	then
	    # Run with systemd whenever it is available now
	    #
	    if [ -z "$__pat" ]
	    then
		# unconditionally "on", or no such option
		#
		echo on
	    else
		if systemctl -q is-enabled $__pat.service >/dev/null 2>&1
		then
		    echo on
		else
		    echo off
		fi
	    fi
	elif which chkconfig >/dev/null 2>&1
	then
	    # Try the older RedHat and SuSE way ..
	    #
	    if [ -z "$__pat" ]
	    then
		# unconditionally "on", or no such option
		#
		echo on
	    else
		if chkconfig $__pat >$tmp.__tmp 2>&1
		then
		    # success from chkconfig is only useful if no output
		    # was generated ... in the latter case, grep the output
		    # for hints (this is for SuSE SLES9 in particular)
		    #
		    if [ -s $tmp.__tmp ]
		    then
			if grep ' on$' $tmp.__tmp >/dev/null
			then
			    echo on
			elif grep ' off$' $tmp.__tmp >/dev/null
			then
			    echo off
			else
			    echo off
			fi
		    else
			echo on
		    fi
		else
		    echo off
		fi
	    fi
	elif [ -x /usr/sbin/sysv-rc-conf ]
	then
	    # Try the Debian and Ubuntu way ..
	    #
	    if [ -z "$__pat" ]
	    then
		# unconditionally "on", or no such option
		#
		echo on
	    else
		if [ -f /etc/rc5.d/S*$__pat ]
		then
		    echo on
		else
		    echo off
		fi
	    fi
	elif which rc-update >/dev/null 2>&1
	then
	    # Try the Gentoo way ..
	    #
	    if [ -z "$__pat" ]
	    then
		# unconditionally "on", or no such option
		#
		echo on
	    else
		if rc-update show default | grep $__pat >/dev/null
		then
		    echo on
		else
		    echo off
		fi
	    fi
	elif which update-rc.d >/dev/null 2>&1
	then
	    # Try the Linux MX way ...
	    #
	    if [ -n "$__pat" ]
	    then
		case $2
		in
		    on)		__act=enable ;;
		    off)	__act=disable ;;
		    *)
			echo "_get_config: Error: \$2=$2 not \"on\" or \"off\" as expected"
			return 1
			;;
		esac
		$sudo update-rc.d $__pat $__act >$tmp._tmp 2>&1
		[ $? -eq 0 ] || cat $tmp._tmp
	    fi
	elif [ -f /etc/slackware-version ]
	then
	    # Slackware uses BSD-style init, so no control
	    :
	else
	    echo "_get_config: Error: don't know how to change config for Linux"
	    return 1
	fi
    elif [ $PCP_PLATFORM = solaris ]
    then
	if which svcs >/dev/null 2>&1
	then
	    case $1 
	    in
		pmlogger|pmcd) __pat=$1 ;;
		verbose) __pat="" ;;
		*) __pat=$1 ;;
	    esac
	    if [ -z "$__pat" ]
	    then
		# unconditionally "on", or no such option
		#
		echo on
	    else
		__state=`svcs -H svc:/application/pcp/$__pat | sed -e 's/[ 	].*//'`
		case "$__state"
		in
		    online|maintenance)
			echo on
			;;
		    offline|disabled)
			echo off
			;;
		    *)
			echo "_get_config: Error: smf ($__state) from svcs not expected"
			return 1
			;;
		esac
	    fi
	else
	    echo "_get_config: Error: cannot find svcs for Solaris"
	    return 1
	fi
    elif [ $PCP_PLATFORM = darwin ]
    then
	case $1 
	in
	    pmcd)	__pat=PMCD ;;
	    pmlogger)	__pat=PMLOGGER ;;
	    pmie)	__pat=PMIE ;;
	    verbose)	__pat="" ;;
	    *)		__pat=$1 ;;
	esac
	if [ -n "$__pat" ]
	then
	    __state=`sed -n -e "/^$__pat=/{"'
s/.*=//
s/-//g
p
}' /etc/hostconfig`
	    if [ -z "$__state" ]
	    then
		echo "_get_config: Error: No $__pat control line in /etc/hostconfig" >&2
		echo "You need to add a $__pat=-YES- line to this file" >&2
		return 1
	    fi
	    if [ "$__state" = "YES" ]
	    then
		echo on
	    elif [ "$__state" = "NO" ]
	    then
		echo off
	    else
		echo "_get_config: Error: bad state ($__state) should be YES or NO" >&2
		return 1
	    fi
	fi
    elif [ $PCP_PLATFORM = freebsd ]
    then
	# no control for FreeBSD
	:
    elif [ $PCP_PLATFORM = netbsd ]
    then
	# no control for NetBSD
	:
    elif [ $PCP_PLATFORM = openbsd ]
    then
	# no control for OpenBSD
	:
    else
	echo "_get_config: Error: cannot \"$1 $2\" for $PCP_PLATFORM"
    fi
    return 0
}

# This used to disable all system pmloggers running via the control
# file(s), but that was problematic and so now replaces the control
# file with a simple one that starts a primary pmlogger with /dev/null
# as the configuration file so pmlogger makes not requests to pmcd.
# 
# See _restore_loggers() below for the logic to re-instate the original
# control file(s).
#
_disable_loggers()
{
    [ -z "$PCP_PMLOGGERCONTROL_PATH" ] && \
		PCP_PMLOGGERCONTROL_PATH="$PCP_VAR_DIR/config/pmlogger/control"
    if [ -f $PCP_PMLOGGERCONTROL_PATH.$seq ]
    then
	echo "_disable_loggers: Botch: $PCP_PMLOGGERCONTROL_PATH.$seq already exists"
	date
	ls -l $PCP_PMLOGGERCONTROL_PATH.$seq
	return 1
    fi
    if [ -f $PCP_PMLOGGERCONTROL_PATH ]
    then
	$sudo mv $PCP_PMLOGGERCONTROL_PATH $PCP_PMLOGGERCONTROL_PATH.$seq
    else
	echo "_disable_loggers: Botch: $PCP_PMLOGGERCONTROL_PATH missing"
    fi
    if [ -d $PCP_PMLOGGERCONTROL_PATH.$seq.d ]
    then
	echo "_disable_loggers: Botch: $PCP_PMLOGGERCONTROL_PATH.$seq.d already exists"
	date
	ls -ld $PCP_PMLOGGERCONTROL_PATH.$seq.d
	return 1
    fi
    if [ -d $PCP_PMLOGGERCONTROL_PATH.d ]
    then
	$sudo mv $PCP_PMLOGGERCONTROL_PATH.d $PCP_PMLOGGERCONTROL_PATH.$seq.d
    else
	echo "_disable_loggers: Botch: $PCP_PMLOGGERCONTROL_PATH.d missing"
    fi
    cat <<End-of-File >$tmp._tmp
# Installed by PCP QA test $seq on `date`
# The goal here is to have a controlled primary logger that does
# not make requests to pmcd!
\$version=1.1
LOCALHOSTNAME y n PCP_ARCHIVE_DIR/LOCALHOSTNAME -c /dev/null
End-of-File
    $sudo cp $tmp._tmp $PCP_PMLOGGERCONTROL_PATH
    return 0
}

_restore_loggers()
{
    [ -z "$PCP_PMLOGGERCONTROL_PATH" ] && \
		PCP_PMLOGGERCONTROL_PATH="$PCP_VAR_DIR/config/pmlogger/control"
    if [ -d $PCP_PMLOGGERCONTROL_PATH.$seq.d ]
    then
	$sudo rm -fr $PCP_PMLOGGERCONTROL_PATH.d
	$sudo mv $PCP_PMLOGGERCONTROL_PATH.$seq.d $PCP_PMLOGGERCONTROL_PATH.d
    else
	echo "_restore_loggers: Botch: $PCP_PMLOGGERCONTROL_PATH.$seq.d missing"
    fi
    if [ -f $PCP_PMLOGGERCONTROL_PATH.$seq ]
    then
	$sudo mv $PCP_PMLOGGERCONTROL_PATH.$seq $PCP_PMLOGGERCONTROL_PATH
    else
	echo "_restore_loggers: Botch: $PCP_PMLOGGERCONTROL_PATH.$seq missing"
    fi
}

# _check_core [dir]
# checks for core files in dir (defaults to .)
#
_check_core()
{
    if [ -z "$1" ]
    then
	__dir=""
    else
	if [ -d $1 ]
	then
	    __dir=$1/
	else
	    echo "_check_core: aaargh $1 is not a directory!"
	    return
	fi
    fi
    if [ "`echo ${__dir}core*`" != "${__dir}core*" ]
    then
	[ -z "$here" ] && here=/tmp
	[ -z "$seq" ] && seq=9999
	$PCP_ECHO_PROG $PCP_ECHO_N "Dumped core! (saved in $here as""$PCP_ECHO_C"
	for __c in ${__dir}core*
	do
	    __d=`basename $__c`
	    $sudo mv $__c $here/$seq.$__d
	    $PCP_ECHO_PROG $PCP_ECHO_N " $seq.$__d""$PCP_ECHO_C"
	done
	echo ")"
	status=0
    fi
}

# is a pre-existing mmv directory in place?  if so, move it aside
# and prepare a directory with write access by unprivileged users
#
_prepare_pmda_mmv()
{
    if [ -d "$PCP_TMP_DIR/mmv" ]
    then
	_save_config "$PCP_TMP_DIR/mmv"
	$sudo rm -rf $PCP_TMP_DIR/mmv
    fi
    $sudo mkdir -p -m 1777 "$PCP_TMP_DIR/mmv"
    $sudo chown $PCP_USER:$PCP_GROUP "$PCP_TMP_DIR/mmv"
    __cullmmv=true
}

_restore_pmda_mmv()
{
    [ -n "$__cullmmv" ] && $__cullmmv && _restore_config "$PCP_TMP_DIR/mmv"
}

# prepare for a save-able pmcd and pmda configuration.
#
_prepare_pmda_install()
{
    [ -z "$1" ] && echo "Error: bad _prepare_pmda_install call"
    iam=$1

    # copy the pmcd config file to restore state later.
    _save_config $PCP_PMCDCONF_PATH

    cd $PCP_PMDAS_DIR/$iam
    if [ -f Makefile -o -f GNUmakefile ] ; then
	if $sudo ${MAKE:-make} clobber >$tmp._tmp 2>&1 ; then
	    :
	else
	    cat $tmp._tmp
	    echo "Arrgh, ${MAKE:-make} clobber failed"
	    return 1
	fi
    fi

    # start from a known starting point
    $sudo ./Remove >/dev/null 2>&1
    [ -f $PCP_VAR_DIR/config/$iam/$iam.conf ] && \
	$sudo mv $PCP_VAR_DIR/config/$iam/$iam.conf $tmp.$iam.conf
    return 0
}

# restore a saved pmcd configuration and ensure pmda back in place.
# $1 is PMDA name
# $2 is optional, and if not empty the (default) Install is not done
# (because the Install may be interactive or may fail without some
# user-provided information)
#
_restore_pmda_install()
{
    [ -z "$1" ] && echo "Error: bad _restore_pmda_install call"
    iam=$1
    __signal=$PCP_BINADM_DIR/pmsignal

    [ -f $PCP_VAR_DIR/config/$iam/$iam.conf.$seq ] \
	&& $sudo mv $PCP_VAR_DIR/config/$iam/$iam.conf.$seq $PCP_VAR_DIR/config/$iam/$iam.conf

    if diff $PCP_PMCDCONF_PATH.$seq $PCP_PMCDCONF_PATH > /dev/null 2>&1
    then
	_restore_config $PCP_PMCDCONF_PATH
    else

	if [ -z "$2" ]
	then
	    # do a default install which ensures the pmns and any
	    # views are installed
	    #
	    __here=`pwd`
	    cd $PCP_PMDAS_DIR/$iam
	    $sudo ./Install < /dev/null > /dev/null 2>&1
	    cd $__here
	fi

# PMDA may have been installed differently to default. As everything is
# installed we can use the old pmcd.conf file to restore state.

        if diff $PCP_PMCDCONF_PATH.$seq $PCP_PMCDCONF_PATH > /dev/null 2>&1
        then
	    _restore_config $PCP_PMCDCONF_PATH
        else
	    _restore_config $PCP_PMCDCONF_PATH
            $sudo $__signal -a -s HUP pmcd
        fi
    fi

    # Potential problem here if pmcd's PMDA config has changed for
    # metrics that the primary pmlogger is logging, then that pmlogger
    # may exit and not have been restarted before the QA test bumps
    # into either (a) an assumption that the primary pmlogger is
    # running (unlikely, as we usually get here as a result an exit
    # trap at the end of QA test, or (b) check is running with
    # a check.callback that checks if the primary pmlogger is alive
    # ... it is (b) this is the real problem.
    #
    # Only safe thing to do is to silently force a restart of the
    # primary pmlogger.
    #
    if [ -n "$here" -a -n "$seq" ]
    then
	echo "_restore_pmda_install: restarting primary pmlogger" >>$seq_full
	_service pmlogger restart >>$seq_full 2>&1
    else
	_service pmlogger restart >/dev/null 2>&1
    fi
    _wait_for_pmlogger || _exit 1
}

# find a local port that is not in use, optionally starting from a suggested port
# ... give up after 100 probes
#
_find_free_port()
{
    __count=0
    __base=$1
    [ -z "$__base" ] && __base=54321

    while $PCP_BINADM_DIR/telnet-probe -c localhost $__base
    do
	__base=`expr $__base + 1`
	if [ $__count -ge 100 ]
	then
	    echo >&2 "Error: _find_free_port: failed after 100 attempts (last port tried $__base)"
	    _exit 1
	fi
	__count=`expr $__count + 1`
    done
    echo $__base
}

# wait until an application is listening on a specified port
_wait_for_port()
{
    __port=$1
    __wait=50 # 5 seconds

    for __i in `seq $__wait`
    do
	if $PCP_BINADM_DIR/telnet-probe -c localhost $__port; then
	    break
	fi
	pmsleep 0.1
    done
    if [ $__i -ge $__wait ]; then
        echo Timeout waiting for an application to bind to port $__port
        return 1
    fi
    return 0
}

# wait for pmlogctl's pmlogger lock to clear
#
_wait_pmlogctl()
{
    __i=0
    while [ $__i -lt 61 ]; do
    	[ ! -f $PCP_ETC_DIR/pcp/pmlogger/lock ] && break
	__i=`expr $__i + 1`
	pmsleep 1
    done
}

# get pid for the running primary pmlogger
#
_get_primary_logger_pid()
{
    __root="$PCP_TMP_DIR/pmlogger/primary"
    if [ ! -L "$__root" ]
    then
	echo ""
    elif which realpath >/dev/null 2>&1
    then
	__symroot=`realpath "$__root"`
	basename "$__symroot"
    else
	__symroot=`ls -l "$__root" | sed -e 's/.*-> //'`
	if [ -z "$__symroot" ]
	then
	    echo "Arrgh, cannot get symlink for device for root fs ..."
	    ls -l "$__root"
	    return 1
	fi
	basename "$__symroot"
    fi
    return 0
}

_get_libpcp_config()
{
    $PCP_BINADM_DIR/pmconfig -L -s > $tmp.config
    . $tmp.config
    rm $tmp.config
}

# core dumping deep inside the libvirt Python wrapper layer ...
# not much we can do here other than skip libvirtpmda stuff`
#
_libvirt_is_ok()
{
    if [ -f /etc/lsb-release ]
    then
	if grep -q 'DISTRIB_ID=Ubuntu' /etc/lsb-release
	then
	    # OK, Ubuntu ... now let's check the libvirt version
	    #
	    cat <<'End-of-File' >$tmp.py
import sys
import libvirt
conn = libvirt.open('qemu:///system')
if conn != None:
    ver = conn.getVersion()
    print(str(ver))
    conn.close()
exit(0)
End-of-File
	    __libvirt_ver=`$PCP_PYTHON_PROG $tmp.py 2>/dev/null`
	    echo "Ubuntu and libvirt version $__libvirt_ver" >>$seq_full
	    case "$__libvirt_ver"
	    in
		2005000)
		    return 1
		    ;;
	    esac
	fi
    fi
    return 0
}

# send pmcd SIGHUP and reliably check that it received (at least) one
#
_sighup_pmcd()
{
    __signal_delay=0.05
    __sighups_before=-1

    eval `pmprobe $@ -v pmcd.sighups 2>/dev/null \
	  | $PCP_AWK_PROG '{ printf "__sighups_before=%d\n", $3 }'`
    if [ $__sighups_before -lt 0 ]
    then
	echo _sighup_pmcd called but pmcd not running
	return 1
    fi

    $sudo $PCP_BINADM_DIR/pmsignal -a -s HUP pmcd >/dev/null 2>&1

    # first make sure pmcd has received SIGHUP
    #
    __sts=1
    for __delay in 0.01 0.05 0.1 0.15 0.25 0.5 1 2
    do
	$PCP_BINADM_DIR/pmsleep $__delay
	__sighups=-1
	eval `pmprobe $@ -v pmcd.sighups 2>/dev/null | \
	    $PCP_AWK_PROG '{ printf "__sighups=%d\n", $3 }'`
	if [ $__sighups -gt $__sighups_before ]
	then
	    __sts=0
	    break
	fi
    done
    if [ $__sts -ne 0 ]
    then
	echo _sighup_pmcd caused no change in pmcd sighup count
	return 1
    fi

    # delay for $__signal_delay while pmcd actually does post-SIGHUP work
    #
    $PCP_BINADM_DIR/pmsleep $__signal_delay
    return 0
}

# Discover the platform-specific DSO suffix, at set $(DSOSUFFIX)
#
_set_dsosuffix()
{
    cat <<End-of-File >$tmp._mk
include $PCP_INC_DIR/builddefs
default:
	@echo \$(DSOSUFFIX)
End-of-File
    if $PCP_MAKE_PROG -f $tmp._mk >$tmp._out 2>$tmp._err
    then
	:
    else
	cat $tmp._err
	echo "Error: $PCP_MAKE_PROG failed establishing DSO suffix"
	return 1
    fi
    if [ -s $tmp._err ]
    then
	cat $tmp._err
	echo "Warning: unexpected stderr from $PCP_MAKE_PROG when establishing DSO suffix"
    fi
    DSOSUFFIX=`cat $tmp._out`
    if [ -z "$DSOSUFFIX" ]
    then
	echo "Error: cannot establish the DSO suffix from $PCP_INC_DIR/builddefs"
	return 1
    fi
    rm -f $tmp._mk $tmp._out $tmp._err
    return 0
}

# common webapi header filtering
#
_webapi_header_filter()
{
    tee -a $seq_full \
    | col -b \
    | sed \
	-e 's/^\(Content-Length:\) [1-9][0-9]*/\1 SIZE/g' \
	-e 's/^\(User-Agent: curl\).*/\1 VERSION/g' \
	-e 's/^\(Date:\).*/\1 DATE/g' \
	-e 's/\(\"context\":\) [0-9][0-9]*/\1 CTXID/g' \
	-e '/^Connection: Keep-Alive/d' \
    | LC_COLLATE=POSIX sort
}

_webapi_response_filter()
{
    tee -a $seq_full \
    | col -b \
    | sed -e 's,#[0-9]*,####,g' \
        -e 's/ connected$//g' \
        -e '/^\* Adding handle: /d' \
        -e '/^\* Closing connection ####/d' \
        -e '/- Conn .*_pipe.*_pipe/d' \
        -e '/Curl_[\.a-zA-Z0-9]*:/d' \
        -e 's/::1/LOCALADDR/g' \
        -e 's/127\.0\.0\.1/LOCALADDR/g' \
        -e 's/localhost[\.a-zA-Z0-9]*/LOCALHOST/g' \
        -e 's/\(Date:\).*/\1 DATE/' \
        -e 's,\(i>pmproxy\)/.*<.i,\1/VERSION<\\i,g' \
	-e 's/\[[0-9][0-9]* bytes data]/[data not shown]/' \
        -e '/> User-Agent: /d' \
	-e '/Connection: Keep-Alive/d' \
    # end
}

# May have to try and retry pmlc before we succeed ... failure requiring
# a retry is indicated by "Address already in use" on stderr
#
# If the optional argument is "expect-failure", then we're expecting this
# to fail, otherwise we're expecting this to succeed (eventually).
#
# pmlc commands are already in $tmp.pmlc
# before returning pmlc output is stashed in $tmp.out and $tmp.err and
# appended to $seq_full
#
_try_pmlc()
{
    __max_tries=10
    __i=0
    while [ $__i -lt $__max_tries ]
    do
	pmlc -ie <$tmp.pmlc >$tmp.out 2>$tmp.err
	if grep -E "(Address already in use)|(Connection refused)" $tmp.err >/dev/null
	then
	    __i=`expr $__i + 1`
	    pmsleep 0.1
	else
	    echo "_try_pmlc: pass @ iter $__i" >>$seq_full
	    cat $tmp.out $tmp.err >>$seq_full
	    break
	fi
    done
    if [ $__i -eq $__max_tries ]
    then
	if [ $# -eq 1 -a "$1" = expect-failure ]
	then
	    echo "_try_pmlc: fail as expected" >>$seq_full
	else
	    echo "_try_pmlc: failed"
	    cat $tmp.out $tmp.err
	fi
    fi
}

# Start a new private non-daemon pmcd with only the sampledso PMDA
# and pmcd PMDA running in the $tmp directory (all pmcd's files in
# and out end up here).
# on entry ...
# $pmcd_args	extra command line args to pmcd
#
# on return ...
# $pmcd_pid		PID of the new pmcd
# $pmcd_port		port the new pmcd is listening on
#			also set in the environment as $PMCD_PORT
# $tmp/pmcd.socket	Unix domain socket for pmcd requests
# $tmp/pmcd.conf	pmcd's config file
# $tmp/pmcd.out		pmcd's stdout
# $tmp/pmcd.err		pmcd's stderr
# $tmp/pmcd.log		pmcd's log file
#
# to cleanup ...
# kill -TERM $pmcd_pid
#
_private_pmcd()
{
    [ -d $tmp ] || mkdir $tmp || return 1
    cd $tmp
    grep -E '^(sampledso|pmcd)[ 	]' $PCP_PMCDCONF_PATH >pmcd.conf
    echo "=== private pmcd ..." >>$seq_full
    cat pmcd.conf >>$seq_full
    pmcd_port=`_find_free_port`
    echo "pmcd_port=$port" >>$seq_full
    $PCP_BINADM_DIR/pmcd $pmcd_args -f -c ./pmcd.conf -s ./pmcd.socket -p $pmcd_port  >pmcd.out 2>pmcd.err &
    pmcd_pid=$!
    PMCD_PORT=$pmcd_port; export PMCD_PORT
    _wait_for_pmcd || _exit 1
    cd $here
    return 0
}

# Usage: path_readable user path
#
_path_readable()
{
    # users like pcp (as in pmcd, or pmlogger control scripts) may have
    # to be able to read some QA-specific path ... which is an issue if
    # $HOME for the person running QA is mode 700
    #
    if [ $# -ne 2 ]
    then
	echo >&2 "Usage: _path_readable user path"
	return 1
    fi
    user="$1"
    path="$2"
    if $sudo -u $user id >/dev/null 2>&1
    then
	:
    else
	echo >&2 "_path_readable: cannot run commands as user $user"
	return 1
    fi
    if [ ! -f "$path" ]
    then
	echo >&2 "_path_readable: bozo, $path does not exist!"
	return 1
    fi
    if $sudo -u "$user" [ -r "$path" ]
    then
	return 0
    else
	echo >&2 "_path_readable: $path is not readable by the user $user"
	tpath=''
	for dir in `echo "$path" | sed -e 's/\// /g'`
	do
	    tpath="$tpath/$dir"
	    if $sudo -u "$user" [ -r "$tpath" ]
	    then
		:
	    else
		echo >&2 "_path_readable: directory $tpath is unreadable"
		break
	    fi
	done
    fi
    return 1
}

# wait for a process to exit ...
# Usage:
#   _wait_process_end [tag] pid
#
_wait_process_end()
{
    if [ $# -eq 2 -a -n "$2" ]
    then
	__tag="$1"
	__pid="$2"
    elif [ $# -eq 1 -a -n "$1" ]
    then
	__tag="_wait_process_end"
	__pid="$1"
    else
	echo >&2 "Usage: _wait_process_end [tag] pid \$#=$# \$1=\"$1\""
	return 1
    fi
    __i=0
    while true
    do
	if $sudo kill -s 0 $__pid 2>/dev/null
	then
	    # process still exists
	    __i=`expr $__i + 1`
	    if [ "$__i" -ge 100 ]
	    then
		# may exist, but is defunct so we should treat this
		# as "exited" ...
		# <defunct> for Linux-style ps(1)
		# (command) for BSD-style ps(1)
		#
		__ppid=`$PCP_PS_PROG $PCP_PS_ALL_FLAGS \
		        | $PCP_AWK_PROG '
$2 == '$__pid' && $NF ~ /<defunct>/ { print $3 }
$2 == '$__pid' && $NF ~ /^([^)]*)$/ { print $3 }
'`
		if [ -n "$__ppid" ]
		then
		    echo "$__tag: pid $__pid is defunct" >>$seq_full
		    if which pstree >/dev/null 2>&1
		    then
			pstree -s -A -pua $__pid >>$seq_full 2>&1
		    else
			echo "$__tag: parent pid $__ppid" >>$seq_full
			$PCP_PS_PROG $PCP_PS_ALL_FLAGS -p $__pid $__ppid >>$seq_full
		    fi
		    return 0
		else
		    echo "$__tag: failed to see pid $__pid exit after 100 iterations"
		fi
		echo "$__tag: failed to see pid $__pid exit after 100 iterations"
		return 1
	    fi
	    pmsleep 0.1
	else
	    # process has really exited
	    return 0
	fi
    done
}

# exit wrapper ... set $status if $1 present
#
_exit()
{
    [ $# -gt 0 ] && status="$1"
    exit
}

# borrowed from rc-proc.sh in ../src/pmcd
# run pstree(1) with the focus on PID $1
# ... Linux and *BSD (at least) have _totally_ different implementations
# and command line arguments
#
_pstree_all()
{
    if which pstree >/dev/null 2>&1
    then
	if pstree -\? 2>&1 | grep ' --show-parents ' >/dev/null
	then
	    # Linux version
	    #
	    echo "Called from:"
	    pstree -asp "$1"
	    echo "--- end of pstree output ---"
	elif pstree -\? 2>&1 | grep ' -s string' >/dev/null
	then
	    # *BSD version
	    #
	    echo "Called from:"
	    pstree -p "$1" \
	    | $PCP_AWK_PROG '
/pstree -p/	{ exit }
		{ print }'
	    echo "--- end of pstree output ---"
	else
	    # don't know what sort of pstree this is ...
	    #
	    :
	fi
    fi
}

# borrowed from rc-proc.sh in ../src/pmcd
# pstree(1) one-line for parents of PID $1
# ... Linux and *BSD (at least) have _totally_ different implementations
# and command line arguments
#
_pstree_oneline()
{
    if which pstree >/dev/null 2>&1
    then
	if pstree -\? 2>&1 | grep ' --show-parents ' >/dev/null
	then
	    # Linux version
	    #
	    pstree -lsp "$1"
	elif pstree -\? 2>&1 | grep ' -s string' >/dev/null
	then
	    # *BSD version
	    #
	    pstree -p "$1" \
	    | sed -e 's/^[ 	|\\=+-][ 	|\\=+-]*//' \
	    | $PCP_AWK_PROG '
$3 == "pstree"	{ exit }
		{ print $3 "(" $1 ")--" }' \
	    | ( tr '\012' '-' ; echo ) \
	    | sed -e 's/---$//'
	else
	    # don't know what sort of pstree this is ...
	    #
	    :
	fi
    fi
}

# check if the primary pmlogger is writing to a local file (as opposed
# to logpush via http to a remote pmproxy
#
_check_local_primary_archive()
{
    _archive=`pminfo -f pmcd.pmlogger.archive 2>>$seq_full | sed -n -e '/"primary"]/{
s/"$//
s/.*"//
p
}'`
    case "$_archive"
    in
    	/*)		# local file
			return 0
			;;

	http://*)	# logpush
			echo "_check_local_primary_archive: false: $_archive" >>$seq_full
			return 1
			;;

	*)		# botch
			echo "_check_local_primary_archive: botch: _archive=\"$_archive\"" >>$seq_full
			return 1
			;;
    esac
}