#!/bin/sh
# PCP QA Test No. 326
# Check PMCD state change notification, aka PMCD reset
#
# Copyright (c) 1995-2002 Silicon Graphics, Inc.  All Rights Reserved.
# Copyright (c) 2016 Red Hat, Inc.
#

seq=`basename $0`
echo "QA output created by $seq"

# get standard filters
. ./common.product
. ./common.filter
. ./common.check

if [ -f /etc/redhat-release ]
then
    # Don't even bother on CentOS release 6.10 (Final) ... we've tried
    # repeatedly to make this test work on this platform and it is not worth
    # investing any more effort on.
    #
    if grep -E -q '^CentOS .* 6\.[0-9]|^Red Hat .* 6\.[0-9]' </etc/redhat-release
    then
	_notrun "don't bother on CentOS/RHEL 6"
    fi
fi

signal=$PCP_BINADM_DIR/pmsignal
status=1
done_clean=false
LOCALHOST=`hostname`
iam=`id -un`

_cleanup()
{
    if $done_clean
    then
	:
    else
	[ -n "$pmcd_pid" ] && $signal -s TERM $pmcd_pid
	_restore_config $PCP_SYSCONF_DIR/labels
	rm -f $tmp.*
	done_clean=true
    fi
    exit $status
}

trap "_cleanup" 0 1 2 3 15
timezone=`pmprobe -v pmcd.timezone | $PCP_AWK_PROG '{print $3}'`

_filter_pmval_live()
{
    # filter hostname, as we now fetch pmcd.hostname, sneakily
    # also, if the pmda goes away between the pmFetch() and the pmGetDesc()
    # then the -Dfetch diag has to report, for example
    #    value 2 2.8025969e-45 0x2
    # instead of
    #    value 2
    # so fix this as well
    #
    sed \
	-e "/   value \"$REALHOST\"/d" \
	-e "/   value \"$LOCALHOST\"/d" \
	-e '/ [0-9]\..*e.*0x/s/ [0-9]\..*//' \
    # end
}

_filter_pmval_arch()
{
    # strip preamble, mask timestamp, deal with arithmetic imprecision
    # and map this
    # 07:59:58.062                    !
    # to this
    # 07:59:58.062  No values available
    #
    sed \
	-e '1,/^interval:/d' \
	-e 's/[0-9][0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9][0-9]*[0-9]/TIMESTAMP/g' \
	-e 's/                  !/No values available/' \
    | $PCP_AWK_PROG '
/No values available/	{ print; next }
0 <= $2 && $2 < 1.1	{ $2 = "non-negative number, less than 1.1" }
			{  print }' \
    | uniq
}

trap "_cleanup" 0 1 2 3 15


# real QA test starts here
_save_config $PCP_SYSCONF_DIR/labels
$sudo rm -rf $PCP_SYSCONF_DIR/labels/*

PMDA_PMCD_PATH=$PCP_PMDAS_DIR/pmcd/pmda_pmcd.$DSO_SUFFIX

cat <<End-of-File >$tmp.pmcd.config
# Installed by PCP QA test $seq on `date`
pmcd	2	dso	pmcd_init	$PMDA_PMCD_PATH
sample	29	pipe	binary 		$PCP_PMDAS_DIR/sample/pmdasample -d 29 -U $iam -l $tmp.pmda.log
End-of-File

PMCD_PORT=`_find_free_port`
echo "PMCD_PORT=$PMCD_PORT" >>$seq_full
export PMCD_PORT

$PCP_BINADM_DIR/pmcd -f -c $tmp.pmcd.config -l $tmp.pmcd.log -U $iam -s $tmp.socket &
pmcd_pid=$!
echo "pmcd_pid=$pmcd_pid" >>$seq_full
_wait_for_pmcd || _exit 1

cat <<End-of-File >$tmp.config
log mandatory on default { pmcd.numagents sample.milliseconds }
End-of-File

# run pmlogger and pmval for about 30 seconds
pmlogger -h localhost -s 90 -t 0.33 -l $tmp.log -c $tmp.config $tmp &
pmlogger_pid=$!
pmval -h localhost -s 90 -t 0.33 -Z "$timezone" -D fetch pmcd.numagents >$tmp.pmval 2>&1 &
pmval_pid=$!

sleep 3
# need PPID in the 3rd field of the ps(1) output
#
pmda_pid=`$PCP_PS_PROG $PCP_PS_ALL_FLAGS \
	  | grep '/[p]mdasample' \
	  | $PCP_AWK_PROG '$3 == "'$pmcd_pid'"	{ print $2 }'`
if [ -z "$pmda_pid" ]
then
    echo "Arrgh ... lost pmdasample process?"
    echo "See $seq.full for details"
    $PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '([P]PID)|(/[p]mcd)|(/[p]mda)' >>$seq_full
    [ -n "$pmlogger_pid" ] && $signal -s TERM $pmlogger_pid
    [ -n "$pmval_pid" ] && $signal -s TERM $pmval_pid
    cat $tmp.pmcd.log >>$seq_full
    cat $tmp.pmda.log >>$seq_full
    exit
fi

echo "=== kill sample PMDA process ==="
$signal -s TERM $pmda_pid
sleep 3 # enough time for pmcd to restart it - once only

pmda_pid=`$PCP_PS_PROG $PCP_PS_ALL_FLAGS \
	  | grep '/[p]mdasample' \
	  | $PCP_AWK_PROG '$3 == "'$pmcd_pid'"	{ print $2 }'`
if [ -z "$pmda_pid" ]
then
    echo "Arrgh ... lost pmdasample process?"
    echo "See $seq.full for details"
    $PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID|/[p]m[cd]( |$)' >>$seq_full
    [ -n "$pmlogger_pid" ] && $signal -s TERM $pmlogger_pid
    [ -n "$pmval_pid" ] && $signal -s TERM $pmval_pid
    cat $tmp.pmcd.log >>$seq_full
    cat $tmp.pmda.log >>$seq_full
    exit
fi
$signal -s TERM $pmda_pid
sleep 3 # and again

echo "=== SIGHUP PMCD ==="
$signal -s HUP $pmcd_pid
sleep 3

echo "=== drop sample PMDA, like Remove ==="
cat <<End-of-File >$tmp.pmcd.config
# Installed by PCP QA test $seq on `date`
pmcd	2	dso	pmcd_init	$PMDA_PMCD_PATH
End-of-File
$signal -s HUP $pmcd_pid
sleep 3

echo "=== add sample PMDA, like Install ==="
cat <<End-of-File >$tmp.pmcd.config
# Installed by PCP QA test $seq on `date`
pmcd	2	dso	pmcd_init	$PMDA_PMCD_PATH
sample	29	pipe	binary 		$PCP_PMDAS_DIR/sample/pmdasample -d 29 -U $iam -l $tmp.pmda.log
End-of-File
$signal -s HUP $pmcd_pid
sleep 3

sleep 3
$signal -s TERM $pmcd_pid
pmcd_pid=''

wait

echo >>$seq_full
echo "pmcd log ..." >>$seq_full
cat $tmp.pmcd.log >>$seq_full

echo
echo "Trace of observed state changes and PMDA count ..."
grep -E '(PMCD state)|( value )' $tmp.pmval \
| _filter_pmval_live \
| uniq

echo >>$seq_full
echo "pmlogger log ..." >>$seq_full
cat $tmp.log >>$seq_full

echo
echo "archive contents ..."
pmdumplog $tmp >$tmp.out 2>&1
grep -E '(<mark>)|(pmcd.numagents)' $tmp.out \
| _filter_pmdumplog \
| uniq

echo >>$seq_full
pmdumplog -m $tmp sample.milliseconds >>$tmp.dump 2>&1
nr=`grep '^[012]' <$tmp.dump | wc -l | sed -e 's/  */ /g'`
echo "archive dump ($nr records)" >>$seq_full
cat $tmp.dump >>$seq_full

# note clip after 6 filtered lines to avoid extra values that sometimes
# appear due to non-determinism in pmlogger fetch samples
#
echo
echo "sanity check with pmval | filter ..."
pmval -t 1 -a $tmp sample.milliseconds 2>&1 \
| tee -a $seq_full \
| _filter_pmval_arch \
| sed -e 6q

echo >>$seq_full
echo "unfiltered pmval live output" >>$seq_full
cat $tmp.pmval >>$seq_full

echo >>$seq_full
echo "unfiltered pmval archive output" >>$seq_full
pmval -t 1 -a $tmp sample.milliseconds >>$seq_full 2>&1

status=0
exit