#!/bin/sh
# PCP QA Test No. 1773
# Exercise multi-host pmseries expressions and functions
# via the REST API.
#
# Copyright (c) 2020 Red Hat.  All Rights Reserved.
#

seq=`basename $0`
echo "QA output created by $seq"

# get standard environment, filters and checks
. ./common.product
. ./common.filter
. ./common.check
. ./common.keys

_check_series

_cleanup()
{
    cd $here
    [ -n "$pmproxy_pid" ] && $signal -s TERM $pmproxy_pid
    [ -n "$options" ] && $keys_cli $options shutdown
    if $need_restore
    then
	need_restore=false
	_restore_config $PCP_SYSCONF_DIR/pmseries
	_restore_config $PCP_SYSCONF_DIR/pmproxy
    fi
    $sudo rm -rf $tmp $tmp.*
}

status=0	# success is the default!
signal=$PCP_BINADM_DIR/pmsignal

userid=`id -u`
username=`id -u -n`
hostname=`hostname`
machineid=`_machine_id`
domainname=`_domain_name`

need_restore=false
trap "_cleanup; exit \$status" 0 1 2 3 15

_filter_source()
{
    sed \
	-e "s,$here,PATH,g" \
	-e "s,$tmp.farm,TMP,g"
    # end
}

_filter_url()
{
    sed \
	-e "s,localhost:$proxyport/,localhost:PROXYPORT/," \
    # end
}

_curl_get()
{
    url="$1"
    echo "$url" | tee -a $seq_full | _filter_url
    curl --get --silent "$url" | pmjson | tee $tmp.out |\
    awk '$1 == "{" {n++} END { printf(" .. %d lines with %d json { objects }\n", NR, n); }'
    if grep -q -s '}$' $tmp.out; then src/pmjson_array_sort <$tmp.out; else cat $tmp.out; fi \
    | tee -a $seq_full
}

# some platforms show values in different notation ...
# unclear if this is curl or pmjson or Python, but try to fix that
#
_filter_values()
{
    sed \
	-e '/"value":/{
s/"83.486139"/"8.348614e+01"/
s/"998.387463"/"9.983875e+02"/
s/"59967170"/"5.996717e+07"/
s/"993.002565"/"9.930026e+02"/
s/"60069950"/"6.006995e+07"/
s/"993.356645"/"9.933566e+02"/
}'
    # end
}

# real QA test starts here
_save_config $PCP_SYSCONF_DIR/pmproxy
_save_config $PCP_SYSCONF_DIR/pmseries
$sudo rm -f $PCP_SYSCONF_DIR/pmseries/*
$sudo rm -f $PCP_SYSCONF_DIR/pmproxy/*
need_restore=true

mkdir -p $tmp.farm
tar -C $tmp.farm -xf archives/farm.tar.xz

echo "Start test key server ..."
key_server_port=`_find_free_port`
options="-p $key_server_port"
$key_server --port $key_server_port --save "" > $tmp.keys 2>&1 &
_check_key_server_ping $key_server_port
_check_key_server $key_server_port
echo

_check_key_server_version $key_server_port

# import multi-host test data into key server
for node in node80 node81; do
    pmseries $options --load $tmp.farm/farm/$node/20201124.0 | _filter_source
done

# start pmproxy 
proxyport=`_find_free_port`
echo "proxyport=$proxyport" >>$seq_full
proxyopts="-p $proxyport -r $key_server_port -t"  # -Dseries,http,af
pmproxy -f -U $username -x $tmp.err -Dseries,query -l $tmp.pmproxy.log $proxyopts &
pmproxy_pid=$!

# check pmproxy has started and is available for requests
pmcd_wait -h localhost@localhost:$proxyport -v -t 5sec

if [ -f $tmp.err ]
then
    echo "Arrgh: errors from key server startup ..."
    cat $tmp.err
fi

$PCP_PS_PROG $PCP_PS_ALL_FLAGS | grep -E '[P]PID|/[p]mproxy( |$)' >> $seq_full
$keys_cli $options keys pcp:* >>$seq_full
cat $tmp.pmproxy.log >> $seq_full

series1=`pmseries $options kernel.all.load`
[ -z "$series1" ] && _fail "FAIL cannot find any timeseries matching kernel.all.load"

echo;echo == NOTE: Due to json sorting issues, see $seq.full.ok for full curl output ==

echo;echo;echo == verify basic two host pmseries checks
echo == pmseries hinv.ncpu : should return 2 SIDs
pmseries $options hinv.ncpu
echo == pmseries kernel.all.load : should return 2 SIDs
pmseries $options kernel.all.load
echo == pmseries kernel.all.load+kernel.all.load : should return one fabricated SID
pmseries $options 'kernel.all.load+kernel.all.load'


echo;echo "== verify two host series sum (a + b) SID lookup" | tee -a $seq_full
echo "kernel.all.load + kernel.all.load"
expr="kernel.all.load%20%2B%20kernel.all.load"
_curl_get "http://localhost:$proxyport/series/query?expr=$expr"

echo;echo "== verify two host series sum (a + b) with samples" | tee -a $seq_full
echo "kernel.all.load[samples:1] + kernel.all.load[samples:1]"
expr="kernel.all.load%5Bsamples%3A1%5D%20%2B%20kernel.all.load%5Bsamples%3A1%5D"
_curl_get "http://localhost:$proxyport/series/query?expr=$expr"

echo;echo "== verify two host series sum (a + b) with samples by SID" | tee -a $seq_full
SID=`pmseries $options "kernel.all.load + kernel.all.load"`
echo "kernel.all.load + kernel.all.load, SID=$SID"
_curl_get "http://localhost:$proxyport/series/values?series=$SID&samples=1"
_curl_get "http://localhost:$proxyport/series/descs?series=$SID"
_curl_get "http://localhost:$proxyport/series/metrics?series=$SID"
_curl_get "http://localhost:$proxyport/series/instances?series=$SID"

# For debugging
# echo === pause for gdb -p $pmproxy_pid Log is $tmp.pmproxy.log ===
# read something


echo;echo "== verify two host series difference (a - b) SID lookup" | tee -a $seq_full
echo "kernel.all.load - kernel.all.load"
expr="kernel.all.load%20-%20kernel.all.load"
_curl_get "http://localhost:$proxyport/series/query?expr=$expr"

echo;echo "== verify two host series difference (a - b) with samples" | tee -a $seq_full
echo "kernel.all.load[samples:1] - kernel.all.load[samples:1]"
expr="kernel.all.load%5Bsamples%3A1%5D%20-%20kernel.all.load%5Bsamples%3A1%5D"
_curl_get "http://localhost:$proxyport/series/query?expr=$expr"

echo;echo "== verify two host series difference (a - b) with samples by SID" | tee -a $seq_full
SID=`pmseries $options kernel.all.load-kernel.all.load`
echo "kernel.all.load - kernel.all.load, SID=$SID"
_curl_get "http://localhost:$proxyport/series/values?series=$SID&samples=1"
_curl_get "http://localhost:$proxyport/series/descs?series=$SID"
_curl_get "http://localhost:$proxyport/series/metrics?series=$SID"
_curl_get "http://localhost:$proxyport/series/instances?series=$SID"


echo;echo "== verify two host series function (floor(a)) SID lookup" | tee -a $seq_full
echo "floor(kernel.all.load)"
expr="floor%28kernel.all.load%29"
_curl_get "http://localhost:$proxyport/series/query?expr=$expr"

echo;echo "== verify two host series function (floor(a)) with samples" | tee -a $seq_full
echo "floor(kernel.all.load[samples:2])"
expr="floor%28kernel.all.load%5Bsamples%3A2%5D%29"
_curl_get "http://localhost:$proxyport/series/query?expr=$expr"

echo;echo "== verify two host series function (floor(a)) with samples by SID" | tee -a $seq_full
SID=`pmseries $options 'floor(kernel.all.load)'`
echo "floor(kernel.all.load), SID=$SID"
_curl_get "http://localhost:$proxyport/series/values?series=$SID&samples=2"
_curl_get "http://localhost:$proxyport/series/descs?series=$SID"
_curl_get "http://localhost:$proxyport/series/metrics?series=$SID"
_curl_get "http://localhost:$proxyport/series/instances?series=$SID"


echo;echo '== verify two host series function rescale(rate(kernel.percpu.cpu.idle), "millisec/s") SID lookup' | tee -a $seq_full
echo 'rescale(rate(kernel.percpu.cpu.idle), "millisec/s")'
expr='rescale%28rate%28kernel.percpu.cpu.idle%29,%22millisec/s%22%29'
_curl_get "http://localhost:$proxyport/series/query?expr=$expr"

echo;echo '== verify two host series function rescale(rate(kernel.percpu.cpu.idle), "millisec/s") with samples' | tee -a $seq_full
echo 'rescale(rate(kernel.percpu.cpu.idle[samples:2]),"millisec/s")'
expr='rescale%28rate%28kernel.percpu.cpu.idle%5Bsamples%3A2%5D%29,%22millisec/s%22%29'
_curl_get "http://localhost:$proxyport/series/query?expr=$expr" | _filter_values

echo;echo '== verify two host series function rescale(kernel.percpu.cpu.idle, "millisec/s") with samples by SID' | tee -a $seq_full
SID=`pmseries $options 'rescale(rate(kernel.percpu.cpu.idle), "millisec/s")'`
echo 'rescale(rate(kernel.percpu.cpu.idle), "millisec/s"),' SID=$SID
_curl_get "http://localhost:$proxyport/series/values?series=$SID&samples=2" | _filter_values
_curl_get "http://localhost:$proxyport/series/descs?series=$SID" | _filter_values
_curl_get "http://localhost:$proxyport/series/metrics?series=$SID" | _filter_values
_curl_get "http://localhost:$proxyport/series/instances?series=$SID" | _filter_values

# success, all done
exit