#!/usr/bin/env perl

# this software is Copyright Harry Mangalam <hjmangalam@gmail.com> 2019 and on.
# parsync, parsyncfp, parsyncfp2 and my derivatives are released under the GPLv3 License.
# fpart is released under the BSD license at the time of this writing.
#      <https://www.gnu.org/licenses/gpl-3.0.en.html>
# see also: <https://en.wikipedia.org/wiki/GNU_General_Public_License>
# Please see the file "COPYRIGHT" that should accompany this program.
# If not it is available at the parsyncfp github site:
#      https://github.com/hjmangalam/parsyncfp

# Change $PATH to include /usr/libexec/parsyncfp2, since the Debian package
# ships scut and stats there
$ENV{PATH} = "/usr/libexec/parsyncfp2:$ENV{PATH}";

##: == COMMON TO MASTER & SEND hosts ==
##: Lib Requirements
use strict;
use Getopt::Long;    # for std option handling: -h --yadda=badda, etc
# use Socket; # soon!
use Env qw(HOME PATH USER);
use File::Path qw(rmtree make_path);
use File::Basename;
use Term::ANSIColor;    # for alarms

##: Dev/github/update gunk
# strip ##NTS and #dd lines from release version
# fpart update 
#   'git clone https://github.com/martymac/fpart.git; 'cd fpart; aclocal; automake --add-missing; autoconf; ./configure; make -j4'

#  perltidy cmd to format uniformly: perltidy -ce -i=2 -l=100 parsyncfp2

# copy to all the local hosts for local testing
# fn="/home/hjm/bin/parsyncfp2"; scp $fn bridgit:~/bin;
# ssh bridgit "scp $fn harry@fibre:~/bin"; ssh bridgit "scp $fn harry@starklab:~/bin";
# ssh bridgit "scp $fn hmangala@hpc2:~/bin"; ssh bridgit "scp $fn hmangala@hpc3:~/bin";
# rsync -av ~/bin/* bigben:~/bin ; rsync -av ~/bin/* pooki:~/bin ;
# rsync -av ~/bin/* bridgit:~/bin; ssh bridgit 'rsync -av ~/bin/* hmangalam@fibre:~/bin'; ssh bridgit 'rsync -av ~/bin/* hmangalam@fibre-compute:~/bin';

# update github
# cd ~/gits/parsyncfp2/
# git add parsyncfp2 README.md pfp-changes.txt scut stats
# git commit -m 'commit comment'
# git push

# if hit a conflict (accept a PR, merge on github, the continue editing on stunted), try
# git reset --merge
# git add <conflicting files>
# git commit -m 'comment'
# git pull # to check, should be up to date
# git push
# if above doesn't work, try a git clone in another dir, then kompare the 2 files, 
#   apply the conflicting diffs, THEN try the above again.

##: ITER notes
# cd ~/bin; scp parsyncfp2.246  bridgit:~/iter; cd - ;  # from stunted, also scut stats
# cd ~/bridgit; scp hjm@bridgit.mmg.uci.edu:~/iter/* . ; cd - ;# from iter login host

# for centos: need 'yum install perl-Env.noarch  perl-Statistics-Descriptive wireless-tools'

# Add changes to changelog in the README.md file
# check github for bug reports.
##: Global Vars
use vars qw($allPIDs $ALL_SYS_RSYNC_PIDS $ch  $cmd
  $crr $CUR_FP_FLE  $DATE $MASTERDATE $dcnt $DEBUG $DBC @DIRS @DIRS2SYNC $dirtmp
  $EMAIL $Filecnt %FILES $fl $fn $fnd2r $FOUT $FPART_LOGFILE 
  $FPARTSIZE $FPARTSIZE_N $FP_PIDFILE $FP_ROOT $FP_ROOT_DIR
  $FP_HOLD_ROOT $FP_HOLD_DIR $OTHERFPRNG $MAX_BG_JOBS
    $HELP $IF_SPEED  $FP_SAVE_DIR $PFP2STOPSCRIPT
  $LOAD1mratio $loadavg $logfile  $MAXBW $MAXLOAD 
   $NDIRS  $NOWAIT $NP $NP_chunk $ALTCACHE $MSTR_SHELL
  $parsync_dir $PARSYNCVER  $prev_cache $lenPID $DISPOSE
  $rem_host $remote $rem_path $rem_user  $rootdir   $pfplog
  $ROOTDIR $RSYNC_CMD $RSYNCOPTS  $STILLRSYNCS $DFLT_RSYNCOPTS
  @SYSLOAD $TARGET $tmp $Totlsiz %UTILS $VERSION $OS $Linux $MacOSX $NETFILE $myIP
   $avgTCPrecv $avgTCPsend $avgRDMArecv $avgRDMAsend $CHECKHOST
  $WARN_FPART_FILES $MAX_FPART_FILES $FILESFROM $TRIMPATH $tf $rPIDs
  $TRUSTME $N @A $bytefiles  $IB_PRSNT $CFL $rHOSTNAME $MD5SUM $MSTR_MD5
  $PFP_DONE $bytesxf $rprtnbr $sfx $ALLBYTES $SENDHOSTSEQ  $RTT $RTT_DELAY $SLOWDOWN
  @NETDEVLIST $NETDEVADDRLIST @spinner $defREMOTEPATH $HOSTLIST $COMMONDIR $UDR
  $SKIPFPART $MULTIHOST @SENDHOSTS @RECHOSTS $nbrhosts $REMOTECMD  
  $RPATH $RPATHPFX @UTILITIES $SKIPCOMPRESS $rUSER $rPATH $SKIPTO $BIGFILES $SPLITDIR $BF_OUT
  $FPART_BF_STDOUT $CHUNK_CUTOFF $R2 $T2    
  $avgTCPsend @rsyncs_2_rpt $RDMA @DEL_SPLITS $ZOTFILES @POST_FP_ZOT_CMDS 
  @LOC_POST_FP_ZOT_CMDS @REM_POST_FP_ZOT_CMDS @ZF_TO_XFER @SPLIT_PIDs
  @RUNNING @WAITQ $HOSTNAME
  );
  
  our ($NCPUs, $sPIDs, $RSYNC_PIDFILE, $hdr_cnt, $hdr_rpt, $FPSTART, $R1, $T1,
  $VERBOSE, $hdr_pr, $CUR_FPI, $FPSTRIDE, $NBR_FP_FLES, $RDMA_T1, $RDMA_T2, 
  $RDMA_R1, $RDMA_R2, $NETIF, $PERFQUERY, $CHECKPERIOD);

##: Pre-Getopt var declarations
my @argv = @ARGV;
# $SKIPFPART = 0; # SKIPFPART is used below as *defined or not*, so don't define it
#$FPSTART = 1;   # trivial check to print only one header; 
                # also defines where $CUR_FPI starts 
                # (fpart now starts at 1, not 0, to support bigfiles)
$PARSYNCVER  = << "VERSION";
parsyncfp2 version 2.59A (Apologies)
June 1, 2025
by Harry Mangalam <hjmangalam\@gmail.com>

parsyncfp2 is a Perl script that wraps the near-miraculous Tridgell/
Mackerras 'rsync' to provide load balancing and parallel operation across
network connections to increase the amount of bandwidth it can use.
The 'fp' variant uses 'fpart' to bypass the need for a full recursive 
descent of the dir trees before the actual transfer starts.
Versions >2 allow multihost sends to increase bandwidth saturation
WARNING: Do NOT use any '--delete*' options in the rsync options ('--ro').  
parsyncfp2 is distributed under the Gnu Public License (GPL) v3.
VERSION

##: Getopt options & Setup
&GetOptions(
  "startdir|sd=s"    => \$ROOTDIR,        # Have to be able to set rootdir -> SRC in rsync
  "altcache|ac=s"    => \$ALTCACHE,       # alternative cache instead of ~/.pfp2
  "ro=s"             => \$RSYNCOPTS,      # passthru to rsync as a string
  "NP|np=i"          => \$NP,             # number of rsync processes to start
  "chunksize|cs=s"   => \$FPARTSIZE,      # the size that fpart chunks (allow PpTtGgMmKk)
  "checkperiod|cp=i" => \$CHECKPERIOD,    # # of sec between system load checks
  "filesfrom|fl=s" => \$FILESFROM,    # take list of input files from file instead of fpart recursion.
  "trimpath|tp=s" => \$TRIMPATH,    # trim the string from the front of the file path.
  "trustme|tm!"   => \$TRUSTME,     # sizes in listfile are correct; don't bother w/ stat
  "maxbw=i"       => \$MAXBW,       # max bw to use (--bwlimit=KBPS passthru to rsync)
  "maxload|ml=f"  => \$MAXLOAD,     # max system load - if > this, sleep rsyncs
  "email=s"       => \$EMAIL,       # email to notify when finished
  "interface|I=s" => \$NETIF,       # network interface to use if multiple ones
  "rdma!"         => \$RDMA,        # user decides about RDMA bytes or not
  "verbose|v=i"   => \$VERBOSE,     # how chatty it should be.
  "nowait|nw!"    => \$NOWAIT,      # sleep a few s rather than wait for a user ack
  "slowdown|sd=f" => \$SLOWDOWN,    # introduce x seconds between commands to other
                                    # SEND & REC hosts; crudely set by ping time in
                                    # checkhost() now but use this to make explicit.
  "help!"         => \$HELP,        # dump usage, tips
  "version|V!"    => \$VERSION,     # duh..
  "dispose=s"     => \$DISPOSE,     # what to do with the cache (compress, delete, leave untouched)
  "debug|d!"      => \$DEBUG,       # developer-level info; (historical) alias for '-v 3'
  "udr!"          => \$UDR,         # non-user/expt'l; call for the use of UDR/UDT if available
  "hosts=s"       => \$HOSTLIST,    # series of SEND and REC hosts, optionally with REC paths
  "maxbgjobs=i"   => \$MAX_BG_JOBS, #  Max # background jobs to run simultaneously' (see zotfiles,
                                    # bigfiles below). If not set, defaults to some number,
                                    # automatically increments --maxload by 1.5x$MAX_BG_JOBS if not
                                    # explicitly set.
  "zotfiles=s"    => \$ZOTFILES,    # opt string is 'Max Avg file size
                                    # so its a 2 part string that has to be parsed into $ZOTFILES and 
                                    # $MAX_BG_JOBS. Shared with --bigfiles.  (see bigfiles below)
                                    # zillions of tiny files opt; the opt string is the avg # of files
                                    # in the chunk that if exceeded, triggers the tar/LZ4 of all files
                                    # in the chunk into a tarball to send and then untar at remote end.
  "bigfiles|bf!"  => \$BIGFILES,    # $BIGFILES uses $MAX_BG_JOBS = Max # of bg split commands to run
                                    # simultaneously
                                    # see above for cooperation with --zotfiles, with which it shares a
                                    # common Q'ing system.  $MAX_BG_JOBS = the total for BOTH
                                    # zotfiles and bigfiles, so if it's 4 and there are already 4
                                    # zotfile jobs running, bigfiles jobs will just go into the Q until 
                                    # a zotfile job finishes. These jobs will also increase the system 
                                    # load so consider this when setting --maxload altho setting the 
                                    # job number will automatically increase the maxload to 
                                    
  "commondir=s"   => \$ALTCACHE,    # alias for ALTCACHE. If it's the multihost version, you have to
                                    # 'mkdir $ALTCACHE/hostname' & write all their logs, etc in there,
                                    # read the fpart chunks from $ALTCACHE
  "reusechunks:i"  => \$SKIPTO,     # reuse the fpart chunk cache; don't run fpart again, very useful
                                    # for failed runs that need to be restarted; the optional integer 
                                    # is the step to skipto.  If no int passed, it defaults to 0 (incr to 1)
  "fpstart=i"     => \$FPSTART,     # non-user: per-host chunk START number
  "fpstride=i"    => \$FPSTRIDE,    # non-user: chunk STRIDE number - same for all hosts, but needs to be passed
  "skipfpart!"    => \$SKIPFPART,   # non-user: tell SEND hosts to skip their own fpart launches
  "date=s"        => \$MASTERDATE,  # non-user: pass the master date to slaves for so they'll be in sync.
  "mstr_md5=s"    => \$MSTR_MD5,    # non-user: the master parsyncfp2 checksum to compare with slave $MD5SUM
  "checkhost!"    => \$CHECKHOST,   # request that pfp check remote send and rec hosts to make sure that all the rsyncs have finished
  "rpathpfx=s"    => \$RPATH,       # the remote PATH prefix on the SEND hosts for the bits needed to run this.
);

##: Var declarations
# Vars to allow calling exe to be renamed/aliased willy-nilly (or anything else) and still work.
my $CALLING_PROGRAM = $0;
my $BN_CALLING_PROGRAM = basename($CALLING_PROGRAM);
$hdr_pr = 1; # repeat indicator for printing the header. 
my $NrPIDs = my $NsPIDs = 0;

# Required Utility checks
# ip fpart ethtool iwconfig perfquery ibstat udr omitted since they're either shared lib exe's
# or they're everywhere.  Instead test for interfaces and then dump a list of packages that
# include the required ones for this package.
@UTILITIES = ( $BN_CALLING_PROGRAM, "scut", "stats" );    #only push-copy the Perl scripts that I supply

##: Reset colors
print STDERR color('reset');
print STDOUT color('reset');

##: Declare run-permanent vars
@spinner = ( '-', '\\', '|', '/' );    # for spinner symbol 
$HOSTNAME = `hostname -s`; chomp $HOSTNAME;    # which host we're on

if ($SKIPFPART) {
  $DATE = $MASTERDATE;                         # take date from master; don't set your own
} else {
  $DATE = `date +"%T_%F" | sed 's/:/./g' `; chomp $DATE;    # master sets the date.
}
if   ( !defined $ALTCACHE ) { $parsync_dir = "${HOME}/.pfp2"; }
else                        { $parsync_dir = "${ALTCACHE}/.pfp2"; }
if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
##: Set Master Shell
# get the MASTER shell to set local shell vars if needed (export vs setenv)
$MSTR_SHELL = `echo \$SHELL`; chomp $MSTR_SHELL;

# get md5sum of executing program to pass to slave to check identical programs running on both ends.
##: MD5 checks of executable
$MD5SUM = `md5sum $0 | awk '{print \$1}'`; chomp $MD5SUM;
if ( defined $SKIPFPART && defined $MSTR_MD5 ) {
  if ( $MD5SUM != $MSTR_MD5 ) {
    FATAL("The master parsyncfp2 (invoked as [$CALLING_PROGRAM]) 
  and the SEND host parsyncfp2 [$0] are 
  not the same version (non-matching MD5 checksums).  This will end badly.  
  Please assure that your PATH allows the correct parsyncfp2 (or whatever you've named it) 
  to be found.  The SEND host parsyncfp2 program should be in [${parsync_dir}]"
    );
  }
}

# if -reusechunks not used, it's undefined, then set to -1 below, if doesn't have an int arg, set to 0
# in any case i fit's used, it either starts at 0 (--reusechunk alone) or is 
# decremented by 1 which is the correct handling.

if (! defined $SKIPTO) { $SKIPTO = -1; }  # only way that $SKIPTO can be <0
if ($SKIPTO == 0) { $SKIPTO = 1; } # used to define $CUR_FPI below
if ($SKIPTO > 0) { $SKIPTO--; } # the printed index is CURFPI+1 (skipto=13 prints index 14.)
elsif ($SKIPTO < -1) { FATAL("'--reusechunks' value must be > 1. Please try again."); }

$DBC       = 0;
$MULTIHOST = 0;
$FPART_BF_STDOUT = "";
$NETFILE   = "/proc/net/dev";
$OS        = `uname -s`; chomp $OS;
$Linux     = $MacOSX = 0;
if   ( $OS =~ /Linux/ ) { $Linux  = 1; }
else                    { $MacOSX = 1; }
my $host = `hostname -s`; chomp $host;
$PERFQUERY        = 0;
$WARN_FPART_FILES = 2000;    # issue warning at this point.
$MAX_FPART_FILES  = 5000;    # die at this point; 'ls' listing will overflow
$IB_PRSNT         = 0;
# Bonus optimization for uncompressible files once I decide how to offer it.
# $SKIPCOMPRESS =
#     "--skip-compress=/arc/arj/arw/asf/avi/bz2/cab/cr2/crypt[5678]/dat/dcr/deb/dmg/drc/ear/erf/flac"
#   . "/flv/gif/gpg/gz/iiq/iso/jar/jp2/jpeg/jpg/k25/kdc/lz/lzma/lzo/m4[apv]/mef/mkv/mos"
#   . "/mov/mp[34]/mpeg/mp[gv]/msi/nef/oga/ogg/ogv/opus/orf/pef/png/qt/rar/rpm/rw2/rzip"
#   . "/s7z/sfx/sr2/srf/svgz/t[gb]z/tlz/txz/vob/wim/wma/wmv/xz/zip/";
$DFLT_RSYNCOPTS = "-asl ";    # the default options to pass to rsync; blanked if define $RSYNCOPTS

##: Define cache and log dirs
# the fpart files have to be in the top-level parsync_dir, not in the per-hostname subdirs
# so defined here.  After these are defined, can set the per-hostname subdirs just below
$FP_SAVE_DIR   = $parsync_dir;
$FP_ROOT_DIR   = "${parsync_dir}/fpcache";
$FP_HOLD_DIR   = "${FP_ROOT_DIR}/hold";
$PFP2STOPSCRIPT = "${parsync_dir}/pfp2stop";
if ( defined $ALTCACHE && defined $SKIPFPART ) {
  $ALTCACHE .= "/.pfp2/${HOSTNAME}";    # stuff everything into a .pfp subdir to keep it away from user files.
  $parsync_dir = $ALTCACHE;
  if ($DEBUG) { DEBUG( __LINE__,  "parsync_dir = $parsync_dir" ); }
}
# Define the root name of the fpart chunk files f.1, etc.  
# Held in HOLD dir until complete and then moved to $FP_ROOT_DIR
$FP_HOLD_ROOT  = "${FP_HOLD_DIR}/f";
$FP_ROOT       = "${FP_ROOT_DIR}/f";
$FPART_LOGFILE = $FP_ROOT_DIR . '/' . "FPART.log." . $DATE;
$FP_PIDFILE    = $FP_ROOT_DIR . '/' . "FP_PIDFILE" . $DATE;
$hdr_rpt       = 20;                                          # nbr of lines to repeat the header
$hdr_cnt = $hdr_rpt + 1;    # header counter; > $hdr_rpt so it gets printed 1st time
if ( !-d "$parsync_dir" ) {
  make_path "$parsync_dir"
    or FATAL("Can't create the required parsyncfp logging dir [$parsync_dir]");
}

# set up a special file to log suspend and unsuspends PIDs to see where/if they get mixed up.
# number the suspended PIDs to see when / if they get unsuspended.
open( SUSLOG, "> $parsync_dir/suspend.log" )
  or FATAL("Can't open SUSLOG [$parsync_dir/suspend.log].");
my $susp_cnt   = 0;
my $unsusp_cnt = 0;
# includes the $HOSTNAME subdir from $ALTCACHE
$RSYNC_PIDFILE = "${parsync_dir}/rsync-PIDs-${DATE}";

##: Get current system stats  
# #CPUs, load, bandwidth, etc
$NCPUs = `cat /proc/cpuinfo | grep processor | wc -l`;
chomp $NCPUs;
$loadavg = `cat /proc/loadavg | tr -d '\n'`;
my $pid_max = `cat /proc/sys/kernel/pid_max`;
$lenPID = length $pid_max;    # usually 5 but can go as high as 7
@SYSLOAD = split( /\s+/, $loadavg );    # 1st 3 fields are 1, 5, 15m loads
# so as long as the 1m load / NCPUs < 1, we're fine; if > 1, we may want to start throttling..
$LOAD1mratio = $SYSLOAD[0] / $NCPUs;
my $LowBWMax = 1000;
my $rrcc = 0; # running rsyncs cycle counter

if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}

##: Define & init Getopt flag vars
if ( defined $HELP ) { usage($parsync_dir); }

# UDR not currently working until I can test it on ESNET
if   ( !defined $UDR )        { $UDR = ""; }
else                          { $UDR = "udr -d 10 "; } # the udr prefix (-v = verbose, -d timeout)
if ( !defined $BIGFILES)      { $BIGFILES = 0; $BF_OUT = "";}
else { # insert the -S flag, send STDOUT to the BIGFILES file.
  $FPART_BF_STDOUT = " -S "; 
  $BF_OUT = " > ${parsync_dir}/BIGFILES ";
  @SPLIT_PIDs = (0) x $BIGFILES ; # init @SPLIT_PIDs to 0s
} 

# resolve all the --zotfiles options / conflicts
$TARGET = $ARGV[$#ARGV];
if ( defined $ZOTFILES && $TARGET =~ /POD::/ ) {
  FATAL("Can only use the '--zotfiles' option with the UniHost version 
  (ie, no 'POD::' targets.)  Remove the --zotfiles option or use the UniHost 
  version as the target 'user\@somehost:/path/to/store'");
}
if ( !defined $ZOTFILES )           { $ZOTFILES  = 0; }  
elsif ( $ZOTFILES =~ /\d+.?\d?[PpTtGgMmKk]/ ) { $ZOTFILES  = ptgmk($ZOTFILES); }
elsif ( $ZOTFILES =~ /^\d+.?\d?$/ )         { $ZOTFILES  = $ZOTFILES; }
else { FATAL("Argument for --zotfiles must be a number or a string ending
in [PpTtGgMmKk] to indicate kilo-, mega-, giga-, etc bytes. 
Decimals are OK (45.67M).  Try again.")}

if ( !defined $FPSTART )      { $FPSTART  = 1; }  # value for the unihost version TODO: new fpart -> 1
if ( !defined $FPSTRIDE )     { $FPSTRIDE = 1; }  # value for the unihost version 
if ( defined $VERSION )       { print colored( ['green'], $PARSYNCVER, "\n" ); exit; }
if ( !defined $CHECKPERIOD )  { $CHECKPERIOD = 3; }
if ( !defined $VERBOSE )      { $VERBOSE     = 2; }
if ( !defined $RTT_DELAY )    { $RTT_DELAY   = 0.5; }
if ( !defined $DEBUG )        { $DEBUG       = 0; }
else                          { $VERBOSE     = 3; }
if ( !defined $NP ) { $NP = (int ($NCPUs) + 0.5 ); } # round sqrt(NCPUs) (hyperthreaded if Intel) 8 
if ( !defined $MAXBW ) { $MAXBW = 0; }    # 0 = unlimited according to rsync
else { $MAXBW = int( $MAXBW / $NP + 0.5 ); }    # users expect total maxbw; so have to divide by NP.
if ( !defined $MAXLOAD ) { $MAXLOAD = $NP * 3; } # as per the testing on FS-FS 
if ( !defined $ROOTDIR ) { $ROOTDIR = `pwd`; chomp $ROOTDIR; }    # where all dirs must be rooted.
if ( !defined $FPARTSIZE ) { $FPARTSIZE = "10G"; $FPARTSIZE_N = 104857600; }    # default is 10Gish
elsif ( $FPARTSIZE < 0 ) {
  $FPARTSIZE        = $FPARTSIZE * -1;
  # $SKIP_FPART_CHECK = 1; # not changed anywhere 
}    # Tells check to ignore huge #s of chunkfiles
if   ( $FPARTSIZE =~ /[PpTtGgMmKk]/ ) { $FPARTSIZE_N = ptgmk($FPARTSIZE); }
else                                  { $FPARTSIZE_N = $FPARTSIZE; }
if ($DEBUG) {
  DEBUG( __LINE__,  "FPARTSIZE set to: [$FPARTSIZE]\nFPARTSIZE_N set to [$FPARTSIZE_N]" );
}
$CHUNK_CUTOFF = $FPARTSIZE * 1.2;  # (for --bigfiles) if it's only a little bigger, don't bother.

if ( defined $VERBOSE && ( $VERBOSE < 0 || $VERBOSE > 3 ) ) {
  FATAL("ERROR: --verbose arg must be 0-3. Try again.");
}
# following needs work to allow things like --compression-level, etc
if ( $RSYNCOPTS =~ /-[a-zA-Z]+[vh]/ || $RSYNCOPTS =~ /-[vh]/ ) {
  FATAL("Detected an option in your rsync option string [$RSYNCOPTS] that 
  makes too much noise (probably -v, -h --verbose, --version). Try again.."
  );
}

##: reformat RSYNC_OPTS 
# same-SEND host needs single quotes, MH format needs requoted into '"double quoted"'
if ($TARGET =~ /POD/ && defined $RSYNCOPTS && $RSYNCOPTS !~ /^" && $RSYNCOPTS !~ "$/) {
  $RSYNCOPTS = '\'"' . $RSYNCOPTS . '"\'';
}

if ( !defined $RSYNCOPTS ) { $RSYNCOPTS = " -asl "; } # single word opts don't need the double quoting.
elsif ( $RSYNCOPTS !~ /^'"-/ && $RSYNCOPTS !~ /^-/) {
  FATAL("You didn't have a leading '-' in your '--ro' string. 
  You need to supply the options exactly as you would with rsync, with 
  leading '-' or '--'. Try again..");
} else {    # if def $RSYNCOPTS, then user takes all responsibility
  $DFLT_RSYNCOPTS = "";
  if ( $RSYNCOPTS =~ / -d / || $RSYNCOPTS =~ / --del/ ) {  # user tries to pass in a 'delete' option
    FATAL("It looks like you're trying to pass in a '--delete' option 
  in the '--ro' string.  [$RSYNCOPTS]
  Because parallel rsyncs don't know what the other rsyncs are doing, 
  'delete' options don't work well. If this is what you want to do, 
  omit that option here and follow the parsyncfp2 command with a regular 
  'rsync --delete' command.  It will be slower than a parallel 
  operation but since most of the action will be remote deletes, 
  it should be fairly fast.

  If the operation is to be performed on locally mounted filesystems 
  (not to remote nodes), I'd strongly recommend the 'fpsync' tool, which 
  you should have already received as part of the 'fpart' package necessary 
  to run parsyncfp2. 'fpsync' DOES provide support for a parallel '--delete', 
  and the author provides a good explanation as to how he does this here:
  <https://goo.gl/dtwp3P>.  HOWEVER!! Anytime you use '--delete' in an rsync 
  operation, MAKE SURE you know what you're doing."
    );
  }
}
if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
##: ARGV processing
# this takes care of the last ARGV so that all the rest of the words are target dirs&files
#$TARGET = $ARGV[$#ARGV];    # remote rsync target
if ( !defined $TARGET ) {
  FATAL("No target defined! Where you gonna put this stuff??!?\nTry '--help' for the built-in help.");
}
$#ARGV--;

if (!$SKIPFPART && $NP < 4 ) { 
  WARN("The --NP value is only [$NP]; to take advantage of pfp2, this should be 
  set to at least 4. Continuing inefficiently.."); 
}

##: parse_rsync_target call
# ### TODO move to appro place and reduce the args to what we need. Does the remote 
# target processing once and for all.
# init all the vars needed
my $rem_user        = "";
my $rem_host        = "";
my $rsync_type      = "";
my $rem_fqpath      = "";
my $rem_homepath    = "";
my $RSYNCMODULE     = ""; # naming hiccup
my $pod_remotepath  = "";
my $recv_hoststring = ""; 
my $rat_hoststring  = "";
# $pod_remotepath will be used when I handle rsyncd servers into the target processing


# This call is just to establish some basic params for defining how vars are treated later on.  
# Mostly to see if this is a POD or not. And calling it with the POD string as TARGET and a 
# null string for the 4th arg is fine.
# This call returns only the $rsync_type (rsync, filesystem, server, etc) and the $pod_remotepath

# if TARGET is POD::/path, really want parse_rsync_target() to provide a $rat_hoststring that (optionally) combines the POD::/path with rem_user@rem_host 
# this parse_rsync_target() call is not POD-specific; should work on any pfp2 command, singlehost or multihost

($rem_user, $rem_host, $rsync_type, $rem_fqpath, $rem_homepath, $RSYNCMODULE, $pod_remotepath, $rat_hoststring ) = parse_rsync_target ($USER, $TARGET, $ALTCACHE, "");  

# pass it ($USER, $TARGET, $ALTCACHE, $recv_hoststring) and get back the rat_hoststring that can be used in the RSYNC_CMD, so it can be called before every RSYNC_CMD is composed, on every SEND host.

$defREMOTEPATH = $pod_remotepath; # integration hack for now.
  # define the useful $rat_hoststring for filesystem target, kind of a hack right now.

#print __LINE__, ": [$host]: parse_rsync_target($USER, $TARGET, $ALTCACHE, '') returns:
#     rem_user [$rem_user]
#     rem_host [$rem_host]
#     rsync_type [$rsync_type]
#     rem_fqpath [$rem_fqpath]
#     rem_homepath [$rem_homepath]
#     RSYNCMODULE [$RSYNCMODULE]
#     pod_remotepath [$pod_remotepath]
#     rat_hoststring [$rat_hoststring]\n";  pause(); #exit;

##: Hostlist processing
my $chckd_rec = ""; # string to which rec hosts already checked have been appended to prevent double checking

if ( defined $HOSTLIST ) { # from --hosts=$HOSTLIST, should never be processed by the SEND hosts.
  $MULTIHOST = 1;
# collect the paths of all the pfp-associated utilities on the master.
  # only needs to be done once.
  if ($TARGET !~ /POD::/) { FATAL("Using the '--hosts' option requires the use of the 'POD::' string 
  as the target (last item on the command).  See '--help'.")}
  if (!defined $ALTCACHE) {FATAL("Using the '--hosts' option requires you define '--commondir' as well.
  See '--help'.")};
  
  my $allutils = "";
  for ( my $i = 0 ; $i <= $#UTILITIES ; $i++ ) {
    my $upath = `which $UTILITIES[$i] 2>&1`; chomp $upath;
    if ( $upath ne "" && $upath !~ /which: no/ ) {    # CentOS7 returns long stderr explanation
      $allutils .= "${upath} ";
    }
  }
  if ( !defined $NETIF ) {
    $NETIF = `route | grep default | head -1 | awk '{print \$8}'`; # 'head -1' takes the 1st if there are 2
    chomp $NETIF;   # grabs default IF on localhost
    if ( $VERBOSE >= 2 ) {
      INFO("The Multihost option requires the network interface to /one/ of the 
  SEND hosts be specified with [--interface].  I've guessed [$NETIF].
  If that's not right, please correct it next time.\n\n");
    }
  }
  # split the hostlist into an array and then process into co-indexed send and rec hosts
  # "s1=r1:/path1,s2=r2:/path2,s3=r3:/path3,s4=r4 s5=r5"
  $N = @A = split( /,/, $HOSTLIST );

  my $rhosts = "";
  my ( $checkcmd, $checkout );
  if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
  # if no '~/.pfp-hostchecked' file, warn that you should run with --checkhost at least once 
  # to sync utils. Place out of tree to avoid being deleted at each re-run.
  if (! -e "$HOME/.pfp-hostchecked" && !defined $CHECKHOST ) {
    WARN("Looks like you've never asked for the '--checkhost' option. 
  You should run it at least once to make sure the utilities are synced 
  across send hosts. Especially if you've been messing with them.  Just sayin..");
  }
  # Have @A containing SEND=REC pairs, now process them 
  ##: Process the hostlist array by pairs
  my @b;  # used below
  for ( my $i = 0 ; $i < $N ; $i++ ) { # $N is number of SEND=REC pairs
  my $rtt_delay = 0.5; # lc is internal
    @b = ("");    # reset for test below.
    my $n = my @a = split( /=/, $A[$i] );    # hostlist string split on '='
    if ( $n != 2 ) {
      FATAL("--host argument [$i] isn't formatted correctly; should be 
  [login\@]SendHost=login\@RecHost[:/path], with the 'login\@' & '/path' optional.
  (or for rsync servers: [login\@]SendHost=login\@Rsyncd_Host::module) with 
  the 'login\@' optional but the '::module' required)
  If no 'login\@' is given, the standard assumption is made that the local login is 
  identical to the hosts that are missing it.\n");
    }
    $SENDHOSTS[$i] = $a[0];
    if ($CHECKHOST) { # in following call, $a[0] will contain the $rem_user, so use "" for ruser arg
      $RTT = checkhost( "SEND", $a[0], "", "", "$ALTCACHE", $VERBOSE, $MAXLOAD );
      if ( $RTT > 1 ) { WARN("The RTT to [$a[0]] is > 1ms [$RTT]ms, which is long for a LAN.
  Are you using the correct set of SEND hosts? 
  If you get ssh failures, try using the '--slowdown' option 
  to slow the ssh attempts"); }
    }

    ##: Send pfp2 utils to SEND hosts    
    if ($CHECKHOST) {    # add to checkhost() when thrashed out.
                         # checking utilities required and providing them if not.
                         # this has to be DISallowed when communicating to an rsyncd server.
      if ( $VERBOSE > 2 ) { INFO("Sending utilities [$allutils] to [$a[0]:${parsync_dir}]\n"); }

      # do it to self as well
      my $ucmdo = `rsync $allutils $a[0]:${parsync_dir} 2> /dev/null`;
      chomp $ucmdo;    # $parsync_dir is already added to $RPATHPFX above
      if ( $ucmdo ne "" ) {
        WARN("rsync of required/recommended utilites resulted in: [$ucmdo]"); sleep 2;
      }
      system "echo 'hostcheck verified from parsyncfp2' > \"$HOME/.pfp-hostchecked\" ";
    }

    # Since it's in HOSTLIST block, parse_rsync_target() will only get POD:: args.  ie $TARGET will
    # be a POD::/path string and last param will be a REC host string from the HOSTLIST
    ##: parse_rsync_target in HOSTLIST block, using $TARGET and REC host as last param
    ($rem_user, $rem_host, $rsync_type, $rem_fqpath, $rem_homepath, $RSYNCMODULE, $pod_remotepath, $rat_hoststring ) = parse_rsync_target ($USER, $TARGET, $ALTCACHE, $a[1]); # 
    # so the following line should never have a non-MASTER host label    

       print __LINE__, ": [$host]: parse_rsync_target($USER, $TARGET, $ALTCACHE, $a[1]) returns:
#   rem_user [$rem_user]
#   rem_host [$rem_host]
#   rsync_type [$rsync_type]
#   rem_fqpath [$rem_fqpath]
#   rem_homepath [$rem_homepath]
#   RSYNCMODULE [$RSYNCMODULE]
#   pod_remotepath [$pod_remotepath]
#   rat_hoststring [$rat_hoststring]\n"; pause(); #exit;

##: checkhost block for RSYNC, RSYNCD REC hosts: 
    if ($rsync_type eq "server" && $CHECKHOST && $chckd_rec !~ /$rem_host/) {
      $RTT = checkhost( "RSYNCD", $rem_host, $rem_user, $RSYNCMODULE, "NONE", $VERBOSE, $MAXLOAD);
      $chckd_rec .= "$rem_host"; # $rem_host is fine for this backcheck
    } elsif ($rsync_type eq "rsync" && $CHECKHOST && $chckd_rec !~ /$rem_host/) {
      $RTT = checkhost( "RSYNC", $rem_host, $rem_user, "", $ALTCACHE, $VERBOSE, $MAXLOAD);
      $chckd_rec .= "$rem_host"; 
    }
    $RECHOSTS[$i] = $rat_hoststring;
    $MULTIHOST++;    # use to track hosts
    # let's see how this works out.
    if ( $RTT < 1 ) { $rtt_delay = 0.5; }
    else            { $rtt_delay = $RTT/20; } # based on ITER net tests (02.19.22), 20 is about right.
    if ($rtt_delay > $RTT_DELAY) { $RTT_DELAY = $rtt_delay; }
  }
  $MULTIHOST--;      # since it started at 1
}

if (defined $SLOWDOWN) { $RTT_DELAY = $SLOWDOWN; }  # --slowdown overrides measured RTT_DELAY

#if ( !$SKIPFPART && !@ARGV && !$FILESFROM ) { print __LINE__; usage($parsync_dir); exit;}    # in case someone doesn't know what to do.  This may be more redundant than necessary..? yes
 
if ( !defined $DISPOSE ) { $DISPOSE = 'l'; }    # for leave untouched
if ( defined $HOSTLIST && $DISPOSE ne 'l' ) {
  WARN("You requested the --dispose=[$DISPOSE] option, but you must use the '--dispose=l' option 
  when using the Multihost function.  --dispose has been set to 'l'");
  $DISPOSE = 'l';
}

my ( $nbr_ifs, $rtd_ifs );
if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
# this is where the initial NETIF determination is extracted.
# the multihost (--hosts) stanza is further below
# this stanza is only looking at the $TARGET value to decide what to do
# if its MH, the NETIF must be defined

# stanza below gets skipped on the MH 1st pass, but is executed in the 2nd (slave) MH pass,
# and is the ONLY part that determines a non-MH NETIF, the core of which looks like:
# rsync --bwlimit=0  -aszl --log-file=yaddafile --files-from=/home/hjm/pfp/fpcache/f.1 \
# '/home/hjm' \
#             user@host:/path/to/store    host-based, fq path
#         or  user@host:~/path/to/store   host-based, user-homed path (if allow '~')
#         or  user@host::rsync_module     host-based, rsyncd connection
#         or  /path/on/mounted/fs         same host will run both sides of the rsync.


##: NETIF determination
if ( !defined $NETIF || $NETIF eq "" ) {
  if (0) {    # $MacOSX - don't want to think about this now..
    $NETIF = `netstat -nr | grep "^default" | head -n1 | awk '{print \$6}'`;  chomp $NETIF;
    $myIP = `ifconfig $NETIF | grep 'inet ' | awk '{print \$2}'`;  chomp $myIP;
  } else {    #  Linux stanza
    # this is using  $TARGET to both args 2 & 4 of parse_rsync_target(), processing them a bit differently
    if ($rat_hoststring eq "") {
      ($rem_user, $rem_host, $rsync_type, $rem_fqpath, $rem_homepath, $RSYNCMODULE, $pod_remotepath,
      $rat_hoststring ) = parse_rsync_target($USER, $TARGET, $ALTCACHE,'');  
    }
    
    my ( $nrd, @rd, $nrdu, @rdu );
    # may have to re-assign some var names to smooth things later on..
    $rUSER = $rem_user;
    $rHOSTNAME = $rem_host; # needed anymore??
    $defREMOTEPATH = $rem_fqpath;
    
     if ( $rsync_type eq "filesystem" ) {
       WARN("parsyncfp2 does not work well across mounted filesystems (it's a NETWORK 
  protocol), so I'll honor your request but this will be fairly slow and the 
  transfer bandwidth will be inaccurate or unavailable. 
  There are long explanations why this is the case on the intertubes, 
  especially on the rsync discussion list.");
      sleep 2;
     }
    
    if ( $rsync_type !~ /filesystem/ ) {
      # next line works on any non-local host, but if the master is also using itself
      # as a slave then it will fail due to a different output format depending on
      # whether it's local or 'via' the router.
      # workaround to extract a valid address for a private, unDNS'ed hostname.
      # the head -1 bit below makes sure we only take the 1st entry in output
      my $getrhost = `getent hosts $rHOSTNAME | head -1 | awk '{print \$1}'`; chomp $getrhost;
      if ( $getrhost eq "" ) { $getrhost = $rHOSTNAME; }
      my $iproute = `ip -o route get $getrhost`;
      if ( $iproute =~ /^local/ ) {    # localhost situation
        $rHOSTNAME = 'localhost';
      }

      my @ipr = split( /\s+/, $iproute );
      if ( $rHOSTNAME =~ 'localhost' ) {
        $NETIF = `ip a | grep ' UP ' | head -1 | scut -f=1 -d=': '`; chomp $NETIF;
        if ($VERBOSE > 2) { WARN("'Choosing the 1st active interface [$NETIF] to monitor."); }
      } elsif ( $ipr[1] eq "via" ) {
        $NETIF = $ipr[4];
      }    # address 'via' router
      else { $NETIF = $ipr[2]; }    # local net address NOT 'via' router}
          # so this next line should generate the routable IP# to the target, regardless of
          # which network it's on.  Thanks Ryan Novosielski for the suggestion.
      $myIP = `ip addr show dev $NETIF | grep 'inet ' | scut -f=2 | sed 's/...\$//'`; chomp $myIP;

      if ($DEBUG) { DEBUG( __LINE__,  "NETIF = [$NETIF], myIP = [$myIP]" ); }
    } else {    # guess at the NETIF if there are multiple.
      $rtd_ifs = `ip link show | grep ' UP ' | scut -f=1 --id1=': ' | tr '\n' ' '`; chomp $rtd_ifs;
      my $ni = my @i = split( /\s+/, $rtd_ifs );
      $nbr_ifs = `ip link show | grep ' UP ' | wc -l`; chomp $nbr_ifs;
      if ( $nbr_ifs eq '1' ) {
        $NETIF = $rtd_ifs;
      } elsif ( !defined $NETIF ) {
        WARN(
          "You have >1 active network interfaces and I can't tell which one 
    you want to monitor.  Next time, please use '--interface' to choose one of:
    [$rtd_ifs]. I've randomly chosen [$i[0]].\n"
        );
        $NETIF = $i[0];
      }
    }
  } #  Linux stanza
}  # if ( !defined $NETIF || $NETIF eq "" )

if ( $VERBOSE > 2 ) { INFO("Using [$NETIF] to send data and to monitor\n"); }

##: IB / perfquery 
# reframe the previous if ($NETIF =~ /ib/) stanza #'ed below as 
my $pqpath = "";
if ($RDMA){
  $pqpath   = `which perfquery`;
  INFO("You've specified what looks like an Infiniband interface [$NETIF]...\n");
  if ( $pqpath ne "" ) {
    $PERFQUERY = 1;
    INFO(".. and you have 'perfquery installed, so RDMA bytes will be reported as well.\n");
  } else {
    $PERFQUERY = 0;
    INFO(".. but you don't have 'perfquery' installed, so only TCP bytes will be reported.\n");
  }
}
if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
##: get IF_SPEED 
# Need to bring it up to date with the new naming conventions
# see: https://goo.gl/kDLr8b
# get some network info
if ( $NETIF =~ /eth|en/ ) {
  $IF_SPEED = `ethtool $NETIF 2> /dev/null | grep Speed | cut -f2 -d:`;
} elsif ( $NETIF =~ /wl/ ) {
  $IF_SPEED = `iwconfig $NETIF | grep -i quality`;
} elsif ( $NETIF =~ /ib/ ) {
  $IF_SPEED = `ibstat | grep Rate | head -1 | sed -e 's/^[ \t]*//'`;
  $IF_SPEED = "IB:" . $IF_SPEED;
}
chomp $IF_SPEED;

if ($DEBUG) {
  DEBUG( __LINE__,  "Using network interface [$NETIF] with connection quality [$IF_SPEED]" );
}

##: fix .ssh/config 
fix_ssh_config();

$IF_SPEED = 0;

##: checkhost on SINGLEMASTER, RSYNCD, RSYNC hosts (NOT POD hosts)
# now check SEND and REC hosts if part of a SINGLEHOST launch and if asked
if ( $CHECKHOST && $MULTIHOST == 0 && !defined $SKIPFPART && $rHOSTNAME ne "FILESYSTEM" ) {
  if ( $TARGET =~ /::/ ) { # no user for rsyncd below
    $RTT = checkhost( "RSYNCD", $rem_host , "", $RSYNCMODULE, "", $VERBOSE, $MAXLOAD );# $rHOSTNAME
  } else {
    $RTT = checkhost( "RSYNC", $rem_host, $rem_user, "", "", $VERBOSE, $MAXLOAD );
  }
  # let's see how this works out.
  if ( $RTT < 1 ) { $RTT_DELAY = 0.5; }
  else            { $RTT_DELAY = $RTT/7; } # 10 = 1, 20 = 2, etc
}

##: Check loadavg too high
if ( $SYSLOAD[0] < $MAXLOAD ) {
  if ($DEBUG) {
    DEBUG( __LINE__, 
      "1m load is [$SYSLOAD[0]] and the 1m Load:#CPU ratio is [$LOAD1mratio] ( [$NCPUs] CPU cores).
    OK to continue."
    );
  }
} else {
  FATAL("1m loadavg on [$HOSTNAME] is > [$SYSLOAD[0]].  
  The 1m Load:#CPU ratio is [$LOAD1mratio].
  If this is a SingleHost pfp2, the rsyncs will suspend immediately.  
  You might want to kill off the competing processes, raise the --maxload 
  value, or wait until the other processes have finished.  
  Exiting to allow you to adjust the parameters."
  );
}

##: == MASTER ONLY ==
if ($SKIPTO > -1 && $VERBOSE >= 2) { INFO(" Re-using previously created chunks.\n") }
if ( $SKIPFPART == 0 && $SKIPTO == -1) {    # skip this if it's the multihost slave or --reuse 
  if ( -d $parsync_dir ) {
    if ( $VERBOSE >= 1 && !$NOWAIT) {
      WARN("About to remove all the old cached chunk, log, 
  and PID files from [$FP_ROOT_DIR]
  and the previous rsync log files from [$parsync_dir].       
  Enter ^C to stop this."
      );
    }
    if (defined $HOSTLIST) {      
      INFO("The kill script to stop all pfp2s and rsyncs on all the SEND and REC hosts is:
  [$PFP2STOPSCRIPT].  
  You may want to note or copy this path in case something goes wrong.\n\n");    
    }  
    if    ($NOWAIT)        { sleep 1; }
    elsif ( $VERBOSE > 0 ) { pause(); }
    my $nbr_els_deleted = rmtree("${parsync_dir}");    # ,{safe => 1} # should probably do this recursively rmtree()?
    if ( $VERBOSE >= 2 ) {
      INFO("[$nbr_els_deleted] files from the previous run have been cleared.
  Re-initializing dirs and configuring fpart.. This may take a few secs.\n\n");
    if ($BIGFILES > 0) { INFO("The '--bigfiles' option may require up to a few minutes to
  process the first large files before the rsyncs start.\n" ); }
    }
  }

  # and re-create it afresh
  if ( !-d $parsync_dir ) {
    make_path $parsync_dir or FATAL("Can't create [ $parsync_dir ]");
  }
  if ( !-d $FP_ROOT_DIR ) {
    mkdir $FP_ROOT_DIR
      or FATAL("Can't make 'FP_ROOT_DIR' [$FP_ROOT_DIR]");
  }
  if ( !-d $FP_HOLD_DIR ) {
    mkdir $FP_HOLD_DIR
      or FATAL("Can't make 'FP_HOLD_DIR' [$FP_HOLD_DIR]");
  }
}
if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
##: process Files & Dirs to send
$dcnt  = 0;
$fnd2r = "";                # zero the list of 'files 'n' dirs to rsync'
# only do this next stanza if NOT taking files from the '$FILESFROM
# define $fnd2r only if there's no existing $FILESFROM and this is the master (undefined $SKIPFPART)

if ( !defined $FILESFROM && !defined $SKIPFPART ) {
  $dirtmp = shift;    # should only be dir/files left once getopt finishes (see above)
  if ($DEBUG) { DEBUG( __LINE__,  "Composing the new fpart target dirtmp in a loop." ); }

  # If there are no files or dirs defined, take the current dir
  if ( !defined $dirtmp ) {
    FATAL( "
  You didn't define the files or dirs to transfer. 
  You used the --startdir=path option without providing the actual source(s) 
  afterwards separated from the option and each other with whitespace.  
  ie: to move '/usr/local/bin & /usr/local/lib': 
    --startdir=/usr/local bin lib TARGET
                         ^   ^ spaces" );
  }
  while ( defined $dirtmp ) {    # should work on explicitly named dirs as well as globs.
    if ($dirtmp !~ '^/') { $dirtmp = $ROOTDIR . '/' . $dirtmp; }
    if ( !-r $dirtmp ) {         # quick check to see if its readable.
      WARN("[$dirtmp] isn't readable. 
  This could be due to:
    - it's not where you think it is 
    - you're using a glob to specify dirs and you're not where you 
      think you are. Globs are immediately evaluated by the shell - try 
      to escape the regex char(s) so that evaluating the glob is deferred.
      ie 'dir\*' instead of 'dir*').
    - you need to escalate your privs.
  Regardless, [$dirtmp] won't be transferred in this run
  but if you specified other dirs, we'll try them.
        "
      );
      if    ($NOWAIT)        { sleep 3; }
      elsif ( $VERBOSE > 0 ) { pause(); }
    } else {                     # otherwise, add the file to list to be chunked and  transferred.
      $fnd2r .= "\'$dirtmp\'" . " ";
      if ($fnd2r =~ '//') { $fnd2r =~ s'//'/'; }
      if ($DEBUG) { DEBUG( __LINE__,  "Looping to add the fpart target: [$fnd2r]" ); }
    }
    $dirtmp = shift;
  }

  if ( $fnd2r eq "" ) {
    FATAL(
      "None of the dirs you specified were readable.  
  Please check again."
    );
  }
  
##: Process $FILESFROM, how to set up fpart cmd
} else {    # if $FILESFROM is defined, is $TRIMPATH defined? if so, is it valid?  End with a '/'?
  $tf = "${parsync_dir}/frmlst.tmp";
  if ( defined $TRIMPATH ) {
    $TRIMPATH = trim($TRIMPATH);
    if ( substr( $TRIMPATH, -1, 1 ) eq '/' ) { chop $TRIMPATH; }    #$TRIMPATH must not end in '/'
    $ROOTDIR = "$TRIMPATH";
    if ( -e $TRIMPATH && -d $TRIMPATH && -r $TRIMPATH ) {
      INFO("The TRIMPATH you specified exists, is a dir, and is readable.\n");
      # now process the input file to trim the TRIMPATH
      if ( -e $tf ) { unlink $tf or FATAL("Temp file [$tf] exists and can't be deleted.\n"); }
      open( CFL, "<$FILESFROM" ) or FATAL("Can't open FILESFROM [$FILESFROM]'");
      open( NFL, ">$tf" ) or FATAL("Can't open TEMPFILE [$tf]'");    # NFL can can be 'normal' FH
      my $lc = 0;
      while (<CFL>) {
        $lc++;
        if ( $_ =~ /$TRIMPATH/ )
        {    # this will now hit the top-level dir line alone since it will now be (ie) '/home/hjm'
          $_ =~ s%$TRIMPATH%%;    # kill the '/home/hjm'
          my $TT;
          if ($TRUSTME) {
            $N = @A = split( /\t/, $_ );
            my $tt = substr( $A[1], 1 );    # trim the remaining '/'
            $TT = $A[0] . "\t" . $tt;
          } else {
            $TT = substr( $_, 1 );
          }    # and now the leftover leading '/' is gone as well
          print NFL $TT;
        } else {    # if $TRIMPATH = '/home/hjm/' subst /home/hjm/nacs/hpc -> nacs/hpc
          chomp;
          print STDERR
            "Warning: [$_] in FILESFROM [$FILESFROM] line [$lc] doesn't have a [$TRIMPATH]\n";
        }
      }    # while (<CFL>)
      close CFL;
      close NFL;    # just close them, don't delete, cp, or mv them.
      if ($DEBUG) { DEBUG( __LINE__,  "# of lines in list: [$lc]" ); }
    }    # if (-e $TRIMPATH && -d $TRIMPATH && -r $TRIMPATH)
  }    # if (defined $TRIMPATH)
}

$#ARGV++;    # now incr to allow the TARGET to be captured.

##: Warn about OTHER FPs running
$OTHERFPRNG = `ps ux | grep ' fpar[t]'`; chomp $OTHERFPRNG;
if ( $OTHERFPRNG ne '' && $SKIPFPART != 1 ) {
  WARN(
    "One or more 'fpart's are already running:
    ======
    [$OTHERFPRNG]
    ======
    Unless you know that these fparts are valid (ie you're running 
    another pfp2 in another shell on this machine) and not 
    left over from previous pfp2's, you should ^C and kill 
    them off before restarting this run.
    
    Pausing for 5s to allow you to read this and take action (or not).
    If you do nothing, I'll continue.
    "
  );
  sleep 5;
}
if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
##: More $FILESFROM proc
my $x = 0;
$fnd2r =~ s/^\s+|\s+$//g;    # trim leading and trailing whitespace
my $FPART_CMD = "";
my $stdin; my $ttlfpcmd;
if ( defined $FILESFROM ) {
  # check to see if it exists & is a file & is readable
  if ( -e $FILESFROM && -f $FILESFROM && -r $FILESFROM ) {
    if ( $VERBOSE >= 2 ) {
      INFO("Alternative file list is readable; converting list to chunks.\n");
    }
  } elsif ( $FILESFROM eq '-' ) {
    $tf = '-';
  } else {
    FATAL(
      "The 'filesfrom input [$FILESFROM] doesn't exist,\nisn't a file (or STDIN), or isn't readable."
    );
  }

# fpart has 3 cmds:
# 1 - files input from a file ($tf) via STDIN which have sizes included.  The -S stdout 
# processing should still work since the sizes still have to be chunked as they would be if scanned.
# (I don't think this is actually tested yet.  TODO - check this more carefully.)
# 2 - files input from a file (again $tf) but this time via the '-i' flag. Ditto above.
# 3 - file info gotten via stat() thru the normal fpart mechanism (full scan). This is the usual
# mechanism that will be used for pfp2.  Start the -S/stdout integration with this one.
# inserting  $FPART_BF_STDOUT into all 3 cmds.  Will be "-S" if --bigfiles, "" if not
# all 3 fpart cmds capture stdout to ${parsync_dir}/BIGFILES which will be processed (if nec) 
# after fpart finishes

# convert to chunks with fpart
# following fpart uses the realtime option (-L) so that the code support should be same as for the original
# and capture the child PID!
  my $AFLAG = "";
  if ($TRUSTME) { $AFLAG = "-a "; }    # if user specs the format that includes sizes
  if ( $tf eq '-' ) {
    # the following cmd now includes the steps to write the in-process chunk files to $FP_ROOT
    #    $FP_HOLD_ROOT = $FP_HOLD_DIR . "/f";
    # and then once the chunk is complete, move them to the $FP_ROOT_DIR where the action takes
    # place after it's found that a chunk file exists there.
    
    ##: Set appro $FPART_CMD (no '&' appended; added when createthe full command)
    ##: fpart 1 - input from STDIN (not well debugged)
    
    $FPART_CMD = qq(fpart -v -L  -W 'mv \$FPART_PARTFILENAME $FP_ROOT_DIR' $FPART_BF_STDOUT -s $FPARTSIZE_N $AFLAG -i '-' -o $FP_HOLD_ROOT < $tf $BF_OUT 2> $FPART_LOGFILE);
    
    if ($DEBUG) { DEBUG( __LINE__,  "FPART_CMD(1) = [$FPART_CMD]\n" ) }
  } else {    # shell variable = $FPART_PARTFILENAME
    ##: fpart 2 - input from an already prepped file (ie from a GPFS prep)
    
    $FPART_CMD = qq(cd $TRIMPATH; fpart -v -L -W 'mv \$FPART_PARTFILENAME $FP_ROOT_DIR'  $FPART_BF_STDOUT -s $FPARTSIZE_N $AFLAG -i $tf -o $FP_HOLD_ROOT $BF_OUT  2> $FPART_LOGFILE);
    
    if ($DEBUG) { DEBUG( __LINE__,  "FPART_CMD(2) = [$FPART_CMD]\n" ); }
  }
} else {    # use the full recursive fpart
            # capture the child PID
            # this is the 'normal' fpart cmd that pfp usually uses
  ##: fpart 3 - input generated fr regular recursive descent (normal fpart usage) 
  
  $FPART_CMD = qq(fpart -vvv -L -W 'mv \$FPART_PARTFILENAME $FP_ROOT_DIR' -z  $FPART_BF_STDOUT -s $FPARTSIZE_N -o $FP_HOLD_ROOT $fnd2r  $BF_OUT  2> $FPART_LOGFILE);
  
} # now fpart sequence works fine.  Files are created in the 'hold' subdir, then mv'ed to the $FP_ROOT_DIR on close.
# now create the rest of the FPART script (now more than a single command)
# when qq(), don't need the & escaped, but it will work OK with it.  DO NEED the \$! escaped
$ttlfpcmd = qq(($FPART_CMD & fpid=\$!; fpr='r'; while [ ! -z "\$fpr" ]; do sleep 1; fpr=`ps -h -p \$fpid`; done; touch $FP_ROOT_DIR/FPART_DONE ) & ); 

# TODO resolve following with --reusechunks - both $SKIPFPART and $SKIPTO want to skip FPART
# but for different reasons.  $SKIPFPART implies that this is a SEND host (and that the MASTER
# has already started the fpart recursive descent, while $SKIPTO says that the chunks have 
# already been generated and we just want to re-use them, starting at $SKIPTO.) 
# Note that the fpart cmd has already been composed at this point.

if ( (defined $SKIPFPART || $SKIPTO >= 0) ) { # && $VERBOSE >= 2  # only actionable thing is if --reusechunks
  if ($SKIPTO >= 0) { INFO("Skipping fpart bc you've requested '--reusechunks'.\n"); }
} else {
  if ($DEBUG) { DEBUG( __LINE__,  "fpart cmd:\n[$ttlfpcmd]" );  }
  ##: Launch $FPART_CMD
 
  # should work same with '&' embedded in cmd or added in system() call but this way works on Ubuntu.
  system "$ttlfpcmd ";
   if ( $VERBOSE >2 ) {
     INFO("fpart launched with command:
     [$ttlfpcmd]\n"); # pause();
     # this next commented stanza check to make sure that fpart has gone into the bg 
     # correctly.  Keep to verify on different Linuxen/shells.  
     # fpart sometimes seems to to write the FPART_DONE instantly..
     # sleep 1;
#     if (-e "$FP_ROOT_DIR/FPART_DONE") {
#       print __LINE__, " : INSTANT WRITE of [$FP_ROOT_DIR/FPART_DONE] exists, something odd with backgrounding fpart.\n";}
#     else { print __LINE__, " : FPART in BACKGROUND, no instant appearance of [$FP_ROOT_DIR/FPART_DONE]\n"; }
   }
   # exit;

  ## remember, the fpart command should use the fqpn,
  ##: Wait for chunkfiles to appear

  if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
  my $ready2start = my $waitcnt = $NBR_FP_FLES = 0;
  my $fp1         = $FP_ROOT . ".1";  # 1st file is 1, not 0 due to latest fpart mod
  my $fp0         = $FP_ROOT . ".0";  # to check for old versions of fpart
  my $done        = 0;
  ##: first (f.1) chunk file detected, can start rsyncs and/or --zotfiles proc
  while ( $ready2start == 0 ) {
    if ( -e $fp1 ) {
      if ( $VERBOSE >= 3 ) { clearline(); INFO("[$fp1] visible.\n"); }
      $NBR_FP_FLES++;
      $ready2start = 1;
      if ( -e $fp0) {
        FATAL("Detected [$fp0] in the chunkfiles.  This indicates you're using an old version
  of fpart that starts writing chunks at '0'.  Please get a newer version (>=1.5)
  that starts writing at '1'. Try: <https://github.com/martymac/fpart>.")
      }
    }
    $waitcnt += 2;
    if ( $VERBOSE >= 3 ) { clearline(); INFO("Waiting [$waitcnt]s for first chunk files to be written.\n"); }
    sleep 2; # or for sub-second sleeps, use select(undef, undef, undef, 0.5); # = sleep 0.5
  }
}

##: Top of the hosts loop processing
# when here, all options should have a value or still be undefined.  The stanza below determines which gets passed to the SEND hosts.

# this loop is itself skipped in the execution path of the pfp that is sent to the SEND hosts

# can interate thru @argv (copy of pre-getopt @ARGV) and if an el has an '=' just after the arg,
# it's a 1liner, but if it doesn't, it's a 2liner (or it's a single arg (--udr --version --help --trustme --nowait)

##: == MASTER ONLY ==
##: reformat orig pfp2 arguments for SEND hosts
# process the option arguments into a form compatible with the MH version if requested.
my $rcmd   = ""; my $i;
my $sdpath = $ROOTDIR;    # as a default.
if ( $argv[$#argv] =~ /POD::/ ) {    # if this is the multihost version..
  my $cmplt_arg = "";
  for ( $i = 0 ; $i < $#argv ; $i++ ) {    # last arg of argv is the target
    $cmplt_arg = '';
    if ( $argv[$i] =~ /^-/ ) {  # was -> /-/
      # special case for rsync options since they have to be requoted to be passed thru to the SEND hosts - write the requote as a sub?
      if ( $argv[$i] =~ '--ro' ) {
        if ( $argv[$i] =~ '=' ) {
          $cmplt_arg = " --ro=$RSYNCOPTS ";
        } else {
          $cmplt_arg = " --ro=$RSYNCOPTS "; $i +=2; # skip the bare ro option
        }
        $rcmd .= $cmplt_arg . " ";  # and then skip this step below...
      }
      if ( $argv[$i] =~ '=' ) {    # don't need the --startdir to be transferred to the SEND client
        if ( $argv[$i] !~ /sta|sd|ro/ ) { 
          $cmplt_arg = $argv[$i]; 
        }
      } elsif ( $argv[$i] =~ /udr|nowait|vers|help|trust|hostch|reu/ ) {
        # $cmplt_arg = $argv[$i]; # none of these need to be sent to SEND hosts. (UDR is a special case for now; the rest are all handled on the MASTER)
      } else {
        $cmplt_arg = $argv[$i] . '=' . $argv[ $i + 1 ]; $i++;
      }

      # don't pass these on to SEND hosts.  
      if ( $cmplt_arg =~ /chu|cs/ || $cmplt_arg =~ /int|I/ || $cmplt_arg =~ /hosts/ || $cmplt_arg =~ /nowait/ || $cmplt_arg =~ /trustme|tm/  || $cmplt_arg =~ /filesfrom|ff/ || $cmplt_arg =~ /trim|tp/   ) { 
        $cmplt_arg = ''; 
      }
      my $nn = my @aa = split( '=', $cmplt_arg );
      if ( $nn == 2 && $aa[0] =~ /sta|sd/ ) {    # then it's --startdir|sd=/path/to/parent
        # and it needs to strip the '--startdir' and supply only the '/path/to/parent'
        $sdpath = $aa[1];
      }
      # have to leave the chunk files for other SEND hosts to use.
      if ( $cmplt_arg =~ /dis/ ) { $cmplt_arg = '--dispose=l'; }
      if ($cmplt_arg !~ /ro/ ) {   $rcmd .= $cmplt_arg . " "; } # cosmetic hack
    }
  }
}  # if ( $argv[$#argv] =~ /POD::/ )


##: Write out pfp2stop script and copy Perl scripts to $parsync_dir
# write out the pfp2stop script which when exec'ed will kill off all parsyncfp2s and rsyncs on all the hosts.
if ( $MULTIHOST > 0 ) {
  my $hosts2hit = "";
  $nbrhosts = $#SENDHOSTS + 1;
  for ( my $i = 0 ; $i < $nbrhosts ; $i++ ) {    # SENDHOSTS are ONLY hostnames
    $hosts2hit .= "${SENDHOSTS[$i]} ";
  }
  $nbrhosts = $#RECHOSTS + 1;                    # RECHOSTS might have trailing paths 'host:/path'
  for ( my $i = 0 ; $i < $nbrhosts ; $i++ ) {
    my @a  = split( /:/, $RECHOSTS[$i] );
    my $hh = $a[0];                       # regardless, the [0] will be the bare hostname (probably)
    if ( $hosts2hit !~ $hh ) { $hosts2hit .= "${hh} "; }
  }
  open( PSS, ">", glob("$PFP2STOPSCRIPT") )
    or FATAL("Can't open PFP2STOP scriptfile [$PFP2STOPSCRIPT] for writing.");
  print PSS
    qq(echo "WARNING: This will kill ALL YOUR rsync and parsyncfp2 processes on
    [$hosts2hit], 
    not just the ones started by parsyncfp2, so please be careful.
    
    Hit 'Enter' to continue, Cntrl+C to abort";
    read ee
    for host in $hosts2hit; do 
      echo [ processing \$host ] ; 
      ssh \$host "kill -9 \\` ps ux | grep -v daemon | grep 'rsyn[c]\\|$BN_CALLING_PROGRAM' | awk '{print \\\$2}' \\` " ; 
done
);
  close PSS; chmod 0700, "$PFP2STOPSCRIPT";

  ##: copy @UTILITIES to $parsync_dir to make sure all hosts are using the same programs
  for (my $dd = 0; $dd <= $#UTILITIES; $dd++) {
    my $utpath = `which $UTILITIES[$dd] 2> /dev/null`; chomp $utpath;
    if ($utpath ne "") { 
      my $cmd = "cp -f $utpath $parsync_dir; sync;";
      system($cmd);
    } else { FATAL("Did not find [$UTILITIES[$dd]] on PATH; [@UTILITIES] need to be on the PATH.");}
  }
}
if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
# set pfplg to 'hostless' string, rewritten below if there are POD SEND hosts
# $pfplog = "${parsync_dir}/pfp-log-${DATE}"; # $pfplog is the copy of the terminal output

##: == SEND hosts only (SH/MH)
##: Compose RSYNC_CMD & send it to all the SEND hosts

my @SH; # @SH is a duplicate of SENDHOSTS but without the user@ part, mostly for MHplotfile.sh below.
my $rem_shell; # what the SEND host uses for its shell, reused to compose the appro RSYNC command
my $export; # set to 'export (bash-like) or 'set' csh-like
if ( $MULTIHOST > 0 ) {
  $nbrhosts = $#SENDHOSTS + 1;
  for ( my $i = 0 ; $i < $nbrhosts ; $i++ ) {
     # compose remotecmd ($rcmd is from the regex tangle above)
     # $RECHOSTS[$i] is the REC host + specific or default target path generated above.
     # need to check if rec host has a 'specific' final resting place and if not, append the default
     # from the POD::/final/resting/place suffix.
     # add the default path if there's not one spec'ed
    my @sh; 
    if ( $RECHOSTS[$i] !~ /:/ ) { $RECHOSTS[$i] .= ':' . $defREMOTEPATH; }
    
    ##: Check 'hostname -s' on all sendhosts to resolve names.
    # resolve diffs between FQDNs, dotted quads, shorthostnames, etc. collapse all to short hostnames by
    # sshing to the given hostname and asking it for it's preferred short name (via short_hostname() )
    my $shortname; 
    # convoluted hack to disallow dirs in .pfp2 named user@host to simplify other things
    # too convoluted.. and doesn't make sense.  it's the $shortname that makes the dir, not the 
    # given $SENDHOSTS[$i], so all the 1st stanza can be deleted.
    ($shortname, $rem_shell) = short_hostname($SENDHOSTS[$i]); 
    $SH[$i] =  $shortname;
    mkdir "${parsync_dir}/${shortname}";
    $pfplog = "${parsync_dir}/${shortname}/pfp-log-${DATE}";
    my $fhverbose = 0;
    if ($i == 0) {$fhverbose = 1;}
    
    # adjust the SEND host cmd for its shell
    if ($rem_shell =~ /csh/) { $export = "set"; }    # csh-like
    else                     { $export = "export"; } # bash-like (default)

    ##: Set up $RPATH depending on whether SHELL is bash-like (default) or csh-like
    if ( !defined $RPATH ) {
      $RPATHPFX = "$export PATH=${parsync_dir}:~/bin:/bin:/usr/sbin:/sbin:/usr/bin:\$PATH;";
    } else { 
      $RPATHPFX = "$export PATH=${RPATH}:${parsync_dir}:/usr/sbin:/sbin:/usr/bin:/bin:\$PATH;" 
    }

    # this is what get sent to all the SEND hosts in quick succession
    # BUT (!!) ${HOME} is based on the issuing user's $HOME.. Should use '~'
    my $fps = $i + 1;
    $REMOTECMD = qq(ssh $SENDHOSTS[$i] "$RPATHPFX \\
    ${parsync_dir}/${BN_CALLING_PROGRAM}  --date=$DATE \\
  --mstr_md5=$MD5SUM \\
  --nowait --verbose=${VERBOSE} --maxload=${MAXLOAD} --slowdown=${RTT_DELAY} \\
  --startdir=$ROOTDIR  --skipfpart --fpstart=$fps --fpstride=$nbrhosts \\
    $rcmd \\
    $RECHOSTS[$i] \\
    |& tee -a $pfplog ");
   
#     $RECHOSTS[$i] 2> /dev/null \\ # from above; no errors to deflect in all recent tests
   
    if ( $VERBOSE > 2 ) { 
      WARN("About to send this REMOTE COMMAND to SENDHOST [$SENDHOSTS[$i]]
  [$REMOTECMD] 
  (also written to [$pfplog])"); #pause();
    }
    open( PFPLOG, ">", $pfplog );
    print PFPLOG "Command to start slave host rsyncs on this host:
  [ $REMOTECMD ]
  \n";
    close PFPLOG;
    sleep 1; # to allow files to be closed, copied, synced.
    # if add '&' below, it backgrounds correctly; if add it to the REMOTECMD above, it doesn't (??)
    system("$REMOTECMD &");
  }
}
if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
# start up NP rsyncs 1st, then cycle every CHECKPERIOD, checking # of rsyncs still going and
# starting new ones as needed until the chunkfiles are exhausted.

my $HL = $HOSTLIST;
$HL =~ s/\@/\\\\\\@/g; # Jesus! to let bash/gnuplot interpret the '@'

##: == MASTER ONLY ==
##: Write feedgnuplot script to viz data xfer; 
# TODO should probably re-write in native gnuplot.
# before exiting, write a bash script to generate a plotfile for feedgnuplot/gnuplot in MH mode
if ( $TARGET =~ /POD::/ ) {
  if ( $VERBOSE >= 2 ) { INFO("Writing plot file to [${parsync_dir}/MHplotfile.sh]\n\n"); }
  open( FGP, ">${parsync_dir}/MHplotfile.sh" )
    or WARN("Can't open the plotfile for writing, but will continue.\n");
  print FGP qq(#!/bin/bash
# this script preps a file for feedgnuplot (a gnuplot wrapper) to generate a quick 
# plot of the bandwidth from all the SEND hosts.
FGP=`which feedgnuplot`
pf="${parsync_dir}/plotfile"
echo -n '#' > \$pf # for vnlog format output
for host in @SH ; do
    hf="${parsync_dir}/\${host}.sbw"
    echo \$host > \$hf
    # have to grep thru the per-host subdirs for the pfp logs
    grep '<>' ${parsync_dir}/\${host}/pfp-log-${DATE} | grep \$host | scut -f=3 >> \$hf
done
paste  ${parsync_dir}/*.sbw >> \$pf
cat \$pf \| feedgnuplot --lines --points --vnlog --autolegend \\
--title "$DATE, [$HL]" \\
--xlabel "Time steps ($CHECKPERIOD s)" \\
--ylabel "MB/s samples"
    );
  close FGP;
  INFO("\nThe Multihost Master process ends here and will exit now. 
  The rest of the visible output is from the SEND hosts.
  Each INFO and WARNING message will be PREFIXED by the SEND hostname.
  Each line of the scrolling log will be SUFFIXED by the SEND hostname.\n\n\n"
  ); 
  exit(0);
}
##: == MASTER EXITS ==

##: == SEND hosts only
if ( $VERBOSE > 2 ) { INFO("Starting the 1st [$NP] rsyncs ..\n"); }
my $sc = 0;
# calculate the chunk file that corresponds to the $FPSTART & $FPSTRIDE that is passed in.
# $CUR_FPI starts at 1 for unihost (for modern fpart) or $FPSTART for MH and needs to increment at the rate of $FPSTRIDE
# UNLESS(!) we're reusing the chunks, in which case it ..starts out at the number of f.* files in the cache.

if ($SKIPTO > 0) { # if $SKIPTO is set, incr all the FPSTARTs
  $CUR_FPI = $SKIPTO + $FPSTART; # maintain MH compatibility - $SKIPTO shifts all fpstarts that far
} else { $CUR_FPI = $FPSTART; }  # so if SKIPTO = 700 & FPSTART was 2, now it's 702

$NBR_FP_FLES = get_nbr_chunk_files( ${FP_ROOT_DIR} );

while ($CUR_FPI > $NBR_FP_FLES) {
  if ($VERBOSE > 1)  {
    clearline();
    print  "[$host]: Chunkindex [$CUR_FPI] > # Chunkfiles [$NBR_FP_FLES], probably due to a slow fpart crawl. Waiting 2s to allow fpart to get ahead.\r";
  }
  sleep 2;
  $NBR_FP_FLES = get_nbr_chunk_files( ${FP_ROOT_DIR} );
}
$rPIDs = $sPIDs = "init";
my $FPART_IS_DONE = 0;
$PFP_DONE = 0;    # overall DONE indicator.  MHs end at different points so need a per-host indicator.
my $start_secs = `date  +"%s"`; chomp $start_secs;
my $day        = `date +"%F"`; chomp $day;
if (! $FPART_IS_DONE && -e "$FP_ROOT_DIR/FPART_DONE")  { $FPART_IS_DONE = 1; }  

##: init Bandwidth vars
$RDMA_T1 = 0;  $RDMA_R1 = 0; $avgRDMAsend = $avgRDMArecv =  0;
$R1 = `cat /sys/class/net/${NETIF}/statistics/rx_bytes`; chomp $R1;
$T1 = `cat /sys/class/net/${NETIF}/statistics/tx_bytes`; chomp $T1;
if ($PERFQUERY) {
  $RDMA_T1 = `perfquery -x | grep XmitData | cut -f2 -d: | sed -e 's/\\.*//g'`; chomp $RDMA_T1;
  $RDMA_R1 = `perfquery -x | grep RcvData  | cut -f2 -d: | sed -e 's/\\.*//g'`; chomp $RDMA_R1;
}
my $start_cp = time();
my $enuf = 0; # hacky flipflop for printing labels
my $printnow = time() + $CHECKPERIOD ; # time-based printing/rsync checking
my $last_CUR_FPI = 44; # 44 is a non-zero placeholder
my $lastone = 44;  
my $BW2low = 0;
$avgTCPsend = 1; # to start things off.

# this loop allows pfp to end while there are still rsyncs running. fix so all rsyncs started by this pfp stop so the stats lines keep refreshing.  The exit test on running rsyncs DOES work, but this loop should end cleanly.
# the problem here is that $PFP_DONE=1 exits the loop.  
# $PFP_DONE prevents more rsyncs from starting but does not prevent entry to this loop.

$crr = 1; # to let the loop start off
$rPIDs = "blah"; $sPIDs = ""; # set $rPIDs to trigger entry to the main loop below
my $WARN_ONCE = 1;
if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}

##: Start the overall common rsync loop
my $NBR_FP_FLESa = $NBR_FP_FLES + 2; # used only at the top of this loop, adj at bottom of loop
# break out of this loop only when there are no more running and suspended PIDs so calc ar bottom of loop as well and test in the initiating while loop below.
my $BF_CNT = 0; # counter to prevent running it 2x
my $REASS_CNT = 0; # counter to prevent multiple headers being printed into the reasemble script.
my $zf_cmd_cntr = 0; # local tgz and corresponding remote host cmd counter for --zotfiles
my $ztx = 0; # internal zotfile counter.
my $zf_fp = 1; # zotfiles_f.# files processed, cf $NBR_FP_FLES to keep up to date.
my $zft = 0; # counter for zot files transferred, used to count thru @ZF_TO_XFER post-loading
my $ZF_END = 0; # value holder for end of the @ZF_TO_XFER list
my $p1 = 0; # print once counter
while ( ( ($CUR_FPI) <= ($NBR_FP_FLES + 1) || !$FPART_IS_DONE ) && !$PFP_DONE  || $rPIDs ne '') { #
  my $start_s = time();  
  if ($SKIPTO == -1)  { # $SKIPTO = -1 means DON'T reuse chunks
    $NBR_FP_FLES = get_nbr_chunk_files( ${FP_ROOT_DIR} );
    #  $NBR_FP_FLES < $NP && 
    while ( ($CUR_FPI + $FPSTRIDE) > $NBR_FP_FLES && !$FPART_IS_DONE) { # TODO: this has to be delayed to give fpart some time to generate the chunks.
      if ($VERBOSE > 2) { # && $repeat++ < 2 
        print "[$host] Waiting for fpart to get ahead; up to [$NBR_FP_FLES]\n"; 
      } 
      sleep 3; 
      $NBR_FP_FLES = get_nbr_chunk_files( ${FP_ROOT_DIR} );
    }
  }
  
  if ( $FPART_IS_DONE && $WARN_ONCE) { 
    $NBR_FP_FLES = get_nbr_chunk_files( ${FP_ROOT_DIR} ) ; 
    if ( $NBR_FP_FLES < $NP ) { WARN("fpart has finished and the # of chunk files [$NBR_FP_FLES] 
    is < the # of parallel rsyncs you've requested, usually a mistake.  
    If so, you may want to kill this run and start another with a smaller chunk size."); }
    $WARN_ONCE = 0;
  }
  
  ##: fpart done, now can proc --bigfiles.
  
  if ($crr >= ($NP - 1) && $FPART_IS_DONE && -e "${FP_ROOT_DIR}/FPART_DONE" && $BIGFILES && -e "${parsync_dir}/BIGFILES") { 
    # insert post-proc loop to read fpart stdout = ${parsync_dir}/BIGFILES
    # whats the current f.#?  Last entry in $FPART_LOGFILE 
    my $fnbr = `grep '^Part #' $FPART_LOGFILE | tail -1 | tr -d [\#\:] | scut -f=1`;
    $fnbr++; # & incr to set the next f index

    open( BF,  "< ${parsync_dir}/BIGFILES " )
    or FATAL("Can't open fpart-generated BIGFILE [${parsync_dir}/BIGFILES].");
    
    open(REASS, ">>", "${parsync_dir}/remote_reassemble.sh") 
    or FATAL("[${parsync_dir}/remote_reassemble.sh] couldn't be opened");
    if ($REASS_CNT == 0){
    print REASS "# writ by pfp2: [$DATE] by [$USER\@$host]
# This script should have been copied to the remote host [$rem_host] and executed by the same 
# user that ran the original pfp2 command (or one authorized to do so).  
# It should have been run automatically by pfp2 on exit, but please retain it until 
# you verify that the reassembly has completed.  
# That you're reading this is an indication that the full process may be running 
# slowly or may have failed at some point, since it should be deleted when pfp2 ends.
"; $REASS_CNT++;

    }
    $NBR_FP_FLES = get_nbr_chunk_files( ${FP_ROOT_DIR} );
    my $REASS_PATH;
    if ($BF_CNT == 0) { # read BIGFILE but just once!!
      # may have to change this to read BF into an array since this will iterate thru BF every loop.
      while (<BF>) { 
        chomp;
        my $split_cnt = 0; # counter for @SPLIT_PIDs
        $BF_CNT++;
        my  $N = my @A = split( /\t/, $_ ); # split only on tabs, not ws/spaces; could be spaces in fns
        if ($N != 3) { WARN("Error - wrong # of args in file [${parsync_dir}/BIGFILES]\n");}
        # DO NOT use the '-z' with basename or dirname; appends a NUL and therefore prevents adding 
        # anything unless you chomp it anyway.
        $A[2] = qq($A[2]); chomp $A[2];
        $REASS_PATH = $A[2];
        my $bn = `basename  \"$A[2]\"`; chomp $bn; 
        my $SPLITDIR =  `dirname  \"$A[2]\"`; chomp $SPLITDIR; 
        $REASS_PATH =~ s/${ROOTDIR}\///;
        # then prefix with $rem_fqpath so that it's either $HOME-based or really FQPN
        $REASS_PATH = $rem_fqpath . '/' . $REASS_PATH;
        my $snbr;
        if ($A[1] > $CHUNK_CUTOFF) {
          # split into chunk_size bits
          
          # NOT going to play with compression at this point.  maybe put it back in later.
          #    if ($COMPRESS) {
          #       DEBUG(__LINE__, "compression loop for file $A[2], size [$A[1]]");
          #       my $cmd = sprintf (" cat $A[2] | lz4 | split -a 4 -d  -b $FPARTSIZE - ${bn}.lz4. ");
          #       print "\n\ncmd = [$cmd]\n\n";
          #       my $cmd_out = `$cmd`; 
          #    } else { 
          #      DEBUG(__LINE__, "Splitting file $A[2] with NO compression, size [$A[1]]");
                # cmd below should split bigfiles in place  file -> file.0000, file.0001, etc
          #my $cmd_out = ` split -a 4 -d -b $FPARTSIZE '$A[2]' '${A[2]}.' `;
          # test whether we can start any more splits
          my $e = 0; 
          # keep looping in while loop until the array is full of PIDs
          # test each el  to see if the PID is still running.  If not, call
          # $reass_str = split_finish($SPLIT_TRGT[$e] (=original filename))
          # an append $reass_str to REASS file handle, then zero the el and start another 
          # split job.
          # so at begin, this should load up $BIGFILES split jobs in bg, then go run rsyncs until one 
          # split finishes, then go re-write the f.# files, then grab the next file to split
          # and send it off to split in the background.
          # so most of the stuff below should be inside the while loop below.
          # it loos thru all els of @SPLIT_PIDs
          my $reass_str; my @SPLIT_TRGT;
          while ($e < $BIGFILES){ #incr until hit a zero
            if ( $SPLIT_PIDs[$e] != 0 ) { # have to check if it;s still running
              my $q = `ps -h -p $SPLIT_PIDs[$e]`; chomp $q; # -h = no headers
              if ($q eq "") { # then the PID is done and  I need to zero the el and start another one 
                # call the split_finish() and set to 0 to re-use the el.
                $reass_str = split_finish($SPLIT_TRGT[$$e]);
                $SPLIT_PIDs[$e] = 0; # set to 0 so it can be re-used
              } else {  # else it's still running
                #print __LINE__, " : PID [$SPLIT_PIDs[$e]] at el [$e] is still running.\n";
              }
            } 
            if ($SPLIT_PIDs[$e] == 0) { # start a new split
              my $split_cmd = "split -a 4 -d -b $FPARTSIZE \"$A[2]\" \"${A[2]}.\" & echo \"\${!}\" "; # >> $SPLIT_PIDFILE  
              # record the PID & the target filename
              $SPLIT_PIDs[$split_cnt] = `$split_cmd`; chomp $SPLIT_PIDs[$split_cnt]; # 
              $SPLIT_TRGT[$split_cnt] = $A[2];  $split_cnt++;
              # system($split_cmd); 
            }
            $e++;
          }
          if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}

          my $trgt = "${A[2]}.";  
          my $hmf = `ls -1 \"${trgt}\"[0-9][0-9][0-9][0-9] | wc -l`; chomp $hmf; # hmf = how many files
          # loop below has to start at the last f.# + 1, not 0.  
          for (my $i=0; $i<$hmf; $i++) { # now copy each chunked section to its own f.# file in 
            $snbr = sprintf ("%04d", $i); # pad to 4 digits, leading zeros
            # don't need to mv them since they're being split in place
            # writes the chunked file name to the next f.# file
            my $echo_out = `echo "${SPLITDIR}/${bn}.${snbr}" > "${FP_ROOT_DIR}/f.${fnbr}"`;
            push @DEL_SPLITS, "${SPLITDIR}/${bn}.${snbr}";            
            # The reassemble cat line has to be formatted into either a $HOME-based path or a FQPN
            # & now write the recombine line into a script to be run on the remote after pfp2 ends.
            if ($i == 0) {
              #if (){}
              print REASS "
  cat   \"${REASS_PATH}.\"[0-9][0-9][0-9][0-9] >  \"${REASS_PATH}\" ; 
  rm -f \"${REASS_PATH}.\"[0-9][0-9][0-9][0-9] ;
  " ;
            }
            $fnbr++; 
          }
        } else { # just copy the filename into an f.# file w/o any more splitting
          $snbr = sprintf ("%04d", $fnbr);
          # no lz4 compression either below, & the redirect char is '>', not '>>'
          my $echo_out = `echo "${SPLITDIR}/${bn}.${snbr}" > "${FP_ROOT_DIR}/f.${fnbr}"`; 
          $fnbr++;    
        }
        my $FP_ROOT_DIR_ls = `ls -lt '${FP_ROOT_DIR}'  | head`;
        my $SPLITDIR_ls = `ls -lt '${SPLITDIR}' | head`;
      }
    } 
  }  # end of BIGFILES loop
  
  # so at this point either we've loaded all the rsyncs up to NP or we've completely finished.
  # If the latter, say good bye.  If the former, then we have to keep launching
  # rsyncs up to NP until we've used up all the fpart chunkfiles.

  my @aprPIDs;    # all recorded parsyncfp2 rsync PIDs ever started
  my @crrPIDs;    # currently RUNNING parsyncfp2 rsync PIDs.
  my @csrPIDs;    # currently SUSPENDED parsyncfp2 rsync PIDs.

  ### FOLLOWING IS THE MAIN PARSYNC-FPART LOOP               HIT
  if (! $FPART_IS_DONE && -e "$FP_ROOT_DIR/FPART_DONE")  { $FPART_IS_DONE = 1; }
  my $IFN        = sprintf( "%7s", $NETIF );

  if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
  
  # check fpart to see if it's still running..                   HIT
  if (! $FPART_IS_DONE && -e "$FP_ROOT_DIR/FPART_DONE")  { $FPART_IS_DONE = 1; }

  # check cycles, print if exceed then reset counter.
  ##: Calc BW since last point

  ##: stats print loop
  my $actual_cp = time() - $start_cp;
  if ( time() > $printnow ) {
    ($start_cp, $printnow) = printdataline( $actual_cp);
  }

  ##: check chunkfile # overflow
  my $warncount = 0;
  ### INSERT test to check that $NBR_FP_FLES is < 20,000.
  if ( $NBR_FP_FLES > $WARN_FPART_FILES && $warncount < 1 ) {
    if ( $VERBOSE >= 2 ) {
      WARN(
        "You've exceeded [$WARN_FPART_FILES] chunk files.  
Are you sure you've set the chunk size (--chunksize) appropriately for this transfer?
If the count goes to [$MAX_FPART_FILES], this transfer will abort. See the help about this.
" );
      $warncount++;
    }
    if ( $NBR_FP_FLES > $MAX_FPART_FILES ) { #  && !$SKIP_FPART_CHECK <- doesn't change anywhere
      FATAL(
        "You've now exceeded [$MAX_FPART_FILES] chunk files, the maximum 
recommended for this utility. Please increase the '--chunksize' 
parameter significantly. If there's a good reason for exceeding it, 
you can force the internal limit to be ignored by specifying it as 
a negative number (--chunksize=-10GB) the next time. However if you 
do this, you may run into the string limit for 'ls'.
");
    }
  }
  ##: Un/Suspend rsyncs for loadbalancing
  my $tmdiff = time() - $printnow; # so if it's +, it's > 
  if ( $SYSLOAD[0] > $MAXLOAD && time() >= $printnow) { # $SYSLOAD[0] = 1m loadavg; only eval on checkperiods
  if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
  
    # suspend a PID; then loop as normal. If still high, continue to suspend PIDs til there's none left.
    if ($DEBUG) {
       DEBUG( __LINE__, 
 "System load [$SYSLOAD[0]] is > MAXLOAD [$MAXLOAD].  Will try to suspend a running rsync to shed load."
       );
    }
    $rPIDs = trim($rPIDs);
    my $rn = my @ra = split( /\s+/, $rPIDs );
    my $sn = my @sa = split( /\s+/, $sPIDs );
    for ( my $r = 0 ; $r < $rn ; $r++ ) {
      for ( my $s = 0 ; $s < $sn ; $s++ ) {
        if ( $ra[$r] eq $sa[$s] ) { $rPIDs =~ s/$ra[$r]//g; }    # delete it from $rPIDs
      }
    }
    # picks up both suspended and running PIDs and the new result has to have something in it as well.
    if ( $rPIDs =~ /\d+/ ) {                                     # if any still left
      my $N = my @raPIDs = split( /\s+/, $rPIDs );
      my $e = 0;    # @raPIDs = temp array to carry currently running PIDs
      while ( $e <= $N && $raPIDs[$e] !~ /\d+/ && $raPIDs[$e] !~ /$sPIDs/) { 
        $e++; 
      }
      
      if ($DEBUG) {
        DEBUG( __LINE__,  " rPIDs=[$rPIDs], raPIDs[$e]=[$raPIDs[$e]]; will now suspend it." );
      }
      kill 'STOP', $raPIDs[$e];

      $susp_cnt++;
      print SUSLOG "Suspend  \t$susp_cnt\t($unsusp_cnt)\t$raPIDs[$e]\n";
      if ( $VERBOSE >= 2 ) { INFO("Load too high; suspended PID [$raPIDs[$e]] ($susp_cnt)\n"); }

      if ( $sPIDs !~ /$raPIDs[$e]/ ) {    # If it's not there already
        $sPIDs = trim( "$sPIDs" . ' ' . "$raPIDs[$e]" );    # transfer rPID to sPID.
        $rPIDs =~ s/$raPIDs[$e]//g;    # only then delete that PID fr the rPID string
        $raPIDs[$e] = undef;
        $rPIDs = trim($rPIDs);
        sleep 2; 
      }
    } else {    # there aren't any more PIDs left - all done or killed off.'
      $rrcc++;
      if ( $VERBOSE >= 2 && $rrcc > 20) { 
        clearline();
        print "[$host] No more running rsync PIDs left. Consider setting --maxload higher. .\r"; 
        $rrcc = 0;
        sleep 10; 
      }
    }
  } elsif ( $sPIDs =~ /\d+/ && $SYSLOAD[0] < $MAXLOAD && time() >= $printnow) {    # if there are sPIDs, unsuspend them one by one
                                   # split em
    my $N = my @saPIDs = split( /\s+/, $sPIDs );
    my $e = 0;
    while ( $e <= $N && $saPIDs[$e] !~ /\d+/ ) { $e++ }
    if ($DEBUG) {
      DEBUG( __LINE__,  "[unsuspend] got one: [$saPIDs[$e]]; will now UNsuspend it." );
    }
    kill 'CONT', $saPIDs[$e];
    $unsusp_cnt++;
    print SUSLOG "UNsuspend\t$unsusp_cnt\t($susp_cnt)\t$saPIDs[$e]\n";
    $rPIDs = "$rPIDs" . ' ' . "$saPIDs[$e]";    # transfer sPID to rPID.
    $sPIDs =~ s/$saPIDs[$e]//g;                 # delete that PID fr the sPID string
  }    # end of 'SUSPEND OR CONTINUE to LOADBALANCE.' test loop
       # and if neither of those conditions are met, then we can launch another rsync.
  elsif ( $crr < $NP ) {    # then launch another rsync with the next fpart chunkfile
    $CUR_FP_FLE = "${FP_ROOT}.${CUR_FPI}";    # generate the next fpart chunk file with $CUR_FPI
    # if fpart is still going, wait for the next chunkfile to show up
    my $cfw = 0;

    if (! $FPART_IS_DONE && -e "$FP_ROOT_DIR/FPART_DONE")  { $FPART_IS_DONE = 1; }
    while ( !-e $CUR_FP_FLE && !$FPART_IS_DONE ) {
      if ( $VERBOSE >= 2 ) { $cfw += 2; INFO("Waiting [$cfw]s for next chunkfile.. \n"); sleep $cfw; } 
      if (! $FPART_IS_DONE && -e "$FP_ROOT_DIR/FPART_DONE")  { $FPART_IS_DONE = 1; }
    }
    if ($DEBUG) { DEBUG( __LINE__,  "sPIDs string = [$sPIDs]" ); }
    if (-e $RSYNC_PIDFILE){
      ( $rPIDs, $crr ) = get_rPIDs( $RSYNC_PIDFILE, $sPIDs );
      $NrPIDs = my @a = split( /\s+/, $rPIDs );
    } 
    # calc the number of rsyncs to start up (currently running + suspended)
    my $R2SU = $NP - ($NrPIDs + $NsPIDs);     
    # OK to call it just before the test below
    $NBR_FP_FLES = get_nbr_chunk_files( ${FP_ROOT_DIR} );
    if (! $FPART_IS_DONE && -e "$FP_ROOT_DIR/FPART_DONE")  { $FPART_IS_DONE = 1; }
    if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
    ##: Spin up more rsyncs if not enough
    # this test is OK if $R2SU above is calculated correctly (consults # suspended PIDs $NsPIDs)
 
    while ((($NrPIDs + $NsPIDs) < $NP) && !$PFP_DONE) {
      # make sure we haven't finished
      if (! $FPART_IS_DONE && -e "$FP_ROOT_DIR/FPART_DONE")  { $FPART_IS_DONE = 1; }
      ( $rPIDs, $crr ) = get_rPIDs( $RSYNC_PIDFILE, $sPIDs );
      my $ee = $CUR_FPI - 1; # + 1; don't need the incr due to fpart change
      
      # if there are still suspended rsyncs at the end of the send, unsuspend them 1 by 1.
      if ($CUR_FPI > $NBR_FP_FLES && $sPIDs =~ /\d+/ ) {
        if ($VERBOSE > 2) {
          print "[$host]: Finishing up; releasing all suspended rsync PIDs [$sPIDs]\n";
        }
        my $n = my @a = split ( /\s+/, $sPIDs );
         #print __LINE__, " : n=[$n]], a=[@a]\n";
        for (my $r=0; $r<$n; $r++) {
          kill 'CONT', $a[$r];
        }          
        $sPIDs = ""; # and blank it since there won't be anymore
      }
      
      if ( $rPIDs eq "" && $sPIDs eq "" && $CUR_FPI > $NBR_FP_FLES && $FPART_IS_DONE ) {
        $PFP_DONE = 1;  # then we're done - exit.
      }
        # slow down loop if we're just waiting for rsyncs to finish at end of the chunk files
        if ($CUR_FPI > $NBR_FP_FLES && $FPART_IS_DONE ) {
          if ( $VERBOSE >=2 && $p1 < 1) {
             INFO("All my rsyncs issued; waiting for them to finish.\n");
             $p1 = 2;
          }
          sleep 2; 
          $R2SU = 0;
        }
        my $spinc = 0;
        while ( ( $CUR_FPI >= $NBR_FP_FLES ) && !$FPART_IS_DONE ) { # was $CUR_FPI >= $NBR_FP_FLES ??
          if ($DEBUG) {
            DEBUG( __LINE__,  "CUR_FPI=$CUR_FPI >= NBR_FP_FLES=$NBR_FP_FLES?" );
          }
          if ( $VERBOSE >= 2 ) {
            clearline();
            INFO("Waiting for fpart. [$NBR_FP_FLES] chunks written. [$spinner[$spinc]]\r"); # 
          }
          $NBR_FP_FLES = get_nbr_chunk_files( ${FP_ROOT_DIR} );
          sleep 2; 
          if (! $FPART_IS_DONE && -e "$FP_ROOT_DIR/FPART_DONE")  { $FPART_IS_DONE = 1; }
          if   ( $spinc > 2 ) { $spinc = 0 }
          else                { $spinc++; }
        }
        
      if ( ! $PFP_DONE && $CUR_FPI <= $NBR_FP_FLES ) { # ???&& $RSYNCS_GOING < $NP
      
        # $logfile is the rsync log (vs the copy of the terminal output in the $pfplog)
        $logfile = "${parsync_dir}/rsync-log-${DATE}_${CUR_FPI}"; # each chunk gets its own log.
        $CUR_FP_FLE = "${FP_ROOT}.${CUR_FPI}";    # generate the next fpart chunk file with $CUR_FPI
        $last_CUR_FPI = $CUR_FPI;
        $NBR_FP_FLES = get_nbr_chunk_files( ${FP_ROOT_DIR} );
        my $nbr_fp_fles = $NBR_FP_FLES; # locally modified version of $NBR_FP_FLES for the zotfile loop
        
        # this stanza should be hit multiple times while fpart is running, but before the master exits.
        ##: Handle --zotfile processing on MASTER only 
        # by using $zf_fp (a $CUR_FPI-like var) to cycle thru the existing f.# files as generated, analyzing  
        # them to see if they meet the zotfile definition and processing them on the MASTER.  
        # Adds more load on the MASTER (~70% of a core per instance - will look into spreading the jobs 
        # to other SEND hosts if the load becomes too onerous on MASTER.

        ##: zotfile processing goes here, close to the RSYNC_CMD composition; check on each SEND host as it takes another ($CUR_FPI + $FPSTRIDE) chunk
        if ($ZOTFILES !=0) { print __LINE__, " : bf ZF stanza: CUR_FPI = [$CUR_FPI] CUR_FP_FLE = [$CUR_FP_FLE] zf_fp = [$zf_fp] NBR_FP_FLES = [$nbr_fp_fles]\n"; }
        
        if ($ZOTFILES != 0) { # want to test the chunks for zf+
          # catchup to where fpart has gotten to; test w/ '-e file', don't invoke get_nbr_chunk_files()
          # since that has a lot more IO than '-e' but when -e fails, will give you the end of where
          # fpart is, if that's of interest.
          
          while ($zf_fp <= $NBR_FP_FLES && -e  "${FP_ROOT}.${zf_fp}") { 
            my $wczf = `wc -l ${FP_ROOT}.${zf_fp}`; chomp $wczf;
            my $cfas =  $FPARTSIZE / $wczf ; # chunkfile average size 
            if ( $cfas < $ZOTFILES ) { # if avg siz is smaller, tgz it
              my $tv = $zf_cmd_cntr; # temp test var ### is $zf_cmd_cntr any diff than $zf_fp ? ###
              $LOC_POST_FP_ZOT_CMDS[$zf_cmd_cntr] = 
                  "echo ${FP_ROOT_DIR}/tar.lz4.${zf_fp} > ${FP_ROOT_DIR}/f.${zf_fp};";
              # the following line is a concatenation of the cmds needed to unpack the tgz'ed zotfiles on the remote end. 
              $REM_POST_FP_ZOT_CMDS[$zf_cmd_cntr++] = "lz4 -d tar.lz4.${zf_fp} -c | tar -C $rem_fqpath  -xf - ; rm -f tar.lz4.${zf_fp};\n"; 

              if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
              fixfilenames( "${FP_ROOT}.${zf_fp}", $ROOTDIR ); # fix those leading paths..
              my $tgz_cmd = "tar -c --files-from=${FP_ROOT}.${zf_fp} -f - 2> /dev/null | lz4 > ${FP_ROOT_DIR}/tar.lz4.${zf_fp}; echo ${FP_ROOT_DIR}/tar.lz4.${zf_fp} > ${FP_ROOT_DIR}/f.${zf_fp}";
              if (!-e "${FP_ROOT_DIR}/tar.lz4.${zf_fp}") {
                system("$tgz_cmd &"); # this should bg the command, allowing pfp2 to continue
              } else { 
                print __LINE__, " : [${FP_ROOT_DIR}/tar.lz4.${zf_fp}] already exists, skipping\n" ;
              }
              $ZF_TO_XFER[$ztx++] = $CUR_FPI;  # loaded with f.#s to be iterated thru at end (again)
              $CUR_FPI++; # to prevent trying to rsync the f.# that we've just tgz'ed
              $CUR_FP_FLE = "${FP_ROOT}.${CUR_FPI}";  # gen the next fpart chunk file with the new $CUR_FPI
              $zf_fp++;
            }
          }
          $ZF_END = $zf_fp - 1;
          }
        
  # ### this works but it places all the tgz files in the rmote dir based on the local placement 
  # which is in ~/pfp2/fpcache and it has to be converted from there to the correct final placement.
  # That is actually not a bad place to end up, but the tar -zf command has to have add a 
  # '-C correct/dir' . have to figure out what that correct dir is by recording where the pfp cmd was executed and what teh target is.
  # executed from ~ :
  # [pfpzot --chunk=20M --NP=2 --zotfiles=3.55k zots tux@minilini-w:~]
  # pwd =  '~'
  # remote dir is ~
  # transfered dir is zots
  # all tgz files end up in remote:~/.pfp2 so tar -C '~'
  # but the fies get tar'ed with the full path minus the leading '/'
  
        
        #####################################    
        # normally when we exit this loop in a normally huge transfer, we have a combo of  zots tgzed as well as 
        # when we come out of the ZF loop for the last time, unless adjusted as above, CUR_FPI and $CUR_FP_FLE all point to the # beyond the end of the F.# series.  At this point, either the seq is finished and (if fpart is done) we need to iterate thru the @ZF_TO_XFER list, or (if not) we need to wait for the next f.#.
        # so if the next f.# doesn't exist and $FPART_IS_DONE, then we should loop thru the existing 
        # @ZF_TO_XFER list to transfer. Can I figure a way to just loop regularly
        # introduce a test at each rsync cmd creation to check if there's a hanging f.# to do?
        
        # $zft = zotfilestranferred ; hmm as we walk thru @ZF_TO_XFER, at each incr, check if it's been 
        # xferred? cuz the index is not going to track completely with the f.#
        # 0=7, 1=8, 2=11, 3=25, 4=26, etc.
        # $ZF_TO_XFER[3] = 25 ; want to insert 25 into CUR_FPI or use another 
        # use the wait (or end) to send a tgz as long as there are still ZFs to send
        # this test will fail as long as there's a waiting, regular $CUR_FP_FLE
        if ( !-e $CUR_FP_FLE && $FPART_IS_DONE && $zft < $ZF_END) { 
          $CUR_FP_FLE = "${FP_ROOT}.$ZF_TO_XFER[$zft]"; # $zft gets set up top as 0
          $CUR_FPI = $ZF_TO_XFER[$zft];
          $zft++;
        }
        if ($VERBOSE >= 2) { clearline(); print "[$host] -> chunk [$CUR_FPI] to be sent.\r";}
        
        if ($FPART_IS_DONE && !-e $CUR_FP_FLE) {
          FATAL("There has been a chunk file overrun after fpart has finished. 
  CUR_FP_FLE = [$CUR_FP_FLE],  NBR_FP_FLES=[$NBR_FP_FLES].  You're probably 
  using an old version of fpart.  Get one that's later than 1.5."); 
        } 
        if ( -e $CUR_FP_FLE ) {
          ##: Compose RSYNC_CMD
          fixfilenames( $CUR_FP_FLE, $ROOTDIR );    # check/fix the path to f.# is OK
          if ($rsync_type =~ /filesystem/ && $rat_hoststring eq "") {
            $rat_hoststring = $rem_host; # support FS to FS rsyncs.. 
          }
          $RSYNC_CMD = qq(cd $TRIMPATH && $UDR rsync  --bwlimit=$MAXBW  $RSYNCOPTS --log-file=$logfile --files-from=$CUR_FP_FLE '$ROOTDIR' $rat_hoststring  & echo \"\${!}\" >> $RSYNC_PIDFILE);
         
          if ($DEBUG) { DEBUG( __LINE__,  "Starting [$RSYNC_CMD]" ); }
        # this loop launches the new rsyncs, but it only launches them at CHECKPERIOD intervals.
        } else {
          WARN("Chunk file [$CUR_FP_FLE] isn't ready; skipping..");
        }
        if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
        if ( -e $CUR_FP_FLE ) { 
          # check/fix the path to f.# is OK
          if ($DEBUG) { DEBUG(__LINE__,  "Starting [$RSYNC_CMD]"); }
          if ( $VERBOSE > 2 ) { clearline(); print "[$host] : Starting [$RSYNC_CMD]\r"; }
          ##: Start RSYNC_CMD
          system("$RSYNC_CMD");    # captures the bg job PID to PIDfile
          $NrPIDs++;
          select(undef, undef, undef, 0.2); # hacky way to sleep < 1s; may be too fast for syncing ops.

          if (($CUR_FPI + $FPSTRIDE) <= $NBR_FP_FLES) {  
            $CUR_FPI += $FPSTRIDE;  # at end of xf, these are equal, so the last loop has to 
            # above is last line in common
          } else {
            $CUR_FPI += $FPSTRIDE;
            if ( $FPART_IS_DONE && $rPIDs !~ /\d+/ && $sPIDs !~ /\d+/) { 
              $PFP_DONE = 1;             
            } # this terminates the loop, which is correct, but ends pfp BEFORE all the rsyncs have finished running. 
          }
        } elsif (!$FPART_IS_DONE) {
          WARN("Chunk file [$CUR_FP_FLE] is not ready @ [$CUR_FPI]; waiting for fpart..");
          sleep 2;
        } else {
          sleep 2;
          $NBR_FP_FLES = get_nbr_chunk_files( ${FP_ROOT_DIR} );
          # final check (if the final f.# hadn't synced)
          if (! -e $CUR_FP_FLE) {
            WARN("fpart has finished and the calculated chunk file [$CUR_FP_FLE] 
  doesn't exist.  At this point, the number of chunk files 
  (NBR_FP_FLES) = [$NBR_FP_FLES]. Unless there's valid reason to continue, you should 
  consider killing this run and filing a bug.");
          } else { # it finally appeared in the nick of time
            INFO("Processing final chunk file [$CUR_FP_FLE] in sequence. NBR_FP_FLES=[$NBR_FP_FLES].\n")
          }
        }
      } #  if ( ! $PFP_DONE && $CUR_FPI < ($NBR_FP_FLES + 1) )

      
      $actual_cp = time() - $start_cp;
      if ( time() > $printnow ) {
        ($start_cp, $printnow) = printdataline( $actual_cp);
      }
    }  # while ($NBR_RSYNCS_RNG < $R2SU && !$PFP_DONE)
    if ( $DEBUG && $sPIDs =~ /\d+/ ) { DEBUG( __LINE__,  "sPIDs string = [$sPIDs]" ); }
    ( $rPIDs, $crr ) = get_rPIDs( $RSYNC_PIDFILE, $sPIDs );
    # need a fpart catchup at this point?
      my $spinc = 0;
      while ( ( $CUR_FPI >= $NBR_FP_FLES ) && !$FPART_IS_DONE ) { # was $CUR_FPI >= $NBR_FP_FLES ??
        if ($DEBUG) {
          DEBUG( __LINE__,  "CUR_FPI=$CUR_FPI >= NBR_FP_FLES=$NBR_FP_FLES?" );
        }
        $NBR_FP_FLES = get_nbr_chunk_files( ${FP_ROOT_DIR} );
        sleep 3;
        if (! $FPART_IS_DONE && -e "$FP_ROOT_DIR/FPART_DONE")  { $FPART_IS_DONE = 1; }
      }
  } 
  
  ##: Detect zero bandwidth at end and kill hung rsyncs 
  #   if BW drop extends over long period (10 low reading in a row)
  # TODO below, going to very low BW only indicates a problem if there are running (not suspended) rsyncs
  # If there are suspended rsyncs, then they can't be sending data, so only if there's low BW and there are still

  # active rsyncs running, enter the loop.
  ( $rPIDs, $crr ) = get_rPIDs( $RSYNC_PIDFILE, $sPIDs );
  if ($avgTCPsend < 0.1 && $rPIDs =~ /\d+/ ) {  
    $BW2low++;
    if ($BW2low == 1) { # low BW measures implies something may be off, but it may also be low BW
# keep these out of the warning until the --zot, --big are ready.
#  - transferring zillions of tiny files, or 
#  - you've used the '--bigfiles'option which can require a long startup to 
#    split the first few big files, or
      
      if ($VERBOSE >= 2 && $rsync_type !~ /filesystem/) {
        WARN("Bandwidth has been < 0.1MB/s for [$LowBWMax] bandwidth checks, 
  usually due to:
  - a 'mostly sync' operation (which exchanges very little data) or 
  - pfp2 isn't sending data bc fpart hasn't produced enough chunkfiles yet or 
  - an rsync has hung.
  
  If the 1st, do nothing.  If you suspect the last, you can allow this to
  continue or you can call the autogenerated kill script:
  [$PFP2STOPSCRIPT] 
  from another terminal and start over again, preferably with the '--reusechunks' 
  option to bypass the chunking process, if it was a large dir structure.
  
  If you think it's only this host [$host] that has failed rsyncs, you can send 
  only the data that was supposed to be sent thru it by executing the 
  host-specific command which is recorded at the top of each host logfile.
  [$logfile]\n");
      }
  
  # $BW2low = 0; # resets on OK BW to kill warning for another cycle.
    sleep 1;
    $BW2low++;
    } elsif ($BW2low < $LowBWMax) {
      clearline();
      print "[$host] : Bandwidth still < 0.1MB/s..\r";
      sleep 1;
      $BW2low++;
    }
  } else {
    $BW2low = 0; # resets on OK BW
  }

  if ( !$PFP_DONE ) {
    # this is where things get dicy if you start generating  huge numbers of chunk files
    $NBR_FP_FLES = get_nbr_chunk_files( ${FP_ROOT_DIR} );
    # need to check both running and suspended PIDs (but $STILLRSYNCS is never used as a test)
  }
  my $end_s = time();
  my $elapsed_s = $end_s - $start_s;
  # select(undef, undef, undef, 0.5); # hacky way to sleep < 1s
  $NBR_FP_FLESa = $NBR_FP_FLES + 2; # a= adjusted to address fpart increment
  
  ( $rPIDs, $crr ) = get_rPIDs( $RSYNC_PIDFILE, $sPIDs );  # duplicated here from a few lines below to get accurate info..
}    # while ($CUR_FPI < $NBR_FP_FLES etc)

( $rPIDs, $crr ) = get_rPIDs( $RSYNC_PIDFILE, $sPIDs );
##: Final rsync log check to verify completions.  
##: Detect failed rsyncs and retransmit.
my $r2ri = 0;
my $log_seq = $FPSTART;
my $r_log_root = "${parsync_dir}/rsync-log-${DATE}_" ;
my $f_fle_root = $FP_ROOT;  
if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
# this loop runs thru all the rsync logs, checking that all of them have correct content
if ( $SKIPTO == 0 ){   # only do this if we're NOT using --skipto.  May revisit this if it becomes nec
  while ($log_seq < $CUR_FPI){ # should this be '<' not '<='?
    my $r_fle_wc; my $r_fle_as;
    if (-e "${r_log_root}${log_seq}" ) {
      $r_fle_wc = `sync && grep '\<f+++'  "${r_log_root}${log_seq}" | wc -l`; chomp $r_fle_wc; # 0 if no files transferred or if there's failure.
      $r_fle_as = `grep 'total size' "${r_log_root}${log_seq}" | wc -l`; # should be 1 if remote = local
    } else { WARN("Odd.  The file [${r_log_root}${log_seq}] was not written. 
    Check the contents of the corresponding chunk file [${FP_ROOT_DIR}/f.${CUR_FPI}].") ; }
    if ($r_fle_wc == 0 && $r_fle_as == 0) {
      if ($VERBOSE > 1) {
        INFO("rsync log [${r_log_root}${log_seq}] 
  has 0 lines of transfer data, indicating failure or all remote files in that chunk are identical.\n");
        } else { 
          clearline();
          print "[${r_log_root}${log_seq}] has 0 lines of transfer data.\r"; 
        }
      $rsyncs_2_rpt[$r2ri++] = $log_seq;
    }
    $log_seq += $FPSTRIDE;
  }
}
my $tt = 0; $r2ri--;
my $resend = "";
if (defined $rsyncs_2_rpt[0] && $SKIPTO == -1) { # $SKIPTO = -1 -> DON'T REUSE chunks.
  WARN("The rsyncs using the following chunk files indicated no files transferred and no syncs, 
  indicating that either:
  a) all remote files in that chunk were identical to the local files (more likely) 
     or 
  b) an rsync failure (less likely):
  [@rsyncs_2_rpt]
  
  Please check the logs in the series: [${r_log_root}#].
  
  If this is a MultiHost send, this question will be coming from a disconnected/remote
  host & the failed rsyncs will not be repeated automatically.
  You'll have to manually repeat this pfp2 session.
  
  If this is a SingleHost send and you would like to resend them, answer 'Y' or 'y'
  to the following question.  
  
  Would you like to re-send the chunkfiles that were noted above? [Ny]");
  if (!$MULTIHOST){ $resend = <STDIN>; }
  $CUR_FP_FLE = "${FP_ROOT}.${rsyncs_2_rpt[$tt]}";
} 

##: Resend failed rsyncs all at once, 
if ( $resend =~ /[Yy]/ ) {
  INFO("Resending the suspect chunk files serially.  This may take some time..\n");
  while ($tt <= $r2ri) { # if $r2ri is -1, fails immed.
    $CUR_FP_FLE = "${FP_ROOT}.${rsyncs_2_rpt[$tt]}";
    $logfile = "${parsync_dir}/rsync-log-${DATE}_${rsyncs_2_rpt[$tt]}"; # each chunk gets its own log.
    $RSYNC_CMD = "cd $TRIMPATH && $UDR rsync  --bwlimit=$MAXBW  $RSYNCOPTS --log-file=$logfile --files-from=$CUR_FP_FLE / $rat_hoststring  & echo \"\${!}\" >> $RSYNC_PIDFILE";
    # since the -files-from is fq, the '$ROOTDIR' needs to be '/'
    DEBUG(__LINE__, "RSYNC_CMD=[$RSYNC_CMD]");
    system("$RSYNC_CMD");    # captures the bg job PID to PIDfile
    clearline();
    print "Re-sending chunkfile [$CUR_FP_FLE]\r";
    $tt++;
  }
}

# if zotfiles, create remote script file, scp and then execute it. or put it into a huge string to be sent as a ssh command.
my $zf_rem_cmds = "";
if ($ZOTFILES) {
  open( RS, "> ${parsync_dir}/untgz.sh" ) or FATAL("Can't open [${parsync_dir}/untgz.sh] for writing");
  print RS "cd ~/.pfp2/fpcache;\n";
  for (my $r=0; $r<@REM_POST_FP_ZOT_CMDS; $r++) {
    $zf_rem_cmds .= $REM_POST_FP_ZOT_CMDS[$r];
  }
  print RS "$zf_rem_cmds";
  close RS or die "Couldn't close ${parsync_dir}/untgz.sh\n";
  my $cmd = "ssh $rem_user\@$rem_host 'mkdir -p ~/.pfp2/fpcache '; scp ${parsync_dir}/untgz.sh $rem_user\@$rem_host:~/.pfp2/fpcache";
  system( "$cmd; ssh $rem_user\@$rem_host 'bash ~/.pfp2/fpcache/untgz.sh'");
}

if ( $SKIPTO < 0 ) { # only do this if it transferred enough bytes to matter. ie no --reuse
  ##: Calc bytes of rsync logs and convert raw bytes to 'human'
  # remind user how much storage the cache takes and to clear the cache files
  # calculate bytes transferred from rsync logs ('$bytefiles')
  #$bytefiles .= "\*";    # to make it a glob
  $bytefiles = "${parsync_dir}/rsync-log-${DATE}_*";    # use this for a glob base
  $bytesxf = `grep 'bytes  total size' $bytefiles | scut -f=11 | stats --quiet --sum`;
  chomp $bytesxf;
if ($DEBUG) {print __LINE__, " : **DEBUG PROGRESS**\n";}
  ## this should be a function.##
  if ( $bytesxf < 1073741824 ) {                        # if < GB, present as MB
    $rprtnbr = $bytesxf / 1048576;
    $sfx     = "MB";                                    # for MB
  } elsif ( $bytesxf < 1.09951162778e+12 ) {            # if < TB, present as GB
    $rprtnbr = $bytesxf / 1073741824;
    $sfx     = "GB";                                    # for GB
  } else {                                              # present in TB
    $rprtnbr = $bytesxf / 1.09951162778e+12;
    $sfx     = "TB";                                    # for TB;
  }
  $ALLBYTES = sprintf( "%9.5f %2s", $rprtnbr, $sfx );
  ##: Print reminders
  my $du_cache = `du -sh $parsync_dir`;
  chomp $du_cache;
  my $tt = $CUR_FPI - $FPSTRIDE;                        # compensate for the autoincr above.
  $logfile = "${parsync_dir}/rsync-log-${DATE}_${tt}";

  if ( $VERBOSE > 2 && $DISPOSE =~ /l/ ) {
    INFO(
      "(Reminder) If you suspect errors, check the rsync logs.
  [${parsync_dir}/rsync-log-${DATE}_*]

    and the fpart log:
  [$FPART_LOGFILE]

    Your fpcache files were written in 
  [${FP_ROOT_DIR}] and take up [$du_cache].
    You rsync'ed about [$bytesxf bytes = $ALLBYTES] via all [$NP] rsyncs on this host.
    
    If you used the multihost mode, the whole pfp2 log is here: 
  [${parsync_dir}/pfp-log-${DATE}]
    (mostly text but identifies as 'binary' due to text coloration strings):
    If you used the singlehost mode and want to log the output, capture it 
    by prefixing the pfp2 command with 'script' ('man script' for details).
    
    The host-specific command used for this parsyncfp2 run (in case the host 
    dies midway or for debugging purposes) was:
  [$0 @argv]\n\n"
    );
  }
}

# POST PROCESSING pfp to clean up details.
##: Exit cleanup: email

if ( defined $EMAIL ) {
  INFO("Mailing completion note to [$EMAIL]\n");
  my $email_cmd = "echo \"Check the pfp2 logs in [${parsync_dir}] 
  for the results of the command: 
  
[$0 @argv]
  
\" | mail -s \"$DATE: parsyncfp2 on host [$host] completed\" $EMAIL";
  system("$email_cmd");
}

if ($DEBUG) { DEBUG( __LINE__,  "DISPOSE=[$DISPOSE]\n" ); }
##: Dispose of cache
# and based on --disposal, (=c(ompress), =d(elete) =l(eave untouched)  all the chunk files.
if ( $DISPOSE =~ /d/ ) {
  if ( $VERBOSE >= 2 ) { INFO("Deleting chunkfile dir as requested. Leaving logs intact.\n"); }
  remove_fp_cache( ${FP_ROOT_DIR}, "[Ff]*" );
} elsif ( $DISPOSE =~ /c/ ) {    # can it just be put into background?
  if ( $VERBOSE >= 2 ) {
    INFO("Tarballing the fpart & rsync log & fpart chunk files.\n");
  }
  $cmd = "tar --remove-files -czf ${HOME}/pfp2_${DATE}.tar.gz ${parsync_dir} 2> /dev/null"; 
  system("$cmd"); 
}

##: Exit message
if ( $VERBOSE >= 2 ) {
  if ( $DISPOSE eq 'c' ) {
    INFO(
      "(Reminder) You requested the '--dispose=c' option, so make sure 
  the tarchive has been processed to completion before you delete anything.  
  The tarchive was saved here:
     [${HOME}/pfp2_${DATE}.tar.gz] 
  and the corresponding pfp dir was deleted afterwards.\n"
    );
  }
}

( $rPIDs, $crr ) = get_rPIDs( $RSYNC_PIDFILE, $sPIDs );

if ($BIGFILES){
  # send the remote_reassemble.sh to the remote and execute it.
  # INFO( "About to send the reassemble script to remote host.\n" ); 
  system("rsync ${parsync_dir}/remote_reassemble.sh ${rem_user}\@${rem_host}:/tmp");
  INFO( "About to execute the remote reassemble script. 
  This may take some time, depending on the size of job." );
  system("ssh ${rem_user}\@${rem_host} 'bash /tmp/remote_reassemble.sh'");

  my $nbr_bigfiles = @DEL_SPLITS;
  # so we used FQPNs in DEL_SPLITS, have to chew off the leader and 
  if ($BIGFILES && $nbr_bigfiles > 0 && !$NOWAIT) {
    WARN("You've used the '--bigfiles' option and there are [$nbr_bigfiles] splits 
  of those files remaining.  If you want to delete them, answer 'Y' to the 
  following query; if you want to save them for some reason answer 'N' (default).");
  print "\nDelete the [$nbr_bigfiles] splitfiles generated from the '--bigfiles option? [yN] "; 
    my $tmp = <STDIN>;
    while ( $tmp !~ /[YyNn]/ ) {
    print "Only accepting [YyNn].  Try again? ";
      $tmp = <STDIN>; chomp $tmp;
    } 
    if ($tmp =~ /[yY]/) {
      my $ulo = unlink @DEL_SPLITS;
      if ($ulo ne "") {INFO("[The 'delete' cmd returns [$ulo].  
  If it's an integer, it's the number of files successfully deleted.")}
    }
  }

  if ($BIGFILES && $nbr_bigfiles > 0 && $NOWAIT){ 
    my $ulo = unlink @DEL_SPLITS ;
    if ($ulo ne "") {INFO("[Deleted [$ulo] local splitfiles successfully.\n")}
  }
}
clearline();
INFO("Thanks for using parsyncfp2. Tell me how to make it better.
                 <hjmangalam\@gmail.com>\n");
exit;

##########################################################################################
##: == Subroutines 
##########################################################################################


##: sub printdataline($$); print out the bandwith/info line

sub printdataline($actual_cp) {  # $start_cp, $printnow
  my $actual_cp = shift;  
  my $rDATE = `date +"%T" | sed 's/:/./g' `; chomp $rDATE;
  my $HOSTNAME = `hostname -s`; chomp $HOSTNAME;    # which host we're on
  
  my ($avgRDMAsend, $avgRDMArecv, $avgTCPsend, $avgTCPrecv, $avgRDMAsend, $loadavg,
  @SYSLOAD, $LOAD1mratio,  $rPIDs, $crr,  $R1, $R2, $T2, $printnow, 
  );
  
  # calc the post-sleep part of the bandwidth connection
  my $ee = time();      
  $R2 = `cat /sys/class/net/$::NETIF/statistics/rx_bytes`; chomp $R2;
  $T2 = `cat /sys/class/net/$::NETIF/statistics/tx_bytes`; chomp $T2;
  $avgTCPsend   = ( $T2 - $::T1 ) / $actual_cp;
  
  if ($PERFQUERY) {
    $RDMA_T2 = `perfquery -x | grep XmitData | cut -f2 -d: | sed -e 's/\\.*//g'`; chomp $RDMA_T2;
    $RDMA_R2 = `perfquery -x | grep RcvData  | cut -f2 -d: | sed -e 's/\\.*//g'`; chomp $RDMA_R2;
    $avgRDMAsend = ( $RDMA_T2 - $::RDMA_T1 ) / $::CHECKPERIOD; 
  }
  # do some basic math on the returned values.
  $avgTCPsend = ( $avgTCPsend / 1048576 );    # convert to MB      
  $avgRDMAsend = ( $avgRDMAsend / 262144 );  # convert to MB; use same divisor as rdma-tct-stats
 
  # now get load, bw, etc, and start rsyncs on new chunkfiles or suspend them to
  # load-balance
  ( $rPIDs, $crr ) = get_rPIDs( $RSYNC_PIDFILE, $::sPIDs );
  # now get load, bw, etc, and start rsyncs on new chunkfiles or suspend them to
  # load-balance
  $loadavg     = `cat /proc/loadavg | tr -d '\n'`;    # What's the system load?
  @SYSLOAD     = split( /\s+/, $loadavg );            # 1st 3 fields are 1, 5, 15m loads
  $LOAD1mratio = $SYSLOAD[0] / $::NCPUs;
  # print out current data with the date
  $rPIDs =~ s/^\s+|\s+$//g;
  $::sPIDs =~ s/^\s+|\s+$//g;                           # trim leading & trailing whitespace
  
  my $cur_secs = `date +"%s"`; chomp $cur_secs;
  my $el_min   = ( $cur_secs - $start_secs ) / 60;
  my $IFN        = sprintf( "%7s", $::NETIF );

  my $NrPIDs = my $NsPIDs = 0;
  if ( $rPIDs ne "" ) { $NrPIDs = my @Lr = split( /\s+/, $rPIDs ); }
  if ( $::sPIDs ne "" ) { $NsPIDs = my @Ls = split( /\s+/, $::sPIDs ); }

   ##: print data header - cycle-based, not entirely time based.
  if ( $::hdr_cnt > $::hdr_rpt) {
    $::hdr_cnt          = 0;
    my $day              = `date +"%F"`;  chomp $day;
    my $hdr = "\n         | Elapsed |   1m   |  [$IFN]   MB/s  | Running || Susp'd  |      Chunks       [$day]
   Time  | time(m) |  Load  |   TCP / RDMA  out  |   PIDs  ||  PIDs   | [UpTo] of [ToDo]    Send Host\n";
 
    if ( $::FPSTART == 1 && $::VERBOSE > 0 && $::hdr_pr == 1) { 
      print $hdr;
      if ( $::VERBOSE == 1 ) { $::hdr_pr = 0; }
    } else { $::hdr_pr = 0; }
  }
  
  ##: Print xfer data line
  my $adjcf;
  if ( $::FPSTART == 0 || $::VERBOSE > 0 ) {
    $adjcf = $::CUR_FPI - $::FPSTRIDE;
    # cosmetic hacks for now. if $adjcf > $NBR_FP_FLES, not going to send any more data.
    if ($adjcf > $::NBR_FP_FLES)  { $adjcf = $::NBR_FP_FLES; } 
    if ($adjcf < 1)               { $adjcf = 1; }
    if ($adjcf <= ($::NBR_FP_FLES + 1) || $rPIDs ne '' || $::sPIDs ne '') { # to prevent a cosmetic overrun of the numbers.
      printf
      "%8s   %5.2f    %5.2f  %9.2f / %-9.2f       %2d    <>  %2d        [%d] of [%d]      < %-10s",
      $rDATE, $el_min, $SYSLOAD[0], $avgTCPsend, $avgRDMAsend, $NrPIDs, $NsPIDs, $adjcf,
      $::NBR_FP_FLES, $HOSTNAME;  
    }  
  }
  # and then over-write it or add a newline for scrolling data.
  if    ( $::VERBOSE == 1 ) { printf "\r"; }
  elsif ( $::VERBOSE >= 2 ) { printf "\n"; }
  $::hdr_cnt++;
  # reset bandwidth counters for the next checkperiod printing.
  # calc the initial values for the BW calc'n
  $::RDMA_T1 = $::RDMA_R1 = $avgRDMAsend = $avgRDMArecv =  0;
  $R1 = `cat /sys/class/net/$::NETIF/statistics/rx_bytes`; chomp $R1;
  $T1 = `cat /sys/class/net/$::NETIF/statistics/tx_bytes`; chomp $T1;
  if ($::PERFQUERY) {
    $::RDMA_T1 = `perfquery -x | grep XmitData | cut -f2 -d: | sed -e 's/\\.*//g'`; chomp $::RDMA_T1;
    $::RDMA_R1 = `perfquery -x | grep RcvData  | cut -f2 -d: | sed -e 's/\\.*//g'`;  chomp $::RDMA_R1;
  }
  $start_cp = time();
  $printnow = time() + $::CHECKPERIOD;
return $start_cp, $printnow;
}  

##: sub clearline()
sub clearline() { print "\e[K"; } # easier to remember this way.

##: sub short_hostname($hostname)
# short_hostname() takes a given host and returns the 
# local short hostname.  If it's an alphanumeric name (bridgit, bridgit.res.ucw.edu), 
# still returns the short name.  Could do more complicated DNS querying, etc, but this is fairly fast 
# (~1s for an 85ms connection and most of these will be on a local net, taking < 0.5s)
sub short_hostname($hostname) {
  my $host = shift;
  my $ret = `ssh $host 'hostname -s ; echo \$SHELL ' `;
  $ret =~ s/\n/#/; trim ($ret);  chomp $ret;
  my $n = my @a = split /#/, $ret;
  my $shortname = $a[0];
  my $shell = $a[1];
  chomp $shortname;
  return $shortname, $shell;
}

#  this sub handles a bunch of cases: must handle the TARGET
#   POD::/remote/path/to/drop - if --hostlist hosts don't have their own path (not MODULE), 
#     this path gets appended.  NB: if sending to a server, each host in hostlist has to specify 
#     their own MODULE (since MODULES will typically differ among all the rsync servers).

#   ACTUALLY, it could also mass-name a rsyncd server by NOT leading with a '/', so that the 
#   pod_remotepath becomes a POD-wide module name so:
# POD::modulename specifies that each SEND target gets a '::modulename' appended.
# --hostlist="stunted=bridgit, bigben=bridgit" ... POD::dumpster
# specifies sending to the rsyncd server with the common modulename 'dumpster'

#   hjm@host.net.uci.edu:/path/to/drop
#   host:/path/to/drop
#   bridgit::module_name
#   user@host::module_name
#   host:~/path/to/dir     # want to handle tilde's?  yes, eventually, so include this processing
#                          # convert to full path?  or to :path/to/dir
#   host:dir = host:~/dir   (in HOME dir)
#   host:path/to/dir = host:~/path/to/dir   (in HOME dir)
# so parse on @ first, then :, then go from there.

 # should be able to mod parse_rsync_target() to also emit a full user@host:/path or user@host::module,
 # for the $RSYNC_CMD given the initial $TARGET and the REC host string.
 # so if $TARGET = user@POD::module and the REC string was 'bridgit', the actual generated hoststring
 # should be user@bridgit::module.
 # if $TARGET = POD::~/dumpster & REC string was 'bridgit', the generated REC hoststring should be 
 # bridgit:dumpster.  So add another input var 'hoststring' and the full created hoststring var gets returned 

# called as: 
# ($rem_user, $rem_host, $rem_shell, $rsync_type, $rem_fqpath, $rem_homepath, $RSYNCMODULE, $pod_remotepath, $rat_hoststring) = parse_rsync_target ($USER, $TARGET, $ALTCACHE);
# where $rat_hoststring = rationalized hoststring
# & $hoststring is the REC hostname from the --host string, often a naked hostname ('bridgit')

# IMPORTANT: parse_rsync_target() call is bimodal.  
# The 1st mode is with the $TARGET only, and the hoststring is '', typically done with a singlehost rsync
# (ie $TARGET='harry@someremotehost.uci.edu:/path/to/storage')
# or the 1st call to the fn().

# ### this is where the current ver is failing now
# The 2nd mode is with the POD variant, with the $TARGET (POD::...) AND the recv_hoststring 
#(the rechost part of the sendhost=rechost string) included. ie  from the phrase 
# [cent@centoo=harry@bigben:/bulkstorage], the recv_hoststring is 'harry@bigben:/bulkstorage'
# In both cases, the '$rat_hoststring' should reflect the 
# the actual target in the RSYNC_CMD string. ie the POD::(string) and the hoststring (from the --hosts options)
# should be combined to form the $rat_hoststring (rationalized_hoststring)

##: sub parse_rsync_target ($LOCALUSER, $TARGET, $ALTCACHE, $recv_hoststring)
sub parse_rsync_target ( $$$$ ){ 
  my $localuser   = shift;
  my $target      = shift;
  my $altcache    = shift;
  my $recv_hs     = shift; # processed at the bottom of the sub()
  my $pod_remote_path = "";
  my $rat_hs = "";
  my $trgtuser; my $host; my $rtype = 'rsync'; 
  my $mod; my $path_fr_HOME; my $fqpath;
  my $n3; my @a3;  # for later   
  my $usage = "The TARGET spec [$target] isn't formatted correctly.
  The TARGET should be one of:
  - {user\@}host::rsyncd_module_name  (for an rsync server) or
  - {user\@}host:/path/to/landing                           or
  - {user\@}host:~  (user's HOME)                           or
  - {user\@}host:   (user's HOME)                           or
  - {user\@}host:~/path/from/HOME                           or
  - {user\@}host:path/from/HOME        ({user\@} optional in all the above)
  
  - /mountedFS/path/name    (ie no host)                    or
    
    in MultiHost (POD) mode:  
  - {user\@}POD::/remote/path                               or
     remote path to append to all '--host' targets that don't supply their own
  - {user\@}POD::~/remote/path 
      HOME-rooted path -> appended as the rsync-friendly ':remote/path'
  - {user\@}POD::modulename 
     (no leading '/' or '~', all the '--host' RECEIVE hosts are rsyncd servers 
     with same module name; 'user' in the POD case is the rsyncd user, NOT the 
     login/ssh user & can be mass-specified via the POD string)
  ";
  if (($target !~ 'POD::/') && ($target !~ /:/ ) && ($target !~ '^/') ) { 
    FATAL("$usage"); # there's another usage call below.  If problems, check both conditions.
  }
  
  # if $target is POD::/ &&  defined $recv_hs as user@host, then $target becomes the remote host
  # if $target is POD::/ && !defined $recv_hs, then $target is still POD::/whatever
  # if $target is a regular target, it should drop thru to be the $rat_hs
  if ($target !~ /POD::/) {
    $recv_hs = $target;  # if it's not a POD string, then it's the actual target
    my $n1 = my @a1 = split('@', $target);
    if ($n1 == 2) { # then there's a user
      $trgtuser = $a1[0];
      $host = $a1[1];
    } 
  } elsif ($recv_hs ne "") {
    my $n1 = my @a1 = split('@', $recv_hs);
    if ($n1 == 2) { # then there's a remote user in the $recv_hs
      $trgtuser = $a1[0];
      $host = $a1[1];  
    } 
  }
  
  # so now target is remotehost
  # if no 'user@', then $target falls thru
  # now proc $target
  
  my $t = $target; # fucking weird.  Without this line (or being referenced in a print), $target fails tests below.??!?
  
  if ($target =~ '^/') { # ie no host, it's a mounted FS
    $rtype = 'filesystem'; 
    $fqpath = $target;  # the $target is the mounted filesystem
  }
    elsif ( $target =~ /POD::/ ) {    # then extract path to suffix all the rsync (not rsyncd) REC hosts
    # the '::' doesn't set the $TARGET as rsyncd - that's the job of the '--hosts' line.
    # this just defines the run as a POD run, as opposed to a singlehost run.
    my $N = my @A = split( '::', $target );  
    if ($A[0] !~ 'POD') {FATAL("The POD spec [$A[0]] is garbled; please correct and try again.")}
    if ( $N == 2 ) {
      if ($A[1] =~ /^~/) { $A[1] =~ s'~/':';}
      $pod_remote_path = $A[1]; 
      $rtype = 'rsync';
      if ($A[0] =~ '@') {
        my $nn = my @aa = split ('@', $A[0]);
        $trgtuser = $aa[0];
      }
    } else {
      FATAL( "The MultiHost default REMOTE path [$target] isn't formatted correctly. 
  It needs something on the right side of the '::'. Minimally a '/' to define a remote path
  (POD::/path/to/dir), or an rsync server module name without a leading '/' (POD::dumpster). 
  This string is appended to any pathless hosts in the '--hosts' option. Please check." );
    }
    if ($pod_remote_path !~ '^[/~:]'){ # if no leading '/' or '~', it's prob a rsyncd server module name.
      $rtype = 'server';
      # but check to see if there are embedded bad chars
      if ( $pod_remote_path =~ '/' ) {FATAL("The rsyncd module name you supplied [$pod_remote_path] has a '/' in it, which is not allowed. Try again.");}
      $mod = $pod_remote_path;
      $pod_remote_path = ""; # null the $pod_remote_path
      
      # should we query the rsyncd to see if that's a valid module?
      # rsync rsync:// returns valid modules..  can do this in the checkhost stanza.
      # just store it for now.
      # now check for an RSYNC_PASSWORD as well
      my $rsyncdpass = $ENV{"RSYNC_PASSWORD"}; chomp $rsyncdpass;
      if ( $rsyncdpass eq "" ) {
        WARN("You haven't set an RSYNC_PASSWORD in the SEND host environment, 
  so the only way to send data is to use the rsyncd server without any security.
  This connection will fail if the rsyncd server requires authentication.
  Continuing, but please see how to set up rsyncd security with 'man rsyncd.conf',
  especially 'auth users' and 'secrets file' lines.
    
  If you see a series of 'Password:' queries, the rsyncd server is expecting 
  authentication and no data will be sent, altho pfp will cycle thru all the
  chunks since the process is now independent.");
      }
    }
     # also check the status of $ALTCACHE - but this could be done in the top error-checking
     # and remove the passed in arg. TODO
    if ( !defined $altcache ) {
      FATAL(
        "If you're going to use the multihost/pod version, 
  you have to explicitly define the shared, common dir for all the SEND hosts; 
  please add the --commondir option."
      );
    }
  }  elsif ($target =~ /::/) { # not POD, but does have '::'
    my $n2 = my @a2 = split(/::/, $target); # if it's a rsyncd module name, there will be a ::
    if ($n2 == 2  && $target !~ /POD/) { # it's std rsyncd module string.
      $rtype = 'server'; # not 'rsyncd' to prevent string overlaps with rsync 
      $host = $a2[0];
      $rat_hs = $target;
      if ( $a2[1] =~ '/' ) {FATAL("The rsyncd module name you supplied [$a2[1]] has a '/' in it.  Try again.")}
      $mod =  $a2[1]; # & the module name should be a single unspaced string
    } 
  }
  elsif ($target =~ ':[/~]?' ) { # host:/path, host:~path, host:path
    # then it's either a ':/' or a single ':' or a ':~' - either way it's a std rsync
    # and if it's neither, then the input target was bad and died above
    $rtype = 'rsync';
    $n3 = @a3 = split(':', $target);
    if ( $n3 == 1 ) { # a terminal ':' is valid to rsync - indicates $HOME
      $path_fr_HOME = ":";  
      if ($a3[0] =~ '@') { # then the $target has a 'user@'
        my $n4 = my @a4 = split('@', $a3[0]);
        if ($n4 == 1) { $host =  $a3[0]; }
        else {$host =  $a4[1]; }
      }
    } elsif ( $n3 == 2 ) {
      $host = $a3[0]; # host is the bit before the ':'
      if ( $a3[1] =~ '^/' ) { # its a fully qualified remote path
        $fqpath = $a3[1];
        $rat_hs = $target; # it was fine as is.
        # if it's a '~', then it could be '~' indicating user's HOME, so just convert to ':'
        #  or ~/path -> :path 
        #  or ~user indicating that the final path should be another user unless the string 
        # after the ~ is = to rem_user or the (unspecified) current local user
      } elsif ( $a3[1] =~ '^~\s?$' ) {  # ie a single '~', it's the *nixy way of describing the remote HOME
                                    # but rsync doesn't like it, so convert it to what rsync likes
                                    # 'host:'
        $path_fr_HOME = ':';
        $rat_hs = "$host" . "$path_fr_HOME"; 
      } elsif ( $a3[1] =~ '^~/' ) {  # then its a path from HOME, so convert the '~' to a ':'
        $a3[1] =~ s'~/':';
        $path_fr_HOME = $a3[1];
        $rat_hs = "$host" . "$path_fr_HOME"; 
      } else { # its a '~user'ish string, so compare the user to both the $trgtuser above.  If it's not the same
                # issue a warning and continue.
        $a3[1] =~ tr/~//d; # delete the leading '~' to leave the user, but ...
        $path_fr_HOME = ":" . "$a3[1]";
        $rat_hs = "$host" . "$path_fr_HOME";
      }
    }
  } elsif ($target =~ /[a-zA-Z0-9\_\-\.]+/)  {  # (plain rsync with a naked hostname) 
    $rtype = "rsync";
    $host = $target;
    }
  else { FATAL("$usage");  }
  # at some point, the naked '$host' has to be extracted out of the recv_host string if it's going to be returned correctly. If its the SH var, the $TARGET will be the real target, from which the $host has to be extracted.  If it's the MH var, the $host has to be extracted out of the right half of the send_host=recv_host pair.
  
  # at this point, now trying to extract $host NOT from $TARGET, but from the recv_hs
  # which we could force here with a couple of splits, re-using $n3, @a3

  $n3 = @a3 = split('@', $recv_hs); 
  my $th; # temp hoststring
  if ($n3 == 1) { $th = $a3[0];}
  else          { $th = $a3[1];}
  $n3 = my @a3 = split(':', $th); 
  $host = $a3[0];
  $fqpath = $a3[1]; # leave '~' in place if it contains one
  if ($fqpath =~ /\~/) { # but if it's a tilde-based path, set the $rem_homepath / $path_fr_HOME var as well.
    $path_fr_HOME = $fqpath;
    $path_fr_HOME =~ s'^~/''; # del the leading '~/'
  }
  
  my $host_is_naked = 0; # 'jack' alone
  my $host_has_user = 0; # hjm@jack
  my $host_has_sufx = 0; # jack:/something
  my $host_is_fq    = 0; # hjm@jack:something
  
  my $nu = my @u = split('\@', $recv_hs);   if ($nu == 2) { $host_has_user = 1; }
  my $ns = my @s = split(':+', $recv_hs);   if ($ns == 2) { $host_has_sufx = 1; }

 # make sure if the remote user is specified, it gets thru to the end.
 # BUT if $recv_host HAS a user, DO NOT give it another one.
      if ($trgtuser ne '' && defined $trgtuser && $host_has_user == 0) {
        $recv_hs = ${trgtuser} . '@' . ${recv_hs} ;
        $rat_hs  = $recv_hs;
      }
  # ### TODO still 
  # now use all the validated vars to create the rationalized hoststring
  # if $recv_hs is a naked host && it came from a --hoststring & the TARGET contains 'POD::', use the 
  # vars above to create the full, correct $TARGET for the $RSYNC_CMD.
  # define naked host if ($recv_hs ) host:
  
#   my $host_is_naked = 0; # 'jack' alone
#   my $host_has_user = 0; # hjm@jack
#   my $host_has_sufx = 0; # jack:/something
#   my $host_is_fq    = 0;  # hjm@jack:something
#   

  if ($nu == 2 && $ns == 2) { # theres a user and a suffix, so no more additions needed
    $host_is_fq    = 1; 
    $rat_hs = $recv_hs;
  }  else {
    if ($nu == 1 && $ns == 1) { # the recv_host is naked w/ no suffix, so it needs a user && suffix
      $host_is_naked = 1; 
      $host = $recv_hs; # ( which has a user by this time )
    }

    if ($target =~ /POD::/ && $host_has_user == 1) {
      # generate the whole string.
      if    ( $trgtuser ne '' )     { $rat_hs = "${trgtuser}\@${host}";  }
      elsif ( $localuser ne '' )    { $rat_hs = "${localuser}\@${host}"; } 
      else                          { $rat_hs = "${host}"; }
      if ($rtype eq 'rsync') {
        if ($fqpath ne '') { $rat_hs .= ":${fqpath}" ; }
        elsif ( $path_fr_HOME ne "" ) { $rat_hs .= ":${path_fr_HOME}"; }
        elsif ( $pod_remote_path ne "" ) { 
          if ($pod_remote_path =~ /^:/) {
            $rat_hs .= "${pod_remote_path}"; # allows ':::' separation
          } else  {$rat_hs .= ":${pod_remote_path}";}  # allows ::/path
        }
      } elsif ($rtype eq 'server') {
        $rat_hs .= "::${mod}"; 
      }
    } elsif ($target =~ /POD::/ ) { $rat_hs = "${recv_hs}"; } # a non-naked hoststring should be the entire thing
  }  
  return ($trgtuser, $host, $rtype, $fqpath, $path_fr_HOME, $mod, $pod_remote_path, $rat_hs); 
  # where $rtype is 'rsync' or 'server' (to prevent regex conflicts), & either $path OR RSYNCMODULE
  # $user can be NULL, in which case it's $USER
} # sub parse_rsync_target

# get rid of all this mumbling after the code is rationalized and running.

#   so hostcheck():
#   Note that hostcheck is distributed both above and below this comment stanza - called 2x,
#   once for SEND, once for REC.  So can do both with a function().
#   - if master && firstrun (!-e ~/.pfp2) checks for installed apps, etc
#      - check distro  - /etc/os-release

#   - if SEND host && firstrun (!-e ${parsync_dir})
#     - based on distro, print packages you need to install
#         - CENTOS - epel-release, fpart, wireless-tools(iwconfig), infiniband-diags (perfquery),
#             perl-Statistics-Descriptive, perl-Env
#         - ubuntu - fpart, iw, infiniband-diags,  libstatistics-descriptive-perl
#         - and remove compiled utils from @UTILITIES
#     - print github location for UDR if wanted.
#     - check for ~/.ssh/config problems and resolv them, also if there are host redefinitions, report them
#     - fix .ssh/config file to eliminate wonky stderrs.
#       check for ~/.ssh/config problems and resolv them,
#       also if there are host redefinitions, report them
#       incorporate all that in fix_ssh_config() and call from here
#     - fix_ssh_config();#

#     - only if detect an rsyncd request
#       - check RSYNC_PASSWWORD is set for the sending USER
#       - check rsyncd reports correct modules

#   - checks for running rsyncs - should be done each time anyway.
#       for master, SEND, and REC hosts (there will alsways be a MASTER/SEND and a REC; there may
#       be multiple SEND AND REC hosts.  Detect which condition and run hostcheck on all of them.
#       may have to call as hostcheck(SINGLEMASTER|| MULTIMASTER || SEND || RSYNC (rec) || RSYNCD (rec)),
#       always have to run MASTER, then detemine what kind of host it is via parsing the -hosts option
#       or the TARGET.  So checkhosts() can be inserted anywhere it's needed.

# all this stuff is inside the MH stanza, so it's run ONLY on the MHmaster, directed TO the other hosts
# (SEND, RSYNC/RSYNCD REC hosts) so the callset would be:
# checkhost(MULTIMASTER, "","",, $ALTCACHE, $VERBOSE)
#   check utilities on firstrun, local ~/.ssh/config,stray rsyncs running
# checkhost(SEND, $HOST, "", $ALTCACHE, $VERBOSE) to all SEND HOSTs
#   check for working ssh, remote ~/.ssh/config, stray rsyncs running, RSYNC_PASSWORD
# checkhost(RSYNC/RSYNCD, $HOST, $RSYNCMODULE, $ALTCACHE, $VERBOSE) to all REC HOSTs (1, most of the time, )
# check for working ssh, remote ~/.ssh/config, stray rsyncs running, rsyncd, requested modules,

# to check the state of the SINGLEMASTER, it has to be run OUTSIDE of the MHmaster stanza(s)
# and the SINGLEMASTER only has to ssh, etc to 1 REC host, either RSYNC/RSYNCD so the callset
# would be:
# checkhost(SINGLEMASTER, "","","", $VERBOSE)
#   check utilities on firstrun, local ~/.ssh/config, stray rsyncs running, RSYNC_PASSWORD
# checkhost(RSYNC/RSYNCD, $HOST, $RSYNCMODULE, $ALTCACHE, $VERBOSE) to check the REC host
#   check for working ssh, remote ~/.ssh/config, stray rsyncs running, rsyncd, requested modules,

# so the only thing that's diff between SINGLE and MULTI is RSYNC_PASSWORD

# also include deps in help file and asciidoc.
# NODETYPE is one of: SINGLEMASTER, MULTIMASTER, SEND, RSYNC, RSYNCD
# returns avg RTT.
##: checkhost ( "NODETYPE", $HOST2CHECK, $REM_USER, $RSYNCMODULE, $ALTCACHE, $VERBOSE, $MAXLOAD )
sub checkhost ( "NODETYPE", $HOST2CHECK, $REM_USER, $RSYNCMODULE, $ALTCACHE, $VERBOSE, $MAXLOAD ) {
  my $ntype      = shift;
  my $host2check = shift;
  my $ruser      = shift;
  my $module     = shift;
  my $altcache   = shift;
  my $verbose    = shift;
  my $maxload    = shift;
  my $distro     = "";
  my $host       = `hostname -s`;
  my $DBC        = 0;
  my $rtt        = 0;
  my @L; my $N;
  my $optional = "Optionally udr (scut & stats provided with parsyncfp2).  
    See '--help' or HTML manual for availability.\n";
  
  if ($VERBOSE >= 2) { 
    my $rec_chk = "";
    if ($ntype =~ "RSYNC") { $rec_chk = "Possible delay waiting for 5 pings.";}
    INFO("HostChecking [$host2check] as [$ntype] host. $rec_chk\n");
  }
    my $fqrhs; # 'fully qualified remote host string' to avoid munging $host2check
    if ($ruser ne '') {$fqrhs = "$ruser\@$host2check"; }
    else { $fqrhs = "$host2check"; }

  if ( $ntype =~ "MASTER|SEND" ) {
    if ( $ntype =~ "SINGLE" ) {

      # detect if it's a firstrun
      if ( -d "$altcache" ) {    # probably NOT a firstrun
                                  #print "not a first run, $altcache exists\n";
      } else {
        first_run_required_utils();
      }
    } elsif ( $ntype =~ "MULTI" ) {    # then check to see if the $COMMONDIR/$ALTCACHE exists
      if ( !-e "$altcache" ) {
        FATAL(
          "You appear to be trying to run the MultiHost version, but the '--commondir'
        shared directory [$altcache] doesn't exist."
        );
      }
      if ( !-e "$altcache" ) {      # probably a firstrun if the ${pfpdir} doesn't exist.
        first_run_required_utils();
      }
    } elsif ( $ntype eq "SEND" ) {

      # use the returned loadavg to make sure the SEND host's loadavg is < maxload.
      my $rem_loadavg = check_ssh_ok($host2check);    # first check if can ssh to it.
      $N = @L = split( /\s+/, $rem_loadavg );

      if ( $L[0] > $maxload ) {
        WARN(
          "The SEND host [$host2check] has a loadavg of [$rem_loadavg]. 
    Sleeping for 5s to allow you to exit and see if you want to continue."
        ); sleep 5;
      }

      if ( !-e "$altcache" ) {                        # probably a first run on the SEND host
        FATAL(
          "You appear to be trying to run the MultiHost version, but the '--commondir'
    shared directory [$altcache] doesn't exist. Please fix this."
        );
        DEBUG( __LINE__,  "(checkhosts) host-specific altcache = [${altcache}/${host}]" );
        if ( !-e "${altcache}/${host}" ) {    # if this doesn't exist, then probably firstrun.
          first_run_required_utils();
        }
      }
    }

    my $checkcmd =
qq(ssh $fqrhs "ps ux | grep rsyn[c]| grep -v '${CALLING_PROGRAM}\\|daemon' | wc -l; mkdir -p ${parsync_dir}" 2> /dev/null);  # should return integer
    my $checkout = `$checkcmd`; chomp $checkout;
    if ( $checkout > 0 ) {
      WARN("Host [$host2check] has [$checkout] rsyncs running already as 
        the login user.  Possible conflicts.
        If you're sure they're related to failed pfp runs, you can kill off all the rsyncs
        and pfp scripts with [$PFP2STOPSCRIPT]\n");
    } else {
      if ( $VERBOSE >= 2 ) { INFO("[$host2check] is free of competing rsyncs.\n") }
    }
    # rewrit to be shell agnostic
    $checkcmd =
    qq(ssh $fqrhs 'test -e .ssh/config && grep Host ~/.ssh/config' 2> /dev/null);
    $checkout = `$checkcmd`; chomp $checkout;
    if ( $checkout =~ "Host" ) {
      WARN("[$fqrhs] has a ~/.ssh/config file containing 'Host' definitions, 
  which might interfere with determining the interface. If it does, 
  please use the FQDN (not the short form) for the corresponding host:    
  host -> host.net.domain.edu");
    } else {
      if ( $VERBOSE > 2 ) { INFO("SEND Host [$fqrhs] No conflicting ~/.ssh/config.\n") }
    }
  }

  if ( $ntype eq "RSYNC" ) {
    # check to see if it responds to ssh login.
    check_ssh_ok($fqrhs);  # was $host2check
  }
  if ( $ntype eq "RSYNCD" ) {    # its RSYNCD
    # check whether the rsyncd is up and has the right module
    my $modulelist = `rsync rsync://$host2check | awk '{print \$1}'`;
    my $na         = my @ma = split( /\s+/, trim($modulelist) );

    my $r = 0;
    while ( $r < $na && $module ne "$ma[$r]" ) {
      $r++;
    }
    if ( $r == $na ) {
      FATAL(
        "The module you supplied [$module] doesn't appear to be offered in the modules 
  offered by [$host2check] which are:
    $modulelist
  To query other rsyncd servers for their modules/descriptions, use: 
    rsync rsync://[host]"
      );
    }
  }
  if ($ntype eq "RSYNCD" || $ntype eq "RSYNC") { # check the rtt
    $rtt = `ping -c 3 $host2check | grep avg | scut -f=7 -d='[\ /]'`; chomp $rtt;
  }
  return $rtt;
}

##: first_run_required_utils ()
sub first_run_required_utils () {

  # if IF =~^en -> ethtool
  # if IF =~^wl -> iwconfig
  # if IF =~^ib -> ibstat, perfquery
  # all should be able to use udr (but that's only needed if you try that option.)
  my $optional = "Optionally 'udr' (get & compile from <https://github.com/martinetd/UDR>)
    'scut' & 'stats' are typically bundled provided with parsyncfp2).  
    See '--help' or HTML manual for availability.\n";

  # the initial values are the REQUIRED utils
  my $recnetutils = "fpart scut stats ";    # utils names; $pkgs are generic names for distro pkgs
  my $pkgs = my $ifstr = my $distro = "";
  my @warn;
  my $wc = 0;

  if ( -e '/etc/os-release' ) {             # detect distro type
    my $distro = `grep PRETTY_NAME /etc/os-release| scut -f=1 -d='='`; chomp $distro;
    if ( $distro =~ /Ubuntu|Debian/ ) {
      $pkgs = "fpart scut stats libstatistics-descriptive-perl ";
      my $ifstr = trim(`ip a  | grep ' UP ' | scut -f=1 -d=': ' | tr '\n' ' '`);
      if ( $ifstr =~ /wl/ )    { $recnetutils .= "iwconfig "; $pkgs .= "iw "; }
      if ( $ifstr =~ /en|em/ ) { $recnetutils .= "ethtool ";  $pkgs .= "ethtool "; }
      if ( $ifstr =~ /ib/ ) { $recnetutils .= "perfquery ibstat "; $pkgs .= "infiniband-diags "; }
    } elsif ( $distro =~ /CentOS|RHEL|Fedora|Amazon|Oracle/ ) {
      $pkgs = "fpart scut stats epel-release perl-Statistics-Descriptive ";
      my $ifstr = `ip a  | grep ' UP ' | scut -f=1 -d=': ' | tr '\n' ' '`;
      if ( $ifstr =~ /wl/ )    { $recnetutils .= "wireless-tools "; $pkgs .= "iw "; }
      if ( $ifstr =~ /en|em/ ) { $recnetutils .= "ethtool ";        $pkgs .= "ethtool "; }
      if ( $ifstr =~ /ib/ ) { $recnetutils .= "perfquery ibstat "; $pkgs .= "infiniband-diags "; }
    }
    # Now test to see whether the actual required utilities are on the path
    my $Nrec = my @RECMD = split( /\s+/, $recnetutils );
    for ( my $i = 0 ; $i < $Nrec ; $i++ ) {
      my $utilpath = `which $RECMD[$i] 2>&1 | tr -d '\n'`;
      if ( $utilpath eq "" || $utilpath =~ /which: no/ ) {    # CentOS7 'which' error
        $warn[ $wc++ ] = $RECMD[$i];                          # util not there
      }
    }
    if ( $wc > 0 ) {
      WARN(
        "The following recommended utilities for your distro were not found:
    [@warn].
    If the utilities aren't on your PATH and parsyncfp2 hits a requirement, it may give faulty
    output or crash."
      );
    }
  }

  # for the debian/ubuntu or RH-based distros..
  INFO(
    "This machine has the following UP network interfaces:
    [$ifstr] and the distro [$distro] requires the following packages to support parsyncfp2:
    [$pkgs].
    $optional"
  );

  # don't know what the names are for other distros yet..
  INFO(
    "This distro [$distro] needs the following utilities and/or packages 
    to support parsyncfp2:
    fpart ip  
    and depending on your networking hardware:
    WiFi - iwconfig (typically in 'iw' or 'wireless-tools')
    Ethernet - ethtool
    Infiniband - perfquery & ibstat (typically in 'infiniband-diags')
    $optional"
  );

  if ( !-e '/etc/os-release' ) {
    WARN(
      "This system doesn't have the '/etc/os-release' file that tells me what 
    distro you're running.  Please see the --help or HTML manual to see what 
    utilities you need."
    );
  }
}

# Call from SINGLEMASTER or MULTIMASTER to verify that SEND & REC hosts are up, online,
# have ssh keys set up, etc.
##: check_ssh_ok ($HOSTNAME)
sub check_ssh_ok ($HOSTNAME) {

  # Check ~/.ssh/config
  my $host     = shift;
  my $rloadavg = `ssh $host 'cat /proc/loadavg' 2> /dev/null`; chomp $rloadavg;
  if ( $rloadavg eq "" ) {
    FATAL(
      "Tried to ssh to [$host] but wasn't successful.  
  Please check that the machine is on, your ssh keys and file 
  permissions are set correctly and that any host stanzas 
  in your local '~/.ssh/config' file don't interfere with the login. "
    );
  }
  return $rloadavg;
}

# get the number of files in a directory based on glob
# passed to grep. This avoids having to use 'ls' and
# gets away from issues where the total number of files
# exceeds the maximum for ls
##: get_nbr_chunk_files () # 1st ver
sub get_nbr_chunk_files {
  my $path = shift;
  my $dh;
  my $num_files = 0;
  opendir( $dh, $path ) or die "Cannot open $path in get_nbr_chunk_files()\n";

  # this = () = this the goatse operator that will convert the result to a scalar
  $num_files = () = grep( /^f\.\d+/, readdir($dh) );
  closedir($dh);
  return $num_files;
}

# same as above but it avoids the use of rm to clean the cache directory
##: remove_fp_cache ()
sub remove_fp_cache {
  my $path = shift;
  my $glob = shift;
  unlink glob "$path/$glob";
}

# check_utils("fpart ethtool scut stats ip", "iwconfig perfquery "); # udr
##: check_utils($required_str, $recommend_str)
sub check_utils($$) {
  my $req = shift;
  my $rec = shift;

  # now break them into bits.
  my $Nreq = my @REQ = split( /\s+/, $req );
  my $Nrec = my @REC = split( /\s+/, $rec );
  my @fatal;
  my $fc = 0;
  my @warn;
  my $wc = 0;

  for ( my $i = 0 ; $i < $Nreq ; $i++ ) {
    my $utilpath = `which $REQ[$i] | tr -d '\n'`;

    #   print "utilpath = [$utilpath]\n";
    if ( $utilpath eq "" ) {
      $fatal[ $fc++ ] = $REQ[$i];

      #WARN("[$REQ[$i]] not found.  Can't continue without it.
      #Check the help page for more info on [$REQ[$i]].");
    }
  }
  for ( my $i = 0 ; $i < $Nrec ; $i++ ) {
    my $utilpath = `which $REC[$i] 2>&1 | tr -d '\n'`;

    #  print "utilpath = [$utilpath]\n";
    if ( $utilpath eq "" || $utilpath =~ /which: no/ ) {    # CentOS7 'which' error
      $warn[ $wc++ ] = $REC[$i];

      #WARN("[$REC[$i]] not found.  This utility is not required but some things may
      #not work. Check the help page for more info on [$REC[$i]]."
      #);
    }
  }
  if ( $wc > 0 ) {
    WARN("The following recommended (but not required) utilities were not found: [@warn].");
  }
  if ( $fc > 0 ) {
    FATAL(
      "The following REQUIRED utilities were not found [@fatal]. 
  Please install them or modify the appropriate \$PATH."
    );
  }
}
##: get_rPIDs ($pidfile, $spids)
# call as ( $rPIDs, $crr ) = get_rPIDs ($pidfile, $suspended_pids)
sub get_rPIDs ($$) {

  # usage:  ($rPIDs, $crr) = get_rPIDs($RSYNC_PIDFILE, $sPIDs);
  # Inputs
  my $pidfile  = shift;    # string name of PIDFILE
  my $spids    = shift;    # suspended PIDs in a string.
  my @aprPIDs  = ();
  my $NSusPIDs = 0;
  my @SusPIDs;
  my $rpids   = "";        # to be generated and returned as a string
  my @crrPIDs = ();        # array that holds the currently running rsync PIDs
  my @ASRP;                # All System Rsync PIDs
  my $NASRP;
  my $crr = 0;             # currently running rsyncs counter
  my $apr = 0;             # all parsyncfp2 rsync PIDs
       # how many rsyncs are running?  Check the PIDFILE against the rsync PIDs that are running
       # if there are other rsyncs running, their PIDs won't be in the PIDFILE.
       # so have to do a diff of the PIDFILE vs all PIDs of rsyncs running.
  my $ALL_SYS_RSYNC_PIDS = `ps ux | grep ' rsyn[c] ' | awk '{print \$2}' | sort -g | tr '\n' ' '`;
  chop $ALL_SYS_RSYNC_PIDS;
  $NASRP = @ASRP = split( /\s+/, $ALL_SYS_RSYNC_PIDS );
  if (-e $pidfile)  {open( PIDFILE, "<$pidfile" ) or FATAL("Can't open PIDFILE [$pidfile]'");}
  else {return ( 0,0 );}
  # PIDs from the PIDFILE to compare system rsyncs (could be multiple going)
  # with pfp2-launched rsyncs
  while (<PIDFILE>) { chomp; $aprPIDs[ $apr++ ] = $_; }    # all pfp2 rsync PIDs
  close PIDFILE;

  # if there are any PIDs in the $spids string, split into an array
  if ( $spids =~ /\d+/ ) { $NSusPIDs = @SusPIDs = split( /\s+/, $spids ); }
  $rpids =~ s/^\s+|\s+$//g;
  $spids =~ s/^\s+|\s+$//g;                                # strip leading/trailing spaces
      # suboptimal I know, but the arrays are so small it doesn't matter.
  for ( my $a = 0 ; $a < $NASRP ; $a++ ) {
    for ( my $b = 0 ; $b < $apr ; $b++ ) {
      # if they match, they're MY rsyncs AND they're running
      if ( $ASRP[$a] eq $aprPIDs[$b] ) {
        $crrPIDs[ $crr++ ] = $aprPIDs[$b];
      }
    }
  }

  # dump @crrPIDs into $rpids
  $rpids = join( " ", @crrPIDs );

  # now mask out the sPIDs from the rPIDs list; works but ugly!
  $spids =~ s/^\s+|\s+$//g;
  if ( $spids =~ /\d+/ ) {    # if there are any spids
    $NSusPIDs = @SusPIDs = split( /\s+/, $spids );
    for ( my $r = 0 ; $r < $NSusPIDs ; $r++ ) {
      for ( my $b = 0 ; $b < $apr ; $b++ ) {
        # if a sPID == rPID, delete the PID from the $rPIDs string
        if ( $SusPIDs[$r] eq $aprPIDs[$b] ) { $rpids =~ s/$aprPIDs[$b]//g; }
      }
    }
  }
  $rpids = trim ( $rpids); $crr = trim ( $crr );
  return ( $rpids, $crr );
}

##: trim ($string)
sub trim {    # ($)
  my $string = shift;
  $string =~ s/^\s+//;
  $string =~ s/\s+$//;
  return $string;
}

# call as  (my $avgTCPrecv, $avgTCPsend, $avgRDMArecv, $avgRDMAsend) = getavgnetbw($NETIF, $CHECKPERIOD, $PERFQUERY);
# this can be cleanly cut in half to do time-based checking. DONE.
# now not used but leave in place for a decent sub
##: getavgnetbw ($NETIF, $CHECKPERIOD, $PERFQUERY)
sub getavgnetbw ($$$) { 
  my (
    $avgrec,      $avgtrans,    $R1,      $T1,      $R2,
    $T2,          $RDMA_T1,     $RDMA_T2, $RDMA_R1, $RDMA_R2,
    $avgRDMAsend, $avgRDMArecv, $PQ
  );
  $avgRDMAsend = $avgRDMArecv = 0;
  my $NETIF       = shift;
  my $CHECKPERIOD = shift;
  $PQ = shift;
  $R1 = `cat /sys/class/net/${NETIF}/statistics/rx_bytes`;
  $T1 = `cat /sys/class/net/${NETIF}/statistics/tx_bytes`;
  if ($PQ) {
    $RDMA_T1 = `perfquery -x | grep XmitData | cut -f2 -d: | sed -e 's/\\.*//g'`; chomp $RDMA_T1;
    $RDMA_R1 = `perfquery -x | grep RcvData  | cut -f2 -d: | sed -e 's/\\.*//g'`;  chomp $RDMA_R1;
  }
  # now sleep
  sleep $CHECKPERIOD;

  $R2 = `cat /sys/class/net/${NETIF}/statistics/rx_bytes`;
  $T2 = `cat /sys/class/net/${NETIF}/statistics/tx_bytes`;
  if ($PQ) {
    $RDMA_T2 = `perfquery -x | grep XmitData | cut -f2 -d: | sed -e 's/\\.*//g'`; chomp $RDMA_T2;
    $RDMA_R2 = `perfquery -x | grep RcvData  | cut -f2 -d: | sed -e 's/\\.*//g'`; chomp $RDMA_R2;

    $avgRDMAsend = ( $RDMA_T2 - $RDMA_T1 ) / $CHECKPERIOD;
    $avgRDMArecv = ( $RDMA_R2 - $RDMA_R1 ) / $CHECKPERIOD;
  }
  $avgrec   = ( $R2 - $R1 ) / $CHECKPERIOD;
  $avgtrans = ( $T2 - $T1 ) / $CHECKPERIOD;
  return ( $avgrec, $avgtrans, $avgRDMArecv, $avgRDMAsend );
}

sub pause {
  print "Press [ENTER] to continue.\n";
  my $tmp = <STDIN>;
}

# colors supported: https://perldoc.perl.org/Term/ANSIColor.html#Supported-Colors
##: INFO($message)
sub INFO ($) {
  my $msg = shift;
  my $hh  = `hostname -s`; chomp $hh;
  print color('bold green');
  print "$hh ";
  print color('bold blue');
  print "INFO: $msg";    # note - no final '\n'.  For INFO, have to supply your own linefeed.
  print color('reset');
}

# color warning string ($) orange
##: WARN($string)
sub WARN ($message) {
  my $msg = shift;
  my $hh  = `hostname -s`; chomp $hh;
  print color('bold green');
  print "\n$hh ";
  print color('bold yellow');
  print "WARN: $msg \n";
  print color('reset');
}

##: WARN($string)
sub BLUEWARN ($message) {
  my $msg = shift;
  my $hh  = `hostname -s`; chomp $hh;
  print color('bold green');
  print "\n$hh ";
  print color('bold yellow on_blue');
  print "WARN: $msg \n";
  print color('reset');
}

# color error string ($) red
sub ERROR ($message) {
  my $msg = shift;
  my $hh  = `hostname -s`; chomp $hh;
  print color('bold green');
  print "$hh ";
  print color('bold magenta');
  print "ERROR: $msg \n";
  print color('reset');
}

##: FATAL($message)
sub FATAL ($) {
  my $msg = shift;
  my $hh  = `hostname -s`; chomp $hh;
  print color('bold green');
  print "\n$hh ";
  print color('bold red');
  print "** FATAL ERROR **: $msg \n\n";
  print color('reset');
  exit(1);
}

# call as [DEBUG(__LINE__, "string")] to print line # and debug string
##: DEBUG (__LINE__, "the message string")
sub DEBUG ($$){    #$
  print STDERR color('magenta');
  my $line = shift;
  #my $dbc  = shift; # not going to use a debug counter
  my $msg  = shift;
  my $hh   = `hostname -s`; chomp $hh;
  #print STDERR "$hh DEBUG($dbc)[$line]: $msg\n";
  print STDERR "$hh DEBUG: [$line]: $msg\n";
 print color('reset');
  pause;
}

# fixfilenames reads in a file of filenames and iterates over them, fixing their
# names and emitting useful warning if something goes odd.
# where $CUR_FP_FLE = current fpart file (fqpn)
#       $ROOTDIR    = pwd, or where all additional dirs are rooted.

##: fixfilenames ($CUR_FP_FLE, $ROOTDIR)
sub fixfilenames ($$) {
  my $FN       = shift;
  my $startdir = shift;
  if ($startdir !~ '/$') { $startdir .= '/'; }    # append a '/'
  my $fpnew = "${FN}.new";
  $fpnew =~ s%fpcache%fpcache/hold%; # put the 'new one in a place where it won't be scanned by 'ls'
  open( FP,  "< $FN" )    or FATAL("Can't open fp file [$FN].");
  open( FPN, "> $fpnew" ) or FATAL("Can't open replacement file [$fpnew].");
  my $lc = my $verified = my $failed = 0;
  while (<FP>) {
    chomp;
    if ( $_ =~ / / ) { s/ /\ /g; }    # subst all spaces with '\ '
    $_ =~ s/^${startdir}//g;  # and also delete off the startdir (Thanks Ken Bass for the missing '^')
    if ( $_ =~ /^${startdir}/ ) {
      # following WARN() call now requires & prefix for some reason.  Didn't previously..
      &WARN("WTF?? in chunk file [$FN], [$startdir] didn't get substituted off.
  Here's the post-attempted substitution line:
  [$_].");
    }
    print FPN "$_\n";
  }
  close FP;
  close FPN;
  rename $fpnew, $FN;       # and then rename the new one to the original
}

# ptgmk converts values suffixed with [PpTtGgMmKk] to bytes correctly
# uses the 1024 bytes/kb as oppo to 1000
##: ptgmk ("154.32M")
sub ptgmk ($) {
  my $instr = shift;

  # trim spaces from back and front
  $instr =~ s/^\s+|\s+$//g;
  my $abbr = chop $instr;
  my $nbr  = $instr;
  if ( $abbr !~ /[PpTtGgMmKk]/ ) {
    FATAL("ptgmk() input doesn't contain [PpTtGgMmKk], so nothing to convert.");
  }
  if ( $abbr =~ /[Kk]/ ) { $nbr *= 1024;              return $nbr; }
  if ( $abbr =~ /[Mm]/ ) { $nbr *= 1048576;           return $nbr; }
  if ( $abbr =~ /[Gg]/ ) { $nbr *= 1073741824;        return $nbr; }
  if ( $abbr =~ /[Tt]/ ) { $nbr *= 1.09951162778e+12; return $nbr; }
  if ( $abbr =~ /[Pp]/ ) { $nbr *= 1.12589990684e+15; return $nbr; }
}

##: fix_ssh_config ()
sub fix_ssh_config () {
  $HOME = $ENV{"HOME"};
  my $append_fxt = 0;
  if ( -e "$HOME/.ssh/config" ) {    # if it exists, fix it.
    open( CF, "<$HOME/.ssh/config" )
      or FATAL("Can't open $HOME/.ssh/config, even tho it exists.. WTF??");
    while (<CF>) {
      if ( $_ =~ /ForwardX11Trusted\s+yes/i ) { $append_fxt = 0; }
      if ( $_ =~ /ForwardX11Trusted\s+no/i )  { $append_fxt = 1; }
    }
    close CF;
  } else {
    $append_fxt = 1;
  }
  if ( $append_fxt && !$NOWAIT ) {
    INFO(
      "parsyncfp2 would like to append 'ForwardX11Trusted yes' & 'ForwardX11 yes' 
        to your ~/.ssh/config.
        Skipping this may result in a lot of odd ssh warnings being emitted during 
        the run if you don't have ssh set correctly for the remote system, but the 
        transfer should still work.)
   
If this mod of your ~/.ssh/config file is OK, hit [Enter].  Otherwise hit [s] to skip.\n "
    );
    my $tmp = <STDIN>;
    if ( $tmp !~ /[sS]/ ) {
      system(
"echo -n \"#Next 2 lines added by parsyncfp2\nForwardX11Trusted       yes\nForwardX11              yes\n\" >> $HOME/.ssh/config"
      );
      system("chmod 600 $HOME/.ssh/config");
      INFO("Your ~/.ssh/config file is set correctly.\n");
    } else {
      INFO("Your ~/.ssh/config was not changed.");
    }
  }
}

# --bigfiles|bf ..... processes files larger than the chunckfiles by splitting
#                      them into chunk-sized bits and sending them as separate
#                          rsync jobs to be recombined at the remote end. This
#                        prevents an enormous file from dragging out the send, 
#                                              but introduces some gotchas(*).

# * '--bigfile' gotchas: This option is useful if your filesystem is fast and 
# your network is slow since you have to 'split' the file which
# involves reading it end to end and then writing it into smaller chunks.
# Also you need the extra disk space to double the size of the large files being
# processed, both locally and remotely.  So if you have a 7TB log file, in
# order to use it with --bigfiles, you'll have to have an extra 7TB on the 
# sending side AND an extra 7TB on the receiving side. This is temporary 
# since the splitfiles (filename.0000, filename.0001, filename.0002, etc) 
# will be recombined on the receiving side and then both sets of splitfiles
# will be deleted. This option also prevents rsyncs from doing a true rsync
# since it chops up the file into chunks before sending it.  If you
# reconstitute it on  the remote end (the default), it will of course be
# identical to the original and will re-rsync rapidly, so the --bigfiles
# option is best used for initial  sends, not re-rsyncs.



# usage: usage($parsync_dir);
sub usage ($) {
  my $pfpdir = shift;
  my $helpfile = "${pfpdir}/parsyncfp-help.tmp";
  if ( !-d ${pfpdir} ) { mkdir $pfpdir; }
  open HLP, ">$helpfile" or die "Can't open the temp help file [$helpfile]\n";
  my $helptxt = <<HELP;

$PARSYNCVER
The only native rsync options that parsyncfp2 (pfp2) uses are '-a -s -l 
(archive, protect-args, copy symlinks as symlinks).  If you need more, 
then it's up to you to provide them ALL via '--ro'. pfp2 checks to see 
if the current system load is too heavy and tries to throttle the rsyncs 
during the run by monitoring and suspending / continuing them as needed. 

pfp2 uses fpart <http://goo.gl/K1WwtD> to create chunkfiles for rsync
to read, bypassing the need to wait for a complete recursive scan. ie, it 
starts the transfer immediately. For large deep trees, this can be useful.
Also see the 'filelist' options.

It appropriates rsync's bandwidth throttle mechanism, using '--maxbw'
as a passthru to rsync's 'bwlimit' option, but divides it by NP so
as to keep the total bw the same as the stated limit.  It monitors and
shows network bandwidth, but can't change the bw allocation mid-job.
It can only suspend rsyncs until the load decreases below the cutoff.
If you suspend parsyncfp2 (^Z), all rsync children will suspend as well,
regardless of current state (in SINGLEHOST mode).  In MULTIHOST, the children
will continue to run because they're running on different hosts (hence the 
admonition to copy the autogenerated killscript call before launching them.) 

pfp2 can send to anything with a standard rsync on the other end. In 'normal' 
client mode (the remote rsync starts up on demand via ssh) the target syntax 
is either host:/fully/qualified/path or host:path (implying a dir off the 
user's HOME dir, usually spec'ed as host:~/path, but unacceptable to rsync)
pfp can also talk to an rsyncd server.  The rsyncd target syntax requires
a module name to send to (host::module).  The user must be pre-registered 
in the server's /etc/rsyncd.conf and /etc/rsyncd.secrets file - see 
'man rsyncd.conf'.


Unless changed by '--interface', pfp2 assumes and monitors the routable interface.  
The transfer will use whatever interface normal routing provides, normally
set by the name of the target.  It can also be used for non-host-based
transfers (between mounted filesystems) but the network bandwidth continues
to be (usually pointlessly) shown.

[NB: Between mounted filesystems, rsync (& therefore pfp2) and will work 
poorly for reasons noted on the intertubes. That said, across 2 same-host-
mounted GPFS parallel filesystems, I see asymptotic increases in speed to 
about NP=12 vs a single rsync.  From ~30m for 1 rsync to ~6m for 12. YMMV.

pfp2 only works on dirs and files that originate from the current dir (or
specified via "--startdir").  You cannot include dirs and files from
discontinuous or higher-level dirs.  parsyncfp2 also does not use rsync's 
sophisticated/idiosyncratic treatment of trailing '/'s to direct where 
files vs dirs are sent; dirs are treated as dirs regardless of the 
trailing '/'.

** the [.pfp2] files **
The [${pfpdir}] dir contains the cache dir (fpcache), and the time-
stamped rsync log files, which are stored in a host-specific subdir. These
are not deleted after the run, but will be removed on the next run. The 
STDERR/STDOUT of the entire transfer (the text that's written to the screen) 
from each of the SEND hosts is captured in the host-specific dir named 
pfp-log-(time)_(date).  Due to the terminal text coloration strings, the 
pfp* files are best viewed by cat'ing them to the terminal and then copy-pasting 
them from the terminal to use as further data.

** Odd characters in names **
parsyncfp2 will refuse to transfer some oddly named files (tho it should copy
filenames with spaces and single quotes fine.  Filenames with embedded newlines, 
DOS EOLs, and some other odd chars will be recorded in the log files in the 
[${pfpdir}] dir.
You should be able to specify dirs and files with either/both escaped spaces 
or with quotes: [file\\ with\\ spaces] or ['file with spaces']

== OPTIONS

[i] = integer number                      [s] = "quoted string"
[f] = floating point number               ( ) = the default if any

--NP|np [i] (sqrt(#CPUs)) ..............  number of rsync processes to start
       optimal NP depends on many vars.   Try the default and incr as needed
--altcache|ac [/path/to/dir] ..... alternative cache dir for placing it on a
               another FS or for running multiple parsyncfp2s simultaneously
--startdir|sd [s] (`pwd`)  ..................  the directory it starts at(*)
--maxbw [i] (unlimited) ...........  in KB/s max bandwidth to use (--bwlimit
       passthru to rsync).  maxbw is the total BW to be used, NOT per rsync.
--maxload|ml [f] (NP*2)  ...........  max system load - if loadavg > maxload
                        an rsync proc will be suspended for the checkperiod,
                       then checked again. If the loadavg is still too high, 
                      another rsync process will be suspended, until no more
                     running rsyncs are left.  If the loadavg goes below the
                             maxload,  an rsync process will be unsuspended.
                              This is independent on each of the Send hosts.
--chunksize|cs [s] (10G) .... aggregate size of files allocated to one rsync
                      process.  Can specify in 'human' terms [100M, 50K, 1T]
                       as well as integer bytes. pfp2 will warn once when/if
                     you exceed the WARN # of chunkfiles [$WARN_FPART_FILES] and abort if
                     you exceed the FATAL # of chunkfiles [$MAX_FPART_FILES]. You CAN force 
                        it to use very high numbers of chunkfiles by setting 
                       the number negative (--chunksize -50KB), but this can
                be risky.  You typically want between 100 - 1000 chunkfiles.
                  (enough so that fpart can collect the starter set quickly,
                         but not so many that doing file ops on them will be 
              excessive - they have to be counted a few times at each cycle)
--interface|I [s]  ......  network interface to monitor (not use; see above)
                         Only SENT bytes are displayed and this is the total
                            number of bytes sent thru the interface, not the 
                                    number sent ONLY by pfp2 (another ToDo).
--rdma ................ report rdma bytes thru the IB or IB-bonded interface
                              otherwise, only TCP/IP bytes will be reported.
--ro [s]  .................  options passed to rsync as single quoted string
                              as follows: --ro='rsync options'
                                        ie -ro='-aslz -I --size-only'
--checkperiod|cp [i] (3) ........ sets the period in seconds between updates
                This is a best effort attempt.  If chunksize is set so small
         so 1000s of chunkfiles are created, file IO may lengthen this time.
--reusechunks [i] (1) ........... re-use the chunking data collected for the
                         previous run, using the same chunk size. Useful for
                              restarting a run that was mistakenly ended w/o
                                waiting for fpart to recalculate the chunks.
               The integer argument is the chunk to start at, so rather than 
               running thru all the (possibly 100s of) chunks, you can start 
                      at the one closest to where the interruption occurred.
--verbose|v [0-3] (2) ....sets chattiness. 3=debug; 2=normal; 1=less; 0=none
                        Some warning & error messages will still be printed.
                                                        '0' implies --nowait
--dispose [s] (l) .  ... what to do with the cache files. (l)eave untouched,
                                          (c)ompress to a tarball, (d)elete.
--email [s] (none)................  email address to send completion message
                           email address should not need escaping or quoting
                       but should also work with them as well (joe\\\@go.com).
--nowait  .............  for scripting, sleep for a few s instead of pausing
--slowdown [f] (0.5) .... introduces delays between ssh-mediated commands if
                               the RTT is too long.  It's increased in steps
                    automatically for large RTTs, but this option allows you
                   to explicitly slow down the speed at which ssh connection
              are made. Increment in integer seconds if you see errors like:
     'rsync error: unexplained error (code 255) at io.c(xxx) [sender=x.x.x]'
--version|V  ...............................  dumps version string and exits
--help|h  .......................................................  this help


== Options for MultiHost transfers (also see below)
The MultiHost (MH) version allows you to rsync multiple streams of data via 
multiple send hosts to the same or multiple receive hosts, including different 
filesystems on the different receive hosts.  The receive hosts can be:
1: rsyncd servers with multiple modules and as such, can define different 
auth for different users and different endpoints for the data.  The 
comprehensive description of how this works is described in rsyncd.conf(5)
2: standard servers which launch matching rsyncs via the usual mechanism. 
These can also have the same or different endpoints.  

Both types can be mixed in the same hosts string.  The MH version requires 
that the initiator and all the send hosts (which can include the initiator)
have access to a common filesystem for both data and configuration info.
 
--hosts [s] ........... specify the SEND and REC hosts, optionally supplying
                    REC hosts with individual alternate paths to store data.
                    If you specify /different/ REC paths, the SEND data will 
                   be split over those host:/path combinations, so they will
                   have to be manually combined afterwards. This is to allow
                       different remote filesystems to accept high bandwidth 
                         transmission without impacting other FS operations.

               The SEND=REC couplets follow ssh rules so that if the user at
                    one of the hosts is different than the one being used to 
           initiate the process, you'll have to specify the user.  Similarly
         for the REC host if the user is different than the initiating USER.  
               ie: in the following option string:
                    --hosts="cooper=ben,tux\@chinstrap=hjm\@ben,nash=ben"
           'hjm' is the initiating user and is the mediating user on cooper, 
              ben, and nash, while 'tux' is the mediating user on chinstrap.  
              Because 'tux\@chinstrap' is mediating the command, ssh assumes
         the same user on ben, so 'hjm\@ben' has to be explicitly specified.
                         
                         The required last element in a multihost command is
                          'POD::/path' which is the default path for any REC
                    hosts that haven't been defined in the '--hosts' option.
                    You can prefix POD with the USER to specify the user for 
               all REC hosts, and append the PATH for all REC hosts as well.
                                       ie    mike\@POD::/where/to/drop/files
                              (More info below and see Good Example 4 below)
--checkhost  ............. pre-check to make sure that the SEND & REC hosts
                    specified with '--hosts' do not have any rsyncs running.
                If they do, the number of them is reported. Those rsyncs may 
                 be valid and independent of pfp but it may be evidence of a 
                       failed pfp which may interfere with a new pfp launch.
                       This option also pushes the required utilities to the 
                            SEND hosts to make sure that they have the exe's
                                   necessary to run with full functionality.
--commondir [s] ............ the shared, common dir in which all chunk files 
                        and rsync logs will stored.  Similar to '--altcache'
                                     but MUST be readable by all SEND hosts.
--rpath [s] ............. the remote PATH prefix on the SEND hosts to check for 
                         the bits needed to run this. Prefixed to the remote 
                             ssh cmd as 'export PATH=<rpath string>:\$PATH;' 
                         The rpath string can contain as many paths as you'd 
                          like, separated by ':',tho vars have to be escaped
                                               --rpath="~/bin:\\\$HOME/pfp/bin"
                   (default is ~/bin:${parsync_dir}, and ':\$PATH is also appended so
                            --rpath="~/bin:\\\$HOME/pfp2/bin"
                                        is transmitted as:
                            --rpath="~/bin:\\\$HOME/pfp/bin:\\\$PATH"
                        (This option may be removed as the --hostcheck option
                                has obviated most of the utility of --rpaths)
                        
The '--hosts' string format is a comma-delimited set of 'Send=Receive' hosts.
example: "s1=r1:/path1,s2=r2:/path2,s3=r3:/path3,s4=r4,s5=r5"
where each 's#' and 'r#' imply a full "user\@host" string.  s# and r# obey
the std Linux rules that they are either long or short hostnames that are 
resolvable by your DNS or by an entry in the '/etc/hosts' file or a numeric 
address (113.42.23.56).
Also, each 'r#' can have a storage path appended (r2:/path2).  If the REC 
path is not given, the path from the final 'POD::/path' target is appended.
ie pfp [option option option..] POD::/common/default/receive/target.

For rsyncd targets, you can specify the REC hosts as:
r1::module_name 
r2::module_name etc", and you can mix rsyncd targets with regular 
rsync targets so a valid hostlist string could be:
"s1=r1:/path1,s2=r2::mod2,s3=r3:/path3,s4=r4::mod4,s5=r5"
However, unless the rsyncd server is open (without authorization) you must 
export your RSYNC_PASSWORD in the SEND host's ~/.bashrc for 
this to work, or use '--ro='--password-file=FILE' to point to a 
permission-protected file containing the appropriate credentials.
Otherwise, the responding rsyncd will query for your rsync user password (not
your login password).  This is defined in the rsyncd host's 
/etc/rsyncd.secrets file).

The master parsyncfp2 command will exit once the fpart chunking process is finished 
and leave the rsyncs running independently on the SEND shosts.  They will
continue to send output back to the originating terminal (prefixed or suffixed)  
with the SEND hostname so you can decipher which SEND host is saying what.

However, unlike the single-host version, killing or suspending the originating 
program will have no effect on the SEND hosts; the remote rsyncs will have to be 
killed manually.  This is made easier with a 'kill script' that is generated at 
every invocation of the MH version, called '\\${parsync_dir}/pfp2stop' and will kill off ALL 
YOUR rsync and parsyncfp2 instances running (including ones that were not part of the 
the originating parsyncfp2, so be careful).

This SEND host independence should be addressed shortly via socket-based controls. 

See 'Good Example 4', below.
                           
== Options for using filelists

(thanks to Bill Abbott for the inspiration/guidance).
The following 3 options provide a means of explicitly naming the files
you  wish to transfer by means of filelists, generated by 'find' or other
means. Typically, you will provide a list of files, for example generated
by a DB lookup (GPFS or Robinhood) with full path names.  If you use
this list directly with rsync, it will remove the leading '/' but then
place the file with that otherwise full path inside the target dir. So
'/home/hjm/DL/hello.c' would be placed in '/target/home/hjm/DL/hello.c'.  
If this result is OK, then simply use the '--filesfrom' option to specify 
the file of files.

If the list of files are NOT fully qualified ('fully qualified' means 
they have a leading '/') then you should make sure that the command is 
run from the correct dir so that the rsyncs can find the designated 
dirs & files.

If you want the file 'hello.c' to end up as '/target/DL/hello.c' (ie
remove the original '/home/hjm'), you would use the --trimpath option
as follows: '--trimpath=/home/hjm'.  This will remove that path
before transferring it and assure that the file ends up in the right
place.  This should work even if the command is executed away from the
directory where the files are rooted. If you have already modified the
file list to remove the leading dir path, then of course you don't need
to use '--trimpath' option.

--filesfrom|ff [s] .. take explicit input file list from given file, 
                        1 path name per line.
--trimpath|tp [s] ... path to trim from front of full path name if 
                        '--filesfrom' file contains full path names and 
                        you want to trim them. Don't use a trailing '/'.  
                        It will be removed if you do.
--trustme|tm ........ with '--filesfrom' above allows the use of file lists
                        of the form:
                        size in bytes<tab>/fully/qualified/filename/path  
                        825692            /home/hjm/nacs/hpc/movedata.txt
                        87456826          /home/hjm/Downloads/xme.tar.gz
                        etc
                        
                        This allows lists to be produced elsewhere to be
                        fed directly to pfp2 without a file stat() or
                        complete recursion of the dir tree.  So if
                        you're using an SQL DB to track your filesystem
                        usage like Robinhood or a filesystem like GPFS
                        that can emit such data, it can save some
                        startup time on gigantic file trees.

(*) you can use globs/regexes with --startdir, but only if you're at that
point in the dir tree. ie: if you're not in the dir where the globs can be
found, then the glob will fail; the glob is immediately tested, so it might
pick up matching globs in the dir where the command was executed. 
However, explicit dirs can be set from anywhere if given an existing startdir.
so if the 'startdir' contains the following contents:

bridgit-test/  dir123456-sizes.txt  dir3/  dir5/  hjm/  parts/
dir1/          dir2/                dir4/  dir6/  log/  scatter/

and you wanted to transfer the dirs that begin with 'dir', but not the file 
'dir123456-sizes.txt', you could specify them with 'dir[123456]'.
If you wanted to include that file, you could use 'dir\*'. The '\' defers the 
glob until it's evaluated in the program.

(+) the '--ro' string can pass any rsync option to all the rsyncs
that will be started.  This allows options like '-z' (compression) or
'--exclude-from'  to filter out unwanted files. I've refused to allow 
any of rsync's 'delete' options.  See below.

== Hints & Workarounds

IMPORTANT: rsync '--delete' options will not work with '--ro' bc the 
multiple rsyncs that parsyncfp2 launches are independent and therefore 
don't know about each other (and so cannot exchange info about what should
be deleted or not.  Use a final, separate 'rsync --delete' to clean up the 
transfer if that's your need.

Also, rsync options related to additional output has been disallowed to avoid 
confusing pfp's IO handling.  -v/-verbose, --version, -h/--help are 
caught, and pfp2 will die with an error.  Most of the info desired from these
are captured in the rsync-logfile files in the [${pfpdir}] dir.

A recent addition to pfp2 is a check for number of chunkfiles, resulting in 
a warning if the chunkfile number gets too large.  This is usually due to 
pointing pfp2 at a huge filesystem, with millions of files, with a 
chunksize that's too small.  The easiest solution is to increase the chunksize
('--chunksize=100G) which will result in a smaller number of chunk files to 
process  Note the text above for '--chunksize'.

Unless you want to view them, it's usually a good idea to send all STDERR 
to /dev/null (append '2> /dev/null' to the command) because there are often 
a variety of utilities that get upset by one thing or another.  Generally
silencing the STDERR doesn't hurt anything.

== Required Utilities

=== ethtool - query or control network driver and hardware settings. 
    Install via repository.
=== ip - show / manipulate routing, network devices, interfaces and tunnels.
    Install via repository.
=== fpart - Sort and pack files into partitions. 
    Install from: https://github.com/martymac/fpart; now in the ubuntu Repos.
=== scut - more intelligent cut. Included in the pfp2 github or install from: 
    https://github.com/hjmangalam/scut
=== stats - calculate descriptive stats from STDIN. Included in the pfp2 
    github or install from: https://github.com/hjmangalam/scut

== Recommended Utilities

=== iwconfig - configure a wireless network interface. Needed only for WiFi.
    Install via repository.
=== perfquery - query InfiniBand port counters.  Needed only for InfiniBand.
    Install via repository.
    

== Examples

=== Good example 1

% parsyncfp2  --maxload=5.5 --NP=4 \\
--chunksize=\$((1024 * 1024 * 4)) \\
--startdir='/home/hjm' dir[123]  \\
hjm\@remotehost:~/backups 2> /dev/null

where
  = "--maxload=5.5" will start suspending rsync instances when the 1m system
      load gets to 5.5 and then unsuspending them when it goes below it.
  = "--NP=4" starts 4 instances of rsync
  = "--chunksize=\$((1024 * 1024 * 4))" sets the chunksize, by multiplication
        or by explicit size: 4194304
  = "--startdir='/home/hjm'" sets the working dir of this operation to
      '/home/hjm' and dir1 dir2 dir3 are subdirs from '/home/hjm'
  = the target "hjm\@remotehost:~/backups" is the same target rsync would use
  = '2> /dev/null' silences all STDERR output from any offended utility.

  It uses 4 instances to rsync dir1 dir2 dir3 to hjm\@remotehost:~/backups


=== Good example 2

% parsyncfp2   --checkperiod 6  --NP 3 \\
--interface eth0  --chunksize=87682352 \\
--ro="--exclude='[abc]*'"  nacs/fabio   \\
hjm\@bridgit:~/backups

The above command shows several options used correctly:

--chunksize=87682352 - shows that the chunksize option can be used with explicit
integers as well as the human specifiers (TGMK).

--ro="--exclude='[abc]*'" - shows the correct form for excluding files
based on regexes (note the quoting to protect the regex as it gets passed thru)

nacs/fabio - shows that you can specify subdirs as well as top-level dirs (as
long as the shell is positioned in the dir above, or has been specified via
'--startdir'

=== Good example 3

parsyncfp2 -v 1 --nowait --ac pfpcache1 --NP 4 --cp=5 --cs=50M --ro '-az'  \\
linux-4.8.4 bridgit:~/test

The above command shows:
- short version of several options (-v for --verbose, --cp for checkperiod, etc)
- shows use of --altcache (--ac pfpcache1), writing to relative dir pfpcache1
- again shows use of --ro (--ro '-az') indicating 'archive' & compression'.
- includes '--nowait' to allow unattended scripting of pfp2

                            == Good example 4 ==
parsyncfp2 --NP=8 --chunksize=500M --filesfrom=/home/hjm/dl550 \\
hjm\@bridgit:/home/hjm/testpfp2
                            
The above command shows:
- if you use the '--filesfrom' option, you cannot use explicit source dirs
  (all the files come from the file of files (which require full path names)
- that the '--chunksize' format can use human abbreviations (m or M for Mega).


=== Good example 4 (Multihost)

parsyncfp2 --verbose=2 --ro='-aslz' \\
--hosts="bigben=bridgit.ure.edu:/d1/in, \\
          pooki=bridgit.ure.edu:/d2/in, \\
        stunted=bridgit.ure.edu:/d3/in" \\
--hostcheck --NP 4 --chunk 15G --check 5 --dispo=l \\
--interface=wlp3s0 --commondir=/home/hjm/pfp --startdir /home/hjm/pfp \\
dir1 dir2 dir3 dir4  POD::/

The above multihost command shows 3 SEND hosts (bigben, pooki, stunted) all sending 
data to the REC host bridgit.ure.edu altho the data is being split among 3
filesystems.  

It shows the preferred way of defining the ro with '--ro=-aslz' and the 
'--dispo=l' option requests that the cachefiles be left alone.  In Multihost mode
the chunk files MUST be left, since all the independent SEND hosts need to reference
them until they're finished.
The 'POD::/' terminal element is the (required) default path for any undefined 
REC hosts.  Since all of the REC hosts paths are defined, they aren't affected.

=== Good example 5 (Multihost)

parsyncfp2  --hostcheck --NP=16 --chunk=50G --check 5  \\
--hosts="bigben=tux\@moon1, \\
          pooki=tux\@moon2, \\
        stunted=tux\@moon3  \\
         cooper=gibson\@moon4::circadian" \\
--maxload=20 --ro='-slaz' \\
--commondir=/home/pfp --startdir /home/pfp/incoming \\
dir1 dir2 dir3 dir4  POD::/d1/incoming

The above multihost command shows 4 SEND hosts (bigben, pooki, stunted, cooper) each sending 
16 stream of data to the 4 clustered REC hosts (moon1 - moon4) with the REC data path being provided by the POD default path '/d1/incoming', except for moon4 which is using a rsyncd module as the REC endpoint, with the rsyncd ID 'gibson' as the authorized user (this requires the rsyncd password to be part of the ENV on cooper:
ie the ~/.bashrc must contain 'RSYNC_PASSWORD=whateveritis').  

Thus there are 64 (4x16) rsync streams pushing data to the REC cluster.  This assumes the filesystem on the moon cluster can write that fast and that the intermediate network can provide the bandwidth. It also assumes that the rsync compression requested by the '--ro (--ro='-slaz')
arguments can stay below the individual 1m loadavg of 20 requested  by '--maxload=20'.  If it doesn't, the SEND hosts will start to suspend rsyncs until the loadavg goes below 20.
The '--commondir' and '--startdir' paths define the shared storage and where in it the data to be sent is stored.  '--commondir' and '--startdir' do not have to be identical, but they do have to be R/W available 
to all the SEND hosts.
The '--hostcheck' command makes sure that required utilities are available, that the parsyncfp2 program is identical, and also checks the latency between the SEND and REC hosts.

=== ERROR example 1

% pwd
/home/hjm  # executing parsyncfp2 from here

% parsyncfp2 --NP4  /usr/local  /media/backupdisk

why this is an error:
  = '--NP4' is not an option (parsyncfp2 will say "Unknown option: np4"
    It should be '--NP=4' or '--NP 4'
  = if you were trying to rsync '/usr/local' to '/media/backupdisk', it will
    fail since there is no /home/hjm/usr/local dir to use as a source.
    This will be shown in the log files in ${pfpdir}/rsync-logfile-<datestamp>_#
    as a spew of "No such file or directory (2)" errors

The correct version of the above command is:

% parsyncfp2 --NP=4  --startdir=/usr  local  /media/backupdisk

HELP

  print HLP $helptxt;
  close HLP;
  system("less -S $helpfile");
  unlink $helpfile;
  die "Did that help?
  Send suggestions for improvement to <hjmangalam\@gmail.com>\n";
}