#!/usr/bin/env bash
PROGRAM="backup_tool_script"
PROGRAM_VERSION=0.5.0
PROGRAM_BUILD=2019081901
AUTHOR="(C) 2017-2019 by Orsiris de Jong"
CONTACT="http://www.netpower.fr - ozy@netpower.fr"
IS_STABLE=true

## backup_tool_script - A script to check @name@ backup sanity

## backup_tool_script can verify a given number of backups for each client. It can run verifiy operations in parallel.
## Verify operations are timed in order to stop them after a given amount of time, leaving the system performance ready for backup operations.
## The script can also list clients that have outdated backups. It uses two different methods to list clients in order to detect rogue clients.
## It can also ensure that the @name@ server service is running properly, relaunch it if needed, on a scheduled basis.
## The script can send a warning / error when problems are found, even while operating.
## The script can send a warning / error when disk quotas exceed, even while operating.
## backup_tool_script can also launch vss_strip for each file found in a given directory.
## The script can also send mails directly to the client, if an email address is given in client config file as 'label = email_address : some@example.com'

## When exiting, backup_tool_script ensures that no forked @name@ processes remain, without touching other @name@ processes that didn't belong to backup_tool_script.

## Set an unique identifier for the script which will be used for logs and alert mails
INSTANCE_ID="base"

## Backup verifications timers
## After how much time (in seconds) for a single verification a warning should be logged (defaults to 3 hours)
SOFT_MAX_EXEC_TIME_PER_VERIFY=10800
## After how much time (in seconds) for a single verification the process should be stopped (defaults to 5 hours)
HARD_MAX_EXEC_TIME_PER_VERIFY=64800
## After how much seconds of execution of all steps a warning should be logged (defaults to 10 hours)
SOFT_MAX_EXEC_TIME=64800
## After how much seconds of execution of all steps a verification process should be stopped (defaults to 12 hours)
HARD_MAX_EXEC_TIME=82800

# Verify operations checks
## When a client isn't idle, we can postpone the it's backup verification process. How many times should we retry the verification command. Set this to 0 to disable operation postponing
POSTPONE_RETRY=2
## When postponed, how much time (in seconds) before next try (defauls to 1 hour)
POSTPONE_TIME=3600

## Backup executable (can be set to /usr/sbin/@name@, /usr/local/sbin/@name@, or autodetect via $(type -p @name@))
BACKUP_EXECUTABLE=@sbindir@/@name@ # PROD = @sbindir@/@name@

## @name@ service type (can be "initv" or "systemd")
SERVICE_TYPE=initv # PROD = initv

## How many simultaneous verify operations should be launched (please check I/O and CPU usage before increasing this)
PARELLEL_VERIFY_CONCURRENCY=2

# ------------ Mail alert settings -------------

## General alert mail subject
MAIL_ALERT_MSG="Execution of $PROGRAM instance $INSTANCE_ID on $(date) has warnings/errors."

## Optional change of mail body encoding (using iconv)
## By default, all mails are sent in UTF-8 format without headers
## You may specify an optional encoding here (like "ISO-8859-1" or whatever iconv can handle) for maximal compatibility
MAIL_BODY_CHARSET="ISO-8859-1"

# ------------ Client specific alert email template --------------

## Email subject
CLIENT_ALERT_SUBJECT="@name@ backup - Warning about your backup"

## Valid message body placeholders are
## [NUMBERDAYS] is the number of days we check.
## [QUOTAEXCEED] is the size of the actual backup versus the quota.
## [CLIENT] is the client name
## [INSTANCE] is the current instance name

## Message sent directly to client email address when no recent backups are found.
CLIENT_ALERT_BODY_OUTDATED="Hello,

No valid backup sets found in the last [NUMBERDAYS] day(s) for client [CLIENT].
Please leave your computer online enough time for a backup to occur.
Contact your system administrator for further information.

@name@ backup server [INSTANCE].
"

# Message sent directly to client email address when quota exceeded.
CLIENT_ALERT_BODY_QUOTA="Hello,

Your backup disk quota has been exceeded ([QUOTAEXCEED]) for client [CLIENT].
Contact your system administrator for further information.

@name@ backup server [INSTANCE].
"

# ------------ Do not modify under this line unless you have great cow powers --------------

if ! type "$BASH" > /dev/null; then
	echo "Please run this script only with bash shell. Tested on bash >= 3.2"
	exit 127
fi

export LC_ALL=C

_LOGGER_SILENT=false
_LOGGER_VERBOSE=false
_LOGGER_ERR_ONLY=false
_LOGGER_PREFIX="date"
if [ "$KEEP_LOGGING" == "" ]; then
	KEEP_LOGGING=1801
fi

# Initial error status, logging 'WARN', 'ERROR' or 'CRITICAL' will enable alerts flags
ERROR_ALERT=false
WARN_ALERT=false

LOCAL_USER=$(whoami)
LOCAL_HOST=$(hostname)

SCRIPT_PID=$$

# Get a random number on Windows BusyBox alike, also works on most Unixes that have dd, if dd is not found, then return $RANDOM
# Get a random number of digits length on Windows BusyBox alike, also works on most Unixes that have dd
function PoorMansRandomGenerator {
	local digits="${1}" # The number of digits to generate
	local number

	# Some read bytes can't be used, se we read twice the number of required bytes
	dd if=/dev/urandom bs=$digits count=2 2> /dev/null | while read -r -n1 char; do
		number=$number$(printf "%d" "'$char")
		if [ ${#number} -ge $digits ]; then
			echo ${number:0:$digits}
			break;
		fi
	done
}

# Initial TSTMAP value before function declaration
TSTAMP=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 5)

ALERT_LOG_FILE="$RUN_DIR/$PROGRAM.$SCRIPT_PID.$TSTAMP.last.log"

## Default log file until config file is loaded
if [ -w /var/log ]; then
	LOG_FILE="/var/log/$PROGRAM.log"
elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
	LOG_FILE="$HOME/$PROGRAM.log"
elif [ -w . ]; then
	LOG_FILE="./$PROGRAM.log"
else
	LOG_FILE="/tmp/$PROGRAM.log"
fi

## Default directory where to store temporary run files
if [ -w /tmp ]; then
	RUN_DIR=/tmp
elif [ -w /var/tmp ]; then
	RUN_DIR=/var/tmp
else
	RUN_DIR=.
fi

#### DEBUG SUBSET ####
## allow function call checks                   #__WITH_PARANOIA_DEBUG
if [ "$_PARANOIA_DEBUG" == true ];then          #__WITH_PARANOIA_DEBUG
	_DEBUG=false                           	#__WITH_PARANOIA_DEBUG
fi                                              #__WITH_PARANOIA_DEBUG

## allow debugging from command line with _DEBUG=true
if [ ! "$_DEBUG" == true ]; then
	_DEBUG=false
	_LOGGER_VERBOSE=false
else
	trap 'TrapError ${LINENO} $?' ERR
	_LOGGER_VERBOSE=true
fi

if [ "$SLEEP_TIME" == "" ]; then # Leave the possibity to set SLEEP_TIME as environment variable when runinng with bash -x in order to avoid spamming console
	SLEEP_TIME=.1
fi
#### DEBUG SUBSET END ####

#### Logger SUBSET ####
#### RemoteLogger SUBSET ####

# Array to string converter, see http://stackoverflow.com/questions/1527049/bash-join-elements-of-an-array
# usage: joinString separaratorChar Array
function joinString {
	local IFS="$1"; shift; echo "$*";
}

# Sub function of Logger
function _Logger {
	local logValue="${1}"           # Log to file
	local stdValue="${2}"           # Log to screeen
	local toStdErr="${3:-false}"    # Log to stderr instead of stdout

	if [ "$logValue" != "" ]; then
		echo -e "$logValue" >> "$LOG_FILE"
		# Build current log file for alerts if we have a sufficient environment
		if [ "$RUN_DIR/$PROGRAM" != "/" ]; then
			echo -e "$logValue" >> "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP"
		fi
	fi

	if [ "$stdValue" != "" ] && [ "$_LOGGER_SILENT" != true ]; then
		if [ $toStdErr == true ]; then
			# Force stderr color in subshell
			(>&2 echo -e "$stdValue")

		else
			echo -e "$stdValue"
		fi
	fi
}

# General log function with log levels:

# Environment variables
# _LOGGER_SILENT: Disables any output to stdout & stderr
# _LOGGER_ERR_ONLY: Disables any output to stdout except for ALWAYS loglevel
# _LOGGER_VERBOSE: Allows VERBOSE loglevel messages to be sent to stdout

# Loglevels
# Except for VERBOSE, all loglevels are ALWAYS sent to log file

# CRITICAL, ERROR, WARN sent to stderr, color depending on level, level also logged
# NOTICE sent to stdout
# VERBOSE sent to stdout if _LOGGER_VERBOSE=true
# ALWAYS is sent to stdout unless _LOGGER_SILENT=true
# DEBUG & PARANOIA_DEBUG are only sent to stdout if _DEBUG=true
function Logger {
	local value="${1}"              # Sentence to log (in double quotes)
	local level="${2}"              # Log level
	local retval="${3:-undef}"      # optional return value of command

	if [ "$_LOGGER_PREFIX" == "time" ]; then
		prefix="TIME: $SECONDS - "
	elif [ "$_LOGGER_PREFIX" == "date" ]; then
		prefix="$(date '+%Y-%m-%d %H:%M:%S') - "
	else
		prefix=""
	fi

	if [ "$level" == "CRITICAL" ]; then
		_Logger "$prefix($level):$value" "$prefix\e[1;33;41m$value\e[0m" true
		ERROR_ALERT=true
		# ERROR_ALERT / WARN_ALERT is not set in main when Logger is called from a subprocess. Need to keep this flag.
		echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
		return
	elif [ "$level" == "ERROR" ]; then
		_Logger "$prefix($level):$value" "$prefix\e[91m$value\e[0m" true
		ERROR_ALERT=true
		echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.error.$SCRIPT_PID.$TSTAMP"
		return
	elif [ "$level" == "WARN" ]; then
		_Logger "$prefix($level):$value" "$prefix\e[33m$value\e[0m" true
		WARN_ALERT=true
		echo -e "[$retval] in [$(joinString , ${FUNCNAME[@]})] SP=$SCRIPT_PID P=$$\n$prefix($level):$value" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.warn.$SCRIPT_PID.$TSTAMP"
		return
	elif [ "$level" == "NOTICE" ]; then
		if [ "$_LOGGER_ERR_ONLY" != true ]; then
			_Logger "$prefix$value" "$prefix$value"
		fi
		return
	elif [ "$level" == "VERBOSE" ]; then
		if [ $_LOGGER_VERBOSE == true ]; then
			_Logger "$prefix($level):$value" "$prefix$value"
		fi
		return
	elif [ "$level" == "ALWAYS" ]; then
		_Logger "$prefix$value" "$prefix$value"
		return
	elif [ "$level" == "DEBUG" ]; then
		if [ "$_DEBUG" == true ]; then
			_Logger "$prefix$value" "$prefix$value"
			return
		fi
	elif [ "$level" == "PARANOIA_DEBUG" ]; then                             #__WITH_PARANOIA_DEBUG
		if [ "$_PARANOIA_DEBUG" == true ]; then                         #__WITH_PARANOIA_DEBUG
			_Logger "$prefix$value" "$prefix\e[35m$value\e[0m"      #__WITH_PARANOIA_DEBUG
			return                                                  #__WITH_PARANOIA_DEBUG
		fi                                                              #__WITH_PARANOIA_DEBUG	
	else
		_Logger "\e[41mLogger function called without proper loglevel [$level].\e[0m" "\e[41mLogger function called without proper loglevel [$level].\e[0m" true
		_Logger "Value was: $prefix$value" "Value was: $prefix$value" true
	fi
}
#### Logger SUBSET END ####

# Portable child (and grandchild) kill function tester under Linux, BSD and MacOS X
function KillChilds {
	local pid="${1}" # Parent pid to kill childs
	local self="${2:-false}" # Should parent be killed too ?

	# Paranoid checks, we can safely assume that $pid should not be 0 nor 1
	if [ $(IsInteger "$pid") -eq 0 ] || [ "$pid" == "" ] || [ "$pid" == "0" ] || [ "$pid" == "1" ]; then
		Logger "Bogus pid given [$pid]." "CRITICAL"
		return 1
	fi

	if kill -0 "$pid" > /dev/null 2>&1; then
		if children="$(pgrep -P "$pid")"; then
			if [[ "$pid" == *"$children"* ]]; then
				Logger "Bogus pgrep implementation." "CRITICAL"
				children="${children/$pid/}"
			fi
			for child in $children; do
				Logger "Launching KillChilds \"$child\" true" "DEBUG"   #__WITH_PARANOIA_DEBUG
				KillChilds "$child" true
			done
		fi
	fi

	# Try to kill nicely, if not, wait 15 seconds to let Trap actions happen before killing
	if [ "$self" == true ]; then
		# We need to check for pid again because it may have disappeared after recursive function call
		if kill -0 "$pid" > /dev/null 2>&1; then
			kill -s TERM "$pid"
			Logger "Sent SIGTERM to process [$pid]." "DEBUG"
			if [ $? -ne 0 ]; then
				sleep 15
				Logger "Sending SIGTERM to process [$pid] failed." "DEBUG"
				kill -9 "$pid"
				if [ $? -ne 0 ]; then
					Logger "Sending SIGKILL to process [$pid] failed." "DEBUG"
					return 1
				fi      # Simplify the return 0 logic here
			else
				return 0
			fi
		else
			return 0
		fi
	else
		return 0
	fi
}

function KillAllChilds {
	local pids="${1}" # List of parent pids to kill separated by semi-colon
	local self="${2:-false}" # Should parent be killed too ?

	__CheckArguments 1 $# "$@"      #__WITH_PARANOIA_DEBUG

	local errorcount=0

	IFS=';' read -a pidsArray <<< "$pids"
	for pid in "${pidsArray[@]}"; do
		KillChilds $pid $self
		if [ $? -ne 0 ]; then
			errorcount=$((errorcount+1))
			fi
	done
	return $errorcount
}

# osync/obackup/pmocr script specific mail alert function, use SendEmail function for generic mail sending
function SendAlert {
	local runAlert="${1:-false}" # Specifies if current message is sent while running or at the end of a run
	local attachment="${2:-true}" # Should we send the log file as attachment

	__CheckArguments 0-2 $# "$@"    #__WITH_PARANOIA_DEBUG

	local attachmentFile
	local subject
	local body

	if [ "$DESTINATION_MAILS" == "" ]; then
		return 0
	fi

	if [ "$_DEBUG" == true ]; then
		Logger "Debug mode, no warning mail will be sent." "NOTICE"
		return 0
	fi

	if [ $attachment == true ]; then
		attachmentFile="$LOG_FILE"
		if type "$COMPRESSION_PROGRAM" > /dev/null 2>&1; then
			eval "cat \"$LOG_FILE\" \"$COMPRESSION_PROGRAM\" > \"$ALERT_LOG_FILE\""
			if [ $? -eq 0 ]; then
				attachmentFile="$ALERT_LOG_FILE"
			fi
		fi
	fi

	body="$MAIL_ALERT_MSG"$'\n\n'"Last 1000 lines of current log"$'\n\n'"$(tail -n 1000 "$RUN_DIR/$PROGRAM._Logger.$SCRIPT_PID.$TSTAMP")"

	if [ $ERROR_ALERT == true ]; then
		subject="Error alert for $INSTANCE_ID"
	elif [ $WARN_ALERT == true ]; then
		subject="Warning alert for $INSTANCE_ID"
	else
		subject="Alert for $INSTANCE_ID"
	fi

	if [ $runAlert == true ]; then
		subject="Currently runing - $subject"
	else
		subject="Finished run - $subject"
	fi

	SendEmail "$subject" "$body" "$DESTINATION_MAILS" "$attachmentFile" "$SENDER_MAIL" "$SMTP_SERVER" "$SMTP_PORT" "$SMTP_ENCRYPTION" "$SMTP_USER" "$SMTP_PASSWORD"

	# Delete tmp log file
	if [ "$attachment" == true ]; then
		if [ -f "$ALERT_LOG_FILE" ]; then
			rm -f "$ALERT_LOG_FILE"
		fi
	fi
}

# Generic email sending function.
# Usage (linux / BSD), attachment is optional, can be "/path/to/my.file" or ""
# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file"
# Usage (Windows, make sure you have mailsend.exe in executable path, see http://github.com/muquit/mailsend)
# attachment is optional but must be in windows format like "c:\\some\path\\my.file", or ""
# smtp_server.domain.tld is mandatory, as is smtpPort (should be 25, 465 or 587)
# encryption can be set to tls, ssl or none
# smtpUser and smtpPassword are optional
# SendEmail "subject" "Body text" "receiver@example.com receiver2@otherdomain.com" "/path/to/attachment.file" "senderMail@example.com" "smtpServer.domain.tld" "smtpPort" "encryption" "smtpUser" "smtpPassword"

# If text is received as attachment ATT00001.bin or noname, consider adding the following to /etc/mail.rc
#set ttycharset=iso-8859-1
#set sendcharsets=iso-8859-1
#set encoding=8bit

function SendEmail {
	local subject="${1}"
	local message="${2}"
	local destinationMails="${3}"
	local attachment="${4}"
	local senderMail="${5}"
	local smtpServer="${6}"
	local smtpPort="${7}"
	local encryption="${8}"
	local smtpUser="${9}"
	local smtpPassword="${10}"

	__CheckArguments 3-10 $# "$@"   #__WITH_PARANOIA_DEBUG

	local mail_no_attachment=
	local attachment_command=

	local encryption_string=
	local auth_string=

	local i

	if [ "${destinationMails}" != "" ]; then
		for i in "${destinationMails[@]}"; do
			if [ $(CheckRFC822 "$i") -ne 1 ]; then
				Logger "Given email [$i] does not seem to be valid." "WARN"
			fi
		done
	else
		Logger "No valid email addresses given." "WARN"
		return 1
	fi

	# Prior to sending an email, convert its body if needed
	if [ "$MAIL_BODY_CHARSET" != "" ]; then
		if type iconv > /dev/null 2>&1; then
			echo "$message" | iconv -f UTF-8 -t $MAIL_BODY_CHARSET -o "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP"
			message="$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.iconv.$SCRIPT_PID.$TSTAMP")"
		else
			Logger "iconv utility not installed. Will not convert email charset." "NOTICE"
		fi
	fi

	if [ ! -f "$attachment" ]; then
		attachment_command="-a $attachment"
		mail_no_attachment=1
	else
		mail_no_attachment=0
	fi
	if [ "$LOCAL_OS" == "Busybox" ] || [ "$LOCAL_OS" == "Android" ]; then
		if [ "$smtpPort" == "" ]; then
			Logger "Missing smtp port, assuming 25." "WARN"
			smtpPort=25
		fi
		if type sendmail > /dev/null 2>&1; then
			if [ "$encryption" == "tls" ]; then
				echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -H "exec openssl s_client -quiet -tls1_2 -starttls smtp -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails"
			elif [ "$encryption" == "ssl" ]; then
				echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -H "exec openssl s_client -quiet -connect $smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails"
			elif [ "$encryption" == "none" ]; then
				echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -S "$smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails"
			else
				echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) -f "$senderMail" -S "$smtpServer:$smtpPort" -au"$smtpUser" -ap"$smtpPassword" "$destinationMails"
				Logger "Bogus email encryption used [$encryption]." "WARN"
			fi

			if [ $? -ne 0 ]; then
				Logger "Cannot send alert mail via $(type -p sendmail) !!!" "WARN"
				# Do not bother try other mail systems with busybox
				return 1
			else
				return 0
			fi
		else
			Logger "Sendmail not present. Will not send any mail" "WARN"
			return 1
		fi
	fi

	if type mutt > /dev/null 2>&1 ; then
		# We need to replace spaces with comma in order for mutt to be able to process multiple destinations
		echo "$message" | $(type -p mutt) -x -s "$subject" "${destinationMails// /,}" $attachment_command
		if [ $? -ne 0 ]; then
			Logger "Cannot send mail via $(type -p mutt) !!!" "WARN"
		else
			Logger "Sent mail using mutt." "NOTICE"
			return 0
		fi
	fi

	if type mail > /dev/null 2>&1 ; then
		# We need to detect which version of mail is installed
		if ! $(type -p mail) -V > /dev/null 2>&1; then
			# This may be MacOS mail program
			attachment_command=""
		elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V | grep "GNU" > /dev/null; then
			attachment_command="-A $attachment"
		elif [ "$mail_no_attachment" -eq 0 ] && $(type -p mail) -V > /dev/null; then
			attachment_command="-a$attachment"
		else
			attachment_command=""
		fi

		echo "$message" | $(type -p mail) $attachment_command -s "$subject" "$destinationMails"
		if [ $? -ne 0 ]; then
			Logger "Cannot send mail via $(type -p mail) with attachments !!!" "WARN"
			echo "$message" | $(type -p mail) -s "$subject" "$destinationMails"
			if [ $? -ne 0 ]; then
				Logger "Cannot send mail via $(type -p mail) without attachments !!!" "WARN"
			else
				Logger "Sent mail using mail command without attachment." "NOTICE"
				return 0
			fi
		else
			Logger "Sent mail using mail command." "NOTICE"
			return 0
		fi
	fi

	if type sendmail > /dev/null 2>&1 ; then
		echo -e "Subject:$subject\r\n$message" | $(type -p sendmail) "$destinationMails"
		if [ $? -ne 0 ]; then
			Logger "Cannot send mail via $(type -p sendmail) !!!" "WARN"
		else
			Logger "Sent mail using sendmail command without attachment." "NOTICE"
			return 0
		fi
	fi

	# Windows specific
	if type "mailsend.exe" > /dev/null 2>&1 ; then
		if [ "$senderMail" == "" ]; then
			Logger "Missing sender email." "ERROR"
			return 1
		fi
		if [ "$smtpServer" == "" ]; then
			Logger "Missing smtp port." "ERROR"
			return 1
		fi
		if [ "$smtpPort" == "" ]; then
			Logger "Missing smtp port, assuming 25." "WARN"
			smtpPort=25
		fi
		if [ "$encryption" != "tls" ] && [ "$encryption" != "ssl" ]  && [ "$encryption" != "none" ]; then
			Logger "Bogus smtp encryption, assuming none." "WARN"
			encryption_string=
		elif [ "$encryption" == "tls" ]; then
			encryption_string=-starttls
		elif [ "$encryption" == "ssl" ]:; then
			encryption_string=-ssl
		fi
		if [ "$smtpUser" != "" ] && [ "$smtpPassword" != "" ]; then
			auth_string="-auth -user \"$smtpUser\" -pass \"$smtpPassword\""
		fi
		$(type mailsend.exe) -f "$senderMail" -t "$destinationMails" -sub "$subject" -M "$message" -attach "$attachment" -smtp "$smtpServer" -port "$smtpPort" $encryption_string $auth_string
		if [ $? -ne 0 ]; then
			Logger "Cannot send mail via $(type mailsend.exe) !!!" "WARN"
		else
			Logger "Sent mail using mailsend.exe command with attachment." "NOTICE"
			return 0
		fi
	fi

	# pfSense specific
	if [ -f /usr/local/bin/mail.php ]; then
		echo "$message" | /usr/local/bin/mail.php -s="$subject"
		if [ $? -ne 0 ]; then
			Logger "Cannot send mail via /usr/local/bin/mail.php (pfsense) !!!" "WARN"
		else
			Logger "Sent mail using pfSense mail.php." "NOTICE"
			return 0
		fi
	fi

	# If function has not returned 0 yet, assume it is critical that no alert can be sent
	Logger "Cannot send mail (neither mutt, mail, sendmail, sendemail, mailsend (windows) or pfSense mail.php could be used)." "ERROR" # Is not marked critical because execution must continue
}

#### TrapError SUBSET ####
function TrapError {
	local job="$0"
	local line="$1"
	local code="${2:-1}"

	if [ $_LOGGER_SILENT == false ]; then
		(>&2 echo -e "\e[45m/!\ ERROR in ${job}: Near line ${line}, exit code ${code}\e[0m")
	fi
}
#### TrapError SUBSET END ####

# Quick and dirty performance logger only used for debugging
function _PerfProfiler {                                                                                                #__WITH_PARANOIA_DEBUG
	local perfString                                                                                                #__WITH_PARANOIA_DEBUG
	local i														#__WITH_PARANOIA_DEBUG
															#__WITH_PARANOIA_DEBUG
	perfString=$(ps -p $$ -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan)                                        #__WITH_PARANOIA_DEBUG
															#__WITH_PARANOIA_DEBUG
	for i in $(pgrep -P $$); do                                                                                     #__WITH_PARANOIA_DEBUG
		perfString="$perfString\n"$(ps -p $i -o args,pid,ppid,%cpu,%mem,time,etime,state,wchan | tail -1)       #__WITH_PARANOIA_DEBUG
	done                                                                                                            #__WITH_PARANOIA_DEBUG
															#__WITH_PARANOIA_DEBUG
	if type iostat > /dev/null 2>&1; then                                                                           #__WITH_PARANOIA_DEBUG
		perfString="$perfString\n"$(iostat)                                                                     #__WITH_PARANOIA_DEBUG
	fi                                                                                                              #__WITH_PARANOIA_DEBUG
															#__WITH_PARANOIA_DEBUG
	Logger "PerfProfiler:\n$perfString" "PARANOIA_DEBUG"                                                            #__WITH_PARANOIA_DEBUG
}

# Checks email address validity
function CheckRFC822 {
	local mail="${1}"
	local rfc822="^[a-z0-9!#\$%&'*+/=?^_\`{|}~-]+(\.[a-z0-9!#$%&'*+/=?^_\`{|}~-]+)*@([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+[a-z0-9]([a-z0-9-]*[a-z0-9])?\$"

	if [[ $mail =~ $rfc822 ]]; then
		echo 1
	else
		echo 0
	fi
}

# Function is busybox compatible since busybox ash does not understand direct regex, we use expr
function IsInteger {
	local value="${1}"

	if type expr > /dev/null 2>&1; then
		expr "$value" : '^[0-9]\{1,\}$' > /dev/null 2>&1
		if [ $? -eq 0 ]; then
			echo 1
		else
			echo 0
		fi
	else
		if [[ $value =~ ^[0-9]+$ ]]; then
			echo 1
		else
			echo 0
		fi
	fi
}

# Usage [ $(IsNumeric $var) -eq 1 ]
function IsNumeric {
	local value="${1}"

	if type expr > /dev/null 2>&1; then
		expr "$value" : '^[-+]\{0,1\}[0-9]*\.\{0,1\}[0-9]\{1,\}$' > /dev/null 2>&1
		if [ $? -eq 0 ]; then
			echo 1
		else
			echo 0
		fi
	else
		if [[ $value =~ ^[-+]?[0-9]+([.][0-9]+)?$ ]]; then
			echo 1
		else
			echo 0
		fi
	fi
}

function IsNumericExpand {
	eval "local value=\"${1}\"" # Needed eval so variable variables can be processed

	echo $(IsNumeric "$value")
}

## Modified version of http://stackoverflow.com/a/8574392
## Usage: [ $(ArrayContains "needle" "${haystack[@]}") -eq 1 ]
function ArrayContains () {
        local needle="${1}"
        local haystack="${2}"
        local e

        if [ "$needle" != "" ] && [ "$haystack" != "" ]; then
                for e in "${@:2}"; do
                        if [ "$e" == "$needle" ]; then
                                echo 1
                                return
                        fi
                done
        fi
        echo 0
        return
}


_OFUNCTIONS_SPINNER="|/-\\"
function Spinner {
	if [ $_LOGGER_SILENT == true ] || [ "$_LOGGER_ERR_ONLY" == true ]; then
		return 0
	else
		printf " [%c]  \b\b\b\b\b\b" "$_OFUNCTIONS_SPINNER"
		_OFUNCTIONS_SPINNER=${_OFUNCTIONS_SPINNER#?}${_OFUNCTIONS_SPINNER%%???}
		return 0
	fi
}

## Main asynchronous execution function
## Function can work in:
## WaitForTaskCompletion mode: monitors given pid in background, and stops them if max execution time is reached. Suitable for multiple synchronous pids to monitor and wait for
## ParallExec mode: takes list of commands to execute in parallel per batch, and stops them if max execution time is reahed.

## Example of improved wait $!
## ExecTasks $! "some_identifier" false 0 0 0 0 true 1 1800 false
## Example: monitor two sleep processes, warn if execution time is higher than 10 seconds, stop after 20 seconds
## sleep 15 &
## pid=$!
## sleep 20 &
## pid2=$!
## ExecTasks "some_identifier" 0 0 10 20 1 1800 true true false false 1 "$pid;$pid2"

## Example of parallel execution of four commands, only if directories exist. Warn if execution takes more than 300 seconds. Stop if takes longer than 900 seconds. Exeute max 3 commands in parallel.
## commands="du -csh /var;du -csh /etc;du -csh /home;du -csh /usr"
## conditions="[ -d /var ];[ -d /etc ];[ -d /home];[ -d /usr]"
## ExecTasks "$commands" "some_identifier" false 0 0 300 900 true 1 1800 true false false 3 "$conditions"

## Bear in mind that given commands and conditions need to be quoted

## ExecTasks has the following ofunctions subfunction requirements:
## Spinner
## Logger
## JoinString
## KillChilds

## Full call
##ExecTasks "$mainInput" "$id" $readFromFile $softPerProcessTime $hardPerProcessTime $softMaxTime $hardMaxTime $counting $sleepTime $keepLogging $spinner $noTimeErrorLog $noErrorLogsAtAll $numberOfProcesses $auxInput $maxPostponeRetries $minTimeBetweenRetries $validExitCodes

function ExecTasks {
	# Mandatory arguments
	local mainInput="${1}"				# Contains list of pids / commands separated by semicolons or filepath to list of pids / commands

	# Optional arguments
	local id="${2:-base}"                           # Optional ID in order to identify global variables from this run (only bash variable names, no '-'). Global variables are WAIT_FOR_TASK_COMPLETION_$id and HARD_MAX_EXEC_TIME_REACHED_$id
	local readFromFile="${3:-false}"		# Is mainInput / auxInput a semicolon separated list (true) or a filepath (false)
	local softPerProcessTime="${4:-0}"		# Max time (in seconds) a pid or command can run before a warning is logged, unless set to 0
	local hardPerProcessTime="${5:-0}"		# Max time (in seconds) a pid or command can run before the given command / pid is stopped, unless set to 0
	local softMaxTime="${6:-0}"			# Max time (in seconds) for the whole function to run before a warning is logged, unless set to 0
	local hardMaxTime="${7:-0}"			# Max time (in seconds) for the whole function to run before all pids / commands given are stopped, unless set to 0
	local counting="${8:-true}"			# Should softMaxTime and hardMaxTime be accounted since function begin (true) or since script begin (false)
	local sleepTime="${9:-.5}"			# Seconds between each state check. The shorter the value, the snappier ExecTasks will be, but as a tradeoff, more cpu power will be used (good values are between .05 and 1)
	local keepLogging="${10:-1800}"			# Every keepLogging seconds, an alive message is logged. Setting this value to zero disables any alive logging
	local spinner="${11:-true}"			# Show spinner (true) or do not show anything (false) while running
	local noTimeErrorLog="${12:-false}"		# Log errors when reaching soft / hard execution times (false) or do not log errors on those triggers (true)
	local noErrorLogsAtAll="${13:-false}"		# Do not log any errros at all (useful for recursive ExecTasks checks)

	# Parallelism specific arguments
	local numberOfProcesses="${14:-0}"		# Number of simulanteous commands to run, given as mainInput. Set to 0 by default (WaitForTaskCompletion mode). Setting this value enables ParallelExec mode.
	local auxInput="${15}"				# Contains list of commands separated by semicolons or filepath fo list of commands. Exit code of those commands decide whether main commands will be executed or not
	local maxPostponeRetries="${16:-3}"		# If a conditional command fails, how many times shall we try to postpone the associated main command. Set this to 0 to disable postponing
	local minTimeBetweenRetries="${17:-300}"	# Time (in seconds) between postponed command retries
	local validExitCodes="${18:-0}"			# Semi colon separated list of valid main command exit codes which will not trigger errors

	__CheckArguments 1-18 $# "$@"

	local i

	Logger "${FUNCNAME[0]} id [$id] called by [${FUNCNAME[1]} < ${FUNCNAME[2]} < ${FUNCNAME[3]} < ${FUNCNAME[4]} < ${FUNCNAME[5]} < ${FUNCNAME[6]} ...]." "PARANOIA_DEBUG"	 #__WITH_PARANOIA_DEBUG
                                                                                                                                       #__WITH_PARANOIA_DEBUG

	# Since ExecTasks takes up to 17 arguments, do a quick preflight check in DEBUG mode
	if [ "$_DEBUG" == true ]; then
		declare -a booleans=(readFromFile counting spinner noTimeErrorLog noErrorLogsAtAll)
		for i in "${booleans[@]}"; do
			test="if [ \$$i != false ] && [ \$$i != true ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
			eval "$test"
		done
		declare -a integers=(softPerProcessTime hardPerProcessTime softMaxTime hardMaxTime keepLogging numberOfProcesses maxPostponeRetries minTimeBetweenRetries)
		for i in "${integers[@]}"; do
			test="if [ $(IsNumericExpand \"\$$i\") -eq 0 ]; then Logger \"Bogus $i value [\$$i] given to ${FUNCNAME[0]}.\" \"CRITICAL\"; exit 1; fi"
			eval "$test"
		done
	fi

	# Expand validExitCodes into array
	IFS=';' read -r -a validExitCodes <<< "$validExitCodes"

	# ParallelExec specific variables
	local auxItemCount=0            # Number of conditional commands
	local commandsArray=()          # Array containing commands
	local commandsConditionArray=() # Array containing conditional commands
	local currentCommand            # Variable containing currently processed command
	local currentCommandCondition   # Variable containing currently processed conditional command
	local commandsArrayPid=()       # Array containing commands indexed by pids
	local commandsArrayOutput=()    # Array containing command results indexed by pids
	local postponedRetryCount=0     # Number of current postponed commands retries
	local postponedItemCount=0      # Number of commands that have been postponed (keep at least one in order to check once)
	local postponedCounter=0
	local isPostponedCommand=false  # Is the current command from a postponed file ?
	local postponedExecTime=0       # How much time has passed since last postponed condition was checked
	local needsPostponing		# Does currentCommand need to be postponed
	local temp

	# Common variables
	local pid                       # Current pid working on
	local pidState                  # State of the process
	local mainItemCount=0           # number of given items (pids or commands)
	local readFromFile              # Should we read pids / commands from a file (true)
	local counter=0
	local log_ttime=0               # local time instance for comparaison

	local seconds_begin=$SECONDS    # Seconds since the beginning of the script
	local exec_time=0               # Seconds since the beginning of this function

	local retval=0                  # return value of monitored pid process
	local subRetval=0               # return value of condition commands
	local errorcount=0              # Number of pids that finished with errors
	local pidsArray                 # Array of currently running pids
	local newPidsArray              # New array of currently running pids for next iteration
	local pidsTimeArray             # Array containing execution begin time of pids
	local executeCommand            # Boolean to check if currentCommand can be executed given a condition
	local hasPids=false             # Are any valable pids given to function ?              #__WITH_PARANOIA_DEBUG
	local functionMode
	local softAlert=false           # Does a soft alert need to be triggered, if yes, send an alert once
	local failedPidsList            # List containing failed pids with exit code separated by semicolons (eg : 2355:1;4534:2;2354:3)
	local randomOutputName          # Random filename for command outputs
	local currentRunningPids	# String of pids running, used for debugging purposes only
	# Initialise global variable
	eval "WAIT_FOR_TASK_COMPLETION_$id=\"\""
	eval "HARD_MAX_EXEC_TIME_REACHED_$id=false"

	# Init function variables depending on mode

	if [ $numberOfProcesses -gt 0 ]; then
		functionMode=ParallelExec
	else
		functionMode=WaitForTaskCompletion
	fi

	if [ $readFromFile == false ]; then
		if [ $functionMode == "WaitForTaskCompletion" ]; then
			IFS=';' read -r -a pidsArray <<< "$mainInput"
			mainItemCount="${#pidsArray[@]}"
		else
			IFS=';' read -r -a commandsArray <<< "$mainInput"
			mainItemCount="${#commandsArray[@]}"
			IFS=';' read -r -a commandsConditionArray <<< "$auxInput"
			auxItemCount="${#commandsConditionArray[@]}"
		fi
	else
		if [ -f "$mainInput" ]; then
			mainItemCount=$(wc -l < "$mainInput")
			readFromFile=true
		else
			Logger "Cannot read main file [$mainInput]." "WARN"
		fi
		if [ "$auxInput" != "" ]; then
			if [ -f "$auxInput" ]; then
				auxItemCount=$(wc -l < "$auxInput")
			else
				Logger "Cannot read aux file [$auxInput]." "WARN"
			fi
		fi
	fi

	if [ $functionMode == "WaitForTaskCompletion" ]; then
		# Force first while loop condition to be true because we don't deal with counters but pids in WaitForTaskCompletion mode
		counter=$mainItemCount
	fi

	Logger "Running ${FUNCNAME[0]} as [$functionMode] for [$mainItemCount] mainItems and [$auxItemCount] auxItems." "PARANOIA_DEBUG"              #__WITH_PARANOIA_DEBUG

	# soft / hard execution time checks that needs to be a subfunction since it is called both from main loop and from parallelExec sub loop
	function _ExecTasksTimeCheck {
		if [ $spinner == true ]; then
			Spinner
		fi
		if [ $counting == true ]; then
			exec_time=$((SECONDS - seconds_begin))
		else
			exec_time=$SECONDS
		fi

		if [ $keepLogging -ne 0 ]; then
			# This log solely exists for readability purposes before having next set of logs
			if [ ${#pidsArray[@]} -eq $numberOfProcesses ] && [ $log_ttime -eq 0 ]; then
				log_ttime=$exec_time
				Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue of which $postponedItemCount are postponed. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
			fi
			if [ $(((exec_time + 1) % keepLogging)) -eq 0 ]; then
				if [ $log_ttime -ne $exec_time ]; then # Fix when sleep time lower than 1 second
					log_ttime=$exec_time
					if [ $functionMode == "WaitForTaskCompletion" ]; then
						Logger "Current tasks still running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
					elif [ $functionMode == "ParallelExec" ]; then
						Logger "There are $((mainItemCount-counter+postponedItemCount)) / $mainItemCount tasks in the queue of which $postponedItemCount are postponed. Currently, ${#pidsArray[@]} tasks running with pids [$(joinString , ${pidsArray[@]})]." "NOTICE"
					fi
				fi
			fi
		fi

		if [ $exec_time -gt $softMaxTime ]; then
			if [ "$softAlert" != true ] && [ $softMaxTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
				Logger "Max soft execution time [$softMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]." "WARN"
				softAlert=true
				SendAlert true false
			fi
		fi

		if [ $exec_time -gt $hardMaxTime ] && [ $hardMaxTime -ne 0 ]; then
			if [ $noTimeErrorLog != true ]; then
				Logger "Max hard execution time [$hardMaxTime] exceeded for task [$id] with pids [$(joinString , ${pidsArray[@]})]. Stopping task execution." "ERROR"
			fi
			for pid in "${pidsArray[@]}"; do
				KillChilds $pid true
				if [ $? -eq 0 ]; then
					Logger "Task with pid [$pid] stopped successfully." "NOTICE"
				else
					if [ $noErrorLogsAtAll != true ]; then
						Logger "Could not stop task with pid [$pid]." "ERROR"
					fi
				fi
				errorcount=$((errorcount+1))
			done
			if [ $noTimeErrorLog != true ]; then
				SendAlert true false
			fi
			eval "HARD_MAX_EXEC_TIME_REACHED_$id=true"
			if [ $functionMode == "WaitForTaskCompletion" ]; then
				return $errorcount
			else
				return 129
			fi
		fi
	}

	function _ExecTasksPidsCheck {
		newPidsArray=()

		if [ "$currentRunningPids" != "$(joinString " " ${pidsArray[@]})" ]; then
			Logger "ExecTask running for pids [$(joinString " " ${pidsArray[@]})]." "DEBUG"
			currentRunningPids="$(joinString " " ${pidsArray[@]})"
		fi
		for pid in "${pidsArray[@]}"; do
			if [ $(IsInteger $pid) -eq 1 ]; then
				if kill -0 $pid > /dev/null 2>&1; then
					# Handle uninterruptible sleep state or zombies by ommiting them from running process array (How to kill that is already dead ? :)
					pidState="$(eval $PROCESS_STATE_CMD)"
					if [ "$pidState" != "D" ] && [ "$pidState" != "Z" ]; then

						# Check if pid hasn't run more than soft/hard perProcessTime
						pidsTimeArray[$pid]=$((SECONDS - seconds_begin))
						if [ ${pidsTimeArray[$pid]} -gt $softPerProcessTime ]; then
							if [ "$softAlert" != true ] && [ $softPerProcessTime -ne 0 ] && [ $noTimeErrorLog != true ]; then
								Logger "Max soft execution time [$softPerProcessTime] exceeded for pid [$pid]." "WARN"
								if [ "${commandsArrayPid[$pid]}]" != "" ]; then
									Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN"
								fi
								softAlert=true
								SendAlert true false
							fi
						fi


						if [ ${pidsTimeArray[$pid]} -gt $hardPerProcessTime ] && [ $hardPerProcessTime -ne 0 ]; then
							if [ $noTimeErrorLog != true ] && [ $noErrorLogsAtAll != true ]; then
								Logger "Max hard execution time [$hardPerProcessTime] exceeded for pid [$pid]. Stopping command execution." "ERROR"
								if [ "${commandsArrayPid[$pid]}]" != "" ]; then
									Logger "Command was [${commandsArrayPid[$pid]}]]." "WARN"
								fi
							fi
							KillChilds $pid true
							if [ $? -eq 0 ]; then
								 Logger "Command with pid [$pid] stopped successfully." "NOTICE"
							else
								if [ $noErrorLogsAtAll != true ]; then
								Logger "Could not stop command with pid [$pid]." "ERROR"
								fi
							fi
							errorcount=$((errorcount+1))

							if [ $noTimeErrorLog != true ]; then
								SendAlert true false
							fi
						fi

						newPidsArray+=($pid)
					fi
				else
					# pid is dead, get its exit code from wait command
					wait $pid
					retval=$?
					# Check for valid exit codes
					if [ $(ArrayContains $retval "${validExitCodes[@]}") -eq 0 ]; then
						if [ $noErrorLogsAtAll != true ]; then
							Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "ERROR"
							if [ "$functionMode" == "ParallelExec" ]; then
								Logger "Command was [${commandsArrayPid[$pid]}]." "ERROR"
							fi
							if [ -f "${commandsArrayOutput[$pid]}" ]; then
								Logger "Truncated output:\n$(head -c16384 "${commandsArrayOutput[$pid]}")" "ERROR"
							fi
						fi
						errorcount=$((errorcount+1))
						# Welcome to variable variable bash hell
						if [ "$failedPidsList" == "" ]; then
							failedPidsList="$pid:$retval"
						else
							failedPidsList="$failedPidsList;$pid:$retval"
						fi
					else
						Logger "${FUNCNAME[0]} called by [$id] finished monitoring pid [$pid] with exitcode [$retval]." "DEBUG"
					fi
				fi
				hasPids=true                                    ##__WITH_PARANOIA_DEBUG
			fi
		done

		# hasPids can be false on last iteration in ParallelExec mode
		if [ $hasPids == false ] && [ "$functionMode" = "WaitForTaskCompletion" ]; then                                 ##__WITH_PARANOIA_DEBUG
			Logger "No valable pids given." "ERROR"                 						##__WITH_PARANOIA_DEBUG
		fi                                                              						##__WITH_PARANOIA_DEBUG
		pidsArray=("${newPidsArray[@]}")

		# Trivial wait time for bash to not eat up all CPU
		sleep $sleepTime

		if [ "$_PERF_PROFILER" == true ]; then                         ##__WITH_PARANOIA_DEBUG
			_PerfProfiler                                           ##__WITH_PARANOIA_DEBUG
		fi                                                              ##__WITH_PARANOIA_DEBUG

	}

	while [ ${#pidsArray[@]} -gt 0 ] || [ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]; do
		_ExecTasksTimeCheck
		retval=$?
		if [ $retval -ne 0 ]; then
			return $retval;
		fi

		# The following execution bloc is only needed in ParallelExec mode since WaitForTaskCompletion does not execute commands, but only monitors them
		if [ $functionMode == "ParallelExec" ]; then
			while [ ${#pidsArray[@]} -lt $numberOfProcesses ] && ([ $counter -lt $mainItemCount ] || [ $postponedItemCount -ne 0 ]); do
				_ExecTasksTimeCheck
				retval=$?
				if [ $retval -ne 0 ]; then
					return $retval;
				fi

				executeCommand=false
				isPostponedCommand=false
				currentCommand=""
				currentCommandCondition=""
				needsPostponing=false

				if [ $readFromFile == true ]; then
					# awk identifies first line as 1 instead of 0 so we need to increase counter
					currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$mainInput")
					if [ $auxItemCount -ne 0 ]; then
						currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$auxInput")
					fi

					# Check if we need to fetch postponed commands
					if [ "$currentCommand" == "" ]; then
						currentCommand=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP")
						currentCommandCondition=$(awk 'NR == num_line {print; exit}' num_line=$((postponedCounter+1)) "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP")
						isPostponedCommand=true
					fi
				else
					currentCommand="${commandsArray[$counter]}"
					if [ $auxItemCount -ne 0 ]; then
						currentCommandCondition="${commandsConditionArray[$counter]}"
					fi

					if [ "$currentCommand" == "" ]; then
						currentCommand="${postponedCommandsArray[$postponedCounter]}"
						currentCommandCondition="${postponedCommandsConditionArray[$postponedCounter]}"
						isPostponedCommand=true
					fi
				fi

				# Check if we execute postponed commands, or if we delay them
				if [ $isPostponedCommand == true ]; then
					# Get first value before '@'
					postponedExecTime="${currentCommand%%@*}"
					postponedExecTime=$((SECONDS-postponedExecTime))
					# Get everything after first '@'
					temp="${currentCommand#*@}"
					# Get first value before '@'
					postponedRetryCount="${temp%%@*}"
					# Replace currentCommand with actual filtered currentCommand
					currentCommand="${temp#*@}"

					# Since we read a postponed command, we may decrase postponedItemCounter
					postponedItemCount=$((postponedItemCount-1))
					#Since we read one line, we need to increase the counter
					postponedCounter=$((postponedCounter+1))

				else
					postponedRetryCount=0
					postponedExecTime=0
				fi
				if ([ $postponedRetryCount -lt $maxPostponeRetries ] && [ $postponedExecTime -ge $minTimeBetweenRetries ]) || [ $isPostponedCommand == false ]; then
					if [ "$currentCommandCondition" != "" ]; then
						Logger "Checking condition [$currentCommandCondition] for command [$currentCommand]." "DEBUG"
						eval "$currentCommandCondition" &
						ExecTasks $! "subConditionCheck" false 0 0 1800 3600 true $SLEEP_TIME $KEEP_LOGGING true true true
						subRetval=$?
						if [ $subRetval -ne 0 ]; then
							# is postponing enabled ?
							if [ $maxPostponeRetries -gt 0 ]; then
								Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Postponing command." "NOTICE"
								postponedRetryCount=$((postponedRetryCount+1))
								if [ $postponedRetryCount -ge $maxPostponeRetries ]; then
									Logger "Max retries reached for postponed command [$currentCommand]. Skipping command." "NOTICE"
								else
									needsPostponing=true
								fi
								postponedExecTime=0
							else
								Logger "Condition [$currentCommandCondition] not met for command [$currentCommand]. Exit code [$subRetval]. Ignoring command." "NOTICE"
							fi
						else
							executeCommand=true
						fi
					else
						executeCommand=true
					fi
				else
					needsPostponing=true
				fi

				if [ $needsPostponing == true ]; then
					postponedItemCount=$((postponedItemCount+1))
					if [ $readFromFile == true ]; then
						echo "$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedMain.$id.$SCRIPT_PID.$TSTAMP"
						echo "$currentCommandCondition" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-postponedAux.$id.$SCRIPT_PID.$TSTAMP"
					else
						postponedCommandsArray+=("$((SECONDS-postponedExecTime))@$postponedRetryCount@$currentCommand")
						postponedCommandsConditionArray+=("$currentCommandCondition")
					fi
				fi

				if [ $executeCommand == true ]; then
					Logger "Running command [$currentCommand]." "DEBUG"
					randomOutputName=$(date '+%Y%m%dT%H%M%S').$(PoorMansRandomGenerator 5)
					eval "$currentCommand" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$pid.$randomOutputName.$SCRIPT_PID.$TSTAMP" 2>&1 &
					pid=$!
					pidsArray+=($pid)
					commandsArrayPid[$pid]="$currentCommand"
					commandsArrayOutput[$pid]="$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$id.$pid.$randomOutputName.$SCRIPT_PID.$TSTAMP"
					# Initialize pid execution time array
					pidsTimeArray[$pid]=0
				else
					Logger "Skipping command [$currentCommand]." "DEBUG"
				fi

				if [ $isPostponedCommand == false ]; then
					counter=$((counter+1))
				fi
				_ExecTasksPidsCheck
			done
		fi

	_ExecTasksPidsCheck
	done

	Logger "${FUNCNAME[0]} ended for [$id] using [$mainItemCount] subprocesses with [$errorcount] errors." "PARANOIA_DEBUG" #__WITH_PARANOIA_DEBUG

	# Return exit code if only one process was monitored, else return number of errors
	# As we cannot return multiple values, a global variable WAIT_FOR_TASK_COMPLETION contains all pids with their return value

	eval "WAIT_FOR_TASK_COMPLETION_$id=\"$failedPidsList\""
	
	if [ $mainItemCount -eq 1 ]; then
		return $retval
	else
		return $errorcount
	fi
}

function CleanUp {
	__CheckArguments 0 $# "$@"      #__WITH_PARANOIA_DEBUG

	if [ "$_DEBUG" != true ]; then
		rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP"
		# Fix for sed -i requiring backup extension for BSD & Mac (see all sed -i statements)
		rm -f "$RUN_DIR/$PROGRAM."*".$SCRIPT_PID.$TSTAMP.tmp"
	fi
}

#__BEGIN_WITH_PARANOIA_DEBUG
function __CheckArguments {
	# Checks the number of arguments of a function and raises an error if some are missing

	if [ "$_DEBUG" == true ]; then
		local numberOfArguments="${1}" # Number of arguments the tested function should have, can be a number of a range, eg 0-2 for zero to two arguments
		local numberOfGivenArguments="${2}" # Number of arguments that have been passed

		local minArgs
		local maxArgs

		# All arguments of the function to check are passed as array in ${3} (the function call waits for $@)
		# If any of the arguments contains spaces, bash things there are two aguments
		# In order to avoid this, we need to iterate over ${3} and count

		callerName="${FUNCNAME[1]}"

		local iterate=3
		local fetchArguments=true
		local argList=""
		local countedArguments
		while [ $fetchArguments == true ]; do
			cmd='argument=${'$iterate'}'
			eval $cmd
			if [ "$argument" == "" ]; then
				fetchArguments=false
			else
				argList="$argList[Argument $((iterate-2)): $argument] "
				iterate=$((iterate+1))
			fi
		done

		countedArguments=$((iterate-3))

		if [ $(IsInteger "$numberOfArguments") -eq 1 ]; then
			minArgs=$numberOfArguments
			maxArgs=$numberOfArguments
		else
			IFS='-' read minArgs maxArgs <<< "$numberOfArguments"
		fi

		Logger "Entering function [$callerName]." "PARANOIA_DEBUG"

		if ! ([ $countedArguments -ge $minArgs ] && [ $countedArguments -le $maxArgs ]); then
			Logger "Function $callerName may have inconsistent number of arguments. Expected min: $minArgs, max: $maxArgs, count: $countedArguments, bash seen: $numberOfGivenArguments." "ERROR"
			Logger "$callerName arguments: $argList" "ERROR"
		else
			if [ ! -z "$argList" ]; then
				Logger "$callerName arguments: $argList" "PARANOIA_DEBUG"
			fi
		fi
	fi
}

#__END_WITH_PARANOIA_DEBUG

############################################ END OF OFUNCTIONS CODE

function TrapQuit {
	local exitcode

	# Get ERROR / WARN alert flags from subprocesses that call Logger
	if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
		WARN_ALERT=true
	fi
	if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
		ERROR_ALERT=true
	fi

	if [ $ERROR_ALERT == true ]; then
		Logger "$PROGRAM finished with errors." "ERROR"
		if [ "$_DEBUG" != true ]
		then
			SendAlert false false
		else
			Logger "Debug mode, no alert mail will be sent." "NOTICE"
		fi
		exitcode=1
	elif [ $WARN_ALERT == true ]; then
		Logger "$PROGRAM finished with warnings." "WARN"
		if [ "$_DEBUG" != true ]
		then
			SendAlert false false
		else
			Logger "Debug mode, no alert mail will be sent." "NOTICE"
		fi
		exitcode=2      # Warning exit code must not force daemon mode to quit
	else
		Logger "$PROGRAM finished." "ALWAYS"
		exitcode=0
	fi
	CleanUp
	KillChilds $SCRIPT_PID > /dev/null 2>&1

	Logger "Elapsed [$SECONDS] seconds." "DEBUG"

	exit $exitcode
}

# Takes as many file arguments as needed
function InterleaveFiles {

	local counter=0
	local hasLine=true
	local i

	while [ $hasLine == true ]; do
		hasLine=false
		for i in "$@"; do
			line=$(awk 'NR == num_line {print; exit}' num_line=$((counter+1)) "$i")
			if [ -n "$line" ]; then
				echo "$line"
			hasLine=true
			fi
		done
		counter=$((counter+1))
	done
}

function ListClients {
	local backupDir="${1}"
	local configFile="${2}"
	local clientConfDir="${3}"

	__CheckArguments 0-3 $# "$@"      #__WITH_PARANOIA_DEBUG

	local clientIsIncluded
	local clientIsExcluded
	local excludeArray

	local client
	local clientEmail
	local configString

	local i
	local j

	if [ -f "$configFile" ]; then
		configString="-c \"$configFile\""
	fi

	if [ "$clientConfDir" == "" ]; then
		clientConfDir="/etc/@name@/clientconfdir"
	fi

	if [ -d "$backupDir" ]; then
		# File 'backup_stats' is there only when a backup is finished
		find "$backupDir" -mindepth 3 -maxdepth 3 -type f -name "backup_stats" | grep -e '.*' > /dev/null
		if [ $? -ne 0 ]; then
			Logger "The directory [$backupDir] does not seem to be a @name@ folder. Please check the path. Additionnaly, protocol 2 directores need to specify the dedup group directory and the client subfolder." "ERROR"
			Logger "This message may also show if the current user running this script does not have sufficient privileges on the @name@ folder." "ERROR"
		fi
	fi

	# Autodetect clients or use provided client with --client or -C option
	if [ "$GIVEN_CLIENT" == "" ]; then

		# Using both @name@ -a S list and find method in order to find maximum backup clients
		cmd="$BACKUP_EXECUTABLE $configString -a S | grep \"last backup\" | awk '{print \$1}' > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP\""
		Logger "Running cmd [$cmd]." "DEBUG"
		eval "$cmd" &
		ExecTasks $! "${FUNCNAME[0]}_lastbackup" false 0 0 1800 3600 true $SLEEP_TIME $KEEP_LOGGING
		if [ $? -ne 0 ]; then
			Logger "Enumerating @name@ clients via [$BACKUP_EXECUTABLE $configString -a S] failed. Check provided @name@ client config file." "ERROR"
		else
			Logger "@name@ detection method found the following clients:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP)" "DEBUG"
		fi

		# First exp removes everything before last '/'
		# sed tested on linux and BSD (should work on MacOS too)
		if [ -d "$backupDir" ]; then
			find "$backupDir" -mindepth 1 -maxdepth 1 -type d | sed -e "s/\(.*\)\/\(.*\)/\2/g" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$SCRIPT_PID.$TSTAMP"

			while IFS=$'\n' read -r client; do
				find "$backupDir$client" -mindepth 2 -maxdepth 2 -type f -name "backup_stats" | grep -e '.*' > /dev/null 2>&1
				if [ $? -eq 0 ]; then
					echo "$client" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP"
				fi
			done < "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$SCRIPT_PID.$TSTAMP"
		fi




		if [ ! -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" ]; then
			touch "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP"
		fi


		Logger "Backup file detection method found the following clients:\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP)" "DEBUG"
		# Merge all clients found by @name@ executable and manual check
		sort "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP" "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" | uniq > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-4.$SCRIPT_PID.$TSTAMP"
	else
		echo "$GIVEN_CLIENT" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-4.$SCRIPT_PID.$TSTAMP"
	fi

	while IFS=$'\n' read -r client; do
		clientIsIncluded=false
		clientIsExcluded=false

		IFS=',' read -a includeArray <<< "$INCLUDE_CLIENTS"
		for i in "${includeArray[@]}"; do
			echo "$client" | grep -e "^"$i"$" > /dev/null 2>&1
			if [ $? -eq 0 ]; then
				clientIsIncluded=true
			fi
		done

		IFS=',' read -a excludeArray <<< "$EXCLUDE_CLIENTS"
		for i in "${excludeArray[@]}"; do
			echo "$client" | grep -e "^"$i"$" > /dev/null 2>&1
			if [ $? -eq 0 ]; then
				clientIsExcluded=true
			fi
		done

		if ([ $clientIsIncluded == false ] && [ $clientIsExcluded == true ]); then
			Logger "Ommiting client [$client]." "NOTICE"
		else
			if [ -f "$backupDir$client/current/timestamp" ]; then
				Logger "Found client [$client] backup directory." "NOTICE"
			elif [ -d "$backupDir" ]; then
				Logger "Client [$client] does not seem to have any backups (check provided backup directory)." "WARN"
			fi

			CLIENT_LIST+=("$client")

			# Client email is a label in client config file
			# Check whether we can fetch the default value
			if [ -f "$clientConfDir/$client" ]; then
				clientEmail=$(egrep "^label( )?=( )?email_address( )?:( )?" "$clientConfDir/$client")
				if [ "$clientEmail" != "" ]; then	
					clientEmail="${clientEmail#*:}"
					# Remove eventual spaces
					clientEmail="${clientEmail/ /}"
					for j in $clientEmail; do
						if [ $(CheckRFC822 "$j") -eq 1 ]; then 
							if [ "${CLIENT_EMAIL["$client"]}" == "" ]; then
									CLIENT_EMAIL["$client"]="$j"
							else
								CLIENT_EMAIL["$client"]=${CLIENT_EMAIL["$client"]}" $j"
							fi
						else
							Logger "Client [$client] has a bogus mail address [$j]." "WARN"
						fi
					done
				else
					Logger "Client [$client] has no mail address set." "NOTICE"
				fi
			elif ([ "$CLIENTS_ALERT_QUOTAS" == true ] || [ "$CLIENTS_ALERT_OUTDATED" == true ]); then
				Logger "Cannot find client config file [$clientConfDir/$client] to fetch email address." "ERROR"
			fi
		fi
	done < "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-4.$SCRIPT_PID.$TSTAMP"
}

function IsClientIdle {
	local client="${1}"
	local configFile="${2}"

	__CheckArguments 2 $# "$@"      #__WITH_PARANOIA_DEBUG

	local exitCode
	local configString

	if [ -f "$configFile" ]; then
		configString="-c \"$configFile\""
	fi

	Logger "Checking if client [$client] is currently idle." "DEBUG"

	cmd="$BACKUP_EXECUTABLE $configString -a S -C $client | grep \"Status: idle\" > /dev/null 2>&1"
	ExecTasks $! "${FUNCNAME[0]}_idle" false 0 0 120 300 true $SLEEP_TIME $KEEP_LOGGING
	exitCode=$?



	if [ $exitCode -ne 0 ]; then
		Logger "Client [$client] is currently backing up." "NOTICE"
		return $exitCode
	else
		return $exitCode
	fi
}

function VerifyBackups {
	local backupDir="${1}"
	local numberToVerify="${2}"
	local configFile="${3}"

	__CheckArguments 2-3 $# "$@"      #__WITH_PARANOIA_DEBUG

	local backupNumber
	local exitCode
	local client

	local configString
	local interleaveFileArgs=()
	local interleaveConditionsFileArgs=()

	local safeExitCodes="0;2;3" # 0: success, 2: warnings, 3: timer conditions not met

	if [ -f "$configFile" ]; then
		configString="-c \"$configFile\""
	fi

	for client in "${CLIENT_LIST[@]}"; do
		# Only backups containing file backup_stats are valid
		find "$backupDir$client" -mindepth 2 -maxdepth 2 -type f -name "backup_stats" | sort -nr | head -n $numberToVerify | sed -e 's/.*\([0-9]\{7\}\).*/\1/' > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP"
		Logger "Can check $(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP | wc -l) backups for [$client]." "NOTICE"
		while IFS=$'\n' read -r backupNumber; do
			# sed here removes all lines containing only block logs (64 chars + number)
			# sed regex isn't complete (lacks \+$ because BSD / macOS sed does not like extended regex)
			Logger "Preparing verification of backup [$backupNumber] for client [$client]." "NOTICE"
			echo "$BACKUP_EXECUTABLE $configString -C $client -a v -b $backupNumber | sed '/^[rRSDBWfydlLsmnkceaipwxzbMFqYZGOPQEtvVuU]\{64\} [0-9]/d' >> \"$LOG_FILE\" 2>&1" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$client.$SCRIPT_PID.$TSTAMP"
			echo "$BACKUP_EXECUTABLE $configString -a S -C $client | grep \"Status: idle\" > /dev/null 2>&1" >> "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.1.$client.$SCRIPT_PID.$TSTAMP"
		done < "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-1.$SCRIPT_PID.$TSTAMP"

		if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$client.$SCRIPT_PID.$TSTAMP" ]; then
			interleaveFileArgs+=("$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.$client.$SCRIPT_PID.$TSTAMP")
		fi

		if [ -f "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.1.$client.$SCRIPT_PID.$TSTAMP" ]; then
			interleaveConditionsFileArgs+=("$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-2.1.$client.$SCRIPT_PID.$TSTAMP")
		fi
	done

	InterleaveFiles "${interleaveFileArgs[@]}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP"
	InterleaveFiles "${interleaveConditionsFileArgs[@]}" > "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.1.$SCRIPT_PID.$TSTAMP"

	Logger "Now running backup verifications, concurrency set to $PARELLEL_VERIFY_CONCURRENCY." "NOTICE"
	Logger "Executing parallel commands\n$(cat $RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP)" "DEBUG"
	ExecTasks "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.$SCRIPT_PID.$TSTAMP" "${FUNCNAME[0]}" true $SOFT_MAX_EXEC_TIME_PER_VERIFY $HARD_MAX_EXEC_TIME_PER_VERIFY $SOFT_MAX_EXEC_TIME $HARD_MAX_EXEC_TIME true $SLEEP_TIME $KEEP_LOGGING true false false $PARELLEL_VERIFY_CONCURRENCY "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}-3.1.$SCRIPT_PID.$TSTAMP" $POSTPONE_RETRY $POSTPONE_TIME "$safeExitCodes"
	exitCode=$?
	if [ $exitCode -ne 0 ]; then
	Logger "Client backup verification produced errors [$exitCode]." "ERROR"
	else
		Logger "Client backup verification succeed." "NOTICE"
	fi

	Logger "Backup verification done." "NOTICE"
}

function ListOutdatedClients {
	local backupDir="${1}"
	local oldDays="${2}"

	__CheckArguments 2 $# "$@"      #__WITH_PARANOIA_DEBUG

	local found=false
	local clientAlertBody
	local recentBackupTimestamp
	local CurrentTimestamp
	
	Logger "Checking for outdated clients." "NOTICE"

	for client in "${CLIENT_LIST[@]}"; do
		if [ -f "$backupDir$client/current/backup_stats" ]; then
			# This extracts time_end timestamp from last backup stats
			recentBackupTimestamp=$(awk '/"name": "time_end"/{p=1}{if(p>0){p=p+1}}{if(p==4){print $2}}' "$backupDir$client/current/backup_stats")
				else
			recentBackupTimestamp=0
		fi
		# Current timestamp
		currentTimestamp="$(date +%s)"
		if [ $((currentTimestamp - (24*3600*oldDays))) -gt $recentBackupTimestamp ]; then
			Logger "Client [$client] has no backups newer than [$oldDays] days." "ERROR"
			if [ $CLIENTS_ALERT_OUTDATED == true ]; then
				clientAlertBody="${CLIENT_ALERT_BODY_OUTDATED/"[CLIENT]"/$client}"
				clientAlertBody="${clientAlertBody/"[NUMBERDAYS]"/$oldDays}"
				clientAlertBody="${clientAlertBody/"[INSTANCE]"/$INSTANCE_ID}"
				if [ "${CLIENT_EMAIL[$client]}" != "" ]; then
					SendEmail "$CLIENT_ALERT_SUBJECT" "$clientAlertBody" "${CLIENT_EMAIL[$client]}"
					Logger "Sent outdated client mail to [${CLIENT_EMAIL[$client]}]." "NOTICE"
				else
					Logger "Client [$client] does not have a mail address. Cannot send notification." "ERROR"
				fi
			fi
			found=true
		fi
	done

	if [ $found == false ]; then
		Logger "No outdated clients found." "NOTICE"
	else
		Logger "Outdated client checks done." "NOTICE"
	fi
}

function ListQuotaExceedClients {
	local backupDir="${1}"

	__CheckArguments 1 $# "$@"      #__WITH_PARANOIA_DEBUG

	local found=false
	local lastBackupDir
	local clientAlertBody
	local bytesEstimated
	local quotaExceed
	local quotaDiff

	Logger "Checking for clients with exceeded quota." "NOTICE"

	for client in "${CLIENT_LIST[@]}"; do
		lastBackupDir=$(find "$backupDir$client" -maxdepth 2 -name "*current")
		if [ -f "$backupDir$client/current/log.gz" ]; then
			if zcat "$lastBackupDir/log.gz" | grep quota > /dev/null 2>&1; then
				bytesEstimated=$(zcat "$lastBackupDir/log.gz" | grep "Bytes estimated")
				bytesEstimated="${bytesEstimated##*:}"
				# Remove leading spaces
				bytesEstimated="${bytesEstimated# *}"
				quotaExceed=$(zcat "$lastBackupDir/log.gz" | grep "quota")
				quotaExceed="${quotaExceed##*:}"
				quotaDiff="$bytesEstimated /$quotaExceed)"

				Logger "Client [$client] quota exceed ($quotaDiff)." "WARN"

				if [ $CLIENTS_ALERT_QUOTAS == true ]; then
					clientAlertBody="${CLIENT_ALERT_BODY_QUOTA/"[CLIENT]"/$client}"
					clientAlertBody="${clientAlertBody/"[QUOTAEXCEED]"/$quotaDiff}"
					if [ "${CLIENT_EMAIL[$client]}" != "" ]; then
						SendEmail "$CLIENT_ALERT_SUBJECT" "$clientAlertBody" "${CLIENT_EMAIL[$client]}"
						Logger "Sent quota exceeded mail to [${CLIENT_EMAIL[$client]}]." "NOTICE"
					else
						Logger "Client [$client] does not have a mail address. Cannot send notification" "ERROR"
					fi
				fi
				found=true
			fi
		else
			Logger "No valid log file found for analysis of client [$client]." "WARN"
		fi
	done

	if [ $found == false ]; then
		Logger "No clients with exceeded quota found." "NOTICE"
	else
		Logger "Quota checks done." "NOTICE"
	fi

}

function VerifyLastWarnings {
	local backupDir="${1}"

	__CheckArguments 1 $# "$@"	#__WITH_PARANOIA_DEBUG

	local found=false

	Logger "Checking for warnings in last backups." "NOTICE"

	for client in "${CLIENT_LIST[@]}"; do
		if [ -f "$backupDir$client/current/log.gz" ]; then
			if zcat "$backupDir$client/current/log.gz" | grep "WARNING" > /dev/null 2>&1; then
				Logger "Client [$client] has the following warnings:" "WARN"
				Logger "$(zcat $backupDir$client/current/log.gz | grep WARNING)" "WARN"
				found=true
			fi
		elif [ -f "$backupDir$client/current/log" ]; then
			if cat "$backupDir$client/current/log" | grep "WARNING" > /dev/null 2>&1; then
				Logger "Client [$client] has the following warnings:" "WARN"
				Logger "$(grep WARNING $backupDir$client/current/log)" "WARN"
				found=true
			fi
		else
			Logger "No log file found for warning analysis in [$backupDir$client/current]." "WARN"
		fi
	done

	if [ $found == false ]; then
		Logger "No warnings found in last backups." "NOTICE"
	fi
}

function UnstripVSS {
	local path="${1}"

	__CheckArguments 1 $# "$@"      #__WITH_PARANOIA_DEBUG

	# We need to have a modular temp extension so we will not overwrite potential existing files
	local tempExtension="$SCRIPT_PID.$TSTAMP.old"

	if ! type vss_strip > /dev/null 2>&1; then
		Logger "Could not find vss_strip binary. Please check your path variable." "CRITICAL"
		exit 1
	fi

	find "$path" -type f -print0 | while IFS= read -r -d $'\0' file; do
		Logger "Unstripping file [$file]." "NOTICE"
		mv -f "$file" "$file.$tempExtension"
		if [ $? -ne 0 ]; then
			Logger "Could not move [$file] to [$file.$tempExtension] for processing." "WARN"
			continue
		else
			vss_strip -i "$file.$tempExtension" -o "$file"
			if [ $? -ne 0 ]; then
				Logger "Could not vss_strip [$file.$tempExtension] to [$file]." "WARN"
				mv -f "$file.$tempExtension" "$file"
				if [ $? -ne 0 ]; then
					Logger "Coult not move back [$file.$tempExtension] to [$file]." "WARN"
				fi
			else
				rm -f "$file.$tempExtension"
				if [ $? -ne 0 ]; then
					Logger "Could not delete temporary file [$file.$tempExtension]." "WARN"
				continue
				fi
			fi
		fi
	done

	# Cannot get exitcode since find uses a subshell. Getting exit code from Logger
	if [ -f "$RUN_DIR/$PROGRAM.Logger.warn.$SCRIPT_PID.$TSTAMP" ]; then
		return 2
	fi
	if [ -f "$RUN_DIR/$PROGRAM.Logger.error.$SCRIPT_PID.$TSTAMP" ]; then
		return 1
	fi
	return 0
}

function VerifyService {
	local serviceName="${1}"
	local serviceType="${2}"

	__CheckArguments 2 $# "$@"      #__WITH_PARANOIA_DEBUG

	local serviceNameArray
	local serviceStatusCommand
	local serviceStartCommand

	local i
	
	if [ "$serviceName" == "" ]; then
		Logger "No service name(s) given." "WARN"
		return
	fi

	IFS=',' read -a serviceNameArray <<< "$serviceName"
	for i in "${serviceNameArray[@]}"; do
		if [ "$serviceType" == "initv" ]; then
			serviceStatusCommand="service $i status"
			serviceStartCommand="service $i start"
		elif [ "$serviceType" == "systemd" ]; then
			serviceStatusCommand="systemctl status $i"
			serviceStartCommand="systemctl start $i"
		else
			serviceStatusCommand="service $i status"
			serviceStartCommand="systemctl start $i"
			Logger "No valid service type given [$serviceType]. Trying default initV style." "ERROR"
		fi
		eval "$serviceStatusCommand" > /dev/null 2>&1 &
		ExecTasks $! "${FUNCNAME[0]}_statuscmd" false 0 0 120 300 true $SLEEP_TIME $KEEP_LOGGING
		if [ $? -ne 0 ]; then
			Logger "Service [$i] is not started. Trying to start it." "WARN"
			eval "$serviceStartCommand" > /dev/null 2>&1 &
			ExecTasks $! "${FUNCNAME[0]}_startcmd" false 0 0 120 300 true $SLEEP_TIME $KEEP_LOGGING
			if [ $? -ne 0 ]; then
				Logger "Cannot start service [$i]." "CRITICAL"
				SendAlert false false
			else
				Logger "Service [$i] was successfuly started." "WARN"
				SendAlert false false
			fi
		else
			Logger "Service [$i] is running." "NOTICE"
		fi
	done
}

function DisplayBackupCalendar {
	local configFile="${1}"
	
	__CheckArguments 1 $# "$@"      #__WITH_PARANOIA_DEBUG

	local client
	local cmd
	local backups
	local months
	local days
	local days_expr
	local cdate
	local month_count=1

	if [ -f "$configFile" ]; then
		configString="-c \"$configFile\""
	fi

	for client in "${CLIENT_LIST[@]}"; do
		cmd="$BACKUP_EXECUTABLE $configString -a l -C \"$client\" | grep \"^Backup: \" |awk '{print \$3}' > \"$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP\""
		Logger "Running cmd [$cmd]." "DEBUG"
		eval "$cmd" &
		ExecTasks $! "${FUNCNAME[0]}" false 0 0 1800 3600 true $SLEEP_TIME $KEEP_LOGGING
		if [ $? -ne 0 ]; then
			Logger "Failed to enumerate backups for client [$client]." "ERROR"
		elif [ -s "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP" ]; then
			backups="$(cat "$RUN_DIR/$PROGRAM.${FUNCNAME[0]}.$SCRIPT_PID.$TSTAMP")"
			months=$(echo "$backups" | cut -d'-' -f1-2 | sort -V | uniq | tail -n $month_count)
			Logger "Calendar for client [$client]:" "NOTICE"
			for month in $months; do
				days=$(echo "$backups" | grep "$month" | cut -d'-' -f3 | sort | uniq )
				days_expr=$(echo "$days" | tr '\n' '|'|rev|cut -c2-|rev)
				cdate="$(date --date="$month-01" +"%m %Y")"
				cal $cdate | head -1
				cal $cdate | tail -n +2 | sed -r 's/\x5f\x08//g;s/ ([0-9]( |$))/0\1/g' | grep --color -EC10 "((^| )$days_expr( |$))"
			done
		else
			Logger "No backups were found for client [$client]" "NOTICE"
		fi
	done
}

function Init {
	# Set error exit code if a piped command fails
	set -o pipefail
	set -o errtrace

	trap TrapQuit TERM EXIT HUP QUIT
}

function Usage {

	if [ "$IS_STABLE" != true ]; then
		echo -e "\e[93mThis is an unstable dev build. Please use with caution.\e[0m"
	fi
	echo "$PROGRAM $PROGRAM_VERSION $PROGRAM_BUILD"
	echo "$AUTHOR"
	echo "$CONTACT"
	echo ""
	echo "Usage:"
	echo "$0 [OPTIONS]"
	echo ""
	echo "[OPTIONS]"
	echo "-d, --backup-dir=\"\"                The directory where the client backup directories are"
	echo "-o, --check-outdated-clients=n     Check for clients that don't have backups newer than n days"
	echo "-v, --verify-last-backups=n        Verify the last n backups of all clients"
	echo "-q, --verify-quotas                Check for client quotas"
	echo "-i, --include-clients=\"\"           Comma separated list of clients to include. This list takes grep -e compatible regular expressions, includes prevail excludes"
	echo "-e, --exclude-clients=\"\"           Comma separated list of clients to exclude. This list takes grep -e compatible regular expressions"
	echo "-c, --config-file=\"\"               Path to optional @name@ client configuration file (defaults to /etc/@name@/@name@.conf)"
	echo "-s, --vss-strip-path=\"\"            Run vss_strip for all files in given path"
	echo "-j, --verify-service=\"\"            Comma separated list of @name@ services to check and restart if they aren't running"
	echo "-w, --verify-warnings              Check for warnings in last backup logs"
	echo "-A, --clients-alert-quotas         Use email defined in client config to alert users about exceeded disk quotas (see email template in header)"
	echo "-a, --clients-alert-outdated       Use email defined in client config to alert users about outdated backups (see email template in header)"
	echo "-z, --clientconfdir=\"\"             Path of clientconfdir in order to fetch email addresses when client alerts are used (defaults to /etc/@name@/clientconfdir)"
	echo "-C, --client=\"\"                    Specify specific client instead of detecting clients."
	echo "--calendar                         Show backup calendar"
	echo ""
	echo "Examples:"
	echo "$0 -d /path/to/@name@ -v 3 -c /etc/@name@/@name@.conf"
	echo "$0 --vss-strip-path=/path/to/restored/files"
	echo "$0 -j @name@.service" 
	echo "Exclude via regex all clients beginning with 'cli' and otherclient1/2:"
	echo "$0 --backup-dir=/path/to/@name@/protocol1 --exclude-clients=cli.*,otherclient1,otherclient2"
	echo ""
	echo "Additionnal options"
	echo "--no-maxtime                       Don't stop checks after the configured maximal time in script"
	echo "-s, --silent                       Don't output to stdout, log file only"
	echo "--errors-only                      Don't output anything but errors."
	echo ""
	echo "--destination-mails=\"\"             Space separated list of email adresses where to send warning and error mails"
	echo "--instance-id=\"\"                   Arbitrary identifier for log files and alert mails"
	exit 128
}


#### SCRIPT ENTRY POINT
DESTINATION_MAILS=""
no_maxtime=false
ERROR_ALERT=false
WARN_ALERT=false
CONFIG_FILE=""
BACKUP_DIR=""
VERIFY_BACKUPS=""
INCLUDE_CLIENTS=""
EXCLUDE_CLIENTS=""
OUTDATED_DAYS=""
CLIENT_LIST=()
GIVEN_CLIENT=""
declare -A CLIENT_EMAIL
VSS_STRIP_DIR=""
VERIFY_SERVICE=false
VERIFY_WARNINGS=false
VERIFY_QUOTAS=false
CLIENTS_ALERT_QUOTAS=false
CLIENTS_ALERT_OUTDATED=false
CLIENT_CONF_DIR=""
SHOW_BACKUP_CALENDAR=false

function GetCommandlineArguments {
	local isFirstArgument=true
	if [ $# -eq 0 ]
	then
		Usage
	fi
	while [ $# -gt 0 ]; do
		## Options name is $1, argument is $2 unless there is a separator other than space
		case $1 in
			--instance-id=*)
			INSTANCE_ID="${1##*=}"
			;;
			--silent)
			_LOGGER_SILENT=true
			;;
			--verbose)
			_LOGGER_VERBOSE=true
			;;
			--no-maxtime)
			no_maxtime=true
			;;
			--help|-h|--version)
			Usage
			;;
			--backup-dir=*)
			BACKUP_DIR="${1##*=}"
			;;
			-d)
			BACKUP_DIR="${2}"
			shift
			;;
			--check-outdated-clients=*)
			OUTDATED_DAYS="${1##*=}"
			;;
			-o)
			OUTDATED_DAYS="${2}"
			shift
			;;
			--verify-last-backups=*)
			VERIFY_BACKUPS="${1##*=}"
			;;
			-v)
			VERIFY_BACKUPS="${2}"
			shift
			;;
			-q|--verify-quotas)
			VERIFY_QUOTAS=true
			;;
			--include-clients=*)
			INCLUDE_CLIENTS="${1##*=}"
			;;
			-i)
			INCLUDE_CLIENTS="${2}"
			shift
			;;
			--exclude-clients=*)
			EXCLUDE_CLIENTS="${1##*=}"
			;;
			-e)
			EXCLUDE_CLIENTS="${2}"
			shift
			;;
			--config-file=*)
			CONFIG_FILE="${1##*=}"
			;;
			-c)
			CONFIG_FILE="${2}"
			shift
			;;
			-C)
			GIVEN_CLIENT="${2}"
			shift
			;;
			--client=*)
			GIVEN_CLIENT="${1##*=}"
			;;
			--calendar)
			SHOW_BACKUP_CALENDAR=true
			;;
			--vss-strip-path=*)
			VSS_STRIP_DIR="${1##*=}"
			;;
			-s)
			VSS_STRIP_DIR="${2}"
			shift
			;;
			-j)
			VERIFY_SERVICE=true
			VERIFY_SERVICES_NAMES="${2}"
			shift
			;;
			--verify-service=*)
			VERIFY_SERVICE=true
			VERIFY_SERVICE_NAMES="${1##*=}"
			;;
			-A|--client-alert-quotas)
			CLIENTS_ALERT_QUOTAS=true
			;;
			-a|--client-alert-outdated)
			CLIENTS_ALERT_OUTDATED=true
			;;
			-z)
			CLIENT_CONF_DIR="${2}"
			shift
			;;
			--clientconfdir=*)
			CLIENT_CONF_DIR="${1##*=}"
			;;
			-w|--verify-warnings)
			VERIFY_WARNINGS=true
			;;
			--errors-only)
			_LOGGER_ERR_ONLY=true
			;;
			--destination-mails=*)
			DESTINATION_MAILS="${1##*=}"
			;;
			--no-maxtime)
			SOFT_MAX_EXEC_TIME=0
			HARD_MAX_EXEC_TIME=0
			;;
			*)
			if [ $isFirstArgument == false ]; then
				Logger "Unknown option '${1}'" "CRITICAL"
				Usage
			fi
			;;
		esac
		shift
		isFirstArgument=false
	done
}

GetCommandlineArguments "$@"
Init

if [ "$LOGFILE" == "" ]; then
	if [ -w /var/log ]; then
		LOG_FILE="/var/log/$PROGRAM.$INSTANCE_ID.log"
	elif ([ "$HOME" != "" ] && [ -w "$HOME" ]); then
		LOG_FILE="$HOME/$PROGRAM.$INSTANCE_ID.log"
	else
		LOG_FILE="./$PROGRAM.$INSTANCE_ID.log"
	fi
else
	LOG_FILE="$LOGFILE"
fi
if [ ! -w "$(dirname $LOG_FILE)" ]; then
	echo "Cannot write to log [$(dirname $LOG_FILE)]."
else
	Logger "Script begin, logging to [$LOG_FILE]." "DEBUG"
fi

DATE=$(date)
Logger "---------------------------------------------------------------------" "NOTICE"
Logger "$DATE - $PROGRAM $PROGRAM_VERSION script begin." "ALWAYS"
Logger "---------------------------------------------------------------------" "NOTICE"
Logger "Instance [$INSTANCE_ID] launched as $LOCAL_USER@$LOCAL_HOST (PID $SCRIPT_PID)" "NOTICE"


if [ $no_maxtime == true ]; then
	SOFT_MAX_EXEC_TIME_PER_VERIFY=0
	HARD_MAX_EXEC_TIME_PER_VERIFY=0
	SOFT_MAX_EXEC_TIME=0
	HARD_MAX_EXEC_TIME=0
fi

if ! type -p "$BACKUP_EXECUTABLE" > /dev/null 2>&1; then
	Logger "Cannot find [$BACKUP_EXECUTABLE]. Please modify binary path in $0 script header." "CRITICAL"
	exit 126
fi

if [ "$VSS_STRIP_DIR" != "" ]; then
	if [ -d "$VSS_STRIP_DIR" ]; then
		UnstripVSS "$VSS_STRIP_DIR"
		exit $?
	else
		Logger "Bogus path given to unstrip [$VSS_STRIP_DIR]." "CRITICAL"
		exit 1
	fi
fi

if [ "$VERIFY_SERVICE" == true ]; then
	VerifyService "$VERIFY_SERVICES_NAMES" "$SERVICE_TYPE"
fi

if [ "$BACKUP_DIR" != "" ]; then
	if [ ! -d "$BACKUP_DIR" ]; then
		Logger "Backup dir [$BACKUP_DIR] doesn't exist." "CRITICAL"
		exit 1
	else
		# Make sure there is only one trailing slash on path
		BACKUP_DIR="${BACKUP_DIR%/}/"
	fi
fi

if [ "$CONFIG_FILE" != "" ]; then
	if [ ! -f "$CONFIG_FILE" ]; then
		Logger "Bogus configuration file [$CONFIG_FILE] given." "CRITICAL"
		exit 1
	fi
fi

if [ "$CLIENT_CONF_DIR" != "" ]; then
	if [ ! -d "$CLIENT_CONF_DIR" ]; then
		Logger "Bogus clientconfdir [$CLIENT_CONF_DIR] given." "CRITICAL"
		exit 1
	fi
fi

ListClients "$BACKUP_DIR" "$CONFIG_FILE" "$CLIENT_CONF_DIR"

if [ $SHOW_BACKUP_CALENDAR == true ]; then
	DisplayBackupCalendar "$CONFIG_FILE"
fi

if [ $VERIFY_WARNINGS == true ]; then
	VerifyLastWarnings "$BACKUP_DIR"
fi

if [ $VERIFY_QUOTAS == true ]; then
	ListQuotaExceedClients "$BACKUP_DIR"
fi

if [ "$OUTDATED_DAYS" != "" ]; then
	if [ $(IsInteger "$OUTDATED_DAYS") -ne 0 ]; then
		ListOutdatedClients "$BACKUP_DIR" $OUTDATED_DAYS
	else
		Logger "Bogus --check-outdated-clients value [$OUTDATED_DAYS]." "CRITICAL"
		exit 1
	fi
fi

if [ "$VERIFY_BACKUPS" != "" ]; then
	if [ $(IsInteger "$VERIFY_BACKUPS") -ne 0 ]; then
		VerifyBackups "$BACKUP_DIR" $VERIFY_BACKUPS "$CONFIG_FILE"
	else
		Logger "Bogus --verify-last-backups value [$VERIFY_BACKUPS]." "CRITICAL"
		exit 1
	fi
fi

# v0.5.0
# - Outdated backups are now chekced against backup_stats file instead backup dir ctime (resolves no outdated backup after copy issue)
# - Check against missing service names for service verification
# - Ported minor fixes from osync project
#       - Prettier logs
#       - Fixed run files cleanup
#       - Fixed RFC822 mail checks
#       - Start speedup (changed PoorMansRandomGenerator)
#       - Fixed potential bash buffer overflow when logging very large file outputs

# v0.4.8
# - Removed attachment sending to prevent mailbox clogging
# - Fixed possible missing characters from log results (see https://github.com/grke/burp/issues/801 and burp/src/cmd.h)
# - Fixed quota warning when no client log file exists
# - Smaller fixes ported from ofunctions project

# v0.4.6
# - Added command output to logs
# - Fixed not using [INSTANCE] placeholder in example text
# - Raised default HARD_MAX_EXEC_TIME_PER_VERIFY to 64800 seconds
# - Replaced yes/no with true/false booleans

# v0.4.4
# - More explicit log messages
# - Fix for missing date %N in BSD / MacOS
# - Ported some minor fixes from osync project

# v0.4.2
# - Added Hakong's backup calendar display (using --calendar)
# - Added [INSTANCE] placeholder for email sending
# - Client detection now also happens when no data directory is set
# - Fixed warning not shown when email address cannot be fetched and client alerts are enabled
# - Fixed multiple email adresses not being checked against RFC822

# v0.4.0 first public release, merged into @name@ codebase