#!/usr/bin/env bash
# No, we can not deal with sh alone.

set -e
set -u
# ERR traps should be inherited from functions too. (And command
# substitutions and subshells and whatnot, but for us the function is
# the important part here)
set -E

# A pipeline's return status is the value of the last (rightmost)
# command to exit with a non-zero status, or zero if all commands exit
# success fully.
set -o pipefail

# ftpsync script for Debian
# Based losely on a number of existing scripts, written by an
# unknown number of different people over the years.
#
# Copyright (C) 2008-2016 Joerg Jaspert <joerg@debian.org>
# Copyright (C) 2016 Peter Palfrader
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.

BINDIR=$(dirname $(readlink -f "$0")); . "${BINDIR}/include-git" ## INCLUDE COMMON

########################################################################
########################################################################
## functions                                                          ##
########################################################################
########################################################################
check_commandline() {
    while [[ $# -gt 0 ]]; do
        case "$1" in
            sync:stage1)
                SYNCSTAGE1="true"
                SYNCALL="false"
                ;;
            sync:stage2)
                SYNCSTAGE2="true"
                SYNCALL="false"
                ;;
            sync:callback)
                SYNCCALLBACK="true"
                ;;
            sync:archive:*)
                ARCHIVE=${1##sync:archive:}
                ;;
            sync:all)
                SYNCALL="true"
                ;;
            sync:mhop)
                SYNCMHOP="true"
                ;;
            *)
                echo "Unknown option ${1} ignored"
                ;;
        esac
        shift  # Check next set of parameters.
    done
}

# All the stuff we want to do when we exit, no matter where
cleanup() {
    rc=$?

    trap - ERR TERM HUP INT QUIT EXIT
    # all done. Mail the log, exit.

    if [[ $rc -gt 0 ]]; then
        log "Mirrorsync done with errors"
    else
        log "Mirrorsync done"
    fi

    if [[ -n ${MAILTO} ]]; then
        local args=()
        local send=
        local subject="SUCCESS"

        # In case rsync had something on stderr
        if [[ -s $LOG_RSYNC_ERROR ]]; then
            args+=(-a $LOG_RSYNC_ERROR -a $LOG)
            subject="ERROR: rsync errors"
            send=1
        # In case of direct errors
        elif [[ $rc -gt 0 ]]; then
            args+=(-a $LOG)
            subject="ERROR"
            send=1
        # In case admin want all logs
        elif [[ ${ERRORSONLY} = false ]]; then
            args+=(-a $LOG)
            if [[ ${LOG_ERROR:-} ]]; then
                subject="ERROR"
            fi
            send=1
        fi
        if [[ $send ]]; then
            # Someone wants full logs including rsync
            if [[ ${FULLLOGS} = true ]]; then
                args+=(-a $LOG_RSYNC)
            fi
            mailf "${args[@]}" -s "[${PROGRAM}@$(hostname -s)] ${subject}" ${MAILTO}
        fi
    fi

    savelog "${LOG_RSYNC}"
    savelog "${LOG_RSYNC_ERROR}"
    savelog "$LOG" > /dev/null

    rm -f "${LOCK}"

    exit $rc
}

run_rsync() {
  local t=$1
  shift

  log "Running $t:" "${_RSYNC[@]}" "$@"

  "${_RSYNC[@]}" "$@" \
    >>"${LOG_RSYNC_ERROR}" 2>&1 || return $?
}

# Check rsyncs return value
check_rsync() {
    ret=$1
    msg=$2

    # Lets get a statistical value
    if [[ -f ${LOG_RSYNC} ]]; then
        SPEED=$(tail -n 2 ${LOG_RSYNC} | sed -Ene 's#.* ([0-9.,]+) bytes/sec#\1#p')
        if [[ ${SPEED} ]]; then
            SPEED=${SPEED%%.*}
            SPEED=${SPEED//,}
            SPEED=$(( SPEED / 1024 ))
            log "Latest recorded rsync transfer speed: ${SPEED} KB/s"
        fi
    fi

    # 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
    # and us re-running. If it's not, uplink is broken anyways.
    case "${ret}" in
        0) return 0;;
        24) return 0;;
        23) return 2;;
        30) return 2;;
        *)
            error "ERROR: ${msg}"
            return 1
            ;;
    esac
}

function tracefile_content() {
    set +e

    LC_ALL=POSIX LANG=POSIX date -u
    rfc822date=$(LC_ALL=POSIX LANG=POSIX date -u -R)
    echo "Date: ${rfc822date}"
    echo "Date-Started: ${DATE_STARTED}"

    if [[ -e $TRACEFILE_MASTER ]]; then
        echo "Archive serial: $(extract_trace_serial $TRACEFILE_MASTER || echo unknown )"
    fi

    echo "Used ftpsync version: ${VERSION}"
    echo "Creator: ftpsync ${VERSION}"
    echo "Running on host: ${TRACEHOST}"

    if [[ ${INFO_MAINTAINER:-} ]]; then
        echo "Maintainer: ${INFO_MAINTAINER}"
    fi
    if [[ ${INFO_SPONSOR:-} ]]; then
        echo "Sponsor: ${INFO_SPONSOR}"
    fi
    if [[ ${INFO_COUNTRY:-} ]]; then
        echo "Country: ${INFO_COUNTRY}"
    fi
    if [[ ${INFO_LOCATION:-} ]]; then
        echo "Location: ${INFO_LOCATION}"
    fi
    if [[ ${INFO_THROUGHPUT:-} ]]; then
        echo "Throughput: ${INFO_THROUGHPUT}"
    fi
    if [[ ${INFO_TRIGGER:-} ]]; then
        echo "Trigger: ${INFO_TRIGGER}"
    fi

    if [[ -d ${TO}/dists ]]; then
        ARCH=$(find ${TO}/dists \( -name 'Packages.*' -o -name 'Sources.*' \) 2>/dev/null |
            sed -Ene 's#.*/binary-([^/]+)/Packages.*#\1#p; s#.*/(source)/Sources.*#\1#p' |
            sort -u | tr '\n' ' ')
        if [[ $ARCH ]]; then
            echo "Architectures: ${ARCH}"
        fi
    fi
    if [[ ${ARCH_INCLUDE} ]]; then
        echo "Architectures-Configuration: INCLUDE $(tr ' ' '\n' <<< ${ARCH_INCLUDE} | sort -u | tr '\n' ' ')"
    elif [[ ${ARCH_EXCLUDE} ]]; then
        echo "Architectures-Configuration: EXCLUDE $(tr ' ' '\n' <<< ${ARCH_EXCLUDE} | sort -u | tr '\n' ' ')"
    else
        echo "Architectures-Configuration: ALL"
    fi
    echo "Upstream-mirror: ${RSYNC_HOST:-unknown}"
    echo "Rsync-Transport: ${RSYNC_TRANSPORT}"
    total=0
    if [[ -e ${LOG_RSYNC} ]]; then
        for bytes in $(sed -Ene 's/(^|.* )sent ([0-9]+) bytes  received ([0-9]+) bytes.*/\3/p' "${LOG_RSYNC}"); do
            total=$(( total + bytes ))
        done
        if [[ $total -gt 0 ]]; then
            echo "Total bytes received in rsync: ${total}"
        fi
    fi
    total_time=$(( STATS_TOTAL_RSYNC_TIME1 + STATS_TOTAL_RSYNC_TIME2 ))
    echo "Total time spent in stage1 rsync: ${STATS_TOTAL_RSYNC_TIME1}"
    echo "Total time spent in stage2 rsync: ${STATS_TOTAL_RSYNC_TIME2}"
    echo "Total time spent in rsync: ${total_time}"
    if [[ 0 != ${total_time} ]]; then
        rate=$(( total / total_time ))
        echo "Average rate: ${rate} B/s"
    fi

    set -e
}

# Write a tracefile
tracefile() {
    local TRACEFILE=${1:-"${TO}/${TRACE}"}
    local TRACEFILE_MASTER="${TO}/${TRACEDIR}/master"

    tracefile_content > "${TRACEFILE}.new"
    mv "${TRACEFILE}.new" "${TRACEFILE}"

    {
        if [[ -e ${TO}/${TRACEHIERARCHY}.mirror ]]; then
            cat ${TO}/${TRACEHIERARCHY}.mirror
        fi
        echo "$(basename "${TRACEFILE}") ${MIRRORNAME} ${TRACEHOST} ${RSYNC_HOST:-unknown}"
    } > "${TO}/${TRACEHIERARCHY}".new
    mv "${TO}/${TRACEHIERARCHY}".new "${TO}/${TRACEHIERARCHY}"
    cp "${TO}/${TRACEHIERARCHY}" "${TO}/${TRACEHIERARCHY}.mirror"

    (cd "${TO}/${TRACEDIR}" && ls -1rt $(find * -type f \! -name "_*")) > "${TO}/${TRACELIST}"
}

arch_imexclude() {
    local param="$1" arch="$2"
    if [[ $arch = source ]]; then
        _RSYNC+=(
            "--filter=${param}_/dists/**/source/"
            "--filter=${param}_/pool/**/*.tar.*"
            "--filter=${param}_/pool/**/*.diff.*"
            "--filter=${param}_/pool/**/*.dsc"
        )
    else
        _RSYNC+=(
            "--filter=${param}_/dists/**/binary-${arch}/"
            "--filter=${param}_/dists/**/installer-${arch}/"
            "--filter=${param}_/dists/**/Contents-${arch}.gz"
            "--filter=${param}_/dists/**/Contents-udeb-${arch}.gz"
            "--filter=${param}_/dists/**/Contents-${arch}.diff/"
            "--filter=${param}_/indices/**/arch-${arch}.files"
            "--filter=${param}_/indices/**/arch-${arch}.list.gz"
            "--filter=${param}_/pool/**/*_${arch}.deb"
            "--filter=${param}_/pool/**/*_${arch}.udeb"
            "--filter=${param}_/pool/**/*_${arch}.changes"
        )
    fi
}

arch_exclude() {
    arch_imexclude exclude "$1"
}

arch_include() {
    arch_imexclude include "$1"
}

# Learn which archs to include/exclude based on ARCH_EXCLUDE and ARCH_INCLUDE
# settings.
# Sets EXCLUDE (which might also have --include statements
# followed by a --exclude *_*.<things>.
set_exclude_include_archs() {
    if [[ -n "${ARCH_EXCLUDE}" ]] && [[ -n "${ARCH_INCLUDE}" ]]; then
        echo >&2 "ARCH_EXCLUDE and ARCH_INCLUDE are mutually exclusive.  Set only one."
        exit 1
    fi

    if [[ -n "${ARCH_EXCLUDE}" ]]; then
        for ARCH in ${ARCH_EXCLUDE}; do
            arch_exclude ${ARCH}
        done
        arch_include '*'
        arch_include source
    elif [[ -n "${ARCH_INCLUDE}" ]]; then
        local include_arch_all=false
        for ARCH in ${ARCH_INCLUDE}; do
            arch_include ${ARCH}
            if [[ ${ARCH} != source ]]; then
                include_arch_all=true
            fi
        done
        if [[ true = ${include_arch_all} ]]; then
            arch_include all
        fi
        arch_exclude '*'
        arch_exclude source
    fi
}

########################################################################
########################################################################


# As what are we called?
NAME="$(basename $0)"

# What should we do?
ARCHIVE=
# Do we sync stage1?
SYNCSTAGE1=false
# Do we sync stage2?
SYNCSTAGE2=false
# Do we sync all?
SYNCALL=true
# Do we have a mhop sync?
SYNCMHOP=false
# Do we callback? (May get changed later)
SYNCCALLBACK=false

while getopts T: option; do
    case $option in
        T) INFO_TRIGGER=$OPTARG;;
        ?) exit 64;;
    esac
done
shift $(($OPTIND - 1))

# Now, check if we got told about stuff via ssh
if [[ -n ${SSH_ORIGINAL_COMMAND:-} ]]; then
    INFO_TRIGGER=${INFO_TRIGGER:-ssh}
    check_commandline ${SSH_ORIGINAL_COMMAND}
fi

# Now, we can locally override all the above variables by just putting
# them into the .ssh/authorized_keys file forced command.
if [[ $# -gt 0 ]]; then
    check_commandline "$@"
fi

# If we have been told to do stuff for a different archive than default,
# set the name accordingly.
if [[ -n ${ARCHIVE} ]]; then
    NAME="${NAME}-${ARCHIVE}"
fi

# Now source the config for the archive we run on.
# (Yes, people can also overwrite the options above in the config file
# if they want to)
read_config "${NAME}.conf"

create_logdir

########################################################################
# Config defaults                                                      #
########################################################################
MIRRORNAME=${MIRRORNAME:-$(hostname -f)}
TO=${TO:-"/srv/mirrors/debian/"}
MAILTO=${MAILTO:-${LOGNAME:?Environment variable LOGNAME unset, please check your system or specify MAILTO}}
HUB=${HUB:-"false"}

# Connection options
if [[ -z ${RSYNC_SOURCE:-} ]]; then
    RSYNC_HOST=${RSYNC_HOST:?Missing a host to mirror from, please set RSYNC_HOST variable in ${CURRENT_CONFIG}}
    RSYNC_PATH=${RSYNC_PATH:-"debian"}
    RSYNC_USER=${RSYNC_USER:-""}
fi
RSYNC_PASSWORD=${RSYNC_PASSWORD:-""}
if [[ ${RSYNC_SSL:-} = true ]]; then
    RSYNC_TRANSPORT=${RSYNC_TRANSPORT:-"ssl"}
else
    RSYNC_TRANSPORT=${RSYNC_TRANSPORT:-"undefined"}
fi
RSYNC_SSL_PORT=${RSYNC_SSL_PORT:-"1873"}
RSYNC_SSL_CAPATH=${RSYNC_SSL_CAPATH:-"/etc/ssl/certs"}
RSYNC_SSL_METHOD=${RSYNC_SSL_METHOD:-"stunnel"}
RSYNC_PROXY=${RSYNC_PROXY:-""}

# Include and exclude options
ARCH_INCLUDE=${ARCH_INCLUDE:-""}
ARCH_EXCLUDE=${ARCH_EXCLUDE:-""}
EXCLUDE=${EXCLUDE:-""}

# Log options
LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
ERRORSONLY=${ERRORSONLY:-"true"}
FULLLOGS=${FULLLOGS:-"false"}
LOGROTATE=${LOGROTATE:-14}
LOG_RSYNC="${LOGDIR}/rsync-${NAME}.log"
LOG_RSYNC_ERROR="${LOGDIR}/rsync-${NAME}.error"

# Other options
LOCKTIMEOUT=${LOCKTIMEOUT:-3600}
UIPSLEEP=${UIPSLEEP:-1200}
UIPRETRIES=${UIPRETRIES:-3}
TRACEHOST=${TRACEHOST:-$(hostname -f)}
RSYNC=${RSYNC:-rsync}
RSYNC_PROTOCOL=$(rsync_protocol)
RSYNC_EXTRA=${RSYNC_EXTRA:-""}
RSYNC_BW=${RSYNC_BW:-0}
if [[ $RSYNC_PROTOCOL -ge 31 ]]; then
    RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --safe-links --chmod=D755,F644 --timeout 120 --stats --no-human-readable --no-inc-recursive"}
else
    RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --safe-links --timeout 120 --stats --no-human-readable --no-inc-recursive"}
fi
RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--include=*.diff/ --exclude=*.diff/Index --exclude=Packages* --exclude=Sources* --exclude=Release* --exclude=InRelease --include=i18n/by-hash --exclude=i18n/* --exclude=ls-lR*"}
RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-delay --delete-excluded"}
CALLBACKUSER=${CALLBACKUSER:-"archvsync"}
CALLBACKHOST=${CALLBACKHOST:-"none"}
CALLBACKKEY=${CALLBACKKEY:-"none"}

# Hooks
HOOK1=${HOOK1:-""}
HOOK2=${HOOK2:-""}
HOOK3=${HOOK3:-""}
HOOK4=${HOOK4:-""}
HOOK5=${HOOK5:-""}
########################################################################
########################################################################

# used by log() and error()
PROGRAM=${PROGRAM:-"${NAME}"}

# Our trace and lock files
LOCK_NAME="Archive-Update-in-Progress-${MIRRORNAME}"
LOCK="${TO}/${LOCK_NAME}"
UPDATEREQUIRED_NAME="Archive-Update-Required-${MIRRORNAME}"
UPDATEREQUIRED="${TO}/${UPDATEREQUIRED_NAME}"
TRACEDIR=project/trace
TRACE="${TRACEDIR}/${MIRRORNAME}"
TRACE_STAGE1="${TRACEDIR}/${MIRRORNAME}-stage1"
TRACEHIERARCHY="${TRACEDIR}/_hierarchy"
TRACELIST="${TRACEDIR}/_traces"

_TRACE_FILES=(
  "${LOCK_NAME}"
  "${UPDATEREQUIRED_NAME}"
  "${TRACE}"
  "${TRACE_STAGE1}"
  "${TRACEHIERARCHY}"
  "${TRACELIST}"
)

_RSYNC=(
  $RSYNC
  --quiet
  --log-file "${LOG_RSYNC}"
)

# Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete
# excluded files
for i in ${_TRACE_FILES[@]}; do
  _RSYNC+=("--filter=exclude_/${i}" "--filter=protect_/${i}")
done
_RSYNC+=(
  "--filter=include_/project/"
  "--filter=protect_/project/"
  "--filter=include_/project/trace/"
  "--filter=protect_/project/trace/"
  "--filter=include_/project/trace/*"
)

# Default rsync options for *every* rsync call
# Now add the bwlimit option. As default is 0 we always add it, rsync interprets
# 0 as unlimited, so this is safe.
_RSYNC+=(${RSYNC_EXTRA} --bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS} ${EXCLUDE})

# collect some stats
STATS_TOTAL_RSYNC_TIME1=0
STATS_TOTAL_RSYNC_TIME2=0

# The temp directory used by rsync --delay-updates is not
# world-readable remotely. Always exclude it to avoid errors.
_RSYNC+=("--exclude=.~tmp~/")

if [[ ${RSYNC_TRANSPORT} = undefined ]]; then
    :
elif [[ ${RSYNC_TRANSPORT} = ssh ]]; then
    _RSYNC+=(-e "ssh")
elif [[ ${RSYNC_TRANSPORT} = ssl ]]; then
    export RSYNC_SSL_PORT
    export RSYNC_SSL_CAPATH
    export RSYNC_SSL_METHOD
    _RSYNC+=(-e "${BINDIR:+${BINDIR}/}rsync-ssl-tunnel")
else
    echo "Unknown rsync transport configured (${RSYNC_TRANSPORT})" >&2
    exit 1
fi

# Exclude architectures defined in $ARCH_EXCLUDE
set_exclude_include_archs

########################################################################
# Really nothing to see below here. Only code follows.                 #
########################################################################
########################################################################
DATE_STARTED=$(LC_ALL=POSIX LANG=POSIX date -u -R)

# Some sane defaults
cd "${BASEDIR:-}"
umask 022

# If we are here for the first time, create the
# destination and the trace directory
mkdir -p "${TO}/${TRACEDIR}"

# Used to make sure we will have the archive fully and completly synced before
# we stop, even if we get multiple pushes while this script is running.
# Otherwise we can end up with a half-synced archive:
# - get a push
# - sync, while locked
# - get another push. Of course no extra sync run then happens, we are locked.
# - done. Archive not correctly synced, we don't have all the changes from the second push.
touch "${UPDATEREQUIRED}"

# Check to see if another sync is in progress
if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then
    if [[ ${BASH_VERSINFO[0]} -gt 3 ]] || [[ -L /proc/self ]]; then
        # We have a recent enough bash version, lets do it the easy way,
        # the lock will contain the right pid, thanks to $BASHPID
        if ! $(kill -0 $(< ${LOCK}) 2>/dev/null); then
            # Process does either not exist or is not owned by us.
            echo "$$" > "${LOCK}"
        else
            echo "Unable to start rsync, lock file still exists, PID $(< ${LOCK})"
            exit 1
        fi
    else
        # Old bash, means we dont have the right pid in our lockfile
        # So take a different way - guess if it is still there by comparing its age.
        # Not optimal, but hey.
        stamptime=$(date --reference="${LOCK}" +%s)
        unixtime=$(date +%s)
        difference=$(( $unixtime - $stamptime ))
        if [[ ${difference} -ge ${LOCKTIMEOUT} ]]; then
            # Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock
            echo "$$" > "${LOCK}"
        else
            echo "Unable to start rsync, lock file younger than one hour"
            exit 1
        fi
    fi
fi

# We want to cleanup always
trap cleanup EXIT TERM HUP INT QUIT

# Open log and close stdin
open_log $LOG
exec 2>&1 <&-
log "Mirrorsync start"

# Look who pushed us and note that in the log.
SSH_CONNECTION=${SSH_CONNECTION:-""}
PUSHFROM="${SSH_CONNECTION%%\ *}"
if [[ -n ${PUSHFROM} ]]; then
    log "We got pushed from ${PUSHFROM}"
fi

if [[ true = ${SYNCCALLBACK} ]]; then
    if [[ none = ${CALLBACKHOST} ]] || [[ none = ${CALLBACKKEY} ]]; then
        SYNCCALLBACK="false"
        error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback"
    fi
fi

HOOK=(
    HOOKNR=1
    HOOKSCR=${HOOK1}
)
hook $HOOK

# Now, we might want to sync from anonymous too.
# This is that deep in this script so hook1 could, if wanted, change things!
if [[ -z ${RSYNC_SOURCE:-} ]]; then
    if [[ -z ${RSYNC_USER:-} ]]; then
        RSYNC_SOURCE="${RSYNC_HOST}::${RSYNC_PATH}"
    else
        RSYNC_SOURCE="${RSYNC_USER}@${RSYNC_HOST}::${RSYNC_PATH}"
    fi
fi

_RSYNC+=("${RSYNC_SOURCE}" "$TO")

# Now do the actual mirroring, and run as long as we have an updaterequired file.
export RSYNC_PASSWORD
export RSYNC_PROXY

UPDATE_RETRIES=0

while [[ -e ${UPDATEREQUIRED} ]]; do
    log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"

    # if we want stage1 *or* all
    if [[ true = ${SYNCSTAGE1} ]] || [[ true = ${SYNCALL} ]]; then
        while [[ -e ${UPDATEREQUIRED} ]]; do
            rm -f "${UPDATEREQUIRED}"
            # Step one, sync everything except Packages/Releases
            rsync_started=$(date +%s)
            result=0
            run_rsync "stage1" ${RSYNC_OPTIONS1} || result=$?
            rsync_ended=$(date +%s)
            STATS_TOTAL_RSYNC_TIME1=$(( STATS_TOTAL_RSYNC_TIME1 + rsync_ended - rsync_started ))

            log "Back from rsync with returncode ${result}"
        done
    else
        time1=$(extract_trace_field 'Total time spent in stage1 rsync' "${TO}/${TRACE_STAGE1}" || :)
        if [[ $time1 ]]; then
            STATS_TOTAL_RSYNC_TIME1="$time1"
        fi
        # Fake a good resultcode
        result=0
    fi # Sync stage 1?
    rm -f "${UPDATEREQUIRED}"

    set +e
    check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}"
    GO=$?
    set -e
    if [[ ${GO} -eq 2 ]] && [[ -e ${UPDATEREQUIRED} ]]; then
        log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
    elif [[ ${GO} -ne 0 ]]; then
        exit 3
    fi

    HOOK=(
        HOOKNR=2
        HOOKSCR=${HOOK2}
    )
    hook $HOOK

    # if we want stage2 *or* all
    if [[ true = ${SYNCSTAGE2} ]] || [[ true = ${SYNCALL} ]]; then
        upstream_uip=false
        for aupfile in "${TO}/Archive-Update-in-Progress-"*; do
            case "$aupfile" in
                "${TO}/Archive-Update-in-Progress-*")
                    error "Lock file is missing, this should not happen"
                    ;;
                "${LOCK}")
                    :
                    ;;
                *)
                    if [[ -f $aupfile ]]; then
                        # Remove the file, it will be synced again if
                        # upstream is still not done
                        rm -f "$aupfile"
                    else
                        log "AUIP file '$aupfile' is not really a file, weird"
                    fi
                    upstream_uip=true
                    ;;
            esac
        done

        if [[ true = ${upstream_uip} ]]; then
            log "Upstream archive update in progress, skipping stage2"
            if [[ ${UPDATE_RETRIES} -lt ${UIPRETRIES} ]]; then
                log "Retrying update in ${UIPSLEEP}"
                touch "${UPDATEREQUIRED}"
                UPDATE_RETRIES=$(($UPDATE_RETRIES+1))
                sleep "${UIPSLEEP}"
                result=0
            else
                error "Update has been retried ${UPDATE_RETRIES} times, aborting"
                log "Perhaps upstream is still updating or there's a stale AUIP file"
                result=1
            fi
        else
            # We are lucky, it worked. Now do step 2 and sync again, this time including
            # the packages/releases files
            rsync_started=$(date +%s)
            result=0
            run_rsync "stage2" ${RSYNC_OPTIONS2} || result=$?
            rsync_ended=$(date +%s)
            STATS_TOTAL_RSYNC_TIME2=$(( STATS_TOTAL_RSYNC_TIME2 + rsync_ended - rsync_started ))

            log "Back from rsync with returncode ${result}"
        fi
    else
        # Fake a good resultcode
        result=0
    fi # Sync stage 2?

    set +e
    check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}"
    GO=$?
    set -e
    if [[ ${GO} -eq 2 ]] && [[ -e ${UPDATEREQUIRED} ]]; then
        log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
    elif [[ ${GO} -ne 0 ]]; then
        exit 4
    fi

    HOOK=(
        HOOKNR=3
        HOOKSCR=${HOOK3}
    )
    hook $HOOK
done

# We only update our tracefile when we had a stage2 or an all sync.
# Otherwise we would update it after stage1 already, which is wrong.
if [[ true = ${SYNCSTAGE2} ]] || [[ true = ${SYNCALL} ]]; then
    tracefile
    if [[ true = ${SYNCALL} ]]; then
        rm -f "${TO}/${TRACE_STAGE1}"
    fi
elif [[ true = ${SYNCSTAGE1} ]]; then
    tracefile "${TO}/${TRACE_STAGE1}"
fi


HOOK=(
    HOOKNR=4
    HOOKSCR=${HOOK4}
)
hook $HOOK

if [[ true = ${SYNCCALLBACK} ]]; then
    set +e
    callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}"
    set -e
fi

# Remove the Archive-Update-in-Progress file before we push our downstreams.
rm -f "${LOCK}"

declare -f -F send_mail_new_version > /dev/null && send_mail_new_version || :

if [[ ${HUB} = true ]]; then
    # Trigger slave mirrors if we had a push for stage2 or all, or if its mhop
    if [[ true = ${SYNCSTAGE2} ]] || [[ true = ${SYNCALL} ]] || [[ true = ${SYNCMHOP} ]]; then
        RUNMIRRORARGS=""
        if [[ -n ${ARCHIVE} ]]; then
            # We tell runmirrors about the archive we are running on.
            RUNMIRRORARGS="-a ${ARCHIVE}"
        fi
        # We also tell runmirrors that we are running it from within ftpsync, so it can change
        # the way it works with mhop based on that.
        RUNMIRRORARGS="${RUNMIRRORARGS} -f"

        if [[ true = ${SYNCSTAGE1} ]]; then
            # This is true when we have a mhop sync. A normal multi-stage push sending stage1 will
            # not get to this point.
            # So if that happens, tell runmirrors we are doing mhop
            RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop"
        elif [[ true = ${SYNCSTAGE2} ]]; then
            RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2"
        elif [[ true = ${SYNCALL} ]]; then
            RUNMIRRORARGS="${RUNMIRRORARGS} -k all"
        fi
        log "Trigger slave mirrors using ${RUNMIRRORARGS}"
        ${BINDIR:+${BINDIR}/}runmirrors ${RUNMIRRORARGS}
        log "Trigger slave done"

        HOOK=(
            HOOKNR=5
            HOOKSCR=${HOOK5}
        )
        hook $HOOK
    fi
fi