File: .%23apt-proxy.1.29

package info (click to toggle)
apt-proxy 1.3.0
links: PTS
area: main
in suites: woody
size: 188 kB
ctags: 10
sloc: sh: 236; makefile: 39
file content (1327 lines) | stat: -rwxr-xr-x 34,494 bytes
#!/bin/ash
# (You may safely change the above to /bin/sh if you do not have ash available)
#
# Copyright Paul Russell <rusty@samba.org>
# Released under the GPL version 2 or later
#	Fixed by Stephen Rothwell <sfr@canb.auug.org.au>
#       Stuff by Martin Schwenke <martin@meltin.net>
#	Maintained and packaged for Debian by Chris Halls <chris.halls@gmx.de>
#
#	`perl!' -- Anton Blanchard
#	`I was joking!' -- Paul Mackerras
#	`Note that Rusty's name is at the top' -- Stephen Rothwell.
#	`Perl!!!  Please?' -- Martin Schwenke
APT_PROXY_VERSION=1.2.9.3

# Default values
APT_PROXY_LOGFILE=/dev/null
CONFIG_FILE=/etc/apt-proxy/apt-proxy.conf
RSYNC=rsync
WGET=wget

# Parse commandline
while [ $# -gt 0 ]; do
	case "$1" in
	  '-c')
 		shift
		CONFIG_FILE="$1"
		shift
		;;
	  '-l')
		shift
		APT_PROXY_LOGFILE="$1"
		shift
		;;
	  *)
		APT_PROXY_LOGFILE="$1"
		shift
		break
		;;
	esac
done

# Before anything, save output in fd 3, and redirect others to arg1 if avail
exec 3>&1 1>>$APT_PROXY_LOGFILE 2>&1


# Write a debug message to the log if debug is enabled
#	debug([string ...])
debug()
{
    [ -n "$DEBUG" ] && echo "[$$ `date +%X`]": "$@" >> $APT_PROXY_LOGFILE
}

# Write a message to the log.
#	log([string ...])
log()
{
    echo "[$$ `date +%X`] " "$@" >> $APT_PROXY_LOGFILE
}

# Determine the size of a file
#	file_size(file name)
#
if [ -x "/usr/bin/stat" ]; then
    file_size()
    {
	[ -z "$1" -o ! -f "$1" ] && return 1
	set -- `/usr/bin/stat -tl "$1"`
	[ -z "$2" ] && return 1
	echo $2
    }
else
    file_size()
    {
	[ -z "$1" -o ! -f "$1" ] && return 1
	echo `wc -c < "$1"`
    }
fi

# Dump the chunk of a file skipping $1 bytes for an
# optional length (to end of file if missing)
#	dump_part_file(file name, start[, length])
#
if tail -c +10 /dev/null 2>/dev/null; then
    dump_part_file()
    {
	dpf_start=`expr $2 + 1`
	if [ -n "$3" ]; then
	    # The tail version doesn't seem to return an error code
	    # if the file no longer exists
	    # tail -c +$dpf_start "$1" | head -c $3
	    dd ibs=1 obs=16384 skip=$2 count=$3 if="$1" 2>/dev/null
	else
	    tail -c +$dpf_start "$1"
	fi
    }
else
    dump_part_file()
    {
	if [ -n "$3" ]; then
	    dd ibs=1 obs=16384 skip=$2 count=$3 if="$1" 2>/dev/null
	else
	    (dd bs=1 count=$2 of=/dev/null 2>/dev/null && cat) < "$1"
	fi
    }
fi
	
# Write a http reply header
#	write_header(response code, keep-alive, connection type, [content type] [content length]
#			[last modified time] [content encoding]
#			[filename])
#
write_header()
{
    debug "write_header $1, keep-alive:$2 content:$3 size:$4 date:$5 enc:$6 request:$7"
    
    # apt sometimes barfs if this isn't in a single packet.
    # That cost me about a week of debugging 8(.
    WH_HEADER="HTTP/1.0 $1\r\nDate: `date -u -R`\r\nServer: Apt-proxy $APT_PROXY_VERSION\r\n"
    # Say whether we are keeping the connection open or not
    WH_HEADER="${WH_HEADER}Connection: $2\r\n"
    [ -n "$3" ] && WH_HEADER="${WH_HEADER}Content-Type: $3\r\n"
    [ -n "$4" ] && WH_HEADER="${WH_HEADER}Content-Length: $4\r\n"
    [ -n "$5" ] && WH_HEADER="${WH_HEADER}Last-Modified: $5\r\n"
    [ -n "$6" ] && WH_HEADER="${WH_HEADER}Content-Encoding: $6\r\n"
    [ -n "$7" ] && [ -n "$DEBUG" ] &&
	WH_HEADER="${WH_HEADER}X-You-Wanted: $7\r\n"

    # dd forces it to sane blocks.
    printf "%b\r\n" "$WH_HEADER" | dd 2>/dev/null >&3

}

# Write out bad config file HTTP response, and exits.
#	bad_config(args...)
bad_config()
{
    log Bad configuration "$@"
    mesg="<HTML><HEAD>\r
<TITLE>500 Bad configuration</TITLE>\r
</HEAD><BODY>\r
<H1>Bad configuration file</H1>\r
The configuration file $CONFIG_FILE is $@.<P>\r
<P>\r
</BODY></HTML>\r\n"
    write_header "500 Bad configuration" close text/html `printf %b "$mesg" | wc -c`
    printf "%b" "$mesg" 1>&3
    exit 1
}

# Called from config script to add back ends.
# APT_PROXY_BACKENDS is a list of backends seperated by spaces.
#  Whitespace from the add_backend command is replaced by commas
#	add_backend(url-prefix, file-prefix, rsync-prefix...)
add_backend()
{
    [ $# -ge 3 ] || bad_config Bad add_backend "$@".
    [ -d "$2" ] || mkdir -p "$2" || bad_config apt-proxy directory "$2" doesn\'t exist\!
    APT_PROXY_BACKENDS="$APT_PROXY_BACKENDS`echo $@ | tr -s ' \t' ,` "
}

# Download a file.  -d means url is a directory
#  download_url([-d] url, destinationDirectory, destinationFile)
download_url()
{
    unset RET DL_DIR

    if [ $1 = "-d" ]; then
      DL_DIR=y
      shift
    fi

    # DL_URL: URL to download				(http://ftp.debian.org/pool/a/aaa-a.deb)
    # DL_DESTDIR: Destination directory for file	(/var/cache/apt-proxy/debian/pool/a)
    # DL_DESTFILE: Destination filename			(aaa-a.deb)
    # DL_FILE: Complete pathname for destination file	(/var/cache/apt-proxy/debian/pool/a/aaa-a.deb)
    DL_URL=$1
    DL_DESTDIR=$2
    DL_DESTFILE=$3
    DL_FILE=$DL_DESTDIR/$DL_DESTFILE

    debug "download_url:$DL_URL dest:$DL_DESTDIR file:$DL_DESTFILE IsDir:[$DL_DIR]"

    DL_LOG=$DL_FILE.log

    case "$DL_URL" in
        http://*|ftp://*)
	    #WGET_CMD="$WGET --non-verbose --timestamping --no-host-directories \
	    WGET_CMD="$WGET --timestamping --no-host-directories \
			--no-directories -P $DL_DESTDIR"
	    if [ -z "$DL_DIR" ];then
	    	# File
	    	DL_TEMPFILE=$DL_DESTDIR/.`echo $DL_DESTFILE | tr -s / _`.wget

		debug $WGET_CMD -O $DL_TEMPFILE $DL_URL
		$WGET_CMD -O $DL_TEMPFILE $DL_URL > $DL_LOG 2>&1
	        RET=$?
		if [ $RET = 0 ];then
		    # Success

		    # If the remote file is not newer than the local file, the
		    # output file is zero length
		    if [ -s $DL_TEMPFILE ]; then
			# move file to final location
			mv $DL_TEMPFILE $DL_FILE
		    else
			rm -f $DL_TEMPFILE
		    fi

		else
		    # Remove empty file created by wget
		    rm -f $DL_TEMPFILE
		fi
	    else
	    	# Directory

		# Note: ftp using wget will create a .listing file in the parent
		#  directory.  Theoretically, two concurrent wgets could conflict
		#  with this file, even though the contents would be the same.
		#  (I couldn't make this happen, but this is just a note in
		#  case this actually happens sometime)
		debug $WGET_CMD -O /dev/null $DL_URL/
		$WGET_CMD -O /dev/null $DL_URL/ > $DL_LOG 2>&1
	        RET=$?
		if [ $RET = 0 ];then
		    # Success - create directory
		    mkdir $DL_FILE && rm $DL_LOG
		fi
	    fi
	    debug "wget: `cat $DL_LOG`"
	    debug "got file: `ls -l $DL_FILE`"
	    ;;
	*::*)
	    RSYNC_CMD="$RSYNC -v --perms --links --times --partial"

	    if [ -z "$DL_DIR" ];then
	        # File

		# Try to rsync uncompressed version?
		case $DL_URL in
		*Packages.gz)
		    debug "Rsyncing uncompressed version ****************************************"
		    # Extract filename for uncompressed file from URL
		    TMP_DESTFILE=`echo $DL_URL | sed 's|^.*/\(.*\)\.gz$|\1|'`
		    TMP_FILE=$DL_DESTDIR/$TMP_DESTFILE
		    TMP_URL=`echo $DL_URL | sed 's/\.gz$//'`
		    
		    [ $TMP_FILE -ot $DL_FILE ] && gunzip -c < $DL_FILE > $TMP_FILE 

		    if [ -f $FILE ]; then
		        # We have an uncompressed file to rsync against
			debug $RSYNC_CMD -z $TMP_URL $TMP_FILE
			if $RSYNC_CMD -z $TMP_URL $TMP_FILE > $DL_LOG 2>&1 ;then
			    # Success.  Now compress file.
			    debug Succeeded - compressing $TMP_FILE
			    gzip -9n < $TMP_FILE > $DL_FILE
			    touch -r $TMP_FILE $DL_FILE
			    RET=0
			fi
	    		[ -n "$KEEP_STATS" ] && log rsync $TMP_DESTFILE: `tail -2 $DL_LOG`
		    fi
		    debug "*** END Rsyncing uncompressed version ****************************************"
		    ;;
		esac

		if [ -z "$RET" ]; then
		    # Retrieve normally
		    debug $RSYNC_CMD $DL_URL $DL_FILE 
		    $RSYNC_CMD $DL_URL $DL_FILE > $DL_LOG 2>&1
		    RET=$?
		fi

            else
	        # Directory
	        debug $RSYNC_CMD $DL_URL
	        $RSYNC_CMD $DL_URL > $DL_LOG 2>&1
	        RET=$?
		[ $RET = 0 ] && mkdir $DL_FILE

	    fi
	    
	    [ -n "$KEEP_STATS" ] && log $DL_DEST `tail -2 $DL_LOG`

	    ;;
	*)
	    log Unknown backend type:$DL_URL
	    RET=1
    esac

    rm -f $DL_LOG
    return $RET
}



# Takes URL, reason, spits out 404 and exits if keep-alive=close.
#	bad_url(keep-alive, url, reason...)
bad_url()
{
    keepalive=$1
    shift

    log Bad URL "$@"
    URL="$1"
    shift
    mesg="<HTML><HEAD>\r
<TITLE>404 Not Found</TITLE>\r
</HEAD><BODY>\r
<H1>Not Found</H1>\r
While serving the requested URL $URL: $@.<P>\r
Really.<P>\r
</BODY></HTML>\r\n"
    write_header "404 `echo \"$@\" | tail -1`" $keepalive text/html `printf %b "$mesg" | wc -c`
    printf "%b\n" "$mesg" 1>&3                                                                                   

    # Exit if no keep alive
    [ "$keepalive" = "close" ] && exit 1
}

# Relative or same directory is OK.
#	is_sane(symlink)
is_sane()
{
    case "$1" in ..*) return 0 ;; */*) return 1 ;; *) return 0; esac
}

# Tries all backends for a link.
#	rsync_link(linkname, backends...)
rsync_link_old()
{
    RSL_LINK=$1
    shift

    for rsl_back in $@; do
	log "rync_link: $RSYNC -l $rsl_back$RSL_LINK"
	if rsl_link="`$RSYNC -l $rsl_back$RSL_LINK`"; then
	    # Ignore server messages.
	    echo "$rsl_link" | tail -2 | grep '^l.*->' | sed 's/.*-> //'
	    return 0
	fi
    done
    return 1
}

rsync_link()
{
    RSL_LINK=$1
    RSL_SUBDIR=$2
    shift 2
echo RSYNC_LINK: Link:$RSL_LINK Subdir:$RSL_SUBDIR, backends:$@
    for rsl_back in $@; do
	if download_url -d $rsl_back$RSL_LINK $RSL_SUBDIR $RSL_LINK; then
	    return 0
	fi
    done
    return 1

}


# Given a directory, print out all the parents in forwards order.
#	directories(pathname)
directories()
{
    DR_TOTAL=""
    for dr_d in `echo $1 | tr / ' '`; do
	DR_TOTAL="$DR_TOTAL/$dr_d"
	echo "$DR_TOTAL"
    done
}

# Make a directory tree, checking that the counterpart are real dirs
#	mkdir_maybe_symlinks(frontend base, subdir, backends...)
mkdir_maybe_symlinks_old()
{
    [ -d "$1$2" ] && return
    MMS_FRONTBASE="$1"
    MMS_SUBDIR="$2"
    shift 2

    for mms_d in `directories $MMS_SUBDIR`; do
	if [ ! -d "$MMS_FRONTBASE$mms_d" ]; then
	    MMS_L="`rsync_link \"$mms_d\" \"$@\"`" ||
		bad_url close "$mms_d" "directory does not exist on any server"
	    if [ -n "$MMS_L" ]; then
		if is_sane "$MMS_L"; then
		    log Making symbolic link "$MMS_FRONTBASE$mms_d" '->' "$MMS_L"
		    rm -f "$MMS_FRONTBASE$mms_d"
		    ln -s "$MMS_L" "$MMS_FRONTBASE$mms_d"
		else
		    bad_url close "$mms_d" "is a bad symlink $MMS_L on server"
		fi
		# Don't support nested directory symlinks.  Mail rusty.
		[ -d "$MMS_FRONTBASE$mms_d" ] ||
		    mkdir -m 755 "`dirname $MMS_FRONTBASE$mms_d`/$MMS_L"
	    else
		log Making directory "$MMS_FRONTBASE$mms_d"
		mkdir -m 755 "$MMS_FRONTBASE$mms_d"
	    fi
	    [ ! -d "$MMS_FRONTBASE$mms_d" ] &&
		bad_config "$MMS_FRONTBASE$mms_d" \
		    "is not a directory, and mkdir failed"
	fi
   done
}

# Make a directory tree, checking that the counterpart are real dirs
#	mkdir_maybe_symlinks(frontend base, subdir, backends...)
mkdir_maybe_symlinks()
{
    [ -d "$1$2" ] && return
    echo mkdir_maybe_symlinks base:$1 subdir:$2
    MMS_FRONTBASE="$1"
    MMS_SUBDIR="$2"
    shift 2

    for mms_d in `directories $MMS_SUBDIR`; do
	if [ ! -d "$MMS_FRONTBASE$mms_d" ]; then
	    rsync_link $mms_d $MMS_FRONTBASE $@ ||
	     bad_url close "$mms_d" "directory does not exist on any server"

	    [ ! -d "$MMS_FRONTBASE$mms_d" ] &&
		bad_config "$MMS_FRONTBASE$mms_d" \
		    "is not a directory, and mkdir failed"
	fi
   done
}



# Make sure that any symlinks in the current path haven't changed.
#	check_symlinks(urlrest, frontend base, backends...)
check_symlinks()
{
    CS_SUBDIR="`basename $1`"
    CS_FRONTBASE="$2"
    shift 2

    for cs_d in `directories "$CS_SUBDIR"`; do
	if [ -L "$CS_FRONTBASE$cs_d" ]; then
	    CS_FRONTL="`readlink $CS_FRONTBASE$cs_d`"
	    # If this fails, simply abort check.
	    CS_BACKL="`rsync_link $cd_d $@`" || return
	    debug Link "$cs_d": mine "$CS_FRONTL" back "$CS_BACKL"
	    if [ "$CS_FRONTL" != "$CS_BACKL" ]; then
		log Updating link "$cs_d" from "$CS_FRONTL" to "$CS_BACKL"
		rm -f "$CS_FRONTBASE$cs_d"
		ln -s "$CS_BACKL" "$CS_FRONTBASE$cs_d"
	    fi
	fi
   done
}

# Given URL, sets $FRONT (local frontend file)
# Also FRONT_BASE, URL_REST, URL_SUBDIR and BACKS
# Creates directories up to $FRONT if neccessary.
# If iscontrolfile is set, then return backend they want control files
# to come from.
#	resolve_url(url, iscontrolfile)
resolve_url()
{
    for try in $APT_PROXY_BACKENDS; do
	PREFIX=`echo $try | cut -d, -f1`
	case "$1" in
	$PREFIX*)
	    # +1 for \n.
	    PREFIXLEN=`echo $PREFIX | wc -c`
	    URL_REST="`echo $1 | cut -c${PREFIXLEN}-`"
	    FRONT_BASE="`echo $try | cut -d, -f2`"
	    BACKS="`echo $try|cut -d, -f3-|tr , \\\n`"
	    if [ -n "$2" ] && echo "$BACKS" | grep -q '^+'; then
		BACKS="`echo \"$BACKS\" | grep '^+'` `echo \"$BACKS\" | grep -v '^+'`"
	    fi
	    # Remove bogus +'s.
	    BACKS="`echo \"$BACKS\" | sed s/^+//`"
	    FRONT="$FRONT_BASE$URL_REST"
	    URL_SUBDIR="`dirname \"$URL_REST\"`"
	    mkdir_maybe_symlinks "$FRONT_BASE" "$URL_SUBDIR" $BACKS
	    return
	;;
	esac
    done
    bad_url close "$1" "is not serviced by this server"
}

# Prints file-prefix that corresponds to the module containing the
# given file.
#	resolve_file(file)
resolve_file()
{
    for try in $APT_PROXY_BACKENDS; do
	PREFIX=`echo $try | cut -d, -f2`
	case "$1" in
	$PREFIX*)
	    echo $PREFIX
	    return
	;;
	esac
    done
    log "$1" "is bogus file in resolve_file"
}

# Test for lockfile with stale lock detection
# return 1 if no lockfile
#	test_lockfile(lockfile)
test_lockfile()
{
    LOCKFILE=$1

    [ -f $LOCKFILE ] || return 1

    # Stale lock detection.
    if kill -0 "`cat $LOCKFILE`" 2>/dev/null; then
        # Lockfile is current
	return 0
    else
	debug Lock $LOCKFILE is stale
	if ! ln $LOCKFILE $LOCKFILE.stale 2>/dev/null; then
	    # Someone may have cleaned this up between above lines.
	    if kill -0 "`cat $LOCKFILE`"; then
		# Someone else cleaned up the lockfile and created a new one
		return 0
	    fi
	fi
	rm -f $LOCKFILE $LOCKFILE.stale
	return 1
    fi
}

# Try to create a lockfile with our process ID
# Return 0 if successful
#	my_lockfile(lockfile)
my_lockfile()
{
    # noclobber
    set -C
    echo $$ > $1
    ret=$?
    set +C

    if [ $ret -eq 0 ]; then
        debug -- Created lock $1
    else
    	debug Lockfile detected: $1
	# Run test_lockfile to clean up lockfile if it is stale
	test_lockfile $1
    fi

    return $ret
}

# Wait for lockfile with stale lock detection
#	wait_for_lock(lockfile)
wait_for_lock()
{
    LOCKFILE=$1
    if test_lockfile $LOCKFILE  ; then
        while
	   sleep 1
	   test_lockfile $LOCKFILE
	do : ; done
        debug Lock was released: $LOCKFILE
    fi
}

# General a list of files from the given Packages file.
# This should be called every time a Packages file is downloaded.
# create_packages_filelist (file)
create_packages_filelist ()
{
    debug create_packages_filelist "$1"
    PREFIX=`resolve_file $1`
    [ -n "$PREFIX"  ] || return

    # +1 for \n.
    PREFIXLEN=`echo $PREFIX | wc -c`
    FILE_REST="`echo $1 | cut -c${PREFIXLEN}- | sed 's/\.gz$//'`"

    OUTFILE="${PREFIX}.apt-proxy-filelists/`echo $FILE_REST | tr -s / _`"

    # Don't bother if OUTFILE has already been created and is newer
    [ $OUTFILE -nt $1 ] && return

    OUTDIR=`dirname $OUTFILE`
    [ -d $OUTDIR ] || mkdir -p $OUTDIR

    CPF_LOCKFILE=${OUTDIR}/.lock
    if my_lockfile $CPF_LOCKFILE ; then
	# Get package filenames and sizes, adding a trailing blank line.
	case $1 in
	    *.gz)
		gzip -cd $1
		;;
	    *)
		cat $1
	esac | \
	(/usr/bin/grep-dctrl -s Filename,Size -n . - ; echo) | \
	while read filename ; do
	    read size
	    echo "${PREFIX}${filename}" $size
	    # Drop interspersed blank lines and the trailing one.
	    read xxx
	done >$OUTFILE
	touch -r $1 $OUTFILE
	debug "Created filelist for $1"
	rm -f $CPF_LOCKFILE
    fi
}

# Generate filelists for all of the Packages files.
# This should only be called if no filelists exist in the desired directory.
# create_all_packages_filelists ()
create_all_packages_filelists ()
{
    debug create_all_packages_filelists

    for try in $APT_PROXY_BACKENDS; do
	PREFIX=`echo $try | cut -d, -f2`
	find $PREFIX -name Packages
    done | \
    sort -u | \
    while read f ; do
	create_packages_filelist $f
    done
}

# Echo the version compenent of a deb filename.
# deb_version(file)
deb_version ()
{
    b=`basename $1`
    ver=`expr $b : '[^_]*_\([^_]*\)_[^_]*\.deb'`
    [ -n "$ver" ] || ver=`expr $b : '[^_]*_\([^_]*\)\.deb'`
    echo $ver
}

# Like dpkg --compare-versions,but works on full filenames.
# deb_compare_versions (file op file)
deb_compare_versions ()
{
    dpkg --compare-versions `deb_version $1` $2 `deb_version $3`
}

# Insertion sort given files by version in descending order, removing dupes.
# FIXME!  This could be fixed to do 1 filename per line internally,
# but then extra checks will need to be added to avoid an initial
# blank line.
# debs_sort_uniq () - file list is on stdin.
debs_sort_uniq ()
{
    ofiles=""
    while read i ; do
	this=$i
	new_ofiles=""
	for j in $ofiles ; do
	    if [ "$this" = "$j" ] ; then
	    	# Remove duplicate.
		new_ofiles="${new_ofiles} $j"
		this=""
	    elif [ -n "$this" ] && deb_compare_versions "$this" gt "$j" ; then
		new_ofiles="${new_ofiles} $this $j"
		this=""
	    else
		new_ofiles="${new_ofiles} $j"
	    fi
	done
	if [ -n "$this" ] ; then
	    new_ofiles="${new_ofiles} $this"
        fi
	ofiles="$new_ofiles"
    done

    for o in $ofiles ; do
	echo $o
    done
}

# Echo the architecture compenent of an absolute deb filename.
# deb_arch(file)
deb_arch ()
{
    b=`basename $1`
    arch=`expr $b : '[^_]*_[^_]*_\([^_]*\)\.deb'`
    [ -n "$arch" ] || arch=`expr $1 : '.*/binary-\([^/]*\)/.*'`
    echo $arch
}

# Echo the filenames givenon stdin if they have the given arch.
# debs_filter_arch(arch)
debs_filter_arch ()
{
    arch=$1
    while read i ; do
	if [ $arch = `deb_arch $i` ] ; then
	    echo $i
	fi
    done
}

# Echo the base part of an absolute deb filename.
# The trailing _ is important, since it acts as an anchor.
# deb_base(file)
deb_base ()
{
    echo `dirname $1`/`basename $1 | cut -d_ -f1`_
}

# Echo a list of all version of given file that are located in the
# same directory.
# deb_all_versions(file)
deb_all_versions ()
{
    case $1 in
    *.deb)     DAV_EXT=.deb ;;
    *.tar.gz)  DAV_EXT=.tar.gz ;;
    *.dsc)     DAV_EXT=.dsc ;;
    *.diff.gz) DAV_EXT=.diff.gz ;;
    esac

    DAV_BASEDEB=`deb_base $1`
    DAV_ARCH=`deb_arch $1`

    if [ "`echo ${DAV_BASEDEB}*$DAV_EXT`" != "${DAV_BASEDEB}*$DAV_EXT" ] ; then
	ls ${DAV_BASEDEB}*$DAV_EXT | debs_filter_arch $DAV_ARCH | debs_sort_uniq
    fi
}
	

# Find a decent match to base rsync on. 
#	copy_best_match(front partial)
copy_best_match()
{
    # Get 1st version that's less than or get the last.
    for CBM_BEST in `deb_all_versions $1` ; do
	deb_compare_versions $CBM_BEST lt $1 && break
    done

    debug Found best basis for ${1}: $CBM_BEST
    if [ -n "$CBM_BEST" ] ;then
        # Save the access time of file because we use it during cache cleaning
    	touch --reference $CBM_BEST $CBM_BEST.timestamp
        cp $CBM_BEST $2
	touch --reference $CBM_BEST.timestamp $CBM_BEST
	rm $CBM_BEST.timestamp
    fi
}

# Do rsync, drop lock.
#	rsync_unlock(front url-rest lockfile backends...)
rsync_unlock()
{
    RSU_FRONT=$1
    RSU_REST=$2
    RSU_LOCK=$3
    shift 3
    RSU_PART=$RSU_FRONT.partial
    RSU_FAIL=$RSU_FRONT.fail
    RSU_OPTS="-v --perms --links --times --partial"

    # If file exists (eg. Packages file), copy it to partial.
    # In that case, don't get Packages if it's NEWER.
    if [ -f "$RSU_FRONT" ]; then
	cp -p $RSU_FRONT $RSU_PART
	RSU_OPTS="$RSU_OPTS --update"
    elif [ ! -f "$RSU_PART" ]; then
	copy_best_match $RSU_FRONT $RSU_PART
    fi
    for rsu_b in $@; do
	debug Trying $rsu_b...
	#debug $RSYNC $RSU_OPTS $rsu_b$RSU_REST $RSU_PART
	if download_url $rsu_b$RSU_REST `dirname $RSU_FRONT` `basename $RSU_PART` ;then
	#if $RSYNC $RSU_OPTS $rsu_b$RSU_REST $RSU_PART >$RSU_FAIL 2>&1; then
	    mv $RSU_PART $RSU_FRONT
	    #[ -n "$KEEP_STATS" ] && log $RSU_REST `tail -2 $RSU_FAIL`
	    #rm -f $RSU_FAIL
	    break;
	fi
	#debug `tail -2 < $RSU_FAIL`
    done
    rm -f $RSU_LOCK
}

# Given filename, return wildcard which rsync would use.
#	rsync_wildcard(filename)
rsync_wildcard()
{
    echo `dirname $1`/.`basename $1`.partial."*"
}

# Returns when actual file (not symlink!) has begun rsync.
# If it has found a stream, returns stream name.
#	fetch_file_start(url-rest frontend-base backends...)
fetch_file_start()
{
    FF_URL_REST=$1
    FF_FRONT_BASE=$2
    shift 2

    FF_FRONT=$FF_FRONT_BASE$FF_URL_REST
    FF_LOCK=$FF_FRONT.lock
    while true; do
	debug Fetching $FF_FRONT from $@

	# Start rsync process in the background so we can watch its progress
	# Need >/dev/null otherwise our caller (waiting for stdout)
	# waits for rsync_unlock to finish.
	my_lockfile $FF_LOCK &&
	    (rsync_unlock $FF_FRONT $FF_URL_REST $FF_LOCK $@ &) >/dev/null

	FF_WILD="`rsync_wildcard $FF_FRONT`"
	while test_lockfile $FF_LOCK ; do
	    # A stream?
	    FF_STREAM="`echo $FF_WILD`"
	    if [ "$FF_STREAM" != "$FF_WILD" ]; then
		echo $FF_STREAM
		return 0
	    fi
	    sleep 1
	done

	# File, or non-dangling symlink?
	[ -f $FF_FRONT ] && return 0

	# Dangling symlink?  Follow.
	LINK="`readlink $FF_FRONT`"
	if [ -n "$LINK" ]; then
	    debug rsync gave a link...
	    if is_sane $LINK; then :
	    else
		log Bogus symlink $LINK from $@ rejected.
		return 1
	    fi
	    # Simply append to dirname.
	    FF_FRONT=`dirname $FF_FRONT`/$LINK
	    FF_URL_REST=`dirname $FF_URL_REST`/$LINK
	    mkdir_maybe_symlinks $FF_FRONT_BASE `dirname $FF_URL_REST` $@
	else
	    # rsync failed to give a result.
	    return 1
	fi
    done
}

# Can I suppress getting this file (too recent).
#	can_suppress(file timeout)
can_suppress()
{
    find $1 -cmin -$2 2>/dev/null | fgrep -q -x $1 && return 0
    find $1.nonexist -mmin -$2 2>/dev/null|fgrep -q -x $1.nonexist && return 0
    return 1
}

# Like fetch_file_start, but doesn't bother if within last BACKEND_FREQ.
#	fetch_file_maybe(url-rest frontend-base backends...)
fetch_file_maybe()
{
    if can_suppress $2$1 ${BACKEND_FREQ:-0}; then debug Suppressing $2$1
    else
	# This is expensive, only do for Packages (first file apt asks for)
	echo $1 | fgrep -q Packages && check_symlinks "$@"
	# Refresh nonexist.
	touch -c $2$1.nonexist 2>/dev/null
	fetch_file_start "$@"
    fi
}

# Wait for lock file to vanish; basically turns into synchronous mode.
#	wait_for_stream(file)
wait_for_stream()
{
    wait_for_lock $1.lock 
    # Return value based on whether file exists.
    [ -f $1 ]
}

# $1 may be null; if not, spool it out: it may turn into $2 at some stage.
#	stream_file(temporary-file, final-file)
stream_file()
{
    SF_OFFSET=0
    debug stream_file: $1, $2

    while SF_LEN=`file_size "$1"`; do
	SF_COUNT=`expr $SF_LEN - $SF_OFFSET`
	if [ "$SF_COUNT" -gt 4096 ]; then
	    debug Doing $SF_COUNT bytes at $SF_OFFSET...
	    dump_part_file "$1" $SF_OFFSET $SF_COUNT >&3 || break
	    SF_OFFSET=$SF_LEN
	fi
	sleep 1
    done 2>/dev/null

    # Race: wait for lock to vanish (may have to move file)
    wait_for_lock $2.lock 

    # Dump the rest.
    dump_part_file "$2" $SF_OFFSET >&3 && {
	log Delivered partial file "$2", from offset $SF_OFFSET.
    }
}

# Remove extra versions of given package file.  Filename must be absolute.
# Only works for .deb files for now.
# remove_extra_versions(file)
remove_extra_versions ()
{
    [ -n "$MAX_VERSIONS" ] || return

    debug "Planning to remove extra versions of $1"

    PREFIX=`resolve_file $1`

    # We're after dirname/foo_*.$CL_EXT
    REC_BASEDEB=`deb_base $1`
    REC_ARCH=`deb_arch $1`
    REC_FILES=`deb_all_versions $1`

    # Is this our first time?
    [ -d ${PREFIX}.apt-proxy-filelists ] || create_all_packages_filelists

    REC_LOCKFILE=${PREFIX}.apt-proxy-filelists/.lock
    my_lockfile ${REC_LOCKFILE} || return
    REC_CURRENT=`grep -r -F -h "${REC_BASEDEB}" ${PREFIX}.apt-proxy-filelists | awk '{print $1}' | debs_filter_arch $REC_ARCH | debs_sort_uniq`
    rm -f ${REC_LOCKFILE}
    [ -n "$REC_CURRENT" ] || return  # Or should this remove all of them?  :-)

    # Place sorted list of current .debs in positional parameters so we can use shift.
    set -- $REC_CURRENT

    # Now try to match them.
    COUNT=0
    for i in $REC_FILES; do
	if [ $# -gt 0 ] ; then
	    # Have current file to compare to.
	    if [ "$i" = "$1" ] ; then
		# This is the next current file, so keep it.
		COUNT=1
		# Pop off next item
		shift
	    elif deb_compare_versions $i lt "$1" ; then
		# Older than next current file: current is missing so pop it.
		COUNT=1
		# Pop off next item
		shift
	    elif [ $COUNT -ge 1 ] ; then
		# Newer than next current file and previous current
		# file exists, so this is an intermediate version that
		# we might want to delete.
		COUNT=$(($COUNT + 1))
		if [ $COUNT -gt $MAX_VERSIONS ] ; then
		    log "Removing surplus version: $i"
		    rm -f $i
		fi
	    else
		# Newer than first next current file.  Should not happen!
		log "remove_extra_copies: Ignoring dodgy file \"$i\""
	    fi
	else
	    # Files left over after last current one, so maybe delete.
	    COUNT=$(($COUNT + 1))
	    if [ $COUNT -gt $MAX_VERSIONS ] ; then
		log "Removing surplus version: $i"
		rm -f $i
	    fi
	fi
    done
}

# Delete any expired files older than that just served.
#	cleanup_old(newfile,days)
cleanup_old()
{
    case $1 in
    *.deb) CL_EXT=.deb ;;
    *.tar.gz) CL_EXT=.tar.gz ;;
    *.dsc) CL_EXT=.dsc ;;
    *.diff.gz) CL_EXT=.diff.gz ;;
    esac

    # We're after dirname/foo_*.$CL_EXT
    CO_BASEDEB=`deb_base $1`
    for CO_F in $CO_BASEDEB*$CL_EXT; do
	if dpkg --compare-versions $CO_F lt $1 && 
	    [ x"`find $CO_F -atime +$2 2>/dev/null`" != x"" ]
	then
	    log Cleaning up $CO_F
	    rm $CO_F
	fi
    done
}

# Clean up any debs which haven't been accessed in this many days.
#	sweep_clean_unlock(basedir days)
sweep_clean_unlock()
{
    SW_FILE=$1/.apt-proxy

    log Doing sweep of $1 in background...
    find $1 \( -name '*.deb' -o -name '*.tar.gz' -o -name '*.dsc' -o -name '*.diff.gz' \) -a -atime +$2 |
    while read f; do
	log Sweeping clean $f
	rm $f
    done

    touch $SW_FILE
    rm -f $SW_FILE.lock
}

# Look back directories up to front-basedir until filename or
# filename.gz is found, and cat it.
#	find_and_cat(filename front-basedir subdir)
find_and_cat()
{
    # Could be a /binary-all/ (no Packages/Sources file).
    FAC_DIR=`echo $3 | sed s:/binary-all/:/binary-*/:`
    FAC_DIR=`echo $FAC_DIR | cut -d' ' -f1`
    
    for d in `directories $FAC_DIR | sort -r`; do
	debug Looking for $2$d/$1
	if [ -f $2/$d/$1 ]; then
	    debug Found $2/$d/$1
	    cat $2/$d/$1
	    return 0
	elif [ -f $2/$d/$1.gz ]; then
	    debug Found $2/$d/$1.gz
	    zcat $2/$d/$1.gz
	    return 0
        fi
    done
}

# Find .deb size and echo.  Return true(0) if found
# find_filesize(cache_directory URL_rest)
find_filesize()
{
   debug find_filesize $1 $2
   FILESZ="`grep $2 $1/.apt-proxy-filelists/*Packages |head -1|cut -d' ' -f2`"
   [ -n "$FILESZ" ] || return 1

   echo "$FILESZ"
   return 0
}

# Verify cache file integrity
# Returns 0 if file OK, or file exists but we don't know how to verify it
# 1 if not found, 2 if corrupted
verify_file()
{
    debug verify_file $1

    [ -f "$1" ] || return 1 # file not found

    case $1 in
    *.deb)
        MSG=`ar -t "$1" 2>&1` && return 0
	debug "verify_file $1 corrupted:$MSG"
	return 2
	;;
    *.gz)
    	MSG=`gunzip -t "$1" 2>&1` && return 0
	debug "verify_file $1 corrupted:$MSG"
	return 2
	;;
    *)
    	return 0
	;;
    esac
    
}

################################# MAIN START ##############################
# Read our configuration
#. /etc/apt-proxy/apt-proxy.conf
. $CONFIG_FILE

debug Backends:$APT_PROXY_BACKENDS

# Set rsync/wget timeouts if specified in config file
[ -n "$RSYNC_TIMEOUT" ] && RSYNC="$RSYNC --timeout=$RSYNC_TIMEOUT"
[ -n "$WGET_TIMEOUT" ] && WGET="$WGET --timeout=$WGET_TIMEOUT"

# For all the requests in a particular session
while true; do
    unset DO_CLEANUP DO_REMOVE_EXTRA_VERSIONS DO_UPDATE_PACKAGES_LIST \
	SIZE SIZE_FILE IMS_DATE UNCOMPRESS REQUEST KEEP_ALIVE
    KEEP_ALIVE=close

    while read PREFIX LINE
    do
    	debug "$PREFIX $LINE"
	LC_PREFIX="`echo $PREFIX | tr -d '\015' | tr '[A-Z]' '[a-z]'`"
	case "$LC_PREFIX" in
	get)
	    REQUEST="`echo $LINE | cut -d\  -f1 | sed 's/%5[Ff]/_/g'`"
	    ;;
	head)
	    REQUEST="`echo $LINE | cut -d\  -f1 | sed 's/%5[Ff]/_/g'`"
	    ;;
	if-modified-since:)
	    IMS_DATE=$LINE
	    debug "If-Modified-Since $IMS_DATE"
	    IMS_DATE=`date -u -d "$IMS_DATE" '+%s'`
	    ;;
	connection:)
	    KEEP_ALIVE="`echo $LINE | tr -d '\015' | tr '[A-Z]' '[a-z]'`"
	    debug "Connection $KEEP_ALIVE"
	    ;;
	'')
	    debug Finished processing request
	    break
	    ;;
	*)
#	    debug Got line "$PREFIX $LINE"
	    ;;
	esac
    done

    INSANE="[^A-Za-z0-9_./+=:-]"
    echo "$REQUEST" | grep -q "$INSANE" &&
	bad_url close "$REQUEST" Malformed chars:`echo \"$REQUEST\"|tr -dc "$INSANE"`

    [ -n "$REQUEST" ] && log `date` Request "$REQUEST"

    echo "$REQUEST" | grep -q "Packages\(.gz\)\?$" && DO_UPDATE_PACKAGES_LIST=1

    case $REQUEST in
    '')
	# Otherwise we send RSTs to some clients.
	sleep 1
	exit 0
	;;
    *..*)
	log Bad file requested $REQUEST
	exit 0
	;;
    # We always keep the Packages and Release files uptodate: derive the .gz.
    *Packages.gz|*Release.gz|*Sources.gz|*Contents-*.gz)
	resolve_url $REQUEST 1
	STREAM="`fetch_file_maybe $URL_REST $FRONT_BASE $BACKS`"

	contype=text/plain
	conenc=x-gzip
	;;

    *Packages|*Release|*Sources|*Contents-*)
	debug Updating file \`$REQUEST\'
	resolve_url $REQUEST 1
	STREAM="`fetch_file_maybe $URL_REST $FRONT_BASE $BACKS`"

	contype=text/plain
	conenc=
	;;

    *.deb)
	debug Looking for file \`$REQUEST\'
	resolve_url $REQUEST
	verify_file $FRONT ||
	    STREAM="`fetch_file_start $URL_REST $FRONT_BASE $BACKS`"
	[ -n "$CLEANUP_DAYS" ] && DO_CLEANUP=1
	[ -n "$MAX_VERSIONS" ] && DO_REMOVE_EXTRA_VERSIONS=1

	SIZE_FILE=Packages
	contype=application/dpkg
	conenc=
	;;

    *.diff.gz|*.tar.gz)
	resolve_url $REQUEST
	verify_file $FRONT ||
	    STREAM="`fetch_file_start $URL_REST $FRONT_BASE $BACKS`"
	[ -n "$CLEANUP_DAYS" ] && DO_CLEANUP=1

	SIZE_FILE=Sources
	contype=application/x-gzip
	conenc=
	;;

    *.dsc)
	resolve_url $REQUEST
	[ -f $FRONT ] ||
	    STREAM="`fetch_file_start $URL_REST $FRONT_BASE $BACKS`"
	[ -n "$CLEANUP_DAYS" ] && DO_CLEANUP=1

	SIZE_FILE=Sources
	contype=text/plain
	conenc=
	;;

    *.bin)
	resolve_url $REQUEST
	[ -f $FRONT ] ||
	    STREAM="`fetch_file_start $URL_REST $FRONT_BASE $BACKS`"
	[ -n "$CLEANUP_DAYS" ] && DO_CLEANUP=1

	contype=application/octet-stream
	conenc=
	;;

    *.tgz)
	resolve_url $REQUEST
	[ -f $FRONT ] ||
	    STREAM="`fetch_file_start $URL_REST $FRONT_BASE $BACKS`"
	[ -n "$CLEANUP_DAYS" ] && DO_CLEANUP=1

	contype=application/x-gzip
	conenc=
	;;

    */)
        # A directory
	resolve_url $REQUEST
	[ -f $FRONT ] ||
	    STREAM="`fetch_file_start $URL_REST $FRONT_BASE $BACKS`"
	;;
    *)
	bad_url close $REQUEST Unknown extension.
	;;
    esac

    # Beware race.
    if [ -f "$STREAM" ]; then
	rm -f $FRONT.nonexist
	if [ -n "$SIZE_FILE" ]; then
	    # Damn.  Don't hit server again to get length: backtrace and find.
	    BASEFILE_NAME=`basename $REQUEST`
	    case $SIZE_FILE in
	    Packages)
	    	if ! SIZE=`find_filesize $FRONT_BASE $URL_REST`; then
		SIZE=`find_and_cat Packages $FRONT_BASE $URL_SUBDIR | grep-dctrl -e -F Filename "/$BASEFILE_NAME$" -s Size | cut -d\  -f2 | head -1`
		fi
		;;
	    Sources)
		PACKAGE_NAME="`basename $REQUEST | cut -d_ -f1`"
		SIZE=`find_and_cat Sources $FRONT_BASE $URL_SUBDIR | grep-dctrl -X -P  "$PACKAGE_NAME" -s Files | fgrep $BASEFILE_NAME | cut -d\  -f3 | head -1`
		;;
	    esac
	    debug Derived size: $SIZE
	fi
    else
	# Race: wait for lock to vanish (may have to move file)
        wait_for_lock $FRONT.lock 

	if [ -f $FRONT ]; then
	    # File already exists in entirity.  Give local answers.
	    rm -f $FRONT.nonexist
	    DATE="`date -u -R -r $FRONT`"
	    # Turn date into seconds. 
	    FILE_DATE=`date -u '+%s' -d "$DATE"`
	    if [ -n "$IMS_DATE" ] && [ $IMS_DATE -ge $FILE_DATE ]; then
		write_header "304 HIT" keep-alive
		continue
	    fi
	    SIZE=`file_size $FRONT`
	else
	    # Golly!
	    CAUSE=`fgrep -v -x 'client: nothing to do' $FRONT.fail 2>/dev/null`
	    rm -f $FRONT.fail
	    # Often Release files really don't exist: cache the `miss'.
	    case "$FRONT" in *Release|*Release.gz)
		[ ! -f $FRONT.nonexist ] && touch $FRONT.nonexist;;
	    esac
	fi
    fi

    if [ -n "$STREAM" -o -f $FRONT ]; then
	# We have a file to send back
	# If we do not have a size, close the connection on completion
	[ -z "$SIZE" ] && KEEP_ALIVE=close

	debug Header - 200 OK, keep-alive:$KEEP_ALIVE content:$contype size:$SIZE \
	      date:$DATE enc:$conenc request:$REQUEST
	write_header "200 OK" "$KEEP_ALIVE" "$contype" "$SIZE" "$DATE" "$conenc" "$REQUEST"
	stream_file "$STREAM" "$FRONT"
    else
	# File not found
	bad_url keep-alive $REQUEST "File does not exist on any server"

	# Do not do any post processing
	unset DO_UPDATE_PACKAGES_LIST DO_CLEANUP DO_REMOVE_EXTRA_VERSIONS
    fi


    # Now, derive the uncompressed file.
#    if [ -n "$UNCOMPRESS" ]; then
#	(
	    # close the sockets as this may take some time
#	    exec 0>&- 3>&-
#	    gunzip < $FRONT > $UNCOMPRESS
#	    touch -r $FRONT $UNCOMPRESS
#	) &
#    fi

    if [ -n "$DO_UPDATE_PACKAGES_LIST" ] ; then
	(
	    # close the sockets as this may take some time
	    exec 0>&- 3>&-
	    create_packages_filelist $FRONT
	) &
    fi

    [ -n "$DO_CLEANUP" ] && cleanup_old $FRONT $CLEANUP_DAYS

    if [ -n "$DO_REMOVE_EXTRA_VERSIONS" ] ; then
	(
	    # close the sockets as this may take some time
	    exec 0>&- 3>&-
	    remove_extra_versions $FRONT
	) &
    fi
    

    # Do async once in a very long while.
    if [ -n "$CLEAN_SWEEP" ]; then
	if [ ! -f $FRONT_BASE/.apt-proxy ] ||
	    find $FRONT_BASE/.apt-proxy -mtime +$CLEAN_SWEEP |
		fgrep -q -x $FRONT_BASE/.apt-proxy
	then
	    (
		# close the sockets as this may take some time
		exec 0>&- 3>&-
		my_lockfile $FRONT_BASE/.apt-proxy.lock &&
		    sweep_clean_unlock $FRONT_BASE $CLEAN_SWEEP
	    ) &
	fi
    fi
    if [ "$KEEP_ALIVE" '!=' keep-alive ]; then break; fi
done