File: netselect-apt

package info (click to toggle)
netselect 0.3.ds1-28
  • links: PTS, VCS
  • area: main
  • in suites: bullseye, buster, sid, stretch
  • size: 336 kB
  • ctags: 243
  • sloc: ansic: 1,082; sh: 625; makefile: 43
file content (446 lines) | stat: -rwxr-xr-x 14,885 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
#!/bin/bash
#
# A script to create an apt sources.list file automatically by downloading
# the list of Debian mirrors and choosing the fastest using netselect.
#
# Author: Avery Pennarun <apenwarr@gmail.com>
# Enhancements:
#    - Filippo Giunchedi <filippo@esaurito.net>
#    - Javier Fernandez-Sanguino <jfs@debian.org>
#
# License: public domain.  Please feel free to improve this script.  It
# doesn't really belong in the netselect package, particularly since the
# netselect package doesn't depend on wget and this script does.  If you
# feel like merging this into another package, or creating a new package,
# please do.  Then tell me, so I can remove it from the netselect package.
#
# TO DO:
#   - test the generated files automatically.  Some mirrors in the list
#       are broken.  We should at least verify that the Packages and Sources
#       files exist and aren't ancient.
#   - maybe generate redundant entries, in case one server is missing files?

# our defaults
# RATIONALE for WANT_SOURCES and WANT_NONFREE environmental variables:
# both variables can be system wide and can be useful to not always specify
# them on the commandline

want_sources=${WANT_SOURCES:-0}
want_nonfree=${WANT_NONFREE:-0}
want_security=1
infile=""
tmpinfile="1"
# File to generate
outfile="sources.list"
# Default distribution
distro="stable"
# Default protocol
protocol="HTTP"
# Number of fastest hosts that will be validated
test_hosts="10"
# URL where the mirror list is retrieved from 
url="http://www.debian.org/mirror/mirrors_full"
# Test file retrieve from sites to valide
test_file="README"     
# Country used to filter the list
country=""
# Debug mode?
DEBUG=""
if [ -x /usr/bin/dpkg-architecture ] ; then
    arch=$(/usr/bin/dpkg-architecture -qDEB_HOST_ARCH)
else
    arch=$(dpkg --print-architecture)
fi

options="-o a:dsi:o:O:t:c:nfhu -l arch:,debug,sources,infile:,outfile:,tests:,country:,nonfree,ftp,help"

trap '[ "$tmpinfile" = "1" ] && rm -f "$infile"' TERM INT EXIT

# misc functions
log()
{
	echo "$@" >&2
}

# tests a list of hosts and returns a (filtered) list of those
# that you can connect to using a defined protocol
tests_hosts()
{
    local host_list="$*"
    local host_filtered=""
    [ -n "$DEBUG" ] && log "DEBUG: Filtering host list is $host_list"
    # Do not continue if curl is not installed 
    [ ! -x /usr/bin/curl  ] && echo $host_list 
    host_filtered=""
    for host in $host_list; do
# Test using curl
        [ -n "$DEBUG" ] && log "DEBUG: Testing host $host with curl"
        test_host $host
        if [ $? -eq 0 ]; then
            [ -n "$DEBUG" ] && log "DEBUG: Host $hot is ALIVE"
            if [ -z "$host_filtered" ] ; then
                host_filtered="$host"
            else
                host_filtered="$host_filtered $host"
            fi
        fi
    done
    [ -n "$DEBUG" ] && log "DEBUG: Filtered host list is $host_filtered"
    echo $host_filtered
}

validate_hosts()
{
    for host in $*; do
        [ -n "$DEBUG" ] && log "DEBUG: Testing host $host with curl"
        test_host $host
        if [ $? -eq 0 ]; then
            [ -n "$DEBUG" ] && log "DEBUG: Host $host is valid"
            echo $host
            return 0
            # We return as soon as we have a valid host since netselect
            # returns them in order (fastest to slowest)
        fi
        [ -n "$DEBUG" ] && log "DEBUG: Host $host is INVALID"
    done
    return -1
}

# Test a single host using Curl, if available
test_host()
{
    local host="$1"
    local rv=0
    [ ! -x /usr/bin/curl  ] && return 255
    [ -z "${host}" ]        && return 255

    # First test: does it actually serve anything?
    curl -m 2 -q -s "$host"  >/dev/null 2>&1
    [ $? -ne 0 ] && return $?

    # Second test: do we have the test file we are looking for?
    [ -n "$DEBUG" ] && log "DEBUG: Checking if the file '$test_file' is available in site"
    temp=`tempfile`
    [ ! -e "$temp" ] && return 0 # Return without error if we cannot create
                                 # a temporary file
    curl -m 2 -q -s "$host/$test_file"  >$temp 2>&1
    rv=$?
    if [ $rv -ne 0 ] ; then
        rm -f $temp
        return $rv
    fi

    # Third test: does the file have any reference to 'Debian'
    [ -n "$DEBUG" ] && log "DEBUG: Checking the contents of the '$test_file' file (downloaded to '$temp')"
    grep -qi debian $temp 
    rv=$?
    if [ $rv -ne 0 ] ; then
        rm -f $temp
        return $rv
    fi
    # Fourth test: does it have the content we expected
    head -1 $temp | grep -qi "See http://www.debian.org/"
    rv=$?
    if [ $rv -ne 0 ] ; then
       rm -f $temp
        return $rv
    fi

    # All tests completed succesfully, remove temporary file and exit
    [ -e "$temp" ] && rm -f $temp
    return 0
}

run_netselect()
{
	SEARCH="$1"
	PROTO="$2"
        hosts=$(cat "$infile" \
		| perl -n -e '
			$/="<br><br>";
                        $country_name  = ".*";
                        $country_iso  = "..";
                        $country = "'"$country"'";
                        $my_country = 1;
                        if ( $country ne "" ) {
                          $my_country = 0;
                          if ( $country =~ /^\w{2}$/ ) {
                            $country_iso  = uc($country);
                            $country_name = "";
                          }  else {
                            $country_iso  = "";
                            $country_name = ucfirst(lc($country));
                          } ;
                        }
			while(<>){
                                if ( /<h3><a name="$country_iso">/ ||
                                     /<h3><a name=".*?">$country_name<\/a>/ ) {
                                    $my_country = 1 ;
                                }
                                if ( $_ =~ /<h3>/ && $_ !~ /<h3><a name="$country_iso">/ &&
                                     $_ !~ /<h3><a name=".*?">$country_name<\/a>/ ) {
                                    $my_country = 0 ;
                                }
				next if $_ !~ /Site:/;
				if( ( /Includes architectures:.+'"$arch"'.+/i ||
						$_ !~ /Includes architectures:/ ) &&
						m@<br>'"$SEARCH"':.*<a.*?href="('"$PROTO"'://.*?)">@i && $my_country == 1
					){
					print("$1\n");
				}}')
        [ -z "$hosts" ] && return 200
        [ -n "$DEBUG" ] && log "DEBUG: Running netselect: 'netselect $netselect_options $hosts'"
	out=`netselect $netselect_options  $hosts`
	rv=$?
	# Throw away hosts with negative scores to workaround a netselect bug.
	# (netselect 0.3.ds1-25 seems to have problems pinging round-robin DNS)
	corruptedhosts=$(echo "$out" | egrep '^ *-')
	if [ $? -eq 0 ]; then
		log "Detected corrupt scores from bug in netselect: $corruptedhosts"
		out=$(echo "$out" | sed -e '/^-/d')
	fi


	# Remove the score number leaving just the host.
        echo $out | sed -e 's/[0-9]\+\s\+/ /g'
	return $rv
}

netselect_generic_error()
{
	log "netselect was unable to find a mirror, this probably means that"
	log "you are behind a firewall and it is blocking ICMP and/or"
        log "UDP traceroute. Or the servers test are actively blocking"
        log "ICMP and/or UDP traceroute probes."
}

netselect_permission_error()
{
	log "netselect was unable to operate successfully, please check the errors,"
	log "most likely you don't have enough permission."
}

netselect_parse_error ()
{
    
	log "netselect-apt was unable to obtain a list of valid hosts from"
        if [ "$tmpinfile" = "0" ] ;then
            log "the file provided ($infile)."
        else
            log "the file downloaded from the url '$url'."
        fi
        if [ -n "$country" ] ; then
            log "You told the program to only look for hosts in country '$country' "
            log "and there might no mirrors to chose from that country. Remove the "
            log "option -c and try again."
        else
            log "This might happen because of any of the following reasons: "
            log "   - there was an error in the file "
            log "   - the file is not in the format netselect-apt expected "
            log "   - there is a bug in netselect-apt "
            log "Please manually check the file. If you believe its contents are correct, file "
            log "a bug (hint: use 'reportbug') against netselect-apt and provide the file as "
            log "well as the output generated by the program (hint: use 'script')."
        fi
}

netselect_aborted_error ()
{
	log "netselect aborted before it was able to find a mirror."
}



usage()
{
	log "Usage: netselect-apt [OPTIONS] [ debian_release ]"
	log "       debian_release is one of stable, testing, unstable, experimental"
	log "       or a codename etch, lenny, squeeze, wheezy, jessie, stretch, sid"
	log "Options:"
	log "   -a, --arch ARCH        Use mirrors containing arch ($arch)"
	log "   -s, --sources          Include deb-src lines in generated file (no)"
	log "   -i, --infile INFILE    Use INFILE as the input file (temp file)"
	log "   -o, --outfile OUTFILE  Use OUTFILE as the output file (sources.list)"
	log "   -n, --nonfree          Use also non-free packages in OUTFILE (no)"
	log "   -f, --ftp              Use FTP as the protocol for OUTFILE (HTTP)"
	log "   -t, --tests #          Number of hosts to test ($test_hosts)"
	log "   -c, --country COUNTRY  Restrict search to servers in that country"
	log "   -d, --debug            Enable debugging"
}

# Sanity tests
NETSELECT=`which netselect`
# Does netselect exist?
if [ -z "$NETSELECT" ] ; then
    log "Sorry, I cannot find the 'netselect' binary in my PATH"
    exit 1
fi
# Are we root?
if [ "`id -u`" -gt 0 ] ; then
# If we are not, is netselect setuid?
    if [ ! -u "$NETSELECT" ] ; then
        log "Sorry, you need to be root to run $0 since the netselect"
        log "binary we will use ($NETSELECT) is not setuid."
        exit 1
    fi
fi


# commandline parsing
# TODO: We should do some sanity checking of some values
# (such as test_hosts to ensure it is a number > 1)
temp=$(getopt $options -n 'netselect-apt' -- "$@")
if [ $? != 0 ]; then echo "Terminating..." >&2; exit 2; fi
eval set -- "$temp"
while true; do
	case "$1" in
		-a|--arch) arch=$2; shift 2 ;;
		-s|--sources) want_sources=1; shift ;;
		-i|--infile) infile="$2"; tmpinfile="0"; shift 2 ;;
		-o|--outfile) outfile="$2"; shift 2 ;;
		-O) netselect_options="$netselect_options $2"; shift 2 ;;
		-n|--nonfree) want_nonfree=1; shift ;;
		-c|--country) country=$2; shift 2 ;;
		-t|--tests) test_hosts=$2; shift 2 ;;
		-f|--ftp) protocol="FTP"; shift ;;
		-d|--debug) DEBUG=1; shift ;;
		-h|--help) usage; exit 0;;
		--) shift; break;;
		*) echo "Internal Error!"; echo "args: $@"; exit 1;;
	esac
done

# Default netselect options
netselect_options="-D -I -v -s $test_hosts"
# The '-s number' option can be used to display the top 'number' best
# hosts. This is useful for finding the best host to use in a sources.list file
# that is shared between several hosts on different networks in the same
# geographical location. For example, it would be better to choose a host that
# is #2 on all hosts than #1 on one host but is #50 on all the rest.

# distro is a non-option for backward compatibility
case "$1" in
	# don't forget to update the usage with new codenames
	stable|testing|unstable|experimental|etch|lenny|squeeze|wheezy|jessie|stretch|sid) distro="$1" ;;
	'') ;;
	*) log "Invalid distribution: $1"; exit 1 ;;
esac

if [ "$distro" != "stable" ]; then
	want_security=0
fi

# netselect starting
log "Using distribution $distro."

if [ "$tmpinfile" = "0" -a ! -f "$infile" -a ! -x /usr/bin/wget ]; then
	log "Sorry, this script requires the 'wget' package in order to run."
	log "You can also download the mirrors list yourself and pass it"
	log "with -i option, consult the manpage for further details:"
	log "        $url"
	exit 2
fi

if [ ! -f "$infile" ]; then
	if [ "$tmpinfile" = "1" ]; then
		infile=$(mktemp -t netselect-apt.XXXXXX) ||
			{ log "unable to create tempfile"; exit 2; }
	fi

	log "Retrieving the list of mirrors from www.debian.org..."
	log

	if ! /usr/bin/wget -O "$infile" "$url"; then
		log "$0: wget failed to retrieve $url."
                log "Please try to correct the problem by reading the wget "
                log " messages printed above."
		exit 2
	fi
else
	log "$infile has been found."
	log "I'll use that, rather than downloading it again."
	log
fi

log "Choosing a main Debian mirror using netselect."
if [ -n "$country" ]; then
    log "(will filter only for mirrors in country $country)"
fi
hosts_netselect=$(run_netselect "Packages over $protocol" $protocol)
netselect_rv=$?
if [ $netselect_rv -eq 0 ] ; then
        if [ ! -z "$hosts_netselect" ] ; then
            if [ "$test_hosts" -gt 1 ]; then
                log "The fastest $test_hosts servers seem to be:"
                echo "$hosts_netselect" | sed -e 's/\s\+/\n\t/g'
                log
            else
                log "The fastest server seems to be:"
                log "          $hosts_netselect"
                log
            fi
            main=$(validate_hosts $hosts_netselect)
            if [ ! -z "$main" ] ; then
                    if [ "$test_hosts" -gt 1 ]; then
                        log "Of the hosts tested we choose the fastest valid for $protocol:"
                        log "        $main"
                        log
                    else
                        log "and is a valid server for $protocol."
                    fi
            else
                    log "No valid servers were found that could be reachable"
                    log "through $PROTO. Netselect found these hosts to be "
                    log "closest to you, but we could not connect to any "
                    log "of them using that protocol."
                    exit 2
            fi
        else
	    netselect_generic_error
            exit 2
        fi
elif [ $netselect_rv -eq 6 ]; then
	netselect_permission_error
	exit 2
elif [ $netselect_rv -eq 130 ]; then
	netselect_aborted_error
	exit 2
elif [ $netselect_rv -eq 200 ]; then
	netselect_parse_error
	exit 2
else
	netselect_generic_error
	exit 2
fi

log "Writing $outfile."

if [ -f "$outfile" ]; then
	destfile="$outfile.$(date +%s)"
	log "$outfile exists, moving to $destfile"
	mv $outfile $destfile
fi

sections="main contrib"
if [ "$want_nonfree" -eq 1 ]; then sections="$sections non-free"; fi

(
	echo "# Debian packages for $distro"
	echo "deb $main $distro $sections"
	echo "# Uncomment the deb-src line if you want 'apt-get source'"
	echo "# to work with most packages."
	if [ "$want_sources" -eq 0 ]; then
		echo -n "# "
	fi
	echo "deb-src $main $distro $sections"

	echo
	echo "# Security updates for stable"
	if [ "$want_security" -eq 0 ]; then
		echo -n "# "
	fi
	echo "deb http://security.debian.org/ stable/updates $sections"

) > $outfile

echo "Done."