File: wget-list

package info (click to toggle)
whatweb 0.6.1-1
  • links: PTS
  • area: main
  • in suites: forky, sid
  • size: 23,948 kB
  • sloc: ruby: 43,493; sh: 213; makefile: 41
file content (18 lines) | stat: -rwxr-xr-x 472 bytes parent folder | download | duplicates (6) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
 
#!/bin/bash

AGENT="Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.12) Gecko/2009070811 Ubuntu/9.04 (jaunty) Firefox/3.0.12"

if [ -z $1 ]; then
 echo "Usage: $0 <file with list of urls>"
 echo "downloads each URL's html and headers into the current directory"
 exit
fi


for i in `cat $1` ; do
 u=$i; f=`echo $i | sed 's/http:\/\///g' | sed 's/\//-/g'`
# wget -U "$AGENT" -t 2 -O $f.html $u
# curl -A "$AGENT" -I $u > $f.meta
 curl -A "$AGENT" -k -i $u > $f.http
done