File: wget-list

package info (click to toggle)
whatweb 0.4.8~git20161009-1
  • links: PTS, VCS
  • area: main
  • in suites: stretch
  • size: 8,588 kB
  • ctags: 540
  • sloc: ruby: 33,376; sh: 612; makefile: 42
file content (18 lines) | stat: -rwxr-xr-x 472 bytes parent folder | download | duplicates (5) 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
 
#!/bin/bash

AGENT="Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.12) Gecko/2009070811 Ubuntu/9.04 (jaunty) Firefox/3.0.12"

if [ -z $1 ]; then
 echo "Usage: $0 <file with list of urls>"
 echo "downloads each URL's html and headers into the current directory"
 exit
fi


for i in `cat $1` ; do
 u=$i; f=`echo $i | sed 's/http:\/\///g' | sed 's/\//-/g'`
# wget -U "$AGENT" -t 2 -O $f.html $u
# curl -A "$AGENT" -I $u > $f.meta
 curl -A "$AGENT" -k -i $u > $f.http
done