1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186
|
#!/usr/bin/env python
## Copyright (C) 1998,1999 marduk <marduk@python.net>
## This program is free software; you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation; either version 2 of the License, or
## (at your option) any later version.
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
## You should have received a copy of the GNU General Public License
## along with this program; if not, write to the Free Software
## Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
# $Id: linbot.py,v 1.8 1999/03/12 04:56:01 marduk Exp $
# $Log: linbot.py,v $
# Revision 1.8 1999/03/12 04:56:01 marduk
# Added ability to warn of old versions of linbot/plugins
# Added patch to enable file:// to work with OS/2
#
# Revision 1.7 1999/03/12 01:48:14 marduk
# Preliminary support for authenticating proxies added.
# Added Andrea's redirect-depth patch.
#
# Revision 1.6 1999/03/11 04:51:25 marduk
# Added version module.
#
# Revision 1.5 1999/03/11 02:29:51 marduk
# Added debugio module to handle debugging and IO
#
# Revision 1.4 1999/02/21 16:39:25 marduk
# 1.0b8
#
# Revision 1.3 1999/01/10 01:01:44 marduk
# Linbot 1.0b6
#
# Revision 1.2 1998/12/23 07:34:59 marduk
# Fixed problem in linbot.py "import parser"
#
# Revision 1.1.1.2 1998/12/20 23:27:50 marduk
# This is pre 1.0, I hope
#
USAGE='linbot [-abvq][-l url][-x url]... [-y url]... [-r depth][-o dir][-w sec][-d level] url [location]...'
PYTHON_VERSION=1.5 # not used right now
explored = []
problem_db = {}
linkList = {}
import sys
import time
import os
start_time = time.ctime(time.time())
# importing the config.py file is a real problem if the user did not install
# the files EXACTLY the way I said to... or even using the frozen version is
# becoming a real bitch. I will just have to tell them right out how to fix it.
try:
sys.path = ['.'] + sys.path
import config
except ImportError:
sys.stdout.write('Please verify that PYTHONPATH knows where to find "config.py"\n')
sys.exit(1)
import myUrlLib
Link=myUrlLib.Link
# myUrlLib will be looking for a 'config' module. set it up here.
myUrlLib.config=config
import debugio
debugio.DEBUG_LEVEL = config.DEBUG_LEVEL
import version
def parse_args():
import getopt
global URL
try:
optlist, args = getopt.getopt(sys.argv[1:],'vl:x:y:ar:o:bw:d:q')
except getopt.error, reason:
print reason
print USAGE
sys.exit(1)
for flag,arg in optlist:
if flag=='-v':
print_version()
sys.exit(0)
elif flag=='-x':
config.EXCLUDED_URLS.append(arg)
elif flag=='-y':
config.YANKED_URLS.append(arg)
elif flag=='-a':
config.AVOID_EXTERNAL_LINKS=1
elif flag=='-r':
config.REDIRECT_DEPTH=int(arg)
elif flag=='-o':
config.OUTPUT_DIR=arg
elif flag=='-b':
config.BASE_URLS_ONLY=1
elif flag=='-w':
config.WAIT_BETWEEN_REQUESTS=int(arg)
elif flag=='-l':
config.LOGO_HREF=arg
elif flag=='-d':
debugio.DEBUG_LEVEL=int(arg)
elif flag=='-q':
debugio.DEBUG_LEVEL=0
if len(args)==0:
print USAGE
sys.exit(1)
else: URL = args[0]
config.HOSTS = config.HOSTS + args[1:]
def print_version():
"""Print version information"""
import os
print " Linbot: " + version.linbot
print " Python: " + sys.version
print " OS: " + os.name
print
def warn():
"""Warn the user that something has gone wrong."""
print "*******************************************"
print "* *"
print "* Warning, Linbot has found nothing to *"
print "* report for this site. If you feel this *"
print "* is in error, please contact *"
print "* %s. *" % version.author
print "* and specify the environment that caused *"
print "* this to occur. *"
print "* *"
print "* Linbot %s *" % version.linbot
print "* *"
print "*******************************************"
def link_image(filename):
source = '/usr/share/linbot/' + filename
target = config.OUTPUT_DIR + filename
if os.path.exists(target): return
try:
os.symlink(source, target)
except os.error, (errcode, errtext):
print 'Warning: "%s": %s' % (target, errtext)
print ' Please copy "%s" to "%s".' % (source, target)
# set up the pages
plugins = config.PLUGINS
if __name__ == '__main__':
parse_args()
config.OUTPUT_DIR=config.OUTPUT_DIR + '/'
debugio.write('checking site....')
try:
Link.base = Link(URL,None) # this will take a while
except KeyboardInterrupt:
sys.stderr.write("Interrupted\n")
sys.exit(1)
debugio.write('done.')
if not hasattr(Link.base,"URL"):
warn()
sys.exit(1)
linkList = Link.linkList
# now we can write out the files
# start with the frame-description page
debugio.write('Generating reports...')
from plugins.rptlib import *
main_index()
nav_bar(plugins)
link_image('blackbar.gif')
if config.LOGO_HREF == 'linbot.gif': link_image('linbot.gif')
debugio.write('done.')
|