#!/usr/bin/python
#
#SPIKE Proxy file: spkproxy.py
#
#Usage: python spkproxy.py [port:8080]
###################################################################
#Requires:
#pyOpenSSL v 5.0 pre or >
#python 2.2
###################################################################
#Version 1.1
#Author: Dave Aitel (dave@immunitysec.com)
#License: GPL v 2.0
####################################################################
#Known Bugs:
#1.
#http://www.btinternet.com/~wildfire/reference/httpstatus/500.htm
#for some reason the header is not parsed correctly...possibly TheCounter/2.1
#web server does not use \r\n?
#2. Netscape toolbar (from cnn.com) is not quite right
#FIXED 3. BBC.co.uk news - needs :// joined to work
#####################################################################
#BEGIN IMPORTS
import socket
import sys
from threading import Thread
import string
import os
from OpenSSL import SSL
import getopt
sys.path.append('ntlm')
#need to import aps stuff so we can send NTLM to the remote system
#annoyingly, he named it utils
from ntlmutils import str2unicode
import ntlm_messages
import ntlm_procs
#threadsafe workaround for PyOpenSSL 1.5.1
#import OpenSSL.tsafe
#should use OpenSSL.tsafe.Connection instead of OpenSSL.SSL.Connection!
#but we can't cause it doesn't seem to work
#default UI, could add others.
import spikeProxyUI
import daveutil
import time
#time all sockets out at ten seconds
import timeoutsocket
timeoutsocket.setDefaultSocketTimeout(3)
import versioncheck
#END IMPORTS
#Begin Code!
VERSION="1.4.8"
default404stringlist=["Page Not Found"]
#### you change these to say what hosts and pages are ALLOWED. If they
#are not set, ALL are allowed
restrictedhosts=[]
restrictedpages=[]
denied1="ErrorYou are not allowed to visit that page during this test, sorry. Try unsetting your proxy temporarily."
deniedstring="HTTP/1.1 404 404 Access Denied !\r\nContent-Length: %d"%len(denied1)+"\r\n\r\n"+denied1
#Class myConnection is used to wrap sockets so we can have some basic
#abstraction over which ssl library we use, for example
#we basically wrap a few socket calls here
class MyConnection:
def __init__(self,conn):
self.doSSL=0
self.mysocket=conn
def recv(self,size):
#if self.doSSL:
# print "Reciving data as ssl!"
#print "Recieving %d bytes" % size
#if self.doSSL:
# print "Reading since we are SSL"
# return self.mysocket.read(size)
result=self.mysocket.recv(size)
#print "Returned from recv()"
return result
#reliable send over socket
def send(self,data):
sizetosend=len(data)
sentsize=0
while sentsize
def startSSLserver(self):
debug_ssl=0
dir = os.path.dirname(sys.argv[0])
if dir == '':
dir = os.curdir
self.mysocket.send("HTTP/1.1 200 Connection established\r\n\r\n")
ctx = SSL.Context(SSL.SSLv23_METHOD)
ctx.set_timeout(5)
ctx.set_verify(SSL.VERIFY_NONE, self.verify_cb) # Don't demand a certificate
try:
ctx.use_privatekey_file (os.path.join(dir, 'server.pkey'))
except:
print "Couldn't find file %s"%(os.path.join(dir, 'server.pkey'))
ctx.use_certificate_file(os.path.join(dir, 'server.cert'))
ctx.load_verify_locations(os.path.join(dir, 'CA.cert'))
#normally would be SSL.connection, but we want to be threadsafe
self.mysocket = SSL.Connection(ctx, self.mysocket)
#only works with pyOpenSSL 5.0pre or >
self.mysocket.set_accept_state()
if debug_ssl:
print "State="+self.mysocket.state_string()
#done automatically
#self.mysocket.do_handshake()
self.doSSL=1
if debug_ssl:
print "Now using SSL to talk to client"
#wraps socket.close
def close(self):
#print "calling connection.close"
self.mysocket.close()
return
#we read 0 and on any exception return 1
def gotclosed(self):
#print "Checking if we got closed"
try:
data=self.mysocket.send("")
except:
#print "CAUGHT EXCEPTION CHECKING IF WE WERE CLOSED"
return 1
return 0
###########################################################################
#class header is what we use to store request and reponse headers
class header:
def __init__(self):
self.clear()
#these two functions arn't necessary anymore
#we have to remove any sockets from our header...
#def __getstate__(self):
# odict = self.__dict__.copy() # copy the dict since we change it
# del odict['connection'] # remove filehandle entry
# return odict
#you need to sometimes set the connection's state into SSL
#def setConnection(self,conn):
# self.connection=conn
# return
#clears out the data structure - used for init
def clear(self):
self.data=[]
self.done=0
self.goodHeader=0
self.clientisSSL=0
self.verb=""
#for the first request, we see a CONNECT verb
self.sawCONNECT=0
self.firstline="" #sheesh
#1 if we are reading a response instead of a GET/POST, etc
self.responseHeader=0
self.wasChunked=0
#here is basically what we return from parsing the headers
self.URLargsDict={}
self.headerValuesDict={}
self.useSSL=0
self.connectHost=""
self.URL=""
self.sawsslinit=0
self.connectPort=0
self.mybodysize=0
self.useRawArguments=0
self.allURLargs=""
self.version=""
#set this to not send a content-length
self.doSurpressContentLength=0
#variables for server response headers
self.returncode=""
self.returnmessage=""
self.proxyHeader=""
self.orderlist=[]
return
def getProxyHeader(self):
return self.proxyHeader
#fixes the URL to not have a ? in it if it happens to
def normalize(self):
if self.URL.count("?")>0 and self.URLargsDict=={} and self.useRawArguments==0:
urlbit=self.URL[:]
#if we have a url as well
self.URL=urlbit.split("?")[0]
#if we have arguments too
if len(urlbit.split("?"))>1:
self.allURLargs="?".join(urlbit.split("?")[1:])
#print "SELF.allURLARGS=%s"%self.allURLargs
#print "SELF.URL=%s"%self.URL
self.URLargsDict=daveutil.splitargs(self.allURLargs,orderlist=self.orderlist)
if self.URLargsDict==None:
self.URLargsDict={}
self.useRawArguments=1
return
else:
self.URL+="?"
return
#returns a site tuple (used for VulnXML)
def getSiteTuple(self):
result=(self.connectHost,self.connectPort,self.clientisSSL)
return result
#sets us up from a site tuple
def setSiteTuple(self,site):
self.connectHost=site[0]
self.connectPort=site[1]
self.clientisSSL=site[2]
return
#debug routine
def printme(self):
#print "All my stuff:"
result=""
result+= "Host: "+self.connectHost + "\n"
result+= "Port: "+str(self.connectPort) + "\n"
result+= "SSL : "
if self.clientisSSL:
result+="Yes"
else:
result+="No"
result+="\n\n"
result+=self.verb
for key in self.headerValuesDict.keys():
for value in self.headerValuesDict[key]:
result+=key+": "+value+"\n"
return result
#returns http://www.cnn.com from our header information
def getSite(self):
result=""
if self.useSSL:
result+="https://"
else:
result+="http://"
result+=self.connectHost
if self.connectPort!=80 and self.connectPort!=443:
result+=":"+str(self.connectPort)
return result
#returns 1 if 2 headers (self and other) are basically the same
def issame(self,other):
#we don't compare the header itself. That makes us
#get false negatives with Date: headers and such
#self.headerValuesDict==other.headerValuesDict and \\
if cmp(self.URL,other.URL)==0 and \
self.clientisSSL==other.clientisSSL and \
self.firstline==other.firstline and \
cmp(self.URLargsDict,other.URLargsDict)==0 and \
self.connectPort==other.connectPort and \
self.mybodysize==other.mybodysize and \
daveutil.headerdictcmp(self.headerValuesDict,other.headerValuesDict) and \
self.allURLargs==other.allURLargs:
return 1
return 0
#returns a string that is a "hash"
def genhash(self):
hash=""
hash+=self.verb+self.returncode
hash+=daveutil.hashstring(self.URL+self.allURLargs)
#hash the cookies
if self.headerValuesDict.has_key("Cookie"):
for key in self.headerValuesDict["Cookie"]:
hash+=daveutil.hashstring(key)
if self.headerValuesDict.has_key("Set-Cookie"):
for key in self.headerValuesDict["Set-Cookie"]:
hash+=daveutil.hashstring(key)
#done!
#return it encoded so we get rid of slashes
return daveutil.strencode(hash,"A")
def setSurpressContentLength(self):
self.doSurpressContentLength=1
return
def surpressContentLength(self):
return self.doSurpressContentLength
def setclientSSL(self):
self.useSSL=1
self.clientisSSL=1
return
def addData(self,moredata):
#print "addData "+moredata
self.data.append(moredata)
#print self.data[-4:]
if self.data[-4:]==['\r', '\n', '\r', '\n']:
#print "Got end of header!"
self.done=1
#print "All data="+"".join(self.data)
self.verifyHeader()
#we shouldn't NEED this, but economist.com has a misbehaving
#IIS 5.0 server which does this!!!
if self.data[-2:]==['\n','\n']:
print "Weird \\n\\n in header!"
self.done=1
self.verifyHeader()
return
#keys is a set of values for which we're going to look and
#return an integer associated with them from the headers
#we return the first value in the header list as an int
def getIntValue(self,keys):
#iterate over all the keys in the argument until we have a match
#print "all header keys: "+str(self.headerValuesDict.keys())
for akey in keys:
if self.headerValuesDict.has_key(akey):
#print "Int key: "+akey+" matched "+self.headerValuesDict[akey][0]
#we just return the first one we encounter, sorry
#so multiple headers will just be on a first come
#first serve basis
return int(self.headerValuesDict[akey][0])
return 0
#we return the first value in the header list as a string
#KEYS IS A LIST, NOT A STRING!
def getStrValue(self,keys):
#print "all header keys: "+str(self.headerValuesDict.keys())
for akey in keys:
#print "str: "+akey
if self.headerValuesDict.has_key(akey):
return str(self.headerValuesDict[akey][0])
return "0"
def removeHeaders(self,hstring):
if self.headerValuesDict.has_key(hstring):
del self.headerValuesDict[hstring]
def addHeader(self,newheader,newheadervalue):
#print "Adding header "+newheader+": "+newheadervalue
#now we store it, at last
if not self.headerValuesDict.has_key(newheader):
#intialize it as a list
self.headerValuesDict[newheader]=[]
else:
#print "Duplicate KEY: "+newheader
pass
#just separating them by commas doesn't work for hotmail.com
self.headerValuesDict[newheader].append(newheadervalue)
def verifyHeader(self):
#this little ditty returns a list of lines, without \r\n's
#the -2 is because there were 2 null \r\n thingies on the end
self.allheaders="".join(self.data).split("\r\n")
#print "Self.allheaders="+str(self.allheaders)
firstline=self.allheaders[0]
self.allheaders=self.allheaders[:-2]
#this will fail if we can't parse the first line
if not self.parseFirstLine(firstline):
print "Couldn't parse first line!"
return 0
#did we see a CONNECT?
if self.sawCONNECT:
#print "Saw SSL CONNECT request!"
self.sawsslinit=1
return 1
for headerLine in self.allheaders[1:]:
#print "Doing header line: "+headerLine
tempvalues=headerLine.split(": ")
if len(tempvalues)<2:
#MS hotmail login is lame - uses this header, notice no space:
#P3P:CP="BUS CUR CONo FIN IVDo ONL OUR PHY SAMo TELo"
#so we handle that condition now
tempvalues=headerLine.split(":")
if len(tempvalues)<2:
print "len(tempvalues)!=2 ="+str(len(tempvalues))+" in "+str(tempvalues)
return 0
self.addHeader(tempvalues[0],":".join(tempvalues[1:]))
#print "About to call massageHeaders"
self.massageHeaders()
#print "Headers="+str(self.headerValuesDict)
#print "Got a good header."
self.goodHeader=1
return
#this function takes in
def massageHeaders(self):
#print "Inside massageHeaders"
#non-IE user Agent, for reference
#User-Agent: Mozilla/5.0 Galeon/1.0.3 (X11; Linux i686; U;) Gecko/0
#IE string
IEstring="Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; Bob)"
nonIEstring="Mozilla/5.0 Galeon/1.0.3 (X11; Linux i686; U;) Gecko/0"
#always massage chunked out of the way
#this will cause problems if someone sends over a gig of data
#I doubt that will happen though
if self.getStrValue(["Transfer-Encoding"])=="chunked":
del self.headerValuesDict["Transfer-Encoding"]
self.wasChunked=1
#massage a response differently from a non-response
#uncomment this for dave's cludgy early morning NTLM pass-through
## if self.responseHeader:
## #print "Did not see connection"
## #print "Auth: "+self.getStrValue(["WWW-Authenticate"])
## authenticate=self.getStrValue(["WWW-Authenticate"])
## if authenticate.count("NTLM")>0 or authenticate.count("Negotiate")>0:
## print "Doing band-aide for NTLM"
## self.addHeader("Proxy-Authenticate",authenticate)
## del self.headerValuesDict["WWW-Authenticate"]
## #must replace return code for some reason as well
## #see http://squid.sourceforge.net/ntlm/client_proxy_protocol.html
## self.firstline=self.firstline.replace("401","407")
## return
## #here we replace Proxy-Authentication with Authentication for NTLM
## if self.headerValuesDict.has_key("Proxy-Authorization"):
## self.headerValuesDict["Authorization"]=self.headerValuesDict["Proxy-Authorization"][:]
## del self.headerValuesDict["Proxy-Authorization"]
#by default, use IE 5.0
replaceUserAgent=1
userAgent=IEstring
#change Proxy-Connection to Connection
if self.headerValuesDict.has_key("Proxy-Connection"):
#DEBUG
#print "MassageHeaders: has key proxy-connection"
self.headerValuesDict["Connection"]=self.headerValuesDict["Proxy-Connection"][:]
#print "Connection is now: "+str(self.headerValuesDict["Connection"])
del self.headerValuesDict["Proxy-Connection"]
#replace the User-Agent
if replaceUserAgent:
#just overwrite the damn thing
if self.headerValuesDict.has_key("User-Agent"):
del self.headerValuesDict["User-Agent"]
#comment out the next line for NO user agent
self.addHeader("User-Agent",userAgent)
pass
#save this off before we delete it
self.mybodysize=self.getIntValue(["Content-length","Content-Length"])
#get rid of Content-Length or Content-length - this is
#a requirement since we recalcuate it later for fun!
if self.headerValuesDict.has_key("Content-length"):
del self.headerValuesDict["Content-length"]
if self.headerValuesDict.has_key("Content-Length"):
del self.headerValuesDict["Content-Length"]
#no return value for massageHeaders
return
def parseFirstLine(self,firstline):
#print "firstline="+firstline
templist=firstline.split(" ")
if len(templist)<2:
print "First line of header has less than 2 members!"
return 0
self.verb=templist[0]
if self.verb in [ "HTTP/1.1", "HTTP/1.0" ]:
#print "Response header - not verifying the first line of %s!" % (firstline)
self.responseHeader=1
if len(templist)>1:
self.returncode=templist[1]
if len(templist)>2:
self.returnmessage=templist[2]
self.firstline=firstline
return 1
#TODO: remove this code from the header class out into the spkProxy class
#this is the only place we use self.connection!
#SSL proxy check
if self.verb=="CONNECT":
#WE ARE SSL!
#signifies we connect to server with ssl
self.useSSL=1
#signifies we connect to client with ssl
self.clientisSSL=1
self.sawCONNECT=1
self.connectHost=templist[1].split(":")[0]
#no port would be weird, but maybe it'll happen...
if templist[1].split(":") < 2:
self.connectPort=443
else:
self.connectPort=templist[1].split(":")[1]
return 1
if not self.processProxyUrl(templist[1]):
return 0
#HTTP/1.1 or HTTP/1.0
self.version=templist[2]
#print "VERB="+self.verb+" URL="+self.URL+" version="+self.version
return 1
def processProxyUrl(self, proxyurl):
#here is basically what we return
self.URLargsDict={}
self.useSSL=0
self.connectHost=""
#this might already be set if we got an SSL proxy request
if not self.connectPort:
self.connectPort=80
self.URL=""
#print "processProxyUrl: "+proxyurl
#just in case we ARE doing ssl...
urlbit=proxyurl
#if we're not doing an SSL proxy
if not self.clientisSSL:
#print "proxyURL is not SSL"
#rip the http:// off
urltype=proxyurl.split("://")[0]
if len(proxyurl.split("://")) < 2:
print "Need something after the http:// - exiting this thread"
return 0
#else we are good to go...we reassign urlbit here
#need to do join because of multiple :// in arguments and stuff
#should fix bbc news error
urlbit="://".join(proxyurl.split("://")[1:])
if urltype=="https":
#this is probably broken: REVISIT
self.useSSL=1
elif urltype!="http":
print "unknown url type "+urltype
return 0
#must have http://something
if len(proxyurl.split("://"))<2:
print "must have http://something"
return 0
self.connectHost=urlbit.split("/")[0]
#get rid of the host from urlbit
if len(urlbit.split("/"))<2:
urlbit="/"
else:
urlbit="/".join(urlbit.split("/")[1:])
if urlbit=="":
urlbit="/"
#lame, but should work
if urlbit[0]!="/":
urlbit="/"+urlbit
#print "connectHost="+self.connectHost
if len(self.connectHost.split(":"))>1:
#print "ConnectHost Split: "+str(self.connectHost.split(":"))
self.connectPort=int(self.connectHost.split(":")[1])
self.connectHost=self.connectHost.split(":")[0]
#print "Set self.connectHost to "+self.connectHost
if self.connectHost=="":
print "Error: empty connect host!"
return 0
#end if self.clientisSSL==0:
#TODO: Fix this to work on blah.ng/asdf=asdf&asdf2=asdf2
#this should work, but there's no way for me, as the client
#to really know
if urlbit.count("?")==0 and urlbit.count("=")>0:
indexequal=urlbit.find("=")
if indexequal!=-1:
indexfirstslash=urlbit.rfind("/",0,indexequal)
if indexfirstslash!=-1:
#print "original = "+urlbit
#print "indexequal="+str(indexequal)
#print "indexfirstslash="+str(indexfirstslash)
urlbit=urlbit[:indexfirstslash]+"?"+urlbit[indexfirstslash+1:]
#print "new="+urlbit
#if we have a url as well
self.URL=urlbit.split("?")[0]
#if we have arguments too
if len(urlbit.split("?"))>1:
self.allURLargs="?".join(urlbit.split("?")[1:])
#print "SELF.allURLARGS=%s"%self.allURLargs
#print "SELF.URL=%s"%self.URL
self.URLargsDict=daveutil.splitargs(self.allURLargs,orderlist=self.orderlist)
if self.URLargsDict==None:
self.URLargsDict={}
self.useRawArguments=1
return 1
#got here! success!
#we now have URLargsDict
return 1
def isdone(self):
#print "self.isdone called "+str(self.done)
if self.done==0:
return 0
return 1
def gotGoodHeader(self):
return self.goodHeader
def bodySize(self):
return self.mybodysize
def grabHeader(self,header):
if self.headerValuesDict.has_key(header):
returnstr=""
#iterate over the list and add a line for each
for value in self.headerValuesDict[header]:
returnstr+=header+": "+value+"\r\n"
return returnstr
else:
return ""
def setProxyHeader(self,newheader):
self.proxyHeader=newheader
return
class body:
def __init__(self):
self.mysize=0
self.data=[]
def printme(self):
result= "".join(self.data)
result=daveutil.prettyprint(result)
return result
def setSize(self,size):
self.mysize=size
return
#just compare sizes for speed.
def issame(self,other):
#and self.data==other.data:
if self.mysize==other.mysize :
return 1
return 0
def genhash(self):
hash=""
hash+=daveutil.hashstring("".join(self.data))
return hash
def getArgsDict(self):
argsDict=daveutil.splitargs("".join(self.data))
if argsDict==None:
argsDict={}
return argsDict
def readBlock(self,connection,size):
targetsize=size
tempdata=""
while targetsize > len(tempdata):
#read some data
tempdata+=connection.recv(targetsize-len(tempdata))
#print "Targetsize=%d, len(tempdata)=%d" % (targetsize,len(tempdata))
#print "read "+str(len(tempdata))+" bytes of data in readblock, wanted "+str(size)
self.data+=tempdata
self.mysize+=targetsize
return size
#This handles chunked data cleanly - well, handles it anyways
#this is the cruftiest function ever made.
def read(self,connection,size,waschunked,readtillclosed):
if not waschunked:
if readtillclosed and size==0:
#print "reading till closed"
temp=""
while 1:
#this is a lame way to do it, but hopefully it will work
try:
length=len(temp)
#print "len="+str(length)
temp+=connection.recv(1000)
#print "len2="+str(len(temp))
#WAY crufty here...
if (length==len(temp)):
break
if temp.count("