1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
|
# coding: utf-8
from __future__ import unicode_literals, division, absolute_import, print_function
import sys
import re
import os
from oscrypto import tls, errors as oscrypto_errors, version
from asn1crypto.util import OrderedDict
if sys.version_info < (3,):
from urlparse import urlparse
str_cls = unicode # noqa
else:
from urllib.parse import urlparse
str_cls = str
class HttpsClientException(Exception):
pass
class HttpsClientError(HttpsClientException):
pass
class HttpsClient():
def __init__(self, keep_alive=True, ignore_close=False):
self.socket = None
self.timeout = None
self.url_info = None
self.keep_alive = keep_alive
self.ignore_close = ignore_close
def close(self):
"""
Closes any open connection
"""
if not self.socket:
return
self.socket.close()
self.socket = None
def download(self, url, timeout):
"""
Downloads a URL and returns the contents
:param url:
The URL to download
:param timeout:
The int number of seconds to set the timeout to
:return:
The string contents of the URL
"""
self.setup_connection(url, timeout)
tries = 0
while tries < 2:
tries += 1
try:
self.ensure_connected()
req_headers = OrderedDict()
req_headers['Host'] = self.url_info[0]
if self.url_info[1] != 443:
req_headers['Host'] += ':%d' % self.url_info[1]
req_headers['Connection'] = 'Keep-Alive' if self.keep_alive else 'Close'
req_headers["User-Agent"] = 'oscrypto %s TLS HTTP Client' % version.__version__
request = 'GET '
url_info = urlparse(url)
path = '/' if not url_info.path else url_info.path
if url_info.query:
path += '?' + url_info.query
request += path + ' HTTP/1.1'
self.write_request(request, req_headers)
response = self.read_headers()
if not response:
self.close()
continue
v, code, message, resp_headers = response
data = self.read_body(code, resp_headers, timeout)
if code == 301:
location = resp_headers.get('location')
if not isinstance(location, str_cls):
raise HttpsClientError('Missing or duplicate Location HTTP header')
if not re.match(r'https?://', location):
if not location.startswith('/'):
location = os.path.dirname(url_info.path) + location
location = url_info.scheme + '://' + url_info.netloc + location
return self.download(location, timeout)
if code != 200:
raise HttpsClientError('HTTP error %s downloading %s.' % (code, url))
else:
return data
except (oscrypto_errors.TLSGracefulDisconnectError):
self.close()
continue
def setup_connection(self, url, timeout):
"""
:param url:
The URL to download
:param timeout:
The int number of seconds to set the timeout to
:return:
A boolean indicating if the connection was reused
"""
url_info = urlparse(url)
if url_info.scheme == 'http':
raise HttpsClientException('Can not connect to a non-TLS server')
hostname = url_info.hostname
port = url_info.port
if not port:
port = 443
if self.socket and self.url_info != (hostname, port):
self.close()
self.timeout = timeout
self.url_info = (hostname, port)
return self.ensure_connected()
def ensure_connected(self):
"""
Make sure a valid tls.TLSSocket() is open to the server
:return:
A boolean indicating if the connection was reused
"""
if self.socket:
return True
host, port = self.url_info
session = tls.TLSSession()
self.socket = tls.TLSSocket(host, port, timeout=self.timeout, session=session)
return False
def write_request(self, request, headers):
"""
:param request:
A unicode string of the first line of the HTTP request
:param headers:
An OrderedDict of the request headers
"""
lines = [request]
for header, value in headers.items():
lines.append('%s: %s' % (header, value))
lines.extend(['', ''])
request = '\r\n'.join(lines).encode('iso-8859-1')
self.socket.write(request)
def read_headers(self):
"""
Reads the HTTP response headers from the socket
:return:
On error, None, otherwise a 4-element tuple:
0: A 2-element tuple of integers representing the HTTP version
1: An integer representing the HTTP response code
2: A unicode string of the HTTP response code name
3: An OrderedDict of HTTP headers with lowercase unicode key and unicode values
"""
version = None
code = None
text = None
headers = OrderedDict()
data = self.socket.read_until(b'\r\n\r\n')
string = data.decode('iso-8859-1')
first = False
for line in string.split('\r\n'):
line = line.strip()
if first is False:
if line == '':
continue
match = re.match(r'^HTTP/(1\.[01]) +(\d+) +(.*)$', line)
if not match:
return None
version = tuple(map(int, match.group(1).split('.')))
code = int(match.group(2))
text = match.group(3)
first = True
else:
if not len(line):
continue
parts = line.split(':', 1)
if len(parts) == 2:
name = parts[0].strip().lower()
value = parts[1].strip()
if name in headers:
if isinstance(headers[name], tuple):
headers[name] = headers[name] + (value,)
else:
headers[name] = (headers[name], value)
else:
headers[name] = value
return (version, code, text, headers)
def parse_content_length(self, headers):
"""
Returns the content-length from a dict of headers
:return:
An integer of the content length
"""
content_length = headers.get('content-length')
if isinstance(content_length, str_cls) and len(content_length) > 0:
content_length = int(content_length)
return content_length
def read_body(self, code, resp_headers, timeout):
"""
"""
data = b''
transfer_encoding = resp_headers.get('transfer-encoding')
if transfer_encoding and transfer_encoding.lower() == 'chunked':
while True:
line = self.socket.read_until(b'\r\n').decode('iso-8859-1').rstrip()
if re.match(r'^[a-fA-F0-9]+$', line):
chunk_length = int(line, 16)
if chunk_length == 0:
break
data += self.socket.read_exactly(chunk_length)
if self.socket.read_exactly(2) != b'\r\n':
raise HttpsClientException('Unable to parse chunk newline')
else:
self.close()
raise HttpsClientException('Unable to parse chunk length')
else:
content_length = self.parse_content_length(resp_headers)
if content_length is not None:
if content_length > 0:
data = self.socket.read_exactly(content_length)
elif code == 304 or code == 204 or (code >= 100 and code < 200):
# Per https://tools.ietf.org/html/rfc7230#section-3.3.3 these have no body
pass
else:
# This should only happen if the server is going to close the connection
while self.socket.select_read(timeout=timeout):
data += self.socket.read(8192)
self.close()
if not self.ignore_close and resp_headers.get('connection', '').lower() == 'close':
self.close()
return data
|
