1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
|
from __future__ import annotations
import gzip
import json
import random
from urllib.parse import urlencode
from twisted.internet.task import deferLater
from twisted.web import resource, server
from twisted.web.server import NOT_DONE_YET
from twisted.web.util import Redirect, redirectTo
from scrapy.utils.python import to_bytes, to_unicode
def getarg(request, name, default=None, type_=None):
if name in request.args:
value = request.args[name][0]
if type_ is not None:
value = type_(value)
return value
return default
def close_connection(request):
# We have to force a disconnection for HTTP/1.1 clients. Otherwise
# client keeps the connection open waiting for more data.
request.channel.loseConnection()
request.finish()
# most of the following resources are copied from twisted.web.test.test_webclient
class ForeverTakingResource(resource.Resource):
"""
L{ForeverTakingResource} is a resource which never finishes responding
to requests.
"""
def __init__(self, write=False):
resource.Resource.__init__(self)
self._write = write
def render(self, request):
if self._write:
request.write(b"some bytes")
return server.NOT_DONE_YET
class HostHeaderResource(resource.Resource):
"""
A testing resource which renders itself as the value of the host header
from the request.
"""
def render(self, request):
return request.requestHeaders.getRawHeaders(b"host")[0]
class PayloadResource(resource.Resource):
"""
A testing resource which renders itself as the contents of the request body
as long as the request body is 100 bytes long, otherwise which renders
itself as C{"ERROR"}.
"""
def render(self, request):
data = request.content.read()
contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0]
if len(data) != 100 or int(contentLength) != 100:
return b"ERROR"
return data
class LeafResource(resource.Resource):
isLeaf = True
def deferRequest(self, request, delay, f, *a, **kw):
from twisted.internet import reactor
def _cancelrequest(_):
# silence CancelledError
d.addErrback(lambda _: None)
d.cancel()
d = deferLater(reactor, delay, f, *a, **kw)
request.notifyFinish().addErrback(_cancelrequest)
return d
class Follow(LeafResource):
def render(self, request):
total = getarg(request, b"total", 100, type_=int)
show = getarg(request, b"show", 1, type_=int)
order = getarg(request, b"order", b"desc")
maxlatency = getarg(request, b"maxlatency", 0, type_=float)
n = getarg(request, b"n", total, type_=int)
if order == b"rand":
nlist = [random.randint(1, total) for _ in range(show)]
else: # order == "desc"
nlist = range(n, max(n - show, 0), -1)
lag = random.random() * maxlatency
self.deferRequest(request, lag, self.renderRequest, request, nlist)
return NOT_DONE_YET
def renderRequest(self, request, nlist):
s = """<html> <head></head> <body>"""
args = request.args.copy()
for nl in nlist:
args[b"n"] = [to_bytes(str(nl))]
argstr = urlencode(args, doseq=True)
s += f"<a href='/follow?{argstr}'>follow {nl}</a><br>"
s += """</body>"""
request.write(to_bytes(s))
request.finish()
class Delay(LeafResource):
def render_GET(self, request):
n = getarg(request, b"n", 1, type_=float)
b = getarg(request, b"b", 1, type_=int)
if b:
# send headers now and delay body
request.write("")
self.deferRequest(request, n, self._delayedRender, request, n)
return NOT_DONE_YET
def _delayedRender(self, request, n):
request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n"))
request.finish()
class Status(LeafResource):
def render_GET(self, request):
n = getarg(request, b"n", 200, type_=int)
request.setResponseCode(n)
return b""
class Raw(LeafResource):
def render_GET(self, request):
request.startedWriting = 1
self.deferRequest(request, 0, self._delayedRender, request)
return NOT_DONE_YET
render_POST = render_GET
def _delayedRender(self, request):
raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n")
request.startedWriting = 1
request.write(raw)
request.channel.transport.loseConnection()
request.finish()
class Echo(LeafResource):
def render_GET(self, request):
output = {
"headers": {
to_unicode(k): [to_unicode(v) for v in vs]
for k, vs in request.requestHeaders.getAllRawHeaders()
},
"body": to_unicode(request.content.read()),
}
return to_bytes(json.dumps(output))
render_POST = render_GET
class RedirectTo(LeafResource):
def render(self, request):
goto = getarg(request, b"goto", b"/")
# we force the body content, otherwise Twisted redirectTo()
# returns HTML with <meta http-equiv="refresh"
redirectTo(goto, request)
return b"redirecting..."
class Partial(LeafResource):
def render_GET(self, request):
request.setHeader(b"Content-Length", b"1024")
self.deferRequest(request, 0, self._delayedRender, request)
return NOT_DONE_YET
def _delayedRender(self, request):
request.write(b"partial content\n")
request.finish()
class Drop(Partial):
def _delayedRender(self, request):
abort = getarg(request, b"abort", 0, type_=int)
request.write(b"this connection will be dropped\n")
tr = request.channel.transport
try:
if abort and hasattr(tr, "abortConnection"):
tr.abortConnection()
else:
tr.loseConnection()
finally:
request.finish()
class ArbitraryLengthPayloadResource(LeafResource):
def render(self, request):
return request.content.read()
class NoMetaRefreshRedirect(Redirect):
def render(self, request: server.Request) -> bytes:
content = Redirect.render(self, request)
return content.replace(
b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"'
)
class ContentLengthHeaderResource(resource.Resource):
"""
A testing resource which renders itself as the value of the Content-Length
header from the request.
"""
def render(self, request):
return request.requestHeaders.getRawHeaders(b"content-length")[0]
class ChunkedResource(resource.Resource):
def render(self, request):
from twisted.internet import reactor
def response():
request.write(b"chunked ")
request.write(b"content\n")
request.finish()
reactor.callLater(0, response)
return server.NOT_DONE_YET
class BrokenChunkedResource(resource.Resource):
def render(self, request):
from twisted.internet import reactor
def response():
request.write(b"chunked ")
request.write(b"content\n")
# Disable terminating chunk on finish.
request.chunked = False
close_connection(request)
reactor.callLater(0, response)
return server.NOT_DONE_YET
class BrokenDownloadResource(resource.Resource):
def render(self, request):
from twisted.internet import reactor
def response():
request.setHeader(b"Content-Length", b"20")
request.write(b"partial")
close_connection(request)
reactor.callLater(0, response)
return server.NOT_DONE_YET
class EmptyContentTypeHeaderResource(resource.Resource):
"""
A testing resource which renders itself as the value of request body
without content-type header in response.
"""
def render(self, request):
request.setHeader("content-type", "")
return request.content.read()
class LargeChunkedFileResource(resource.Resource):
def render(self, request):
from twisted.internet import reactor
def response():
for i in range(1024):
request.write(b"x" * 1024)
request.finish()
reactor.callLater(0, response)
return server.NOT_DONE_YET
class DuplicateHeaderResource(resource.Resource):
def render(self, request):
request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"])
return b""
class UriResource(resource.Resource):
"""Return the full uri that was requested"""
def getChild(self, path, request):
return self
def render(self, request):
# Note: this is an ugly hack for CONNECT request timeout test.
# Returning some data here fail SSL/TLS handshake
# ToDo: implement proper HTTPS proxy tests, not faking them.
if request.method != b"CONNECT":
return request.uri
return b""
class ResponseHeadersResource(resource.Resource):
"""Return a response with headers set from the JSON request body"""
def render(self, request):
body = json.loads(request.content.read().decode())
for header_name, header_value in body.items():
request.responseHeaders.addRawHeader(header_name, header_value)
return json.dumps(body).encode("utf-8")
class Compress(resource.Resource):
"""Compress the data sent in the request url params and set Content-Encoding header"""
def render(self, request):
data = request.args.get(b"data")[0]
accept_encoding_header = request.getHeader(b"accept-encoding")
# include common encoding schemes here
if accept_encoding_header == b"gzip":
request.setHeader(b"Content-Encoding", b"gzip")
return gzip.compress(data)
# just set this to trigger a test failure if no valid accept-encoding header was set
request.setResponseCode(500)
return b"Did not receive a valid accept-encoding header"
class SetCookie(resource.Resource):
"""Return a response with a Set-Cookie header for each request url parameter"""
def render(self, request):
for cookie_name, cookie_values in request.args.items():
for cookie_value in cookie_values:
cookie = (cookie_name.decode() + "=" + cookie_value.decode()).encode()
request.setHeader(b"Set-Cookie", cookie)
return b""
|
