1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
|
# coding: utf-8
# -------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
# --------------------------------------------------------------------------
import pytest
from os import path, remove, sys, urandom
import platform
import uuid
from azure.core.pipeline.policies import HTTPPolicy
from azure.storage.blob import (
BlobServiceClient,
BlobBlock
)
from azure.storage.blob._shared.base_client import _format_shared_key_credential
from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer
# ------------------------------------------------------------------------------
TEST_BLOB_PREFIX = 'largestblob'
LARGEST_BLOCK_SIZE = 4000 * 1024 * 1024
LARGEST_SINGLE_UPLOAD_SIZE = 5000 * 1024 * 1024
# ------------------------------------------------------------------------------
if platform.python_implementation() == 'PyPy':
pytest.skip("Skip tests for Pypy", allow_module_level=True)
class StorageLargestBlockBlobTest(StorageTestCase):
def _setup(self, storage_account, key, additional_policies=None, min_large_block_upload_threshold=1 * 1024 * 1024,
max_single_put_size=32 * 1024):
self.bsc = BlobServiceClient(
self.account_url(storage_account, "blob"),
credential=key,
max_single_put_size=max_single_put_size,
max_block_size=LARGEST_BLOCK_SIZE,
min_large_block_upload_threshold=min_large_block_upload_threshold,
_additional_pipeline_policies=additional_policies)
self.config = self.bsc._config
self.container_name = self.get_resource_name('utcontainer')
self.container_name = self.container_name + str(uuid.uuid4())
if self.is_live:
self.bsc.create_container(self.container_name)
def _teardown(self, file_name):
if path.isfile(file_name):
try:
remove(file_name)
except:
pass
# --Helpers-----------------------------------------------------------------
def _get_blob_reference(self):
return self.get_resource_name(TEST_BLOB_PREFIX)
def _create_blob(self):
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
blob.upload_blob(b'')
return blob
# --Test cases for block blobs --------------------------------------------
@pytest.mark.live_test_only
@pytest.mark.skip(reason="This takes really long time")
@GlobalStorageAccountPreparer()
def test_put_block_bytes_largest(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob = self._create_blob()
# Act
data = urandom(LARGEST_BLOCK_SIZE)
blockId = str(uuid.uuid4()).encode('utf-8')
resp = blob.stage_block(
blockId,
data,
length=LARGEST_BLOCK_SIZE)
blob.commit_block_list([BlobBlock(blockId)])
block_list = blob.get_block_list()
# Assert
self.assertIsNotNone(resp)
assert 'content_md5' in resp
assert 'content_crc64' in resp
assert 'request_id' in resp
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list), 2)
self.assertEqual(len(block_list[1]), 0)
self.assertEqual(len(block_list[0]), 1)
self.assertEqual(block_list[0][0].size, LARGEST_BLOCK_SIZE)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_put_block_bytes_largest_without_network(self, resource_group, location, storage_account, storage_account_key):
payload_dropping_policy = PayloadDroppingPolicy()
credential_policy = _format_shared_key_credential(storage_account.name, storage_account_key)
self._setup(storage_account, storage_account_key, [payload_dropping_policy, credential_policy])
blob = self._create_blob()
# Act
data = urandom(LARGEST_BLOCK_SIZE)
blockId = str(uuid.uuid4()).encode('utf-8')
resp = blob.stage_block(
blockId,
data,
length=LARGEST_BLOCK_SIZE)
blob.commit_block_list([BlobBlock(blockId)])
block_list = blob.get_block_list()
# Assert
self.assertIsNotNone(resp)
assert 'content_md5' in resp
assert 'content_crc64' in resp
assert 'request_id' in resp
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list), 2)
self.assertEqual(len(block_list[1]), 0)
self.assertEqual(len(block_list[0]), 1)
self.assertEqual(payload_dropping_policy.put_block_counter, 1)
self.assertEqual(payload_dropping_policy.put_block_sizes[0], LARGEST_BLOCK_SIZE)
@pytest.mark.live_test_only
@pytest.mark.skip(reason="This takes really long time")
@GlobalStorageAccountPreparer()
def test_put_block_stream_largest(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob = self._create_blob()
# Act
stream = LargeStream(LARGEST_BLOCK_SIZE)
blockId = str(uuid.uuid4())
requestId = str(uuid.uuid4())
resp = blob.stage_block(
blockId,
stream,
length=LARGEST_BLOCK_SIZE,
client_request_id=requestId)
blob.commit_block_list([BlobBlock(blockId)])
block_list = blob.get_block_list()
# Assert
self.assertIsNotNone(resp)
assert 'content_md5' in resp
assert 'content_crc64' in resp
assert 'request_id' in resp
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list), 2)
self.assertEqual(len(block_list[1]), 0)
self.assertEqual(len(block_list[0]), 1)
self.assertEqual(block_list[0][0].size, LARGEST_BLOCK_SIZE)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_put_block_stream_largest_without_network(self, resource_group, location, storage_account, storage_account_key):
payload_dropping_policy = PayloadDroppingPolicy()
credential_policy = _format_shared_key_credential(storage_account.name, storage_account_key)
self._setup(storage_account, storage_account_key, [payload_dropping_policy, credential_policy])
blob = self._create_blob()
# Act
stream = LargeStream(LARGEST_BLOCK_SIZE)
blockId = str(uuid.uuid4())
requestId = str(uuid.uuid4())
resp = blob.stage_block(
blockId,
stream,
length=LARGEST_BLOCK_SIZE,
client_request_id=requestId)
blob.commit_block_list([BlobBlock(blockId)])
block_list = blob.get_block_list()
# Assert
self.assertIsNotNone(resp)
assert 'content_md5' in resp
assert 'content_crc64' in resp
assert 'request_id' in resp
self.assertIsNotNone(block_list)
self.assertEqual(len(block_list), 2)
self.assertEqual(len(block_list[1]), 0)
self.assertEqual(len(block_list[0]), 1)
self.assertEqual(payload_dropping_policy.put_block_counter, 1)
self.assertEqual(payload_dropping_policy.put_block_sizes[0], LARGEST_BLOCK_SIZE)
@pytest.mark.live_test_only
@pytest.mark.skip(reason="This takes really long time")
@GlobalStorageAccountPreparer()
def test_create_largest_blob_from_path(self, resource_group, location, storage_account, storage_account_key):
self._setup(storage_account, storage_account_key)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
FILE_PATH = 'largest_blob_from_path.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
largeStream = LargeStream(LARGEST_BLOCK_SIZE, 100 * 1024 * 1024)
chunk = largeStream.read()
while chunk:
stream.write(chunk)
chunk = largeStream.read()
# Act
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, max_concurrency=2)
# Assert
self._teardown(FILE_PATH)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_largest_blob_from_path_without_network(self, resource_group, location, storage_account, storage_account_key):
payload_dropping_policy = PayloadDroppingPolicy()
credential_policy = _format_shared_key_credential(storage_account.name, storage_account_key)
self._setup(storage_account, storage_account_key, [payload_dropping_policy, credential_policy])
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
FILE_PATH = 'largest_blob_from_path.temp.{}.dat'.format(str(uuid.uuid4()))
with open(FILE_PATH, 'wb') as stream:
largeStream = LargeStream(LARGEST_BLOCK_SIZE, 100 * 1024 * 1024)
chunk = largeStream.read()
while chunk:
stream.write(chunk)
chunk = largeStream.read()
# Act
with open(FILE_PATH, 'rb') as stream:
blob.upload_blob(stream, max_concurrency=2)
# Assert
self._teardown(FILE_PATH)
self.assertEqual(payload_dropping_policy.put_block_counter, 1)
self.assertEqual(payload_dropping_policy.put_block_sizes[0], LARGEST_BLOCK_SIZE)
@pytest.mark.skip(reason="This takes really long time")
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_largest_blob_from_stream_without_network(self, resource_group, location, storage_account, storage_account_key):
payload_dropping_policy = PayloadDroppingPolicy()
credential_policy = _format_shared_key_credential(storage_account.name, storage_account_key)
self._setup(storage_account, storage_account_key, [payload_dropping_policy, credential_policy])
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
number_of_blocks = 50000
stream = LargeStream(LARGEST_BLOCK_SIZE*number_of_blocks)
# Act
blob.upload_blob(stream, max_concurrency=1)
# Assert
self.assertEqual(payload_dropping_policy.put_block_counter, number_of_blocks)
self.assertEqual(payload_dropping_policy.put_block_sizes[0], LARGEST_BLOCK_SIZE)
@pytest.mark.live_test_only
@GlobalStorageAccountPreparer()
def test_create_largest_blob_from_stream_single_upload_without_network(self, resource_group, location, storage_account, storage_account_key):
payload_dropping_policy = PayloadDroppingPolicy()
credential_policy = _format_shared_key_credential(storage_account.name, storage_account_key)
self._setup(storage_account, storage_account_key, [payload_dropping_policy, credential_policy],
max_single_put_size=LARGEST_SINGLE_UPLOAD_SIZE)
blob_name = self._get_blob_reference()
blob = self.bsc.get_blob_client(self.container_name, blob_name)
stream = LargeStream(LARGEST_SINGLE_UPLOAD_SIZE)
# Act
blob.upload_blob(stream, length=LARGEST_SINGLE_UPLOAD_SIZE, max_concurrency=1)
# Assert
self.assertEqual(payload_dropping_policy.put_block_counter, 0)
self.assertEqual(payload_dropping_policy.put_blob_counter, 1)
class LargeStream:
def __init__(self, length, initial_buffer_length=1024*1024):
self._base_data = urandom(initial_buffer_length)
self._base_data_length = initial_buffer_length
self._position = 0
self._remaining = length
def read(self, size=None):
if self._remaining == 0:
return b""
if size is None:
e = self._base_data_length
else:
e = size
e = min(e, self._remaining)
if e > self._base_data_length:
self._base_data = urandom(e)
self._base_data_length = e
self._remaining = self._remaining - e
return self._base_data[:e]
def remaining(self):
return self._remaining
class PayloadDroppingPolicy(HTTPPolicy):
def __init__(self):
self.put_block_counter = 0
self.put_block_sizes = []
self.put_blob_counter = 0
self.put_blob_sizes = []
def send(self, request): # type: (PipelineRequest) -> PipelineResponse
if _is_put_block_request(request):
if request.http_request.body:
self.put_block_counter = self.put_block_counter + 1
self.put_block_sizes.append(_get_body_length(request))
replacement = "dummy_body"
request.http_request.body = replacement
request.http_request.headers["Content-Length"] = str(len(replacement))
elif _is_put_blob_request(request):
if request.http_request.body:
self.put_blob_counter = self.put_blob_counter + 1
self.put_blob_sizes.append(_get_body_length(request))
replacement = "dummy_body"
request.http_request.body = replacement
request.http_request.headers["Content-Length"] = str(len(replacement))
return self.next.send(request)
def _is_put_block_request(request):
query = request.http_request.query
return query and "comp" in query and query["comp"] == "block"
def _is_put_blob_request(request):
query = request.http_request.query
return request.http_request.method == "PUT" and not query
def _get_body_length(request):
body = request.http_request.body
length = 0
if hasattr(body, "read"):
chunk = body.read(10*1024*1024)
while chunk:
length = length + len(chunk)
chunk = body.read(10 * 1024 * 1024)
else:
length = len(body)
return length
# ------------------------------------------------------------------------------
|
