1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
|
import sys
from io import BytesIO
from urllib.request import urlopen
def bytes_to_string(nbytes):
if nbytes < 1024:
return '%ib' % nbytes
nbytes /= 1024.
if nbytes < 1024:
return '%.1fkb' % nbytes
nbytes /= 1024.
if nbytes < 1024:
return '%.2fMb' % nbytes
nbytes /= 1024.
return '%.1fGb' % nbytes
def url_content_length(fhandle):
length = dict(fhandle.info())['Content-Length']
return int(length.strip())
def download_with_progress_bar(data_url, return_buffer=False):
"""Download a file, showing progress
Parameters
----------
data_url : string
web address
return_buffer : boolean (optional)
if true, return a BytesIO buffer rather than a string
Returns
-------
s : string
content of the file
"""
num_units = 40
fhandle = urlopen(data_url)
content_length = url_content_length(fhandle)
chunk_size = content_length // num_units
print("Downloading %s" % data_url)
nchunks = 0
buf = BytesIO()
content_length_str = bytes_to_string(content_length)
while True:
next_chunk = fhandle.read(chunk_size)
nchunks += 1
if next_chunk:
buf.write(next_chunk)
s = ('[' + nchunks * '='
+ (num_units - 1 - nchunks) * ' '
+ '] {} / {} \r'.format(bytes_to_string(buf.tell()),
content_length_str))
else:
sys.stdout.write('\n')
break
sys.stdout.write(s)
sys.stdout.flush()
buf.seek(0)
if return_buffer:
return buf
else:
return buf.getvalue()
|
