1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
|
# Django storage using libcloud providers
# Aymeric Barantal (mric at chamal.fr) 2011
#
import io
from urllib.parse import urljoin
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.core.files.base import File
from django.core.files.storage import Storage
from django.utils.deconstruct import deconstructible
from storages.utils import clean_name
try:
from libcloud.storage.providers import get_driver
from libcloud.storage.types import ObjectDoesNotExistError
from libcloud.storage.types import Provider
except ImportError:
raise ImproperlyConfigured("Could not load libcloud")
@deconstructible
class LibCloudStorage(Storage):
"""Django storage derived class using apache libcloud to operate
on supported providers"""
def __init__(self, provider_name=None, option=None):
if provider_name is None:
provider_name = getattr(settings, "DEFAULT_LIBCLOUD_PROVIDER", "default")
self.provider = settings.LIBCLOUD_PROVIDERS.get(provider_name)
if not self.provider:
raise ImproperlyConfigured(
"LIBCLOUD_PROVIDERS %s not defined or invalid" % provider_name
)
extra_kwargs = {}
if "region" in self.provider:
extra_kwargs["region"] = self.provider["region"]
# Used by the GoogleStorageDriver
if "project" in self.provider:
extra_kwargs["project"] = self.provider["project"]
try:
provider_type = self.provider["type"]
if isinstance(provider_type, str):
module_path, tag = provider_type.rsplit(".", 1)
if module_path != "libcloud.storage.types.Provider":
raise ValueError("Invalid module path")
provider_type = getattr(Provider, tag)
Driver = get_driver(provider_type)
self.driver = Driver(
self.provider["user"], self.provider["key"], **extra_kwargs
)
except Exception as e:
raise ImproperlyConfigured(
"Unable to create libcloud driver type %s: %s"
% (self.provider.get("type"), e)
)
self.bucket = self.provider["bucket"] # Limit to one container
def _get_bucket(self):
"""Helper to get bucket object (libcloud container)"""
return self.driver.get_container(self.bucket)
def _get_object(self, name):
"""Get object by its name. ObjectDoesNotExistError will be raised if object not
found"""
return self.driver.get_object(self.bucket, clean_name(name))
def delete(self, name):
"""Delete object on remote"""
try:
obj = self._get_object(name)
return self.driver.delete_object(obj)
except ObjectDoesNotExistError:
pass
def exists(self, name):
try:
_ = self._get_object(name)
except ObjectDoesNotExistError:
return False
return True
def listdir(self, path="/"):
"""Lists the contents of the specified path,
returning a 2-tuple of lists; the first item being
directories, the second item being files.
"""
container = self._get_bucket()
objects = self.driver.list_container_objects(container)
path = clean_name(path)
if not path.endswith("/"):
path = "%s/" % path
files = []
dirs = []
# TOFIX: better algorithm to filter correctly
# (and not depend on google-storage empty folder naming)
for o in objects:
if path == "/":
if o.name.count("/") == 0:
files.append(o.name)
elif o.name.count("/") == 1:
dir_name = o.name[: o.name.index("/")]
if dir_name not in dirs:
dirs.append(dir_name)
elif o.name.startswith(path):
if o.name.count("/") <= path.count("/"):
# TOFIX : special case for google storage with empty dir
if o.name.endswith("_$folder$"):
name = o.name[:-9]
name = name[len(path) :]
dirs.append(name)
else:
name = o.name[len(path) :]
files.append(name)
return (dirs, files)
def size(self, name):
obj = self._get_object(name)
return obj.size if obj else -1
def url(self, name):
provider_type = self.provider["type"].lower()
obj = self._get_object(name)
if not obj:
return None
try:
url = self.driver.get_object_cdn_url(obj)
except NotImplementedError as e:
object_path = "{}/{}".format(self.bucket, obj.name)
if "s3" in provider_type:
base_url = "https://%s" % self.driver.connection.host
url = urljoin(base_url, object_path)
elif "google" in provider_type:
url = urljoin("https://storage.googleapis.com", object_path)
elif "azure" in provider_type:
base_url = "https://%s.blob.core.windows.net" % self.provider["user"]
url = urljoin(base_url, object_path)
elif "backblaze" in provider_type:
url = urljoin("api.backblaze.com/b2api/v1/", object_path)
else:
raise e
return url
def _open(self, name, mode="rb"):
remote_file = LibCloudFile(name, self, mode=mode)
return remote_file
def _read(self, name):
try:
obj = self._get_object(name)
except ObjectDoesNotExistError as e:
raise FileNotFoundError(str(e))
# TOFIX : we should be able to read chunk by chunk
return next(self.driver.download_object_as_stream(obj, obj.size))
def _save(self, name, file):
self.driver.upload_object_via_stream(iter(file), self._get_bucket(), name)
return name
class LibCloudFile(File):
"""File inherited class for libcloud storage objects read and write"""
def __init__(self, name, storage, mode):
self.name = name
self._storage = storage
self._mode = mode
self._is_dirty = False
self._file = None
def _get_file(self):
if self._file is None:
data = self._storage._read(self.name)
self._file = io.BytesIO(data)
return self._file
def _set_file(self, value):
self._file = value
file = property(_get_file, _set_file)
@property
def size(self):
if not hasattr(self, "_size"):
self._size = self._storage.size(self.name)
return self._size
def read(self, num_bytes=None):
return self.file.read(num_bytes)
def write(self, content):
if "w" not in self._mode:
raise AttributeError("File was opened for read-only access.")
self.file = io.BytesIO(content)
self._is_dirty = True
def close(self):
if self._is_dirty:
self._storage._save(self.name, self.file)
self.file.close()
|
