1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
|
"""An implementation of a GCS data source for nsscache."""
import logging
import warnings
from google.cloud import storage
from nss_cache import error
from nss_cache.maps import group
from nss_cache.maps import passwd
from nss_cache.maps import shadow
from nss_cache.sources import source
from nss_cache.util import file_formats
from nss_cache.util import timestamps
warnings.filterwarnings(
"ignore", "Your application has authenticated using end user credentials"
)
def RegisterImplementation(registration_callback):
registration_callback(GcsFilesSource)
class GcsFilesSource(source.Source):
"""Source for data fetched from GCS."""
# GCS Defaults
BUCKET = ""
PASSWD_OBJECT = ""
GROUP_OBJECT = ""
SHADOW_OBJECT = ""
# for registration
name = "gcs"
def __init__(self, conf):
"""Initialize the GcsFilesSource object.
Args:
conf: A dictionary of key/value pairs.
Raises:
RuntimeError: object wasn't initialized with a dict.
"""
super(GcsFilesSource, self).__init__(conf)
self._SetDefaults(conf)
self._gcs_client = None
def _GetClient(self):
if self._gcs_client is None:
self._gcs_client = storage.Client()
return self._gcs_client
def _SetDefaults(self, configuration):
"""Set defaults if necessary."""
if "bucket" not in configuration:
configuration["bucket"] = self.BUCKET
if "passwd_object" not in configuration:
configuration["passwd_object"] = self.PASSWD_OBJECT
if "group_object" not in configuration:
configuration["group_object"] = self.GROUP_OBJECT
if "shadow_object" not in configuration:
configuration["shadow_object"] = self.SHADOW_OBJECT
def GetPasswdMap(self, since=None):
"""Return the passwd map from this source.
Args:
since: Get data only changed since this timestamp (inclusive) or None
for all data.
Returns:
instance of passwd.PasswdMap
"""
return PasswdUpdateGetter().GetUpdates(
self._GetClient(), self.conf["bucket"], self.conf["passwd_object"], since
)
def GetGroupMap(self, since=None):
"""Return the group map from this source.
Args:
since: Get data only changed since this timestamp (inclusive) or None
for all data.
Returns:
instance of group.GroupMap
"""
return GroupUpdateGetter().GetUpdates(
self._GetClient(), self.conf["bucket"], self.conf["group_object"], since
)
def GetShadowMap(self, since=None):
"""Return the shadow map from this source.
Args:
since: Get data only changed since this timestamp (inclusive) or None
for all data.
Returns:
instance of shadow.ShadowMap
"""
return ShadowUpdateGetter().GetUpdates(
self._GetClient(), self.conf["bucket"], self.conf["shadow_object"], since
)
class GcsUpdateGetter(object):
"""Base class that gets updates from GCS."""
def __init__(self):
self.log = logging.getLogger(__name__)
def GetUpdates(self, gcs_client, bucket_name, obj, since):
"""Gets updates from a source.
Args:
gcs_client: initialized gcs client
bucket_name: gcs bucket name
obj: object with the data
since: a timestamp representing the last change (None to force-get)
Returns:
A tuple containing the map of updates and a maximum timestamp
"""
bucket = gcs_client.bucket(bucket_name)
blob = bucket.get_blob(obj)
# get_blob captures NotFound error and returns None:
if blob is None:
self.log.error("GCS object gs://%s/%s not found", bucket_name, obj)
raise error.SourceUnavailable("unable to download object from GCS.")
# GCS doesn't return HTTP 304 like HTTP or S3 sources,
# so return if updated timestamp is before 'since':
if since and timestamps.FromDateTimeToTimestamp(blob.updated) < since:
return []
data_map = self.GetMap(cache_info=blob.open())
data_map.SetModifyTimestamp(timestamps.FromDateTimeToTimestamp(blob.updated))
return data_map
def GetParser(self):
"""Return the approriate parser.
Must be implemented by child class.
"""
raise NotImplementedError
def GetMap(self, cache_info):
"""Creates a Map from the cache_info data.
Args:
cache_info: file-like object containing the data to parse
Returns:
A child of Map containing the cache data.
"""
return self.GetParser().GetMap(cache_info, self.CreateMap())
class PasswdUpdateGetter(GcsUpdateGetter):
"""Get passwd updates."""
def GetParser(self):
"""Returns a MapParser to parse FilesPasswd cache."""
return file_formats.FilesPasswdMapParser()
def CreateMap(self):
"""Returns a new PasswdMap instance to have PasswdMapEntries added to it."""
return passwd.PasswdMap()
class GroupUpdateGetter(GcsUpdateGetter):
"""Get group updates."""
def GetParser(self):
"""Returns a MapParser to parse FilesGroup cache."""
return file_formats.FilesGroupMapParser()
def CreateMap(self):
"""Returns a new GroupMap instance to have GroupMapEntries added to it."""
return group.GroupMap()
class ShadowUpdateGetter(GcsUpdateGetter):
"""Get shadow updates."""
def GetParser(self):
"""Returns a MapParser to parse FilesShadow cache."""
return file_formats.FilesShadowMapParser()
def CreateMap(self):
"""Returns a new ShadowMap instance to have ShadowMapEntries added to it."""
return shadow.ShadowMap()
|
