1"""An implementation of a S3 data source for nsscache."""
2
3__author__ = 'alexey.pikin@gmail.com'
4
5import base64
6import collections
7import logging
8import json
9import datetime
10import boto3
11from botocore.exceptions import ClientError
12
13from nss_cache.maps import group
14from nss_cache.maps import passwd
15from nss_cache.maps import shadow
16from nss_cache.maps import sshkey
17from nss_cache.sources import source
18from nss_cache import error
19
20
21def RegisterImplementation(registration_callback):
22    registration_callback(S3FilesSource)
23
24
25class S3FilesSource(source.Source):
26    """Source for data fetched from S3."""
27
28    # S3 defaults
29    BUCKET = ''
30    PASSWD_OBJECT = ''
31    GROUP_OBJECT = ''
32    SHADOW_OBJECT = ''
33    SSH_OBJECT = ''
34
35    # for registration
36    name = 's3'
37
38    def __init__(self, conf):
39        """Initialise the S3FilesSource object.
40
41        Args:
42          conf: A dictionary of key/value pairs.
43
44        Raises:
45          RuntimeError: object wasn't initialised with a dict
46        """
47        super(S3FilesSource, self).__init__(conf)
48        self._SetDefaults(conf)
49        self.s3_client = None
50
51    def _GetClient(self):
52        if self.s3_client is None:
53            self.s3_client = boto3.client('s3')
54        return self.s3_client
55
56    def _SetDefaults(self, configuration):
57        """Set defaults if necessary."""
58
59        if 'bucket' not in configuration:
60            configuration['bucket'] = self.BUCKET
61        if 'passwd_object' not in configuration:
62            configuration['passwd_object'] = self.PASSWD_OBJECT
63        if 'group_object' not in configuration:
64            configuration['group_object'] = self.GROUP_OBJECT
65        if 'shadow_object' not in configuration:
66            configuration['shadow_object'] = self.SHADOW_OBJECT
67        if 'sshkey_object' not in configuration:
68            configuration['sshkey_object'] = self.SSH_OBJECT
69
70    def GetPasswdMap(self, since=None):
71        """Return the passwd map from this source.
72
73        Args:
74          since: Get data only changed since this timestamp (inclusive) or None
75          for all data.
76
77        Returns:
78          instance of passwd.PasswdMap
79        """
80        return PasswdUpdateGetter().GetUpdates(self._GetClient(),
81                                               self.conf['bucket'],
82                                               self.conf['passwd_object'],
83                                               since)
84
85    def GetGroupMap(self, since=None):
86        """Return the group map from this source.
87
88        Args:
89          since: Get data only changed since this timestamp (inclusive) or None
90          for all data.
91
92        Returns:
93          instance of group.GroupMap
94        """
95        return GroupUpdateGetter().GetUpdates(self._GetClient(),
96                                              self.conf['bucket'],
97                                              self.conf['group_object'], since)
98
99    def GetShadowMap(self, since=None):
100        """Return the shadow map from this source.
101
102        Args:
103          since: Get data only changed since this timestamp (inclusive) or None
104          for all data.
105
106        Returns:
107          instance of shadow.ShadowMap
108        """
109        return ShadowUpdateGetter().GetUpdates(self._GetClient(),
110                                               self.conf['bucket'],
111                                               self.conf['shadow_object'],
112                                               since)
113
114    def GetSshkeyMap(self, since=None):
115        """Return the ssh map from this source.
116
117        Args:
118          since: Get data only changed since this timestamp (inclusive) or None
119          for all data.
120
121        Returns:
122          instance of shadow.SSHMap
123        """
124        return SshkeyUpdateGetter().GetUpdates(self._GetClient(),
125                                               self.conf['bucket'],
126                                               self.conf['sshkey_object'],
127                                               since)
128
129
130class S3UpdateGetter(object):
131    """Base class that gets updates from s3."""
132
133    def __init__(self):
134        self.log = logging.getLogger(__name__)
135
136    def FromTimestampToDateTime(self, ts):
137        """Converts internal nss_cache timestamp to datetime object.
138
139        Args:
140          ts: number of seconds since epoch
141        Returns:
142          datetime object
143        """
144        return datetime.datetime.utcfromtimestamp(ts)
145
146    def FromDateTimeToTimestamp(self, datetime_obj):
147        """Converts datetime object to internal nss_cache timestamp.
148
149        Args:
150          datetime object
151        Returns:
152          number of seconds since epoch
153        """
154        dt = datetime_obj.replace(tzinfo=None)
155        return int((dt - datetime.datetime(1970, 1, 1)).total_seconds())
156
157    def GetUpdates(self, s3_client, bucket, obj, since):
158        """Get updates from a source.
159
160        Args:
161          s3_client: initialized s3 client
162          bucket: s3 bucket
163          obj: object with the data
164          since: a timestamp representing the last change (None to force-get)
165
166        Returns:
167          A tuple containing the map of updates and a maximum timestamp
168
169        Raises:
170          ValueError: an object in the source map is malformed
171          ConfigurationError:
172        """
173        try:
174            if since is not None:
175                response = s3_client.get_object(
176                    Bucket=bucket,
177                    IfModifiedSince=self.FromTimestampToDateTime(since),
178                    Key=obj)
179            else:
180                response = s3_client.get_object(Bucket=bucket, Key=obj)
181            body = response['Body']
182            last_modified_ts = self.FromDateTimeToTimestamp(
183                response['LastModified'])
184        except ClientError as e:
185            error_code = int(e.response['Error']['Code'])
186            if error_code == 304:
187                return []
188            self.log.error('error getting S3 object ({}): {}'.format(obj, e))
189            raise error.SourceUnavailable('unable to download object from S3')
190
191        data_map = self.GetMap(cache_info=body)
192        data_map.SetModifyTimestamp(last_modified_ts)
193        return data_map
194
195    def GetParser(self):
196        """Return the appropriate parser.
197
198        Must be implemented by child class.
199        """
200        raise NotImplementedError
201
202    def GetMap(self, cache_info):
203        """Creates a Map from the cache_info data.
204
205        Args:
206          cache_info: file-like object containing the data to parse
207
208        Returns:
209          A child of Map containing the cache data.
210        """
211        return self.GetParser().GetMap(cache_info, self.CreateMap())
212
213
214class PasswdUpdateGetter(S3UpdateGetter):
215    """Get passwd updates."""
216
217    def GetParser(self):
218        """Returns a MapParser to parse FilesPasswd cache."""
219        return S3PasswdMapParser()
220
221    def CreateMap(self):
222        """Returns a new PasswdMap instance to have PasswdMapEntries added to
223        it."""
224        return passwd.PasswdMap()
225
226
227class GroupUpdateGetter(S3UpdateGetter):
228    """Get group updates."""
229
230    def GetParser(self):
231        """Returns a MapParser to parse FilesGroup cache."""
232        return S3GroupMapParser()
233
234    def CreateMap(self):
235        """Returns a new GroupMap instance to have GroupMapEntries added to
236        it."""
237        return group.GroupMap()
238
239
240class ShadowUpdateGetter(S3UpdateGetter):
241    """Get shadow updates."""
242
243    def GetParser(self):
244        """Returns a MapParser to parse FilesShadow cache."""
245        return S3ShadowMapParser()
246
247    def CreateMap(self):
248        """Returns a new ShadowMap instance to have ShadowMapEntries added to
249        it."""
250        return shadow.ShadowMap()
251
252
253class SshkeyUpdateGetter(S3UpdateGetter):
254    """Get ssh updates."""
255
256    def GetParser(self):
257        """Returns a MapParser to parse FilesSsh cache."""
258        return S3SshkeyMapParser()
259
260    def CreateMap(self):
261        """Returns a new SshMap instance to have SshMapEntries added to
262        it."""
263        return sshkey.SshkeyMap()
264
265
266class S3MapParser(object):
267    """A base class for parsing nss_files module cache."""
268
269    def __init__(self):
270        self.log = logging.getLogger(__name__)
271
272    def GetMap(self, cache_info, data):
273        """Returns a map from a cache.
274
275        Args:
276          cache_info: file like object containing the cache.
277          data: a Map to populate.
278        Returns:
279          A child of Map containing the cache data.
280        """
281        for obj in json.loads(cache_info.read()):
282            key = obj.get('Key', '')
283            value = obj.get('Value', '')
284            if not value or not key:
285                continue
286            map_entry = self._ReadEntry(key, value)
287            if map_entry is None:
288                self.log.warning(
289                    'Could not create entry from line %r in cache, skipping',
290                    value)
291                continue
292            if not data.Add(map_entry):
293                self.log.warning(
294                    'Could not add entry %r read from line %r in cache',
295                    map_entry, value)
296        return data
297
298
299class S3PasswdMapParser(S3MapParser):
300    """Class for parsing nss_files module passwd cache."""
301
302    def _ReadEntry(self, name, entry):
303        """Return a PasswdMapEntry from a record in the target cache."""
304
305        map_entry = passwd.PasswdMapEntry()
306        # maps expect strict typing, so convert to int as appropriate.
307        map_entry.name = name
308        map_entry.passwd = entry.get('passwd', 'x')
309
310        try:
311            map_entry.uid = int(entry['uid'])
312            map_entry.gid = int(entry['gid'])
313        except (ValueError, KeyError):
314            return None
315
316        map_entry.gecos = entry.get('comment', '')
317        map_entry.dir = entry.get('home', '/home/{}'.format(name))
318        map_entry.shell = entry.get('shell', '/bin/bash')
319
320        return map_entry
321
322
323class S3SshkeyMapParser(S3MapParser):
324    """Class for parsing nss_files module sshkey cache."""
325
326    def _ReadEntry(self, name, entry):
327        """Return a sshkey from a record in the target cache."""
328
329        map_entry = sshkey.SshkeyMapEntry()
330        # maps expect strict typing, so convert to int as appropriate.
331        map_entry.name = name
332        map_entry.sshkey = entry.get('sshPublicKey', '')
333
334        return map_entry
335
336
337class S3GroupMapParser(S3MapParser):
338    """Class for parsing a nss_files module group cache."""
339
340    def _ReadEntry(self, name, entry):
341        """Return a GroupMapEntry from a record in the target cache."""
342
343        map_entry = group.GroupMapEntry()
344        # map entries expect strict typing, so convert as appropriate
345        map_entry.name = name
346        map_entry.passwd = entry.get('passwd', 'x')
347
348        try:
349            map_entry.gid = int(entry['gid'])
350        except (ValueError, KeyError):
351            return None
352
353        try:
354            members = entry.get('members', '').split('\n')
355        except (ValueError, TypeError):
356            members = ['']
357        map_entry.members = members
358        return map_entry
359
360
361class S3ShadowMapParser(S3MapParser):
362    """Class for parsing nss_files module shadow cache."""
363
364    def _ReadEntry(self, name, entry):
365        """Return a ShadowMapEntry from a record in the target cache."""
366
367        map_entry = shadow.ShadowMapEntry()
368        # maps expect strict typing, so convert to int as appropriate.
369        map_entry.name = name
370        map_entry.passwd = entry.get('passwd', '*')
371
372        for attr in ['lstchg', 'min', 'max', 'warn', 'inact', 'expire']:
373            try:
374                setattr(map_entry, attr, int(entry[attr]))
375            except (ValueError, KeyError):
376                continue
377
378        return map_entry
379