1"""An implementation of a S3 data source for nsscache.""" 2 3__author__ = 'alexey.pikin@gmail.com' 4 5import base64 6import collections 7import logging 8import json 9import datetime 10import boto3 11from botocore.exceptions import ClientError 12 13from nss_cache.maps import group 14from nss_cache.maps import passwd 15from nss_cache.maps import shadow 16from nss_cache.maps import sshkey 17from nss_cache.sources import source 18from nss_cache import error 19 20 21def RegisterImplementation(registration_callback): 22 registration_callback(S3FilesSource) 23 24 25class S3FilesSource(source.Source): 26 """Source for data fetched from S3.""" 27 28 # S3 defaults 29 BUCKET = '' 30 PASSWD_OBJECT = '' 31 GROUP_OBJECT = '' 32 SHADOW_OBJECT = '' 33 SSH_OBJECT = '' 34 35 # for registration 36 name = 's3' 37 38 def __init__(self, conf): 39 """Initialise the S3FilesSource object. 40 41 Args: 42 conf: A dictionary of key/value pairs. 43 44 Raises: 45 RuntimeError: object wasn't initialised with a dict 46 """ 47 super(S3FilesSource, self).__init__(conf) 48 self._SetDefaults(conf) 49 self.s3_client = None 50 51 def _GetClient(self): 52 if self.s3_client is None: 53 self.s3_client = boto3.client('s3') 54 return self.s3_client 55 56 def _SetDefaults(self, configuration): 57 """Set defaults if necessary.""" 58 59 if 'bucket' not in configuration: 60 configuration['bucket'] = self.BUCKET 61 if 'passwd_object' not in configuration: 62 configuration['passwd_object'] = self.PASSWD_OBJECT 63 if 'group_object' not in configuration: 64 configuration['group_object'] = self.GROUP_OBJECT 65 if 'shadow_object' not in configuration: 66 configuration['shadow_object'] = self.SHADOW_OBJECT 67 if 'sshkey_object' not in configuration: 68 configuration['sshkey_object'] = self.SSH_OBJECT 69 70 def GetPasswdMap(self, since=None): 71 """Return the passwd map from this source. 72 73 Args: 74 since: Get data only changed since this timestamp (inclusive) or None 75 for all data. 76 77 Returns: 78 instance of passwd.PasswdMap 79 """ 80 return PasswdUpdateGetter().GetUpdates(self._GetClient(), 81 self.conf['bucket'], 82 self.conf['passwd_object'], 83 since) 84 85 def GetGroupMap(self, since=None): 86 """Return the group map from this source. 87 88 Args: 89 since: Get data only changed since this timestamp (inclusive) or None 90 for all data. 91 92 Returns: 93 instance of group.GroupMap 94 """ 95 return GroupUpdateGetter().GetUpdates(self._GetClient(), 96 self.conf['bucket'], 97 self.conf['group_object'], since) 98 99 def GetShadowMap(self, since=None): 100 """Return the shadow map from this source. 101 102 Args: 103 since: Get data only changed since this timestamp (inclusive) or None 104 for all data. 105 106 Returns: 107 instance of shadow.ShadowMap 108 """ 109 return ShadowUpdateGetter().GetUpdates(self._GetClient(), 110 self.conf['bucket'], 111 self.conf['shadow_object'], 112 since) 113 114 def GetSshkeyMap(self, since=None): 115 """Return the ssh map from this source. 116 117 Args: 118 since: Get data only changed since this timestamp (inclusive) or None 119 for all data. 120 121 Returns: 122 instance of shadow.SSHMap 123 """ 124 return SshkeyUpdateGetter().GetUpdates(self._GetClient(), 125 self.conf['bucket'], 126 self.conf['sshkey_object'], 127 since) 128 129 130class S3UpdateGetter(object): 131 """Base class that gets updates from s3.""" 132 133 def __init__(self): 134 self.log = logging.getLogger(__name__) 135 136 def FromTimestampToDateTime(self, ts): 137 """Converts internal nss_cache timestamp to datetime object. 138 139 Args: 140 ts: number of seconds since epoch 141 Returns: 142 datetime object 143 """ 144 return datetime.datetime.utcfromtimestamp(ts) 145 146 def FromDateTimeToTimestamp(self, datetime_obj): 147 """Converts datetime object to internal nss_cache timestamp. 148 149 Args: 150 datetime object 151 Returns: 152 number of seconds since epoch 153 """ 154 dt = datetime_obj.replace(tzinfo=None) 155 return int((dt - datetime.datetime(1970, 1, 1)).total_seconds()) 156 157 def GetUpdates(self, s3_client, bucket, obj, since): 158 """Get updates from a source. 159 160 Args: 161 s3_client: initialized s3 client 162 bucket: s3 bucket 163 obj: object with the data 164 since: a timestamp representing the last change (None to force-get) 165 166 Returns: 167 A tuple containing the map of updates and a maximum timestamp 168 169 Raises: 170 ValueError: an object in the source map is malformed 171 ConfigurationError: 172 """ 173 try: 174 if since is not None: 175 response = s3_client.get_object( 176 Bucket=bucket, 177 IfModifiedSince=self.FromTimestampToDateTime(since), 178 Key=obj) 179 else: 180 response = s3_client.get_object(Bucket=bucket, Key=obj) 181 body = response['Body'] 182 last_modified_ts = self.FromDateTimeToTimestamp( 183 response['LastModified']) 184 except ClientError as e: 185 error_code = int(e.response['Error']['Code']) 186 if error_code == 304: 187 return [] 188 self.log.error('error getting S3 object ({}): {}'.format(obj, e)) 189 raise error.SourceUnavailable('unable to download object from S3') 190 191 data_map = self.GetMap(cache_info=body) 192 data_map.SetModifyTimestamp(last_modified_ts) 193 return data_map 194 195 def GetParser(self): 196 """Return the appropriate parser. 197 198 Must be implemented by child class. 199 """ 200 raise NotImplementedError 201 202 def GetMap(self, cache_info): 203 """Creates a Map from the cache_info data. 204 205 Args: 206 cache_info: file-like object containing the data to parse 207 208 Returns: 209 A child of Map containing the cache data. 210 """ 211 return self.GetParser().GetMap(cache_info, self.CreateMap()) 212 213 214class PasswdUpdateGetter(S3UpdateGetter): 215 """Get passwd updates.""" 216 217 def GetParser(self): 218 """Returns a MapParser to parse FilesPasswd cache.""" 219 return S3PasswdMapParser() 220 221 def CreateMap(self): 222 """Returns a new PasswdMap instance to have PasswdMapEntries added to 223 it.""" 224 return passwd.PasswdMap() 225 226 227class GroupUpdateGetter(S3UpdateGetter): 228 """Get group updates.""" 229 230 def GetParser(self): 231 """Returns a MapParser to parse FilesGroup cache.""" 232 return S3GroupMapParser() 233 234 def CreateMap(self): 235 """Returns a new GroupMap instance to have GroupMapEntries added to 236 it.""" 237 return group.GroupMap() 238 239 240class ShadowUpdateGetter(S3UpdateGetter): 241 """Get shadow updates.""" 242 243 def GetParser(self): 244 """Returns a MapParser to parse FilesShadow cache.""" 245 return S3ShadowMapParser() 246 247 def CreateMap(self): 248 """Returns a new ShadowMap instance to have ShadowMapEntries added to 249 it.""" 250 return shadow.ShadowMap() 251 252 253class SshkeyUpdateGetter(S3UpdateGetter): 254 """Get ssh updates.""" 255 256 def GetParser(self): 257 """Returns a MapParser to parse FilesSsh cache.""" 258 return S3SshkeyMapParser() 259 260 def CreateMap(self): 261 """Returns a new SshMap instance to have SshMapEntries added to 262 it.""" 263 return sshkey.SshkeyMap() 264 265 266class S3MapParser(object): 267 """A base class for parsing nss_files module cache.""" 268 269 def __init__(self): 270 self.log = logging.getLogger(__name__) 271 272 def GetMap(self, cache_info, data): 273 """Returns a map from a cache. 274 275 Args: 276 cache_info: file like object containing the cache. 277 data: a Map to populate. 278 Returns: 279 A child of Map containing the cache data. 280 """ 281 for obj in json.loads(cache_info.read()): 282 key = obj.get('Key', '') 283 value = obj.get('Value', '') 284 if not value or not key: 285 continue 286 map_entry = self._ReadEntry(key, value) 287 if map_entry is None: 288 self.log.warning( 289 'Could not create entry from line %r in cache, skipping', 290 value) 291 continue 292 if not data.Add(map_entry): 293 self.log.warning( 294 'Could not add entry %r read from line %r in cache', 295 map_entry, value) 296 return data 297 298 299class S3PasswdMapParser(S3MapParser): 300 """Class for parsing nss_files module passwd cache.""" 301 302 def _ReadEntry(self, name, entry): 303 """Return a PasswdMapEntry from a record in the target cache.""" 304 305 map_entry = passwd.PasswdMapEntry() 306 # maps expect strict typing, so convert to int as appropriate. 307 map_entry.name = name 308 map_entry.passwd = entry.get('passwd', 'x') 309 310 try: 311 map_entry.uid = int(entry['uid']) 312 map_entry.gid = int(entry['gid']) 313 except (ValueError, KeyError): 314 return None 315 316 map_entry.gecos = entry.get('comment', '') 317 map_entry.dir = entry.get('home', '/home/{}'.format(name)) 318 map_entry.shell = entry.get('shell', '/bin/bash') 319 320 return map_entry 321 322 323class S3SshkeyMapParser(S3MapParser): 324 """Class for parsing nss_files module sshkey cache.""" 325 326 def _ReadEntry(self, name, entry): 327 """Return a sshkey from a record in the target cache.""" 328 329 map_entry = sshkey.SshkeyMapEntry() 330 # maps expect strict typing, so convert to int as appropriate. 331 map_entry.name = name 332 map_entry.sshkey = entry.get('sshPublicKey', '') 333 334 return map_entry 335 336 337class S3GroupMapParser(S3MapParser): 338 """Class for parsing a nss_files module group cache.""" 339 340 def _ReadEntry(self, name, entry): 341 """Return a GroupMapEntry from a record in the target cache.""" 342 343 map_entry = group.GroupMapEntry() 344 # map entries expect strict typing, so convert as appropriate 345 map_entry.name = name 346 map_entry.passwd = entry.get('passwd', 'x') 347 348 try: 349 map_entry.gid = int(entry['gid']) 350 except (ValueError, KeyError): 351 return None 352 353 try: 354 members = entry.get('members', '').split('\n') 355 except (ValueError, TypeError): 356 members = [''] 357 map_entry.members = members 358 return map_entry 359 360 361class S3ShadowMapParser(S3MapParser): 362 """Class for parsing nss_files module shadow cache.""" 363 364 def _ReadEntry(self, name, entry): 365 """Return a ShadowMapEntry from a record in the target cache.""" 366 367 map_entry = shadow.ShadowMapEntry() 368 # maps expect strict typing, so convert to int as appropriate. 369 map_entry.name = name 370 map_entry.passwd = entry.get('passwd', '*') 371 372 for attr in ['lstchg', 'min', 'max', 'warn', 'inact', 'expire']: 373 try: 374 setattr(map_entry, attr, int(entry[attr])) 375 except (ValueError, KeyError): 376 continue 377 378 return map_entry 379