1# 2# Copyright (C) 2018 Satoru SATOH <ssato @ redhat.com> 3# License: MIT 4# 5# pylint: disable=unidiomatic-typecheck 6r"""Abstract processor module. 7 8.. versionadded:: 0.9.5 9 10 - Add to abstract processors such like Parsers (loaders and dumpers). 11""" 12from __future__ import absolute_import 13 14import operator 15import pkg_resources 16 17import anyconfig.compat 18import anyconfig.ioinfo 19import anyconfig.models.processor 20import anyconfig.utils 21 22from anyconfig.globals import ( 23 UnknownProcessorTypeError, UnknownFileTypeError, IOInfo 24) 25 26 27def _load_plugins_itr(pgroup, safe=True): 28 """ 29 .. seealso:: the doc of :func:`load_plugins` 30 """ 31 for res in pkg_resources.iter_entry_points(pgroup): 32 try: 33 yield res.load() 34 except ImportError: 35 if safe: 36 continue 37 raise 38 39 40def load_plugins(pgroup, safe=True): 41 """ 42 :param pgroup: A string represents plugin type, e.g. anyconfig_backends 43 :param safe: Do not raise ImportError during load if True 44 :raises: ImportError 45 """ 46 return list(_load_plugins_itr(pgroup, safe=safe)) 47 48 49def sort_by_prio(prs): 50 """ 51 :param prs: A list of :class:`anyconfig.models.processor.Processor` classes 52 :return: Sambe as above but sorted by priority 53 """ 54 return sorted(prs, key=operator.methodcaller("priority"), reverse=True) 55 56 57def select_by_key(items, sort_fn=sorted): 58 """ 59 :param items: A list of tuples of keys and values, [([key], val)] 60 :return: A list of tuples of key and values, [(key, [val])] 61 62 >>> select_by_key([(["a", "aaa"], 1), (["b", "bb"], 2), (["a"], 3)]) 63 [('a', [1, 3]), ('aaa', [1]), ('b', [2]), ('bb', [2])] 64 """ 65 itr = anyconfig.utils.concat(((k, v) for k in ks) for ks, v in items) 66 return list((k, sort_fn(t[1] for t in g)) 67 for k, g 68 in anyconfig.utils.groupby(itr, operator.itemgetter(0))) 69 70 71def list_by_x(prs, key): 72 """ 73 :param key: Grouping key, "type" or "extensions" 74 :return: 75 A list of :class:`Processor` or its children classes grouped by 76 given 'item', [(cid, [:class:`Processor`)]] by default 77 """ 78 if key == "type": 79 kfn = operator.methodcaller(key) 80 res = sorted(((k, sort_by_prio(g)) for k, g 81 in anyconfig.utils.groupby(prs, kfn)), 82 key=operator.itemgetter(0)) 83 84 elif key == "extensions": 85 res = select_by_key(((p.extensions(), p) for p in prs), 86 sort_fn=sort_by_prio) 87 else: 88 raise ValueError("Argument 'key' must be 'type' or " 89 "'extensions' but it was '%s'" % key) 90 91 return res 92 93 94def findall_with_pred(predicate, prs): 95 """ 96 :param predicate: any callable to filter results 97 :param prs: A list of :class:`anyconfig.models.processor.Processor` classes 98 :return: A list of appropriate processor classes or [] 99 """ 100 return sorted((p for p in prs if predicate(p)), 101 key=operator.methodcaller("priority"), reverse=True) 102 103 104def maybe_processor(type_or_id, cls=anyconfig.models.processor.Processor): 105 """ 106 :param type_or_id: 107 Type of the data to process or ID of the processor class or 108 :class:`anyconfig.models.processor.Processor` class object or its 109 instance 110 :param cls: A class object to compare with 'type_or_id' 111 :return: Processor instance or None 112 """ 113 if isinstance(type_or_id, cls): 114 return type_or_id 115 116 if type(type_or_id) == type(cls) and issubclass(type_or_id, cls): 117 return type_or_id() 118 119 return None 120 121 122def find_by_type_or_id(type_or_id, prs): 123 """ 124 :param type_or_id: Type of the data to process or ID of the processor class 125 :param prs: A list of :class:`anyconfig.models.processor.Processor` classes 126 :return: 127 A list of processor classes to process files of given data type or 128 processor 'type_or_id' found by its ID 129 :raises: UnknownProcessorTypeError 130 """ 131 def pred(pcls): 132 """Predicate""" 133 return pcls.cid() == type_or_id or pcls.type() == type_or_id 134 135 pclss = findall_with_pred(pred, prs) 136 if not pclss: 137 raise UnknownProcessorTypeError(type_or_id) 138 139 return pclss 140 141 142def find_by_fileext(fileext, prs): 143 """ 144 :param fileext: File extension 145 :param prs: A list of :class:`anyconfig.models.processor.Processor` classes 146 :return: A list of processor class to processor files with given extension 147 :raises: UnknownFileTypeError 148 """ 149 def pred(pcls): 150 """Predicate""" 151 return fileext in pcls.extensions() 152 153 pclss = findall_with_pred(pred, prs) 154 if not pclss: 155 raise UnknownFileTypeError("file extension={}".format(fileext)) 156 157 return pclss # :: [Processor], never [] 158 159 160def find_by_maybe_file(obj, prs): 161 """ 162 :param obj: 163 a file path, file or file-like object, pathlib.Path object or an 164 'anyconfig.globals.IOInfo' (namedtuple) object 165 :param cps_by_ext: A list of processor classes 166 :return: A list of processor classes to process given (maybe) file 167 :raises: UnknownFileTypeError 168 """ 169 if not isinstance(obj, IOInfo): 170 obj = anyconfig.ioinfo.make(obj) 171 172 return find_by_fileext(obj.extension, prs) # :: [Processor], never [] 173 174 175def findall(obj, prs, forced_type=None, 176 cls=anyconfig.models.processor.Processor): 177 """ 178 :param obj: 179 a file path, file, file-like object, pathlib.Path object or an 180 'anyconfig.globals.IOInfo` (namedtuple) object 181 :param prs: A list of :class:`anyconfig.models.processor.Processor` classes 182 :param forced_type: 183 Forced processor type of the data to process or ID of the processor 184 class or None 185 :param cls: A class object to compare with 'forced_type' later 186 187 :return: A list of instances of processor classes to process 'obj' data 188 :raises: ValueError, UnknownProcessorTypeError, UnknownFileTypeError 189 """ 190 if (obj is None or not obj) and forced_type is None: 191 raise ValueError("The first argument 'obj' or the second argument " 192 "'forced_type' must be something other than " 193 "None or False.") 194 195 if forced_type is None: 196 pclss = find_by_maybe_file(obj, prs) # :: [Processor], never [] 197 else: 198 pclss = find_by_type_or_id(forced_type, prs) # Do. 199 200 return pclss 201 202 203def find(obj, prs, forced_type=None, cls=anyconfig.models.processor.Processor): 204 """ 205 :param obj: 206 a file path, file, file-like object, pathlib.Path object or an 207 'anyconfig.globals.IOInfo' (namedtuple) object 208 :param prs: A list of :class:`anyconfig.models.processor.Processor` classes 209 :param forced_type: 210 Forced processor type of the data to process or ID of the processor 211 class or :class:`anyconfig.models.processor.Processor` class object or 212 its instance itself 213 :param cls: A class object to compare with 'forced_type' later 214 215 :return: an instance of processor class to process 'obj' data 216 :raises: ValueError, UnknownProcessorTypeError, UnknownFileTypeError 217 """ 218 if forced_type is not None: 219 processor = maybe_processor(forced_type, cls=cls) 220 if processor is not None: 221 return processor 222 223 pclss = findall(obj, prs, forced_type=forced_type, cls=cls) 224 return pclss[0]() 225 226 227class Processors(object): 228 """An abstract class of which instance holding processors. 229 """ 230 _pgroup = None # processor group name to load plugins 231 232 def __init__(self, processors=None): 233 """ 234 :param processors: 235 A list of :class:`Processor` or its children class objects or None 236 """ 237 self._processors = dict() # {<processor_class_id>: <processor_class>} 238 if processors is not None: 239 self.register(*processors) 240 241 self.load_plugins() 242 243 def register(self, *pclss): 244 """ 245 :param pclss: A list of :class:`Processor` or its children classes 246 """ 247 for pcls in pclss: 248 if pcls.cid() not in self._processors: 249 self._processors[pcls.cid()] = pcls 250 251 def load_plugins(self): 252 """Load and register pluggable processor classes internally. 253 """ 254 if self._pgroup: 255 self.register(*load_plugins(self._pgroup)) 256 257 def list(self, sort=False): 258 """ 259 :param sort: Result will be sorted if it's True 260 :return: A list of :class:`Processor` or its children classes 261 """ 262 prs = self._processors.values() 263 if sort: 264 return sorted(prs, key=operator.methodcaller("cid")) 265 266 return prs 267 268 def list_by_cid(self): 269 """ 270 :return: 271 A list of :class:`Processor` or its children classes grouped by 272 each cid, [(cid, [:class:`Processor`)]] 273 """ 274 prs = self._processors 275 return sorted(((cid, [prs[cid]]) for cid in sorted(prs.keys())), 276 key=operator.itemgetter(0)) 277 278 def list_by_type(self): 279 """ 280 :return: 281 A list of :class:`Processor` or its children classes grouped by 282 each type, [(type, [:class:`Processor`)]] 283 """ 284 return list_by_x(self.list(), "type") 285 286 def list_by_x(self, item=None): 287 """ 288 :param item: Grouping key, one of "cid", "type" and "extensions" 289 :return: 290 A list of :class:`Processor` or its children classes grouped by 291 given 'item', [(cid, [:class:`Processor`)]] by default 292 """ 293 prs = self._processors 294 295 if item is None or item == "cid": # Default. 296 res = [(cid, [prs[cid]]) for cid in sorted(prs.keys())] 297 298 elif item in ("type", "extensions"): 299 res = list_by_x(prs.values(), item) 300 else: 301 raise ValueError("keyword argument 'item' must be one of " 302 "None, 'cid', 'type' and 'extensions' " 303 "but it was '%s'" % item) 304 return res 305 306 def list_x(self, key=None): 307 """ 308 :param key: Which of key to return from "cid", "type", and "extention" 309 :return: A list of x 'key' 310 """ 311 if key in ("cid", "type"): 312 return sorted(set(operator.methodcaller(key)(p) 313 for p in self._processors.values())) 314 if key == "extension": 315 return sorted(k for k, _v in self.list_by_x("extensions")) 316 317 raise ValueError("keyword argument 'key' must be one of " 318 "None, 'cid', 'type' and 'extension' " 319 "but it was '%s'" % key) 320 321 def findall(self, obj, forced_type=None, 322 cls=anyconfig.models.processor.Processor): 323 """ 324 :param obj: 325 a file path, file, file-like object, pathlib.Path object or an 326 'anyconfig.globals.IOInfo' (namedtuple) object 327 :param forced_type: Forced processor type to find 328 :param cls: A class object to compare with 'ptype' 329 330 :return: A list of instances of processor classes to process 'obj' 331 :raises: ValueError, UnknownProcessorTypeError, UnknownFileTypeError 332 """ 333 return [p() for p in findall(obj, self.list(), 334 forced_type=forced_type, cls=cls)] 335 336 def find(self, obj, forced_type=None, 337 cls=anyconfig.models.processor.Processor): 338 """ 339 :param obj: 340 a file path, file, file-like object, pathlib.Path object or an 341 'anyconfig.globals.IOInfo' (namedtuple) object 342 :param forced_type: Forced processor type to find 343 :param cls: A class object to compare with 'ptype' 344 345 :return: an instance of processor class to process 'obj' 346 :raises: ValueError, UnknownProcessorTypeError, UnknownFileTypeError 347 """ 348 return find(obj, self.list(), forced_type=forced_type, cls=cls) 349 350# vim:sw=4:ts=4:et: 351