1import importlib.machinery 2import inspect 3import logging 4import os 5import secrets 6import sys 7from types import ModuleType 8from typing import Any 9from typing import Callable 10from typing import Dict 11from typing import Iterator 12from typing import List 13from typing import Tuple 14 15import pop.contract 16import pop.dirs 17import pop.exc 18import pop.loader 19import pop.scanner 20import pop.verify 21 22try: 23 import pytest_pop.mods 24 25 HAS_TESTING_LIBS = True 26except ImportError: 27 HAS_TESTING_LIBS = False 28 29EXT_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES) 30log = logging.getLogger(__name__) 31 32 33def ex_path(path: str) -> List[str]: 34 """ 35 Take a path that is sent to the Sub and expand it if it is a string or not 36 """ 37 if path is None: 38 return [] 39 elif isinstance(path, str): 40 return path.split(",") 41 elif isinstance(path, list): 42 return path 43 return [] 44 45 46class Hub: 47 """ 48 The redistributed pop central hub. All components of the system are 49 rooted to the Hub. 50 """ 51 52 def __init__(self): 53 self._subs = {} 54 self._sub_alias = {} 55 self._dynamic = {} 56 self._dscan = False 57 # Add the pop sub to the hub, this should always use pypath and 58 # Should never be made dynamic. This is a core system sub and should 59 # NOT be app-merged 60 pypaths = ["pop.mods.pop"] 61 if HAS_TESTING_LIBS: 62 pypaths.append("pytest_pop.mods") 63 self._subs["pop"] = Sub(self, "pop", pypath=pypaths) 64 self._reverse_sub = None 65 self._iter_subs = sorted(self._subs.keys()) 66 self._iter_ind = 0 67 # Set up the conf OPT structure so it is always available 68 self.OPT = {} 69 70 def __getstate__(self) -> Dict: 71 return dict(_subs=self._subs) 72 73 def __setstate__(self, state: Dict): 74 self.__dict__.update(state) 75 76 def __iter__(self) -> Iterator["Sub"]: 77 def iter(subs: Dict[str, Sub]): 78 for sub in sorted(subs.keys()): 79 yield subs[sub] 80 81 return iter(self._subs) 82 83 def _resolve_this(self, levels: int) -> "Hub": 84 """ 85 This function allows for hub to pop introspective calls. 86 This should only ever be called from within a hub module, otherwise 87 it should stack trace, or return heaven knows what... 88 :param levels: The number of frames to search for a hub reference 89 """ 90 if hasattr( 91 sys, "_getframe" 92 ): # implementation detail of CPython, speeds up things by 100x. 93 desired_frame = sys._getframe(3) 94 contracted = desired_frame.f_locals["self"] 95 else: 96 call_frame = inspect.stack(0)[3] 97 contracted = call_frame[0].f_locals["self"] 98 ref = contracted.ref.split(".") 99 100 # (0=module, 1=module's parent etc.) 101 level_offset = levels - 1 102 traversed = self 103 for i in range(len(ref) - level_offset): 104 traversed = getattr(traversed, ref[i]) 105 return traversed 106 107 def _remove_subsystem(self, subname: str) -> bool: 108 """ 109 Remove the named subsystem 110 :param subname: The name of a subsystem to remove 111 :return True if the subsystem was successfully removed, else False 112 """ 113 if subname in self._subs: 114 # Remove the subsystem 115 self._subs.pop(subname) 116 # reset the iterator 117 self._iter_subs = sorted(self._subs.keys()) 118 self._iter_ind = 0 119 return True 120 return False 121 122 def _scan_dynamic(self): 123 """ 124 Refresh the dynamic roots data used for loading app merge module roots 125 """ 126 self._dynamic = pop.dirs.dynamic_dirs() 127 self._dscan = True 128 129 def __getattr__(self, item: str): 130 if item.startswith("_"): 131 if item == item[0] * len(item): 132 return self._resolve_this(len(item)) 133 else: 134 return self.__getattribute__(item) 135 if "." in item: 136 return self.pop.ref.last(item) 137 if item in self._subs: 138 return self._subs[item] 139 elif item in self._sub_alias: 140 resolved = self._sub_alias[item] 141 if resolved in self._subs: 142 return self._subs[resolved] 143 return self.__getattribute__(item) 144 145 def __getitem__(self, item: str): 146 return getattr(self, item) 147 148 149class Sub: 150 """ 151 The pop object that contains the loaded module data 152 """ 153 154 def __init__( 155 self, 156 hub: Hub, 157 subname: str, 158 root: Hub or "Sub" = None, 159 pypath: List[str] or str = None, 160 static: List[str] or str = None, 161 contracts_pypath: List[str] or str = None, 162 contracts_static: List[str] or str = None, 163 default_contracts: List[str] or str = None, 164 virtual: bool = True, 165 dyne_name: str = None, 166 omit_start: Tuple[str] = ("_",), 167 omit_end: Tuple[str] = (), 168 omit_func: bool = False, 169 omit_class: bool = False, 170 omit_vars: bool = False, 171 mod_basename: str = "", 172 stop_on_failures: bool = False, 173 init: bool = True, 174 is_contract: bool = False, 175 sub_virtual: bool = True, 176 recursive_contracts_static=None, 177 default_recursive_contracts=None, 178 ): 179 """ 180 :param hub: The redistributed pop central hub 181 :param subname: The name that the sub is going to take on the hub 182 if nothing else is passed, it is used as the pypath (TODO make it the dyne_name not the pypath) 183 :param pypath: One or many python paths which will be imported 184 :param static: Directories that can be explicitly passed 185 :param contracts_pypath: Load additional contract paths 186 :param contracts_static: Load additional contract paths from a specific directory 187 :param default_contracts: Specifies that a specific contract plugin will be applied as a default to all plugins 188 :param virtual: Toggle whether or not to process __virtual__ functions 189 :param dyne_name: The dynamic name to use to look up paths to find plugins -- linked to conf.py 190 :param omit_start: Allows you to pass in a tuple of characters that would omit the loading of any object 191 I.E. Any function starting with an underscore will not be loaded onto a plugin 192 (You should probably never change this) 193 :param omit_end:Allows you to pass in a tuple of characters that would omit the loading of an object 194 (You should probably never change this) 195 :param omit_func: bool: Don't load any functions 196 :param omit_class: bool: Don't load any classes 197 :param omit_vars: bool: Don't load any vars 198 :param mod_basename: str: Manipulate the location in sys.modules that the plugin will be loaded to. 199 Allow plugins to be loaded into a separate namespace. 200 :param stop_on_failures: If any module fails to load for any reason, stacktrace and do not continue loading this sub 201 :param init: bool: determine whether or not we process __init__ functions 202 :param is_contract: Specify whether or not this sub is a contract 203 :param sub_virtual: bool: Recursively ignore this sub and it's subs 204 """ 205 self._iter_ind = 0 206 self._hub = hub 207 self._root = root or hub 208 self._subs = {} 209 self._alias = [] 210 self._sub_alias = {} 211 self._subname = subname 212 self._pypath = ex_path(pypath) 213 self._static = ex_path(static) 214 self._contracts_pypath = ex_path(contracts_pypath) 215 self._contracts_static = ex_path(contracts_static) 216 self._recursive_contracts_static = ex_path(recursive_contracts_static) 217 if isinstance(default_contracts, str): 218 default_contracts = [default_contracts] 219 if isinstance(default_recursive_contracts, str): 220 default_recursive_contracts = [default_recursive_contracts] 221 self._reverse_sub = None 222 self._default_recursive_contracts = default_recursive_contracts or [] 223 self._default_contracts = default_contracts or () 224 self._dyne_name = dyne_name 225 self._virtual = virtual 226 self._omit_start = omit_start 227 self._sub_virtual = sub_virtual 228 self._omit_end = omit_end 229 self._omit_func = omit_func 230 self._omit_class = omit_class 231 self._omit_vars = omit_vars 232 self._mod_basename = mod_basename 233 self._stop_on_failures = stop_on_failures 234 self._is_contract = is_contract 235 self._process_init = init 236 self._prepare() 237 238 def _prepare(self): 239 self._dirs = pop.dirs.dir_list( 240 self._subname, 241 "mods", 242 self._pypath, 243 self._static, 244 ) 245 if self._dyne_name: 246 self._load_dyne() 247 self._contract_dirs = pop.dirs.dir_list( 248 self._subname, 249 "contracts", 250 self._contracts_pypath, 251 self._contracts_static, 252 ) 253 self._contract_dirs.extend(pop.dirs.inline_dirs(self._dirs, "contracts")) 254 self._recursive_contract_dirs = pop.dirs.dir_list( 255 self._subname, 256 "recursive_contracts", 257 [], 258 self._recursive_contracts_static, 259 ) 260 self._recursive_contract_dirs.extend( 261 pop.dirs.inline_dirs(self._dirs, "recursive_contracts") 262 ) 263 264 if self._contract_dirs: 265 self._contracts = Sub( 266 hub=self._hub, 267 subname=f"{self._subname}.contracts", 268 static=self._contract_dirs, 269 is_contract=True, 270 ) 271 else: 272 self._contracts = None 273 274 if self._recursive_contract_dirs: 275 self._recursive_contracts = Sub( 276 hub=self._hub, 277 subname=f"{self._subname}.recursive_contracts", 278 static=self._recursive_contract_dirs, 279 is_contract=True, 280 ) 281 else: 282 self._recursive_contracts = getattr( 283 self._root, "_recursive_contracts", None 284 ) 285 self._name_root = self._load_name_root() 286 self._scan = pop.scanner.scan(self._dirs) 287 self._loaded = {} 288 self._vmap = {} 289 self._load_errors = {} 290 self._loaded_all = False 291 292 def _load_dyne(self): 293 """ 294 Load up the dynamic dirs for this sub 295 """ 296 if not self._hub._dscan: 297 self._hub._scan_dynamic() 298 for path in self._hub._dynamic.get(self._dyne_name, {}).get("paths", []): 299 self._dirs.append(path) 300 301 def _load_name_root(self): 302 """ 303 Generate the root of the name to be used to apply to the loaded modules 304 """ 305 if self._pypath: 306 return self._pypath[0] 307 elif self._dirs: 308 return secrets.token_hex() 309 310 def __getstate__(self): 311 return dict( 312 _hub=self._hub, 313 _subname=self._subname, 314 _pypath=self._pypath, 315 _static=self._static, 316 _contracts_pypath=self._contracts_pypath, 317 _contracts_static=self._contracts_static, 318 _default_contracts=self._default_contracts, 319 _virtual=self._virtual, 320 _omit_start=self._omit_start, 321 _omit_end=self._omit_end, 322 _omit_func=self._omit_func, 323 _omit_class=self._omit_class, 324 _omit_vars=self._omit_vars, 325 _mod_basename=self._mod_basename, 326 _stop_on_failures=self._stop_on_failures, 327 ) 328 329 def __setstate__(self, state: Dict): 330 self.__dict__.update(state) 331 self._prepare() 332 333 def __getattr__(self, item: str): 334 """ 335 If the item should be loaded, load it, else serve it 336 """ 337 if item.startswith("_"): 338 return self.__getattribute__(item) 339 elif "." in item: 340 # Get the attribute from one sub higher up until the hub calls "ref.last" 341 return getattr(self._root, f"{self._subname}.{item}") 342 elif item in self._loaded: 343 ret = self._loaded[item] 344 # If this previously errored on load, try it again, 345 # it might be ready to load now 346 if isinstance(ret, pop.loader.LoadError): 347 ret = self._find_mod(item) 348 if isinstance(ret, pop.loader.LoadError): 349 # If this is still a LoadError, process it 350 self._process_load_error(ret) 351 return ret 352 elif item in self._subs: 353 return self._subs[item] 354 elif item in self._sub_alias: 355 resolved = self._sub_alias[item] 356 if resolved in self._subs: 357 return self._subs[resolved] 358 359 mod = self._find_mod(item) 360 if mod is None: 361 if self._reverse_sub: 362 # This sub contains a reverse sub that could contain the result 363 return self._reverse_sub[item] 364 raise AttributeError(f"'{self._subname}' has no attribute '{item}'") 365 return mod 366 367 def __getitem__(self, item: str): 368 return getattr(self, item) 369 370 def __contains__(self, item: str): 371 try: 372 ret = getattr(self, item, None) 373 if isinstance(ret, ReverseSub): 374 return item in self._reverse_sub 375 return ret is not None 376 except pop.exc.PopLookupError: 377 return False 378 379 def __iter__(self) -> Iterator["Sub"]: 380 self._load_all() 381 382 def iterate(loaded): 383 for mod_name in sorted(loaded.keys()): 384 yield loaded[mod_name] 385 386 return iterate(self._loaded) 387 388 def __next__(self) -> "Sub": 389 self._load_all() 390 if self._iter_ind == len(self._iter_keys): 391 self._iter_ind = 0 392 raise StopIteration 393 self._iter_ind += 1 394 return self._loaded[self._iter_keys[self._iter_ind - 1]] 395 396 def _sub_init(self): 397 """ 398 Run load init.py for the sub, running '__init__' function if present 399 """ 400 self._find_mod("init", match_only=True) 401 402 def _process_load_error( 403 self, mod: ModuleType, skip_full_stop: bool = False 404 ) -> bool: 405 if not isinstance(mod, pop.loader.LoadError): 406 # This is not a LoadError, return now! 407 return False 408 409 if mod.edict["verror"]: 410 error = "{0[msg]}: {0[verror]}".format(mod()) 411 if skip_full_stop is False and self._stop_on_failures is True: 412 raise pop.exc.PopError(error) 413 log.log(level=5, msg=error) 414 return False 415 error = "{0[msg]}: {0[exception]!r}".format(mod()) 416 if mod.traceback: 417 error += "\n" + mod.traceback 418 if skip_full_stop is False and self._stop_on_failures is True: 419 raise pop.exc.PopError(error) 420 if mod.traceback: 421 log.warning(error) 422 else: 423 log.info(error) 424 return True 425 426 def _find_mod(self, item: str, match_only: bool = False) -> Dict: 427 """ 428 Find the module named item 429 :param item: The module to search for (then load) from any scanned interface 430 :param match_only: return the loaded module 431 :return a loaded mod_dict 432 """ 433 for iface in self._scan: 434 for bname in self._scan[iface]: 435 if os.path.basename(bname) == item: 436 self._load_item(iface, bname) 437 if item in self._loaded: 438 return self._loaded[item] 439 if not match_only: 440 for iface in self._scan: 441 for bname in self._scan[iface]: 442 if self._scan[iface][bname].get("loaded"): 443 continue 444 self._load_item(iface, bname) 445 if item in self._loaded: 446 return self._loaded[item] 447 # Let's see if the module being lookup is in the load errors dictionary 448 if item in self._load_errors: 449 # Return the LoadError 450 return self._load_errors[item] 451 452 def _load_item(self, iface: str, bname: str): 453 """ 454 Load the named basename 455 :param iface: A scanned directory type 456 :param bname: The base name of the python path of a module 457 """ 458 if iface not in self._scan: 459 raise pop.exc.PopLoadError(f"Bad call to load item, no iface {iface}") 460 if bname not in self._scan[iface]: 461 raise pop.exc.PopLoadError( 462 f"Bad call to load item, no bname {bname} in iface {iface}" 463 ) 464 # The mname is the name to give the module in python's sys.modules 465 # This name must be unique for every loaded module, so we use the full 466 # module path sans the file extention 467 mname = self._scan[iface][bname]["path"].replace(os.sep, ".") 468 mname = mname[mname.index(".") + 1 : mname.rindex(".")].strip(".") 469 mod = pop.loader.load_mod( 470 mname, 471 iface, 472 self._scan[iface][bname]["path"], 473 ) 474 if self._process_load_error(mod): 475 self._load_errors[os.path.basename(bname)] = mod 476 return 477 self._prep_mod(mod, iface, bname) 478 479 def _process_vret(self, vret: Dict[str, Any]) -> bool: 480 """ 481 :param vret: The return from a __virtual__ or __sub_virtual__ function 482 :return: True if there was an error, else false 483 """ 484 if "error" in vret: 485 # Virtual Errors should not full stop pop 486 self._process_load_error(vret["error"], skip_full_stop=True) 487 # Store the LoadError under the __virtualname__ if defined 488 self._load_errors[vret["vname"]] = vret["error"] 489 return True 490 else: 491 return False 492 493 def _prep_mod(self, mod: ModuleType, iface: str, bname: str): 494 """ 495 Prepare the module! 496 :param mod: A python module containing data 497 :param iface: A scanned directory type 498 :param bname: The base name of the python path of a module 499 """ 500 if not self._sub_virtual: 501 return 502 else: 503 vret = pop.loader.load_sub_virtual(self._hub, self._virtual, mod, bname) 504 if self._process_vret(vret): 505 self._sub_virtual = False 506 return 507 vret = pop.loader.load_virtual(self._hub, self._virtual, mod, bname) 508 if self._process_vret(vret): 509 return 510 511 contracts = pop.contract.load_contract( 512 self._contracts, self._default_contracts, mod, vret["name"] 513 ) 514 recursive_contracts = set( 515 pop.contract.load_contract( 516 self._recursive_contracts, 517 self._default_recursive_contracts, 518 mod, 519 vret["name"], 520 ) 521 ) 522 if getattr(self._root, "_recursive_contracts", None): 523 recursive_contracts.update( 524 pop.contract.load_contract( 525 self._root._recursive_contracts, 526 self._root._default_recursive_contracts, 527 mod, 528 vret["name"], 529 ) 530 ) 531 recursive_contracts = list(recursive_contracts) 532 name = vret["name"] 533 if name.endswith(EXT_SUFFIXES): 534 for ext in EXT_SUFFIXES: 535 if name.endswith(ext): 536 name = name.split(ext)[0] 537 break 538 mod_dict = pop.loader.prep_loaded_mod( 539 self, mod, name, contracts, recursive_contracts 540 ) 541 if name != "init": 542 pop.verify.contract(self._hub, contracts + recursive_contracts, mod_dict) 543 self._loaded[name] = mod_dict 544 self._vmap[mod.__file__] = name 545 # Let's mark the module as loaded 546 self._scan[iface][bname]["loaded"] = True 547 if self._process_init: 548 # Now that the module has been added to the sub, call mod_init 549 pop.loader.mod_init(self, mod, name) 550 551 def _load_all(self): 552 """ 553 Load all modules found during the scan. 554 555 .. attention:: This completely disables the lazy loader behavior of pop 556 """ 557 if self._loaded_all is True: 558 return 559 for iface in self._scan: 560 for bname in self._scan[iface]: 561 if self._scan[iface][bname].get("loaded"): 562 continue 563 self._load_item(iface, bname) 564 self._loaded_all = True 565 566 567class ReverseSub(Sub): 568 """ 569 A sub that discovers attributes available to it dynamically through a resolver 570 """ 571 572 def __init__( 573 self, 574 hub: Hub, 575 subname: str, 576 resolver: Callable, 577 context: Any = None, 578 refs: Tuple[str, ...] = None, 579 **kwargs, 580 ): 581 """ 582 :param subname: The name that the sub is going to take on the hub 583 :param resolver: A callable function that retrieves a target based on the current reference and context 584 :param context: Resources that will be passed to the resolver function 585 :param kwargs: Any kwargs that should be passed to the underlying sub 586 """ 587 Sub.__init__(self, subname=subname, hub=hub, **kwargs) 588 self._resolver = resolver 589 self._context = context 590 # self._loaded_all = True 591 if refs is None: 592 self._refs = tuple([*subname.split(".")]) 593 else: 594 self._refs = tuple([*refs, *subname.split(".")]) 595 596 # Imitate the name attribute like a callable should 597 self.__name__ = self._subname 598 599 def __getattr__(self, item: str): 600 """ 601 Build a reference based on the namespace attrs until a reverse sub is called 602 """ 603 if item.startswith("_"): 604 return self.__getattribute__(item) 605 606 return ReverseSub( 607 hub=self._hub, 608 subname=item, 609 # Share a root 610 root=self._root, 611 resolver=self._resolver, 612 context=self._context, 613 pypath=self._pypath, 614 refs=self._refs, 615 static=self._static, 616 contracts_pypath=self._contracts_pypath, 617 contracts_static=self._recursive_contracts_static, 618 default_contracts=self._default_contracts, 619 virtual=self._virtual, 620 dyne_name=self._dyne_name, 621 omit_start=self._omit_start, 622 omit_end=self._omit_end, 623 omit_func=self._omit_func, 624 omit_class=self._omit_class, 625 omit_vars=self._omit_vars, 626 mod_basename=self._mod_basename, 627 stop_on_failures=self._stop_on_failures, 628 init=None, 629 is_contract=self._is_contract, 630 sub_virtual=self._sub_virtual, 631 recursive_contracts_static=self._recursive_contracts_static, 632 default_recursive_contracts=self._default_recursive_contracts, 633 ) 634 635 def __iter__(self): 636 return self 637 638 def __next__(self): 639 # Override the parent Sub's iterator 640 raise StopIteration 641 642 def __contains__(self, item): 643 """ 644 Return true if the item's reference underneath this one is resolvable 645 """ 646 try: 647 if getattr(self, item)._resolve(): 648 return True 649 except Exception: 650 ... 651 return False 652 653 def _resolve(self) -> Callable or None: 654 """ 655 Call the resolver and get back the external call represented by this sub's reference 656 """ 657 ref = ".".join(self._refs) 658 ret = self._resolver(ref, self._context) 659 return ret 660 661 def __call__(self, *args, **kwargs): 662 func = self._resolve() 663 664 if not func: 665 ref = ".".join(self._refs) 666 raise AttributeError(f"ReverseSub could not resolve the reference {ref}") 667 if isinstance(func, pop.contract.Contracted): 668 discovered_contracted_function = func 669 else: 670 discovered_contracted_function = pop.contract.create_contracted( 671 hub=self._hub, 672 contracts=getattr(self._contracts, "_loaded", {}).values(), 673 func=func, 674 ref=".".join(self._refs), 675 name=func.__name__, 676 implicit_hub=False, 677 ) 678 679 if self._recursive_contracts and not all( 680 rc in discovered_contracted_function.contracts 681 for rc in self._recursive_contracts 682 ): 683 discovered_contracted_function.contracts.extend( 684 self._recursive_contracts._loaded.values() 685 ) 686 discovered_contracted_function._load_contracts() 687 688 return discovered_contracted_function(*args, **kwargs) 689