1# -*- coding: utf-8 -*- 2# 3# Copyright (C) 2020 Chris Caron <lead2gold@gmail.com> 4# All rights reserved. 5# 6# This code is licensed under the MIT License. 7# 8# Permission is hereby granted, free of charge, to any person obtaining a copy 9# of this software and associated documentation files(the "Software"), to deal 10# in the Software without restriction, including without limitation the rights 11# to use, copy, modify, merge, publish, distribute, sublicense, and / or sell 12# copies of the Software, and to permit persons to whom the Software is 13# furnished to do so, subject to the following conditions : 14# 15# The above copyright notice and this permission notice shall be included in 16# all copies or substantial portions of the Software. 17# 18# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.IN NO EVENT SHALL THE 21# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 24# THE SOFTWARE. 25 26import os 27import re 28import six 29import yaml 30import time 31 32from .. import plugins 33from ..AppriseAsset import AppriseAsset 34from ..URLBase import URLBase 35from ..common import ConfigFormat 36from ..common import CONFIG_FORMATS 37from ..common import ContentIncludeMode 38from ..utils import GET_SCHEMA_RE 39from ..utils import parse_list 40from ..utils import parse_bool 41from ..utils import parse_urls 42from ..utils import cwe312_url 43from . import SCHEMA_MAP 44 45# Test whether token is valid or not 46VALID_TOKEN = re.compile( 47 r'(?P<token>[a-z0-9][a-z0-9_]+)', re.I) 48 49 50class ConfigBase(URLBase): 51 """ 52 This is the base class for all supported configuration sources 53 """ 54 55 # The Default Encoding to use if not otherwise detected 56 encoding = 'utf-8' 57 58 # The default expected configuration format unless otherwise 59 # detected by the sub-modules 60 default_config_format = ConfigFormat.TEXT 61 62 # This is only set if the user overrides the config format on the URL 63 # this should always initialize itself as None 64 config_format = None 65 66 # Don't read any more of this amount of data into memory as there is no 67 # reason we should be reading in more. This is more of a safe guard then 68 # anything else. 128KB (131072B) 69 max_buffer_size = 131072 70 71 # By default all configuration is not includable using the 'include' 72 # line found in configuration files. 73 allow_cross_includes = ContentIncludeMode.NEVER 74 75 # the config path manages the handling of relative include 76 config_path = os.getcwd() 77 78 def __init__(self, cache=True, recursion=0, insecure_includes=False, 79 **kwargs): 80 """ 81 Initialize some general logging and common server arguments that will 82 keep things consistent when working with the configurations that 83 inherit this class. 84 85 By default we cache our responses so that subsiquent calls does not 86 cause the content to be retrieved again. For local file references 87 this makes no difference at all. But for remote content, this does 88 mean more then one call can be made to retrieve the (same) data. This 89 method can be somewhat inefficient if disabled. Only disable caching 90 if you understand the consequences. 91 92 You can alternatively set the cache value to an int identifying the 93 number of seconds the previously retrieved can exist for before it 94 should be considered expired. 95 96 recursion defines how deep we recursively handle entries that use the 97 `include` keyword. This keyword requires us to fetch more configuration 98 from another source and add it to our existing compilation. If the 99 file we remotely retrieve also has an `include` reference, we will only 100 advance through it if recursion is set to 2 deep. If set to zero 101 it is off. There is no limit to how high you set this value. It would 102 be recommended to keep it low if you do intend to use it. 103 104 insecure_include by default are disabled. When set to True, all 105 Apprise Config files marked to be in STRICT mode are treated as being 106 in ALWAYS mode. 107 108 Take a file:// based configuration for example, only a file:// based 109 configuration can include another file:// based one. because it is set 110 to STRICT mode. If an http:// based configuration file attempted to 111 include a file:// one it woul fail. However this include would be 112 possible if insecure_includes is set to True. 113 114 There are cases where a self hosting apprise developer may wish to load 115 configuration from memory (in a string format) that contains 'include' 116 entries (even file:// based ones). In these circumstances if you want 117 these 'include' entries to be honored, this value must be set to True. 118 """ 119 120 super(ConfigBase, self).__init__(**kwargs) 121 122 # Tracks the time the content was last retrieved on. This place a role 123 # for cases where we are not caching our response and are required to 124 # re-retrieve our settings. 125 self._cached_time = None 126 127 # Tracks previously loaded content for speed 128 self._cached_servers = None 129 130 # Initialize our recursion value 131 self.recursion = recursion 132 133 # Initialize our insecure_includes flag 134 self.insecure_includes = insecure_includes 135 136 if 'encoding' in kwargs: 137 # Store the encoding 138 self.encoding = kwargs.get('encoding') 139 140 if 'format' in kwargs \ 141 and isinstance(kwargs['format'], six.string_types): 142 # Store the enforced config format 143 self.config_format = kwargs.get('format').lower() 144 145 if self.config_format not in CONFIG_FORMATS: 146 # Simple error checking 147 err = 'An invalid config format ({}) was specified.'.format( 148 self.config_format) 149 self.logger.warning(err) 150 raise TypeError(err) 151 152 # Set our cache flag; it can be True or a (positive) integer 153 try: 154 self.cache = cache if isinstance(cache, bool) else int(cache) 155 if self.cache < 0: 156 err = 'A negative cache value ({}) was specified.'.format( 157 cache) 158 self.logger.warning(err) 159 raise TypeError(err) 160 161 except (ValueError, TypeError): 162 err = 'An invalid cache value ({}) was specified.'.format(cache) 163 self.logger.warning(err) 164 raise TypeError(err) 165 166 return 167 168 def servers(self, asset=None, **kwargs): 169 """ 170 Performs reads loaded configuration and returns all of the services 171 that could be parsed and loaded. 172 173 """ 174 175 if not self.expired(): 176 # We already have cached results to return; use them 177 return self._cached_servers 178 179 # Our cached response object 180 self._cached_servers = list() 181 182 # read() causes the child class to do whatever it takes for the 183 # config plugin to load the data source and return unparsed content 184 # None is returned if there was an error or simply no data 185 content = self.read(**kwargs) 186 if not isinstance(content, six.string_types): 187 # Set the time our content was cached at 188 self._cached_time = time.time() 189 190 # Nothing more to do; return our empty cache list 191 return self._cached_servers 192 193 # Our Configuration format uses a default if one wasn't one detected 194 # or enfored. 195 config_format = \ 196 self.default_config_format \ 197 if self.config_format is None else self.config_format 198 199 # Dynamically load our parse_ function based on our config format 200 fn = getattr(ConfigBase, 'config_parse_{}'.format(config_format)) 201 202 # Initialize our asset object 203 asset = asset if isinstance(asset, AppriseAsset) else self.asset 204 205 # Execute our config parse function which always returns a tuple 206 # of our servers and our configuration 207 servers, configs = fn(content=content, asset=asset) 208 self._cached_servers.extend(servers) 209 210 # Configuration files were detected; recursively populate them 211 # If we have been configured to do so 212 for url in configs: 213 214 if self.recursion > 0: 215 # Attempt to acquire the schema at the very least to allow 216 # our configuration based urls. 217 schema = GET_SCHEMA_RE.match(url) 218 if schema is None: 219 # Plan B is to assume we're dealing with a file 220 schema = 'file' 221 if not os.path.isabs(url): 222 # We're dealing with a relative path; prepend 223 # our current config path 224 url = os.path.join(self.config_path, url) 225 226 url = '{}://{}'.format(schema, URLBase.quote(url)) 227 228 else: 229 # Ensure our schema is always in lower case 230 schema = schema.group('schema').lower() 231 232 # Some basic validation 233 if schema not in SCHEMA_MAP: 234 ConfigBase.logger.warning( 235 'Unsupported include schema {}.'.format(schema)) 236 continue 237 238 # CWE-312 (Secure Logging) Handling 239 loggable_url = url if not asset.secure_logging \ 240 else cwe312_url(url) 241 242 # Parse our url details of the server object as dictionary 243 # containing all of the information parsed from our URL 244 results = SCHEMA_MAP[schema].parse_url(url) 245 if not results: 246 # Failed to parse the server URL 247 self.logger.warning( 248 'Unparseable include URL {}'.format(loggable_url)) 249 continue 250 251 # Handle cross inclusion based on allow_cross_includes rules 252 if (SCHEMA_MAP[schema].allow_cross_includes == 253 ContentIncludeMode.STRICT 254 and schema not in self.schemas() 255 and not self.insecure_includes) or \ 256 SCHEMA_MAP[schema].allow_cross_includes == \ 257 ContentIncludeMode.NEVER: 258 259 # Prevent the loading if insecure base protocols 260 ConfigBase.logger.warning( 261 'Including {}:// based configuration is prohibited. ' 262 'Ignoring URL {}'.format(schema, loggable_url)) 263 continue 264 265 # Prepare our Asset Object 266 results['asset'] = asset 267 268 # No cache is required because we're just lumping this in 269 # and associating it with the cache value we've already 270 # declared (prior to our recursion) 271 results['cache'] = False 272 273 # Recursion can never be parsed from the URL; we decrement 274 # it one level 275 results['recursion'] = self.recursion - 1 276 277 # Insecure Includes flag can never be parsed from the URL 278 results['insecure_includes'] = self.insecure_includes 279 280 try: 281 # Attempt to create an instance of our plugin using the 282 # parsed URL information 283 cfg_plugin = SCHEMA_MAP[results['schema']](**results) 284 285 except Exception as e: 286 # the arguments are invalid or can not be used. 287 self.logger.warning( 288 'Could not load include URL: {}'.format(loggable_url)) 289 self.logger.debug('Loading Exception: {}'.format(str(e))) 290 continue 291 292 # if we reach here, we can now add this servers found 293 # in this configuration file to our list 294 self._cached_servers.extend( 295 cfg_plugin.servers(asset=asset)) 296 297 # We no longer need our configuration object 298 del cfg_plugin 299 300 else: 301 # CWE-312 (Secure Logging) Handling 302 loggable_url = url if not asset.secure_logging \ 303 else cwe312_url(url) 304 305 self.logger.debug( 306 'Recursion limit reached; ignoring Include URL: %s', 307 loggable_url) 308 309 if self._cached_servers: 310 self.logger.info( 311 'Loaded {} entries from {}'.format( 312 len(self._cached_servers), 313 self.url(privacy=asset.secure_logging))) 314 else: 315 self.logger.warning( 316 'Failed to load Apprise configuration from {}'.format( 317 self.url(privacy=asset.secure_logging))) 318 319 # Set the time our content was cached at 320 self._cached_time = time.time() 321 322 return self._cached_servers 323 324 def read(self): 325 """ 326 This object should be implimented by the child classes 327 328 """ 329 return None 330 331 def expired(self): 332 """ 333 Simply returns True if the configuration should be considered 334 as expired or False if content should be retrieved. 335 """ 336 if isinstance(self._cached_servers, list) and self.cache: 337 # We have enough reason to look further into our cached content 338 # and verify it has not expired. 339 if self.cache is True: 340 # we have not expired, return False 341 return False 342 343 # Verify our cache time to determine whether we will get our 344 # content again. 345 age_in_sec = time.time() - self._cached_time 346 if age_in_sec <= self.cache: 347 # We have not expired; return False 348 return False 349 350 # If we reach here our configuration should be considered 351 # missing and/or expired. 352 return True 353 354 @staticmethod 355 def parse_url(url, verify_host=True): 356 """Parses the URL and returns it broken apart into a dictionary. 357 358 This is very specific and customized for Apprise. 359 360 Args: 361 url (str): The URL you want to fully parse. 362 verify_host (:obj:`bool`, optional): a flag kept with the parsed 363 URL which some child classes will later use to verify SSL 364 keys (if SSL transactions take place). Unless under very 365 specific circumstances, it is strongly recomended that 366 you leave this default value set to True. 367 368 Returns: 369 A dictionary is returned containing the URL fully parsed if 370 successful, otherwise None is returned. 371 """ 372 373 results = URLBase.parse_url(url, verify_host=verify_host) 374 375 if not results: 376 # We're done; we failed to parse our url 377 return results 378 379 # Allow overriding the default config format 380 if 'format' in results['qsd']: 381 results['format'] = results['qsd'].get('format') 382 if results['format'] not in CONFIG_FORMATS: 383 URLBase.logger.warning( 384 'Unsupported format specified {}'.format( 385 results['format'])) 386 del results['format'] 387 388 # Defines the encoding of the payload 389 if 'encoding' in results['qsd']: 390 results['encoding'] = results['qsd'].get('encoding') 391 392 # Our cache value 393 if 'cache' in results['qsd']: 394 # First try to get it's integer value 395 try: 396 results['cache'] = int(results['qsd']['cache']) 397 398 except (ValueError, TypeError): 399 # No problem, it just isn't an integer; now treat it as a bool 400 # instead: 401 results['cache'] = parse_bool(results['qsd']['cache']) 402 403 return results 404 405 @staticmethod 406 def detect_config_format(content, **kwargs): 407 """ 408 Takes the specified content and attempts to detect the format type 409 410 The function returns the actual format type if detected, otherwise 411 it returns None 412 """ 413 414 # Detect Format Logic: 415 # - A pound/hashtag (#) is alawys a comment character so we skip over 416 # lines matched here. 417 # - Detection begins on the first non-comment and non blank line 418 # matched. 419 # - If we find a string followed by a colon, we know we're dealing 420 # with a YAML file. 421 # - If we find a string that starts with a URL, or our tag 422 # definitions (accepting commas) followed by an equal sign we know 423 # we're dealing with a TEXT format. 424 425 # Define what a valid line should look like 426 valid_line_re = re.compile( 427 r'^\s*(?P<line>([;#]+(?P<comment>.*))|' 428 r'(?P<text>((?P<tag>[ \t,a-z0-9_-]+)=)?[a-z0-9]+://.*)|' 429 r'((?P<yaml>[a-z0-9]+):.*))?$', re.I) 430 431 try: 432 # split our content up to read line by line 433 content = re.split(r'\r*\n', content) 434 435 except TypeError: 436 # content was not expected string type 437 ConfigBase.logger.error( 438 'Invalid Apprise configuration specified.') 439 return None 440 441 # By default set our return value to None since we don't know 442 # what the format is yet 443 config_format = None 444 445 # iterate over each line of the file to attempt to detect it 446 # stop the moment a the type has been determined 447 for line, entry in enumerate(content, start=1): 448 449 result = valid_line_re.match(entry) 450 if not result: 451 # Invalid syntax 452 ConfigBase.logger.error( 453 'Undetectable Apprise configuration found ' 454 'based on line {}.'.format(line)) 455 # Take an early exit 456 return None 457 458 # Attempt to detect configuration 459 if result.group('yaml'): 460 config_format = ConfigFormat.YAML 461 ConfigBase.logger.debug( 462 'Detected YAML configuration ' 463 'based on line {}.'.format(line)) 464 break 465 466 elif result.group('text'): 467 config_format = ConfigFormat.TEXT 468 ConfigBase.logger.debug( 469 'Detected TEXT configuration ' 470 'based on line {}.'.format(line)) 471 break 472 473 # If we reach here, we have a comment entry 474 # Adjust default format to TEXT 475 config_format = ConfigFormat.TEXT 476 477 return config_format 478 479 @staticmethod 480 def config_parse(content, asset=None, config_format=None, **kwargs): 481 """ 482 Takes the specified config content and loads it based on the specified 483 config_format. If a format isn't specified, then it is auto detected. 484 485 """ 486 487 if config_format is None: 488 # Detect the format 489 config_format = ConfigBase.detect_config_format(content) 490 491 if not config_format: 492 # We couldn't detect configuration 493 ConfigBase.logger.error('Could not detect configuration') 494 return (list(), list()) 495 496 if config_format not in CONFIG_FORMATS: 497 # Invalid configuration type specified 498 ConfigBase.logger.error( 499 'An invalid configuration format ({}) was specified'.format( 500 config_format)) 501 return (list(), list()) 502 503 # Dynamically load our parse_ function based on our config format 504 fn = getattr(ConfigBase, 'config_parse_{}'.format(config_format)) 505 506 # Execute our config parse function which always returns a list 507 return fn(content=content, asset=asset) 508 509 @staticmethod 510 def config_parse_text(content, asset=None): 511 """ 512 Parse the specified content as though it were a simple text file only 513 containing a list of URLs. 514 515 Return a tuple that looks like (servers, configs) where: 516 - servers contains a list of loaded notification plugins 517 - configs contains a list of additional configuration files 518 referenced. 519 520 You may also optionally associate an asset with the notification. 521 522 The file syntax is: 523 524 # 525 # pound/hashtag allow for line comments 526 # 527 # One or more tags can be idenified using comma's (,) to separate 528 # them. 529 <Tag(s)>=<URL> 530 531 # Or you can use this format (no tags associated) 532 <URL> 533 534 # you can also use the keyword 'include' and identify a 535 # configuration location (like this file) which will be included 536 # as additional configuration entries when loaded. 537 include <ConfigURL> 538 539 """ 540 # A list of loaded Notification Services 541 servers = list() 542 543 # A list of additional configuration files referenced using 544 # the include keyword 545 configs = list() 546 547 # Prepare our Asset Object 548 asset = asset if isinstance(asset, AppriseAsset) else AppriseAsset() 549 550 # Define what a valid line should look like 551 valid_line_re = re.compile( 552 r'^\s*(?P<line>([;#]+(?P<comment>.*))|' 553 r'(\s*(?P<tags>[^=]+)=|=)?\s*' 554 r'(?P<url>[a-z0-9]{2,9}://.*)|' 555 r'include\s+(?P<config>.+))?\s*$', re.I) 556 557 try: 558 # split our content up to read line by line 559 content = re.split(r'\r*\n', content) 560 561 except TypeError: 562 # content was not expected string type 563 ConfigBase.logger.error( 564 'Invalid Apprise TEXT based configuration specified.') 565 return (list(), list()) 566 567 for line, entry in enumerate(content, start=1): 568 result = valid_line_re.match(entry) 569 if not result: 570 # Invalid syntax 571 ConfigBase.logger.error( 572 'Invalid Apprise TEXT configuration format found ' 573 '{} on line {}.'.format(entry, line)) 574 575 # Assume this is a file we shouldn't be parsing. It's owner 576 # can read the error printed to screen and take action 577 # otherwise. 578 return (list(), list()) 579 580 url, config = result.group('url'), result.group('config') 581 if not (url or config): 582 # Comment/empty line; do nothing 583 continue 584 585 if config: 586 # CWE-312 (Secure Logging) Handling 587 loggable_url = config if not asset.secure_logging \ 588 else cwe312_url(config) 589 590 ConfigBase.logger.debug( 591 'Include URL: {}'.format(loggable_url)) 592 593 # Store our include line 594 configs.append(config.strip()) 595 continue 596 597 # CWE-312 (Secure Logging) Handling 598 loggable_url = url if not asset.secure_logging \ 599 else cwe312_url(url) 600 601 # Acquire our url tokens 602 results = plugins.url_to_dict( 603 url, secure_logging=asset.secure_logging) 604 if results is None: 605 # Failed to parse the server URL 606 ConfigBase.logger.warning( 607 'Unparseable URL {} on line {}.'.format( 608 loggable_url, line)) 609 continue 610 611 # Build a list of tags to associate with the newly added 612 # notifications if any were set 613 results['tag'] = set(parse_list(result.group('tags'))) 614 615 # Set our Asset Object 616 results['asset'] = asset 617 618 try: 619 # Attempt to create an instance of our plugin using the 620 # parsed URL information 621 plugin = plugins.SCHEMA_MAP[results['schema']](**results) 622 623 # Create log entry of loaded URL 624 ConfigBase.logger.debug( 625 'Loaded URL: %s', plugin.url(privacy=asset.secure_logging)) 626 627 except Exception as e: 628 # the arguments are invalid or can not be used. 629 ConfigBase.logger.warning( 630 'Could not load URL {} on line {}.'.format( 631 loggable_url, line)) 632 ConfigBase.logger.debug('Loading Exception: %s' % str(e)) 633 continue 634 635 # if we reach here, we successfully loaded our data 636 servers.append(plugin) 637 638 # Return what was loaded 639 return (servers, configs) 640 641 @staticmethod 642 def config_parse_yaml(content, asset=None): 643 """ 644 Parse the specified content as though it were a yaml file 645 specifically formatted for Apprise. 646 647 Return a tuple that looks like (servers, configs) where: 648 - servers contains a list of loaded notification plugins 649 - configs contains a list of additional configuration files 650 referenced. 651 652 You may optionally associate an asset with the notification. 653 654 """ 655 656 # A list of loaded Notification Services 657 servers = list() 658 659 # A list of additional configuration files referenced using 660 # the include keyword 661 configs = list() 662 663 try: 664 # Load our data (safely) 665 result = yaml.load(content, Loader=yaml.SafeLoader) 666 667 except (AttributeError, 668 yaml.parser.ParserError, 669 yaml.error.MarkedYAMLError) as e: 670 # Invalid content 671 ConfigBase.logger.error( 672 'Invalid Apprise YAML data specified.') 673 ConfigBase.logger.debug( 674 'YAML Exception:{}{}'.format(os.linesep, e)) 675 return (list(), list()) 676 677 if not isinstance(result, dict): 678 # Invalid content 679 ConfigBase.logger.error( 680 'Invalid Apprise YAML based configuration specified.') 681 return (list(), list()) 682 683 # YAML Version 684 version = result.get('version', 1) 685 if version != 1: 686 # Invalid syntax 687 ConfigBase.logger.error( 688 'Invalid Apprise YAML version specified {}.'.format(version)) 689 return (list(), list()) 690 691 # 692 # global asset object 693 # 694 asset = asset if isinstance(asset, AppriseAsset) else AppriseAsset() 695 tokens = result.get('asset', None) 696 if tokens and isinstance(tokens, dict): 697 for k, v in tokens.items(): 698 699 if k.startswith('_') or k.endswith('_'): 700 # Entries are considered reserved if they start or end 701 # with an underscore 702 ConfigBase.logger.warning( 703 'Ignored asset key "{}".'.format(k)) 704 continue 705 706 if not (hasattr(asset, k) and 707 isinstance(getattr(asset, k), 708 (bool, six.string_types))): 709 710 # We can't set a function or non-string set value 711 ConfigBase.logger.warning( 712 'Invalid asset key "{}".'.format(k)) 713 continue 714 715 if v is None: 716 # Convert to an empty string 717 v = '' 718 719 if (isinstance(v, (bool, six.string_types)) 720 and isinstance(getattr(asset, k), bool)): 721 722 # If the object in the Asset is a boolean, then 723 # we want to convert the specified string to 724 # match that. 725 setattr(asset, k, parse_bool(v)) 726 727 elif isinstance(v, six.string_types): 728 # Set our asset object with the new value 729 setattr(asset, k, v.strip()) 730 731 else: 732 # we must set strings with a string 733 ConfigBase.logger.warning( 734 'Invalid asset value to "{}".'.format(k)) 735 continue 736 # 737 # global tag root directive 738 # 739 global_tags = set() 740 741 tags = result.get('tag', None) 742 if tags and isinstance(tags, (list, tuple, six.string_types)): 743 # Store any preset tags 744 global_tags = set(parse_list(tags)) 745 746 # 747 # include root directive 748 # 749 includes = result.get('include', None) 750 if isinstance(includes, six.string_types): 751 # Support a single inline string or multiple ones separated by a 752 # comma and/or space 753 includes = parse_urls(includes) 754 755 elif not isinstance(includes, (list, tuple)): 756 # Not a problem; we simply have no includes 757 includes = list() 758 759 # Iterate over each config URL 760 for no, url in enumerate(includes): 761 762 if isinstance(url, six.string_types): 763 # Support a single inline string or multiple ones separated by 764 # a comma and/or space 765 configs.extend(parse_urls(url)) 766 767 elif isinstance(url, dict): 768 # Store the url and ignore arguments associated 769 configs.extend(u for u in url.keys()) 770 771 # 772 # urls root directive 773 # 774 urls = result.get('urls', None) 775 if not isinstance(urls, (list, tuple)): 776 # Not a problem; we simply have no urls 777 urls = list() 778 779 # Iterate over each URL 780 for no, url in enumerate(urls): 781 782 # Our results object is what we use to instantiate our object if 783 # we can. Reset it to None on each iteration 784 results = list() 785 786 # CWE-312 (Secure Logging) Handling 787 loggable_url = url if not asset.secure_logging \ 788 else cwe312_url(url) 789 790 if isinstance(url, six.string_types): 791 # We're just a simple URL string... 792 schema = GET_SCHEMA_RE.match(url) 793 if schema is None: 794 # Log invalid entries so that maintainer of config 795 # config file at least has something to take action 796 # with. 797 ConfigBase.logger.warning( 798 'Invalid URL {}, entry #{}'.format( 799 loggable_url, no + 1)) 800 continue 801 802 # We found a valid schema worthy of tracking; store it's 803 # details: 804 _results = plugins.url_to_dict( 805 url, secure_logging=asset.secure_logging) 806 if _results is None: 807 ConfigBase.logger.warning( 808 'Unparseable URL {}, entry #{}'.format( 809 loggable_url, no + 1)) 810 continue 811 812 # add our results to our global set 813 results.append(_results) 814 815 elif isinstance(url, dict): 816 # We are a url string with additional unescaped options. In 817 # this case we want to iterate over all of our options so we 818 # can at least tell the end user what entries were ignored 819 # due to errors 820 821 if six.PY2: 822 it = url.iteritems() 823 else: # six.PY3 824 it = iter(url.items()) 825 826 # Track the URL to-load 827 _url = None 828 829 # Track last acquired schema 830 schema = None 831 for key, tokens in it: 832 # Test our schema 833 _schema = GET_SCHEMA_RE.match(key) 834 if _schema is None: 835 # Log invalid entries so that maintainer of config 836 # config file at least has something to take action 837 # with. 838 ConfigBase.logger.warning( 839 'Ignored entry {} found under urls, entry #{}' 840 .format(key, no + 1)) 841 continue 842 843 # Store our schema 844 schema = _schema.group('schema').lower() 845 846 # Store our URL and Schema Regex 847 _url = key 848 849 if _url is None: 850 # the loop above failed to match anything 851 ConfigBase.logger.warning( 852 'Unsupported URL, entry #{}'.format(no + 1)) 853 continue 854 855 _results = plugins.url_to_dict( 856 _url, secure_logging=asset.secure_logging) 857 if _results is None: 858 # Setup dictionary 859 _results = { 860 # Minimum requirements 861 'schema': schema, 862 } 863 864 if isinstance(tokens, (list, tuple, set)): 865 # populate and/or override any results populated by 866 # parse_url() 867 for entries in tokens: 868 # Copy ourselves a template of our parsed URL as a base 869 # to work with 870 r = _results.copy() 871 872 # We are a url string with additional unescaped options 873 if isinstance(entries, dict): 874 if six.PY2: 875 _url, tokens = next(url.iteritems()) 876 else: # six.PY3 877 _url, tokens = next(iter(url.items())) 878 879 # Tags you just can't over-ride 880 if 'schema' in entries: 881 del entries['schema'] 882 883 # support our special tokens (if they're present) 884 if schema in plugins.SCHEMA_MAP: 885 entries = ConfigBase._special_token_handler( 886 schema, entries) 887 888 # Extend our dictionary with our new entries 889 r.update(entries) 890 891 # add our results to our global set 892 results.append(r) 893 894 elif isinstance(tokens, dict): 895 # support our special tokens (if they're present) 896 if schema in plugins.SCHEMA_MAP: 897 tokens = ConfigBase._special_token_handler( 898 schema, tokens) 899 900 # Copy ourselves a template of our parsed URL as a base to 901 # work with 902 r = _results.copy() 903 904 # add our result set 905 r.update(tokens) 906 907 # add our results to our global set 908 results.append(r) 909 910 else: 911 # add our results to our global set 912 results.append(_results) 913 914 else: 915 # Unsupported 916 ConfigBase.logger.warning( 917 'Unsupported Apprise YAML entry #{}'.format(no + 1)) 918 continue 919 920 # Track our entries 921 entry = 0 922 923 while len(results): 924 # Increment our entry count 925 entry += 1 926 927 # Grab our first item 928 _results = results.pop(0) 929 930 # tag is a special keyword that is managed by Apprise object. 931 # The below ensures our tags are set correctly 932 if 'tag' in _results: 933 # Tidy our list up 934 _results['tag'] = \ 935 set(parse_list(_results['tag'])) | global_tags 936 937 else: 938 # Just use the global settings 939 _results['tag'] = global_tags 940 941 for key in list(_results.keys()): 942 # Strip out any tokens we know that we can't accept and 943 # warn the user 944 match = VALID_TOKEN.match(key) 945 if not match: 946 ConfigBase.logger.warning( 947 'Ignoring invalid token ({}) found in YAML ' 948 'configuration entry #{}, item #{}' 949 .format(key, no + 1, entry)) 950 del _results[key] 951 952 ConfigBase.logger.trace( 953 'URL #{}: {} unpacked as:{}{}' 954 .format(no + 1, url, os.linesep, os.linesep.join( 955 ['{}="{}"'.format(k, a) 956 for k, a in _results.items()]))) 957 958 # Prepare our Asset Object 959 _results['asset'] = asset 960 961 try: 962 # Attempt to create an instance of our plugin using the 963 # parsed URL information 964 plugin = plugins.SCHEMA_MAP[_results['schema']](**_results) 965 966 # Create log entry of loaded URL 967 ConfigBase.logger.debug( 968 'Loaded URL: {}'.format( 969 plugin.url(privacy=asset.secure_logging))) 970 971 except Exception as e: 972 # the arguments are invalid or can not be used. 973 ConfigBase.logger.warning( 974 'Could not load Apprise YAML configuration ' 975 'entry #{}, item #{}' 976 .format(no + 1, entry)) 977 ConfigBase.logger.debug('Loading Exception: %s' % str(e)) 978 continue 979 980 # if we reach here, we successfully loaded our data 981 servers.append(plugin) 982 983 return (servers, configs) 984 985 def pop(self, index=-1): 986 """ 987 Removes an indexed Notification Service from the stack and returns it. 988 989 By default, the last element of the list is removed. 990 """ 991 992 if not isinstance(self._cached_servers, list): 993 # Generate ourselves a list of content we can pull from 994 self.servers() 995 996 # Pop the element off of the stack 997 return self._cached_servers.pop(index) 998 999 @staticmethod 1000 def _special_token_handler(schema, tokens): 1001 """ 1002 This function takes a list of tokens and updates them to no longer 1003 include any special tokens such as +,-, and : 1004 1005 - schema must be a valid schema of a supported plugin type 1006 - tokens must be a dictionary containing the yaml entries parsed. 1007 1008 The idea here is we can post process a set of tokens provided in 1009 a YAML file where the user provided some of the special keywords. 1010 1011 We effectivley look up what these keywords map to their appropriate 1012 value they're expected 1013 """ 1014 # Create a copy of our dictionary 1015 tokens = tokens.copy() 1016 1017 for kw, meta in plugins.SCHEMA_MAP[schema]\ 1018 .template_kwargs.items(): 1019 1020 # Determine our prefix: 1021 prefix = meta.get('prefix', '+') 1022 1023 # Detect any matches 1024 matches = \ 1025 {k[1:]: str(v) for k, v in tokens.items() 1026 if k.startswith(prefix)} 1027 1028 if not matches: 1029 # we're done with this entry 1030 continue 1031 1032 if not isinstance(tokens.get(kw), dict): 1033 # Invalid; correct it 1034 tokens[kw] = dict() 1035 1036 # strip out processed tokens 1037 tokens = {k: v for k, v in tokens.items() 1038 if not k.startswith(prefix)} 1039 1040 # Update our entries 1041 tokens[kw].update(matches) 1042 1043 # Now map our tokens accordingly to the class templates defined by 1044 # each service. 1045 # 1046 # This is specifically used for YAML file parsing. It allows a user to 1047 # define an entry such as: 1048 # 1049 # urls: 1050 # - mailto://user:pass@domain: 1051 # - to: user1@hotmail.com 1052 # - to: user2@hotmail.com 1053 # 1054 # Under the hood, the NotifyEmail() class does not parse the `to` 1055 # argument. It's contents needs to be mapped to `targets`. This is 1056 # defined in the class via the `template_args` and template_tokens` 1057 # section. 1058 # 1059 # This function here allows these mappings to take place within the 1060 # YAML file as independant arguments. 1061 class_templates = \ 1062 plugins.details(plugins.SCHEMA_MAP[schema]) 1063 1064 for key in list(tokens.keys()): 1065 1066 if key not in class_templates['args']: 1067 # No need to handle non-arg entries 1068 continue 1069 1070 # get our `map_to` and/or 'alias_of' value (if it exists) 1071 map_to = class_templates['args'][key].get( 1072 'alias_of', class_templates['args'][key].get('map_to', '')) 1073 1074 if map_to == key: 1075 # We're already good as we are now 1076 continue 1077 1078 if map_to in class_templates['tokens']: 1079 meta = class_templates['tokens'][map_to] 1080 1081 else: 1082 meta = class_templates['args'].get( 1083 map_to, class_templates['args'][key]) 1084 1085 # Perform a translation/mapping if our code reaches here 1086 value = tokens[key] 1087 del tokens[key] 1088 1089 # Detect if we're dealign with a list or not 1090 is_list = re.search( 1091 r'^(list|choice):.*', 1092 meta.get('type'), 1093 re.IGNORECASE) 1094 1095 if map_to not in tokens: 1096 tokens[map_to] = [] if is_list \ 1097 else meta.get('default') 1098 1099 elif is_list and not isinstance(tokens.get(map_to), list): 1100 # Convert ourselves to a list if we aren't already 1101 tokens[map_to] = [tokens[map_to]] 1102 1103 # Type Conversion 1104 if re.search( 1105 r'^(choice:)?string', 1106 meta.get('type'), 1107 re.IGNORECASE) \ 1108 and not isinstance(value, six.string_types): 1109 1110 # Ensure our format is as expected 1111 value = str(value) 1112 1113 # Apply any further translations if required (absolute map) 1114 # This is the case when an arg maps to a token which further 1115 # maps to a different function arg on the class constructor 1116 abs_map = meta.get('map_to', map_to) 1117 1118 # Set our token as how it was provided by the configuration 1119 if isinstance(tokens.get(map_to), list): 1120 tokens[abs_map].append(value) 1121 1122 else: 1123 tokens[abs_map] = value 1124 1125 # Return our tokens 1126 return tokens 1127 1128 def __getitem__(self, index): 1129 """ 1130 Returns the indexed server entry associated with the loaded 1131 notification servers 1132 """ 1133 if not isinstance(self._cached_servers, list): 1134 # Generate ourselves a list of content we can pull from 1135 self.servers() 1136 1137 return self._cached_servers[index] 1138 1139 def __iter__(self): 1140 """ 1141 Returns an iterator to our server list 1142 """ 1143 if not isinstance(self._cached_servers, list): 1144 # Generate ourselves a list of content we can pull from 1145 self.servers() 1146 1147 return iter(self._cached_servers) 1148 1149 def __len__(self): 1150 """ 1151 Returns the total number of servers loaded 1152 """ 1153 if not isinstance(self._cached_servers, list): 1154 # Generate ourselves a list of content we can pull from 1155 self.servers() 1156 1157 return len(self._cached_servers) 1158 1159 def __bool__(self): 1160 """ 1161 Allows the Apprise object to be wrapped in an Python 3.x based 'if 1162 statement'. True is returned if our content was downloaded correctly. 1163 """ 1164 if not isinstance(self._cached_servers, list): 1165 # Generate ourselves a list of content we can pull from 1166 self.servers() 1167 1168 return True if self._cached_servers else False 1169 1170 def __nonzero__(self): 1171 """ 1172 Allows the Apprise object to be wrapped in an Python 2.x based 'if 1173 statement'. True is returned if our content was downloaded correctly. 1174 """ 1175 if not isinstance(self._cached_servers, list): 1176 # Generate ourselves a list of content we can pull from 1177 self.servers() 1178 1179 return True if self._cached_servers else False 1180