1# Copyright 2014-2016 OpenMarket Ltd 2# Copyright 2017-2018 New Vector Ltd 3# Copyright 2019 The Matrix.org Foundation C.I.C. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17import argparse 18import errno 19import os 20from collections import OrderedDict 21from hashlib import sha256 22from textwrap import dedent 23from typing import ( 24 Any, 25 Dict, 26 Iterable, 27 List, 28 MutableMapping, 29 Optional, 30 Tuple, 31 Type, 32 TypeVar, 33 Union, 34) 35 36import attr 37import jinja2 38import pkg_resources 39import yaml 40 41from synapse.util.templates import _create_mxc_to_http_filter, _format_ts_filter 42 43 44class ConfigError(Exception): 45 """Represents a problem parsing the configuration 46 47 Args: 48 msg: A textual description of the error. 49 path: Where appropriate, an indication of where in the configuration 50 the problem lies. 51 """ 52 53 def __init__(self, msg: str, path: Optional[Iterable[str]] = None): 54 self.msg = msg 55 self.path = path 56 57 58# We split these messages out to allow packages to override with package 59# specific instructions. 60MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS = """\ 61Please opt in or out of reporting anonymized homeserver usage statistics, by 62setting the `report_stats` key in your config file to either True or False. 63""" 64 65MISSING_REPORT_STATS_SPIEL = """\ 66We would really appreciate it if you could help our project out by reporting 67anonymized usage statistics from your homeserver. Only very basic aggregate 68data (e.g. number of users) will be reported, but it helps us to track the 69growth of the Matrix community, and helps us to make Matrix a success, as well 70as to convince other networks that they should peer with us. 71 72Thank you. 73""" 74 75MISSING_SERVER_NAME = """\ 76Missing mandatory `server_name` config option. 77""" 78 79 80CONFIG_FILE_HEADER = """\ 81# Configuration file for Synapse. 82# 83# This is a YAML file: see [1] for a quick introduction. Note in particular 84# that *indentation is important*: all the elements of a list or dictionary 85# should have the same indentation. 86# 87# [1] https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html 88 89""" 90 91 92def path_exists(file_path: str) -> bool: 93 """Check if a file exists 94 95 Unlike os.path.exists, this throws an exception if there is an error 96 checking if the file exists (for example, if there is a perms error on 97 the parent dir). 98 99 Returns: 100 True if the file exists; False if not. 101 """ 102 try: 103 os.stat(file_path) 104 return True 105 except OSError as e: 106 if e.errno != errno.ENOENT: 107 raise e 108 return False 109 110 111class Config: 112 """ 113 A configuration section, containing configuration keys and values. 114 115 Attributes: 116 section: The section title of this config object, such as 117 "tls" or "logger". This is used to refer to it on the root 118 logger (for example, `config.tls.some_option`). Must be 119 defined in subclasses. 120 """ 121 122 section: str 123 124 def __init__(self, root_config: "RootConfig" = None): 125 self.root = root_config 126 127 # Get the path to the default Synapse template directory 128 self.default_template_dir = pkg_resources.resource_filename( 129 "synapse", "res/templates" 130 ) 131 132 @staticmethod 133 def parse_size(value: Union[str, int]) -> int: 134 if isinstance(value, int): 135 return value 136 sizes = {"K": 1024, "M": 1024 * 1024} 137 size = 1 138 suffix = value[-1] 139 if suffix in sizes: 140 value = value[:-1] 141 size = sizes[suffix] 142 return int(value) * size 143 144 @staticmethod 145 def parse_duration(value: Union[str, int]) -> int: 146 """Convert a duration as a string or integer to a number of milliseconds. 147 148 If an integer is provided it is treated as milliseconds and is unchanged. 149 150 String durations can have a suffix of 's', 'm', 'h', 'd', 'w', or 'y'. 151 No suffix is treated as milliseconds. 152 153 Args: 154 value: The duration to parse. 155 156 Returns: 157 The number of milliseconds in the duration. 158 """ 159 if isinstance(value, int): 160 return value 161 second = 1000 162 minute = 60 * second 163 hour = 60 * minute 164 day = 24 * hour 165 week = 7 * day 166 year = 365 * day 167 sizes = {"s": second, "m": minute, "h": hour, "d": day, "w": week, "y": year} 168 size = 1 169 suffix = value[-1] 170 if suffix in sizes: 171 value = value[:-1] 172 size = sizes[suffix] 173 return int(value) * size 174 175 @staticmethod 176 def abspath(file_path: str) -> str: 177 return os.path.abspath(file_path) if file_path else file_path 178 179 @classmethod 180 def path_exists(cls, file_path: str) -> bool: 181 return path_exists(file_path) 182 183 @classmethod 184 def check_file(cls, file_path: Optional[str], config_name: str) -> str: 185 if file_path is None: 186 raise ConfigError("Missing config for %s." % (config_name,)) 187 try: 188 os.stat(file_path) 189 except OSError as e: 190 raise ConfigError( 191 "Error accessing file '%s' (config for %s): %s" 192 % (file_path, config_name, e.strerror) 193 ) 194 return cls.abspath(file_path) 195 196 @classmethod 197 def ensure_directory(cls, dir_path: str) -> str: 198 dir_path = cls.abspath(dir_path) 199 os.makedirs(dir_path, exist_ok=True) 200 if not os.path.isdir(dir_path): 201 raise ConfigError("%s is not a directory" % (dir_path,)) 202 return dir_path 203 204 @classmethod 205 def read_file(cls, file_path: Any, config_name: str) -> str: 206 """Deprecated: call read_file directly""" 207 return read_file(file_path, (config_name,)) 208 209 def read_template(self, filename: str) -> jinja2.Template: 210 """Load a template file from disk. 211 212 This function will attempt to load the given template from the default Synapse 213 template directory. 214 215 Files read are treated as Jinja templates. The templates is not rendered yet 216 and has autoescape enabled. 217 218 Args: 219 filename: A template filename to read. 220 221 Raises: 222 ConfigError: if the file's path is incorrect or otherwise cannot be read. 223 224 Returns: 225 A jinja2 template. 226 """ 227 return self.read_templates([filename])[0] 228 229 def read_templates( 230 self, 231 filenames: List[str], 232 custom_template_directories: Optional[Iterable[str]] = None, 233 ) -> List[jinja2.Template]: 234 """Load a list of template files from disk using the given variables. 235 236 This function will attempt to load the given templates from the default Synapse 237 template directory. If `custom_template_directories` is supplied, any directory 238 in this list is tried (in the order they appear in the list) before trying 239 Synapse's default directory. 240 241 Files read are treated as Jinja templates. The templates are not rendered yet 242 and have autoescape enabled. 243 244 Args: 245 filenames: A list of template filenames to read. 246 247 custom_template_directories: A list of directory to try to look for the 248 templates before using the default Synapse template directory instead. 249 250 Raises: 251 ConfigError: if the file's path is incorrect or otherwise cannot be read. 252 253 Returns: 254 A list of jinja2 templates. 255 """ 256 search_directories = [] 257 258 # The loader will first look in the custom template directories (if specified) 259 # for the given filename. If it doesn't find it, it will use the default 260 # template dir instead. 261 if custom_template_directories is not None: 262 for custom_template_directory in custom_template_directories: 263 # Check that the given template directory exists 264 if not self.path_exists(custom_template_directory): 265 raise ConfigError( 266 "Configured template directory does not exist: %s" 267 % (custom_template_directory,) 268 ) 269 270 # Search the custom template directory as well 271 search_directories.append(custom_template_directory) 272 273 # Append the default directory at the end of the list so Jinja can fallback on it 274 # if a template is missing from any custom directory. 275 search_directories.append(self.default_template_dir) 276 277 # TODO: switch to synapse.util.templates.build_jinja_env 278 loader = jinja2.FileSystemLoader(search_directories) 279 env = jinja2.Environment( 280 loader=loader, 281 autoescape=jinja2.select_autoescape(), 282 ) 283 284 # Update the environment with our custom filters 285 env.filters.update( 286 { 287 "format_ts": _format_ts_filter, 288 "mxc_to_http": _create_mxc_to_http_filter( 289 self.root.server.public_baseurl 290 ), 291 } 292 ) 293 294 # Load the templates 295 return [env.get_template(filename) for filename in filenames] 296 297 298TRootConfig = TypeVar("TRootConfig", bound="RootConfig") 299 300 301class RootConfig: 302 """ 303 Holder of an application's configuration. 304 305 What configuration this object holds is defined by `config_classes`, a list 306 of Config classes that will be instantiated and given the contents of a 307 configuration file to read. They can then be accessed on this class by their 308 section name, defined in the Config or dynamically set to be the name of the 309 class, lower-cased and with "Config" removed. 310 """ 311 312 config_classes = [] 313 314 def __init__(self): 315 for config_class in self.config_classes: 316 if config_class.section is None: 317 raise ValueError("%r requires a section name" % (config_class,)) 318 319 try: 320 conf = config_class(self) 321 except Exception as e: 322 raise Exception("Failed making %s: %r" % (config_class.section, e)) 323 setattr(self, config_class.section, conf) 324 325 def invoke_all( 326 self, func_name: str, *args: Any, **kwargs: Any 327 ) -> MutableMapping[str, Any]: 328 """ 329 Invoke a function on all instantiated config objects this RootConfig is 330 configured to use. 331 332 Args: 333 func_name: Name of function to invoke 334 *args 335 **kwargs 336 337 Returns: 338 ordered dictionary of config section name and the result of the 339 function from it. 340 """ 341 res = OrderedDict() 342 343 for config_class in self.config_classes: 344 config = getattr(self, config_class.section) 345 346 if hasattr(config, func_name): 347 res[config_class.section] = getattr(config, func_name)(*args, **kwargs) 348 349 return res 350 351 @classmethod 352 def invoke_all_static(cls, func_name: str, *args: Any, **kwargs: any) -> None: 353 """ 354 Invoke a static function on config objects this RootConfig is 355 configured to use. 356 357 Args: 358 func_name: Name of function to invoke 359 *args 360 **kwargs 361 362 Returns: 363 ordered dictionary of config section name and the result of the 364 function from it. 365 """ 366 for config in cls.config_classes: 367 if hasattr(config, func_name): 368 getattr(config, func_name)(*args, **kwargs) 369 370 def generate_config( 371 self, 372 config_dir_path: str, 373 data_dir_path: str, 374 server_name: str, 375 generate_secrets: bool = False, 376 report_stats: Optional[bool] = None, 377 open_private_ports: bool = False, 378 listeners: Optional[List[dict]] = None, 379 tls_certificate_path: Optional[str] = None, 380 tls_private_key_path: Optional[str] = None, 381 ) -> str: 382 """ 383 Build a default configuration file 384 385 This is used when the user explicitly asks us to generate a config file 386 (eg with --generate_config). 387 388 Args: 389 config_dir_path: The path where the config files are kept. Used to 390 create filenames for things like the log config and the signing key. 391 392 data_dir_path: The path where the data files are kept. Used to create 393 filenames for things like the database and media store. 394 395 server_name: The server name. Used to initialise the server_name 396 config param, but also used in the names of some of the config files. 397 398 generate_secrets: True if we should generate new secrets for things 399 like the macaroon_secret_key. If False, these parameters will be left 400 unset. 401 402 report_stats: Initial setting for the report_stats setting. 403 If None, report_stats will be left unset. 404 405 open_private_ports: True to leave private ports (such as the non-TLS 406 HTTP listener) open to the internet. 407 408 listeners: A list of descriptions of the listeners synapse should 409 start with each of which specifies a port (int), a list of 410 resources (list(str)), tls (bool) and type (str). For example: 411 [{ 412 "port": 8448, 413 "resources": [{"names": ["federation"]}], 414 "tls": True, 415 "type": "http", 416 }, 417 { 418 "port": 443, 419 "resources": [{"names": ["client"]}], 420 "tls": False, 421 "type": "http", 422 }], 423 424 tls_certificate_path: The path to the tls certificate. 425 426 tls_private_key_path: The path to the tls private key. 427 428 Returns: 429 The yaml config file 430 """ 431 432 return CONFIG_FILE_HEADER + "\n\n".join( 433 dedent(conf) 434 for conf in self.invoke_all( 435 "generate_config_section", 436 config_dir_path=config_dir_path, 437 data_dir_path=data_dir_path, 438 server_name=server_name, 439 generate_secrets=generate_secrets, 440 report_stats=report_stats, 441 open_private_ports=open_private_ports, 442 listeners=listeners, 443 tls_certificate_path=tls_certificate_path, 444 tls_private_key_path=tls_private_key_path, 445 ).values() 446 ) 447 448 @classmethod 449 def load_config( 450 cls: Type[TRootConfig], description: str, argv: List[str] 451 ) -> TRootConfig: 452 """Parse the commandline and config files 453 454 Doesn't support config-file-generation: used by the worker apps. 455 456 Returns: 457 Config object. 458 """ 459 config_parser = argparse.ArgumentParser(description=description) 460 cls.add_arguments_to_parser(config_parser) 461 obj, _ = cls.load_config_with_parser(config_parser, argv) 462 463 return obj 464 465 @classmethod 466 def add_arguments_to_parser(cls, config_parser: argparse.ArgumentParser) -> None: 467 """Adds all the config flags to an ArgumentParser. 468 469 Doesn't support config-file-generation: used by the worker apps. 470 471 Used for workers where we want to add extra flags/subcommands. 472 473 Args: 474 config_parser: App description 475 """ 476 477 config_parser.add_argument( 478 "-c", 479 "--config-path", 480 action="append", 481 metavar="CONFIG_FILE", 482 help="Specify config file. Can be given multiple times and" 483 " may specify directories containing *.yaml files.", 484 ) 485 486 config_parser.add_argument( 487 "--keys-directory", 488 metavar="DIRECTORY", 489 help="Where files such as certs and signing keys are stored when" 490 " their location is not given explicitly in the config." 491 " Defaults to the directory containing the last config file", 492 ) 493 494 cls.invoke_all_static("add_arguments", config_parser) 495 496 @classmethod 497 def load_config_with_parser( 498 cls: Type[TRootConfig], parser: argparse.ArgumentParser, argv: List[str] 499 ) -> Tuple[TRootConfig, argparse.Namespace]: 500 """Parse the commandline and config files with the given parser 501 502 Doesn't support config-file-generation: used by the worker apps. 503 504 Used for workers where we want to add extra flags/subcommands. 505 506 Args: 507 parser 508 argv 509 510 Returns: 511 Returns the parsed config object and the parsed argparse.Namespace 512 object from parser.parse_args(..)` 513 """ 514 515 obj = cls() 516 517 config_args = parser.parse_args(argv) 518 519 config_files = find_config_files(search_paths=config_args.config_path) 520 521 if not config_files: 522 parser.error("Must supply a config file.") 523 524 if config_args.keys_directory: 525 config_dir_path = config_args.keys_directory 526 else: 527 config_dir_path = os.path.dirname(config_files[-1]) 528 config_dir_path = os.path.abspath(config_dir_path) 529 data_dir_path = os.getcwd() 530 531 config_dict = read_config_files(config_files) 532 obj.parse_config_dict( 533 config_dict, config_dir_path=config_dir_path, data_dir_path=data_dir_path 534 ) 535 536 obj.invoke_all("read_arguments", config_args) 537 538 return obj, config_args 539 540 @classmethod 541 def load_or_generate_config( 542 cls: Type[TRootConfig], description: str, argv: List[str] 543 ) -> Optional[TRootConfig]: 544 """Parse the commandline and config files 545 546 Supports generation of config files, so is used for the main homeserver app. 547 548 Returns: 549 Config object, or None if --generate-config or --generate-keys was set 550 """ 551 parser = argparse.ArgumentParser(description=description) 552 parser.add_argument( 553 "-c", 554 "--config-path", 555 action="append", 556 metavar="CONFIG_FILE", 557 help="Specify config file. Can be given multiple times and" 558 " may specify directories containing *.yaml files.", 559 ) 560 561 generate_group = parser.add_argument_group("Config generation") 562 generate_group.add_argument( 563 "--generate-config", 564 action="store_true", 565 help="Generate a config file, then exit.", 566 ) 567 generate_group.add_argument( 568 "--generate-missing-configs", 569 "--generate-keys", 570 action="store_true", 571 help="Generate any missing additional config files, then exit.", 572 ) 573 generate_group.add_argument( 574 "-H", "--server-name", help="The server name to generate a config file for." 575 ) 576 generate_group.add_argument( 577 "--report-stats", 578 action="store", 579 help="Whether the generated config reports anonymized usage statistics.", 580 choices=["yes", "no"], 581 ) 582 generate_group.add_argument( 583 "--config-directory", 584 "--keys-directory", 585 metavar="DIRECTORY", 586 help=( 587 "Specify where additional config files such as signing keys and log" 588 " config should be stored. Defaults to the same directory as the last" 589 " config file." 590 ), 591 ) 592 generate_group.add_argument( 593 "--data-directory", 594 metavar="DIRECTORY", 595 help=( 596 "Specify where data such as the media store and database file should be" 597 " stored. Defaults to the current working directory." 598 ), 599 ) 600 generate_group.add_argument( 601 "--open-private-ports", 602 action="store_true", 603 help=( 604 "Leave private ports (such as the non-TLS HTTP listener) open to the" 605 " internet. Do not use this unless you know what you are doing." 606 ), 607 ) 608 609 cls.invoke_all_static("add_arguments", parser) 610 config_args = parser.parse_args(argv) 611 612 config_files = find_config_files(search_paths=config_args.config_path) 613 614 if not config_files: 615 parser.error( 616 "Must supply a config file.\nA config file can be automatically" 617 ' generated using "--generate-config -H SERVER_NAME' 618 ' -c CONFIG-FILE"' 619 ) 620 621 if config_args.config_directory: 622 config_dir_path = config_args.config_directory 623 else: 624 config_dir_path = os.path.dirname(config_files[-1]) 625 config_dir_path = os.path.abspath(config_dir_path) 626 data_dir_path = os.getcwd() 627 628 generate_missing_configs = config_args.generate_missing_configs 629 630 obj = cls() 631 632 if config_args.generate_config: 633 if config_args.report_stats is None: 634 parser.error( 635 "Please specify either --report-stats=yes or --report-stats=no\n\n" 636 + MISSING_REPORT_STATS_SPIEL 637 ) 638 639 (config_path,) = config_files 640 if not path_exists(config_path): 641 print("Generating config file %s" % (config_path,)) 642 643 if config_args.data_directory: 644 data_dir_path = config_args.data_directory 645 else: 646 data_dir_path = os.getcwd() 647 data_dir_path = os.path.abspath(data_dir_path) 648 649 server_name = config_args.server_name 650 if not server_name: 651 raise ConfigError( 652 "Must specify a server_name to a generate config for." 653 " Pass -H server.name." 654 ) 655 656 config_str = obj.generate_config( 657 config_dir_path=config_dir_path, 658 data_dir_path=data_dir_path, 659 server_name=server_name, 660 report_stats=(config_args.report_stats == "yes"), 661 generate_secrets=True, 662 open_private_ports=config_args.open_private_ports, 663 ) 664 665 os.makedirs(config_dir_path, exist_ok=True) 666 with open(config_path, "w") as config_file: 667 config_file.write(config_str) 668 config_file.write("\n\n# vim:ft=yaml") 669 670 config_dict = yaml.safe_load(config_str) 671 obj.generate_missing_files(config_dict, config_dir_path) 672 673 print( 674 ( 675 "A config file has been generated in %r for server name" 676 " %r. Please review this file and customise it" 677 " to your needs." 678 ) 679 % (config_path, server_name) 680 ) 681 return 682 else: 683 print( 684 ( 685 "Config file %r already exists. Generating any missing config" 686 " files." 687 ) 688 % (config_path,) 689 ) 690 generate_missing_configs = True 691 692 config_dict = read_config_files(config_files) 693 if generate_missing_configs: 694 obj.generate_missing_files(config_dict, config_dir_path) 695 return None 696 697 obj.parse_config_dict( 698 config_dict, config_dir_path=config_dir_path, data_dir_path=data_dir_path 699 ) 700 obj.invoke_all("read_arguments", config_args) 701 702 return obj 703 704 def parse_config_dict( 705 self, 706 config_dict: Dict[str, Any], 707 config_dir_path: Optional[str] = None, 708 data_dir_path: Optional[str] = None, 709 ) -> None: 710 """Read the information from the config dict into this Config object. 711 712 Args: 713 config_dict: Configuration data, as read from the yaml 714 715 config_dir_path: The path where the config files are kept. Used to 716 create filenames for things like the log config and the signing key. 717 718 data_dir_path: The path where the data files are kept. Used to create 719 filenames for things like the database and media store. 720 """ 721 self.invoke_all( 722 "read_config", 723 config_dict, 724 config_dir_path=config_dir_path, 725 data_dir_path=data_dir_path, 726 ) 727 728 def generate_missing_files( 729 self, config_dict: Dict[str, Any], config_dir_path: str 730 ) -> None: 731 self.invoke_all("generate_files", config_dict, config_dir_path) 732 733 734def read_config_files(config_files: Iterable[str]) -> Dict[str, Any]: 735 """Read the config files into a dict 736 737 Args: 738 config_files: A list of the config files to read 739 740 Returns: 741 The configuration dictionary. 742 """ 743 specified_config = {} 744 for config_file in config_files: 745 with open(config_file) as file_stream: 746 yaml_config = yaml.safe_load(file_stream) 747 748 if not isinstance(yaml_config, dict): 749 err = "File %r is empty or doesn't parse into a key-value map. IGNORING." 750 print(err % (config_file,)) 751 continue 752 753 specified_config.update(yaml_config) 754 755 if "server_name" not in specified_config: 756 raise ConfigError(MISSING_SERVER_NAME) 757 758 if "report_stats" not in specified_config: 759 raise ConfigError( 760 MISSING_REPORT_STATS_CONFIG_INSTRUCTIONS + "\n" + MISSING_REPORT_STATS_SPIEL 761 ) 762 return specified_config 763 764 765def find_config_files(search_paths: List[str]) -> List[str]: 766 """Finds config files using a list of search paths. If a path is a file 767 then that file path is added to the list. If a search path is a directory 768 then all the "*.yaml" files in that directory are added to the list in 769 sorted order. 770 771 Args: 772 search_paths: A list of paths to search. 773 774 Returns: 775 A list of file paths. 776 """ 777 778 config_files = [] 779 if search_paths: 780 for config_path in search_paths: 781 if os.path.isdir(config_path): 782 # We accept specifying directories as config paths, we search 783 # inside that directory for all files matching *.yaml, and then 784 # we apply them in *sorted* order. 785 files = [] 786 for entry in os.listdir(config_path): 787 entry_path = os.path.join(config_path, entry) 788 if not os.path.isfile(entry_path): 789 err = "Found subdirectory in config directory: %r. IGNORING." 790 print(err % (entry_path,)) 791 continue 792 793 if not entry.endswith(".yaml"): 794 err = ( 795 "Found file in config directory that does not end in " 796 "'.yaml': %r. IGNORING." 797 ) 798 print(err % (entry_path,)) 799 continue 800 801 files.append(entry_path) 802 803 config_files.extend(sorted(files)) 804 else: 805 config_files.append(config_path) 806 return config_files 807 808 809@attr.s(auto_attribs=True) 810class ShardedWorkerHandlingConfig: 811 """Algorithm for choosing which instance is responsible for handling some 812 sharded work. 813 814 For example, the federation senders use this to determine which instances 815 handles sending stuff to a given destination (which is used as the `key` 816 below). 817 """ 818 819 instances: List[str] 820 821 def should_handle(self, instance_name: str, key: str) -> bool: 822 """Whether this instance is responsible for handling the given key.""" 823 # If no instances are defined we assume some other worker is handling 824 # this. 825 if not self.instances: 826 return False 827 828 return self._get_instance(key) == instance_name 829 830 def _get_instance(self, key: str) -> str: 831 """Get the instance responsible for handling the given key. 832 833 Note: For federation sending and pushers the config for which instance 834 is sending is known only to the sender instance, so we don't expose this 835 method by default. 836 """ 837 838 if not self.instances: 839 raise Exception("Unknown worker") 840 841 if len(self.instances) == 1: 842 return self.instances[0] 843 844 # We shard by taking the hash, modulo it by the number of instances and 845 # then checking whether this instance matches the instance at that 846 # index. 847 # 848 # (Technically this introduces some bias and is not entirely uniform, 849 # but since the hash is so large the bias is ridiculously small). 850 dest_hash = sha256(key.encode("utf8")).digest() 851 dest_int = int.from_bytes(dest_hash, byteorder="little") 852 remainder = dest_int % (len(self.instances)) 853 return self.instances[remainder] 854 855 856@attr.s 857class RoutableShardedWorkerHandlingConfig(ShardedWorkerHandlingConfig): 858 """A version of `ShardedWorkerHandlingConfig` that is used for config 859 options where all instances know which instances are responsible for the 860 sharded work. 861 """ 862 863 def __attrs_post_init__(self): 864 # We require that `self.instances` is non-empty. 865 if not self.instances: 866 raise Exception("Got empty list of instances for shard config") 867 868 def get_instance(self, key: str) -> str: 869 """Get the instance responsible for handling the given key.""" 870 return self._get_instance(key) 871 872 873def read_file(file_path: Any, config_path: Iterable[str]) -> str: 874 """Check the given file exists, and read it into a string 875 876 If it does not, emit an error indicating the problem 877 878 Args: 879 file_path: the file to be read 880 config_path: where in the configuration file_path came from, so that a useful 881 error can be emitted if it does not exist. 882 Returns: 883 content of the file. 884 Raises: 885 ConfigError if there is a problem reading the file. 886 """ 887 if not isinstance(file_path, str): 888 raise ConfigError("%r is not a string", config_path) 889 890 try: 891 os.stat(file_path) 892 with open(file_path) as file_stream: 893 return file_stream.read() 894 except OSError as e: 895 raise ConfigError("Error accessing file %r" % (file_path,), config_path) from e 896 897 898__all__ = [ 899 "Config", 900 "RootConfig", 901 "ShardedWorkerHandlingConfig", 902 "RoutableShardedWorkerHandlingConfig", 903 "read_file", 904] 905