1#!/usr/local/bin/python3.8 2 3""" 4Configuration program for botan 5 6(C) 2009-2020 Jack Lloyd 7(C) 2015,2016,2017 Simon Warta (Kullo GmbH) 8 9Botan is released under the Simplified BSD License (see license.txt) 10 11This script is regularly tested with CPython 2.7 and 3.5, and 12occasionally tested with CPython 2.6 and PyPy 4. 13 14Support for CPython 2.6 will be dropped eventually, but is kept up for as 15long as reasonably convenient. 16 17CPython 2.5 and earlier are not supported. 18 19On Jython target detection does not work (use --os and --cpu). 20""" 21 22import collections 23import copy 24import json 25import sys 26import os 27import os.path 28import platform 29import re 30import shlex 31import shutil 32import subprocess 33import traceback 34import logging 35import time 36import errno 37import optparse # pylint: disable=deprecated-module 38 39# An error caused by and to be fixed by the user, e.g. invalid command line argument 40class UserError(Exception): 41 pass 42 43 44# An error caused by bugs in this script or when reading/parsing build data files 45# Those are not expected to be fixed by the user of this script 46class InternalError(Exception): 47 pass 48 49 50def flatten(l): 51 return sum(l, []) 52 53def normalize_source_path(source): 54 """ 55 cmake needs this, and nothing else minds 56 """ 57 return os.path.normpath(source).replace('\\', '/') 58 59def parse_version_file(version_path): 60 version_file = open(version_path) 61 key_and_val = re.compile(r"([a-z_]+) = ([a-zA-Z0-9:\-\']+)") 62 63 results = {} 64 for line in version_file.readlines(): 65 if not line or line[0] == '#': 66 continue 67 match = key_and_val.match(line) 68 if match: 69 key = match.group(1) 70 val = match.group(2) 71 72 if val == 'None': 73 val = None 74 elif val.startswith("'") and val.endswith("'"): 75 val = val[1:len(val)-1] 76 else: 77 val = int(val) 78 79 results[key] = val 80 return results 81 82class Version(object): 83 """ 84 Version information are all static members 85 """ 86 data = {} 87 88 @staticmethod 89 def get_data(): 90 if not Version.data: 91 root_dir = os.path.dirname(os.path.realpath(__file__)) 92 Version.data = parse_version_file(os.path.join(root_dir, 'src/build-data/version.txt')) 93 94 suffix = Version.data["release_suffix"] 95 if suffix != "": 96 suffix_re = re.compile('-(alpha|beta|rc)[0-9]+') 97 98 if not suffix_re.match(suffix): 99 raise Exception("Unexpected version suffix '%s'" % (suffix)) 100 return Version.data 101 102 @staticmethod 103 def major(): 104 return Version.get_data()["release_major"] 105 106 @staticmethod 107 def minor(): 108 return Version.get_data()["release_minor"] 109 110 @staticmethod 111 def patch(): 112 return Version.get_data()["release_patch"] 113 114 @staticmethod 115 def suffix(): 116 return Version.get_data()["release_suffix"] 117 118 @staticmethod 119 def packed(): 120 # Used on macOS for dylib versioning 121 return Version.major() * 1000 + Version.minor() 122 123 @staticmethod 124 def so_rev(): 125 return Version.get_data()["release_so_abi_rev"] 126 127 @staticmethod 128 def release_type(): 129 return Version.get_data()["release_type"] 130 131 @staticmethod 132 def datestamp(): 133 return Version.get_data()["release_datestamp"] 134 135 @staticmethod 136 def as_string(): 137 return '%d.%d.%d%s' % (Version.major(), Version.minor(), Version.patch(), Version.suffix()) 138 139 @staticmethod 140 def vc_rev(): 141 # Lazy load to ensure _local_repo_vc_revision() does not run before logger is set up 142 if Version.get_data()["release_vc_rev"] is None: 143 Version.data["release_vc_rev"] = Version._local_repo_vc_revision() 144 return Version.get_data()["release_vc_rev"] 145 146 @staticmethod 147 def _local_repo_vc_revision(): 148 vc_command = ['git', 'rev-parse', 'HEAD'] 149 cmdname = vc_command[0] 150 151 try: 152 vc = subprocess.Popen( 153 vc_command, 154 stdout=subprocess.PIPE, 155 stderr=subprocess.PIPE, 156 universal_newlines=True) 157 (stdout, stderr) = vc.communicate() 158 159 if vc.returncode != 0: 160 logging.debug('Error getting rev from %s - %d (%s)', 161 cmdname, vc.returncode, stderr) 162 return 'unknown' 163 164 rev = str(stdout).strip() 165 logging.debug('%s reported revision %s', cmdname, rev) 166 167 return '%s:%s' % (cmdname, rev) 168 except OSError as e: 169 logging.debug('Error getting rev from %s - %s' % (cmdname, e.strerror)) 170 return 'unknown' 171 172 173 174class SourcePaths(object): 175 """ 176 A collection of paths defined by the project structure and 177 independent of user configurations. 178 All paths are relative to the base_dir, which may be relative as well (e.g. ".") 179 """ 180 181 def __init__(self, base_dir): 182 self.base_dir = base_dir 183 self.doc_dir = os.path.join(self.base_dir, 'doc') 184 self.src_dir = os.path.join(self.base_dir, 'src') 185 186 # dirs in src/ 187 self.build_data_dir = os.path.join(self.src_dir, 'build-data') 188 self.configs_dir = os.path.join(self.src_dir, 'configs') 189 self.lib_dir = os.path.join(self.src_dir, 'lib') 190 self.python_dir = os.path.join(self.src_dir, 'python') 191 self.scripts_dir = os.path.join(self.src_dir, 'scripts') 192 193 # subdirs of src/ 194 self.test_data_dir = os.path.join(self.src_dir, 'tests/data') 195 self.sphinx_config_dir = os.path.join(self.configs_dir, 'sphinx') 196 197 198class BuildPaths(object): # pylint: disable=too-many-instance-attributes 199 """ 200 Constructor 201 """ 202 def __init__(self, source_paths, options, modules): 203 self.build_dir = os.path.join(options.with_build_dir, 'build') 204 205 self.libobj_dir = os.path.join(self.build_dir, 'obj', 'lib') 206 self.cliobj_dir = os.path.join(self.build_dir, 'obj', 'cli') 207 self.testobj_dir = os.path.join(self.build_dir, 'obj', 'test') 208 209 self.doc_output_dir = os.path.join(self.build_dir, 'docs') 210 self.handbook_output_dir = os.path.join(self.doc_output_dir, 'handbook') 211 self.doc_output_dir_doxygen = os.path.join(self.doc_output_dir, 'doxygen') if options.with_doxygen else None 212 213 self.include_dir = os.path.join(self.build_dir, 'include') 214 self.botan_include_dir = os.path.join(self.include_dir, 'botan') 215 self.internal_include_dir = os.path.join(self.botan_include_dir, 'internal') 216 self.external_include_dir = os.path.join(self.include_dir, 'external') 217 218 self.internal_headers = sorted(flatten([m.internal_headers() for m in modules])) 219 self.external_headers = sorted(flatten([m.external_headers() for m in modules])) 220 221 # this is overwritten if amalgamation is used 222 self.lib_sources = [normalize_source_path(s) for s in sorted(flatten([mod.sources() for mod in modules]))] 223 224 self.public_headers = sorted(flatten([m.public_headers() for m in modules])) 225 226 def find_sources_in(basedir, srcdir): 227 for (dirpath, _, filenames) in os.walk(os.path.join(basedir, srcdir)): 228 for filename in filenames: 229 if filename.endswith('.cpp') and not filename.startswith('.'): 230 yield os.path.join(dirpath, filename) 231 232 def find_headers_in(basedir, srcdir): 233 for (dirpath, _, filenames) in os.walk(os.path.join(basedir, srcdir)): 234 for filename in filenames: 235 if filename.endswith('.h') and not filename.startswith('.'): 236 yield os.path.join(dirpath, filename) 237 238 self.cli_sources = [normalize_source_path(s) for s in find_sources_in(source_paths.src_dir, 'cli')] 239 self.cli_headers = [normalize_source_path(s) for s in find_headers_in(source_paths.src_dir, 'cli')] 240 self.test_sources = [normalize_source_path(s) for s in find_sources_in(source_paths.src_dir, 'tests')] 241 242 if options.build_fuzzers: 243 self.fuzzer_sources = list(find_sources_in(source_paths.src_dir, 'fuzzer')) 244 self.fuzzer_output_dir = os.path.join(self.build_dir, 'fuzzer') 245 self.fuzzobj_dir = os.path.join(self.build_dir, 'obj', 'fuzzer') 246 else: 247 self.fuzzer_sources = None 248 self.fuzzer_output_dir = None 249 self.fuzzobj_dir = None 250 251 def build_dirs(self): 252 out = [ 253 self.libobj_dir, 254 self.cliobj_dir, 255 self.testobj_dir, 256 self.botan_include_dir, 257 self.internal_include_dir, 258 self.external_include_dir, 259 self.handbook_output_dir, 260 ] 261 if self.doc_output_dir_doxygen: 262 out += [self.doc_output_dir_doxygen] 263 if self.fuzzer_output_dir: 264 out += [self.fuzzobj_dir] 265 out += [self.fuzzer_output_dir] 266 return out 267 268 def format_include_paths(self, cc, external_includes): 269 dash_i = cc.add_include_dir_option 270 output = dash_i + self.include_dir 271 if self.external_headers: 272 output += ' ' + dash_i + self.external_include_dir 273 for external_include in external_includes: 274 output += ' ' + dash_i + external_include 275 return output 276 277 def src_info(self, typ): 278 if typ == 'lib': 279 return (self.lib_sources, self.libobj_dir) 280 elif typ == 'cli': 281 return (self.cli_sources, self.cliobj_dir) 282 elif typ == 'test': 283 return (self.test_sources, self.testobj_dir) 284 elif typ == 'fuzzer': 285 return (self.fuzzer_sources, self.fuzzobj_dir) 286 else: 287 raise InternalError("Unknown src info type '%s'" % (typ)) 288 289ACCEPTABLE_BUILD_TARGETS = ["static", "shared", "cli", "tests", "bogo_shim"] 290 291def process_command_line(args): # pylint: disable=too-many-locals,too-many-statements 292 """ 293 Handle command line options 294 Do not use logging in this method as command line options need to be 295 available before logging is setup. 296 """ 297 298 parser = optparse.OptionParser( 299 formatter=optparse.IndentedHelpFormatter(max_help_position=50), 300 version=Version.as_string()) 301 302 parser.add_option('--verbose', action='store_true', default=False, 303 help='Show debug messages') 304 parser.add_option('--quiet', action='store_true', default=False, 305 help='Show only warnings and errors') 306 307 target_group = optparse.OptionGroup(parser, 'Target options') 308 309 target_group.add_option('--cpu', help='set the target CPU architecture') 310 311 target_group.add_option('--os', help='set the target operating system') 312 313 target_group.add_option('--cc', dest='compiler', help='set the desired build compiler') 314 315 target_group.add_option('--cc-min-version', dest='cc_min_version', default=None, 316 metavar='MAJOR.MINOR', 317 help='Set the minimal version of the target compiler. ' \ 318 'Use --cc-min-version=0.0 to support all compiler versions. ' \ 319 'Default is auto detection.') 320 321 target_group.add_option('--cc-bin', dest='compiler_binary', metavar='BINARY', 322 help='set path to compiler binary') 323 324 target_group.add_option('--cc-abi-flags', metavar='FLAGS', default='', 325 help='set compiler ABI flags') 326 327 target_group.add_option('--cxxflags', metavar='FLAGS', default=None, 328 help='override all compiler flags') 329 330 target_group.add_option('--extra-cxxflags', metavar='FLAGS', default=[], action='append', 331 help='set extra compiler flags') 332 333 target_group.add_option('--ldflags', metavar='FLAGS', 334 help='set linker flags', default=None) 335 336 target_group.add_option('--extra-libs', metavar='LIBS', 337 help='specify extra libraries to link against', default='') 338 339 target_group.add_option('--ar-command', dest='ar_command', metavar='AR', default=None, 340 help='set path to static archive creator') 341 342 target_group.add_option('--ar-options', dest='ar_options', metavar='AR_OPTIONS', default=None, 343 help='set options for ar') 344 345 target_group.add_option('--msvc-runtime', metavar='RT', default=None, 346 help='specify MSVC runtime (MT, MD, MTd, MDd)') 347 348 target_group.add_option('--compiler-cache', 349 help='specify a compiler cache to use') 350 351 target_group.add_option('--with-endian', metavar='ORDER', default=None, 352 help='override byte order guess') 353 354 target_group.add_option('--with-os-features', action='append', metavar='FEAT', 355 help='specify OS features to use') 356 target_group.add_option('--without-os-features', action='append', metavar='FEAT', 357 help='specify OS features to disable') 358 359 isa_extensions = [ 360 'SSE2', 'SSSE3', 'SSE4.1', 'SSE4.2', 'AVX2', 'BMI2', 'RDRAND', 'RDSEED', 361 'AES-NI', 'SHA-NI', 362 'AltiVec', 'NEON', 'ARMv8 Crypto', 'POWER Crypto'] 363 364 for isa_extn_name in isa_extensions: 365 isa_extn = isa_extn_name.lower().replace(' ', '') 366 367 target_group.add_option('--disable-%s' % (isa_extn), 368 help='disable %s intrinsics' % (isa_extn_name), 369 action='append_const', 370 const=isa_extn.replace('-', '').replace('.', '').replace(' ', ''), 371 dest='disable_intrinsics') 372 373 build_group = optparse.OptionGroup(parser, 'Build options') 374 375 build_group.add_option('--system-cert-bundle', metavar='PATH', default=None, 376 help='set path to trusted CA bundle') 377 378 build_group.add_option('--with-debug-info', action='store_true', default=False, dest='with_debug_info', 379 help='include debug symbols') 380 381 build_group.add_option('--with-sanitizers', action='store_true', default=False, dest='with_sanitizers', 382 help='enable ASan/UBSan checks') 383 384 build_group.add_option('--enable-sanitizers', metavar='SAN', default='', 385 help='enable specific sanitizers') 386 387 build_group.add_option('--with-stack-protector', dest='with_stack_protector', 388 action='store_false', default=None, help=optparse.SUPPRESS_HELP) 389 390 build_group.add_option('--without-stack-protector', dest='with_stack_protector', 391 action='store_false', help='disable stack smashing protections') 392 393 build_group.add_option('--with-coverage', action='store_true', default=False, dest='with_coverage', 394 help='add coverage info and disable opts') 395 396 build_group.add_option('--with-coverage-info', action='store_true', default=False, dest='with_coverage_info', 397 help='add coverage info') 398 399 build_group.add_option('--enable-shared-library', dest='build_shared_lib', 400 action='store_true', default=None, 401 help=optparse.SUPPRESS_HELP) 402 build_group.add_option('--disable-shared-library', dest='build_shared_lib', 403 action='store_false', 404 help='disable building shared library') 405 406 build_group.add_option('--enable-static-library', dest='build_static_lib', 407 action='store_true', default=None, 408 help=optparse.SUPPRESS_HELP) 409 build_group.add_option('--disable-static-library', dest='build_static_lib', 410 action='store_false', 411 help='disable building static library') 412 413 build_group.add_option('--optimize-for-size', dest='optimize_for_size', 414 action='store_true', default=False, 415 help='optimize for code size') 416 417 build_group.add_option('--no-optimizations', dest='no_optimizations', 418 action='store_true', default=False, 419 help='disable all optimizations (for debugging)') 420 421 build_group.add_option('--debug-mode', action='store_true', default=False, dest='debug_mode', 422 help='enable debug info, disable optimizations') 423 424 build_group.add_option('--amalgamation', dest='amalgamation', 425 default=False, action='store_true', 426 help='use amalgamation to build') 427 428 build_group.add_option('--name-amalgamation', metavar='NAME', default='botan_all', 429 help='specify alternate name for amalgamation files') 430 431 build_group.add_option('--with-build-dir', metavar='DIR', default='', 432 help='setup the build in DIR') 433 434 build_group.add_option('--with-external-includedir', metavar='DIR', default=[], 435 help='use DIR for external includes', action='append') 436 437 build_group.add_option('--with-external-libdir', metavar='DIR', default=[], 438 help='use DIR for external libs', action='append') 439 440 build_group.add_option('--define-build-macro', metavar='DEFINE', default=[], 441 help='set compile-time pre-processor definition like KEY[=VALUE]', action='append') 442 443 build_group.add_option('--with-sysroot-dir', metavar='DIR', default='', 444 help='use DIR for system root while cross-compiling') 445 446 build_group.add_option('--with-openmp', default=False, action='store_true', 447 help='enable use of OpenMP') 448 449 link_methods = ['symlink', 'hardlink', 'copy'] 450 build_group.add_option('--link-method', default=None, metavar='METHOD', 451 choices=link_methods, 452 help='choose how links to include headers are created (%s)' % ', '.join(link_methods)) 453 454 build_group.add_option('--with-local-config', 455 dest='local_config', metavar='FILE', 456 help='include the contents of FILE into build.h') 457 458 build_group.add_option('--distribution-info', metavar='STRING', 459 help='distribution specific version', 460 default='unspecified') 461 462 build_group.add_option('--maintainer-mode', dest='maintainer_mode', 463 action='store_true', default=False, 464 help=optparse.SUPPRESS_HELP) 465 466 build_group.add_option('--werror-mode', dest='werror_mode', 467 action='store_true', default=False, 468 help="Prohibit compiler warnings") 469 470 build_group.add_option('--no-store-vc-rev', action='store_true', default=False, 471 help=optparse.SUPPRESS_HELP) 472 473 build_group.add_option('--no-install-python-module', action='store_true', default=False, 474 help='skip installing Python module') 475 476 build_group.add_option('--with-python-versions', dest='python_version', 477 metavar='N.M', 478 default='%d.%d' % (sys.version_info[0], sys.version_info[1]), 479 help='where to install botan2.py (def %default)') 480 481 build_group.add_option('--disable-cc-tests', dest='enable_cc_tests', 482 default=True, action='store_false', 483 help=optparse.SUPPRESS_HELP) 484 485 build_group.add_option('--with-valgrind', help='use valgrind API', 486 dest='with_valgrind', action='store_true', default=False) 487 488 # Cmake and bakefile options are hidden as they should not be used by end users 489 build_group.add_option('--with-cmake', action='store_true', 490 default=False, help=optparse.SUPPRESS_HELP) 491 492 build_group.add_option('--with-bakefile', action='store_true', 493 default=False, help=optparse.SUPPRESS_HELP) 494 495 build_group.add_option('--unsafe-fuzzer-mode', action='store_true', default=False, 496 help='Disable essential checks for testing') 497 498 build_group.add_option('--build-fuzzers', dest='build_fuzzers', 499 metavar='TYPE', default=None, 500 help='Build fuzzers (afl, libfuzzer, klee, test)') 501 502 build_group.add_option('--with-fuzzer-lib', metavar='LIB', default=None, dest='fuzzer_lib', 503 help='additionally link in LIB') 504 505 build_group.add_option('--test-mode', action='store_true', default=False, 506 help=optparse.SUPPRESS_HELP) 507 508 build_group.add_option('--with-debug-asserts', action='store_true', default=False, 509 help=optparse.SUPPRESS_HELP) 510 511 build_group.add_option('--build-targets', default=None, dest="build_targets", action='append', 512 help="build specific targets and tools (%s)" % ', '.join(ACCEPTABLE_BUILD_TARGETS)) 513 514 build_group.add_option('--with-pkg-config', action='store_true', default=None, 515 help=optparse.SUPPRESS_HELP) 516 build_group.add_option('--without-pkg-config', dest='with_pkg_config', action='store_false', 517 help=optparse.SUPPRESS_HELP) 518 build_group.add_option('--boost-library-name', dest='boost_libnames', default=[], 519 help="file name of some boost library to link", action='append') 520 521 docs_group = optparse.OptionGroup(parser, 'Documentation Options') 522 523 docs_group.add_option('--with-documentation', action='store_true', 524 help=optparse.SUPPRESS_HELP) 525 526 docs_group.add_option('--without-documentation', action='store_false', 527 default=True, dest='with_documentation', 528 help='Skip building/installing documentation') 529 530 docs_group.add_option('--with-sphinx', action='store_true', 531 default=None, help='Use Sphinx') 532 533 docs_group.add_option('--without-sphinx', action='store_false', 534 dest='with_sphinx', help=optparse.SUPPRESS_HELP) 535 536 docs_group.add_option('--with-pdf', action='store_true', 537 default=False, help='Use Sphinx to generate PDF doc') 538 539 docs_group.add_option('--without-pdf', action='store_false', 540 dest='with_pdf', help=optparse.SUPPRESS_HELP) 541 542 docs_group.add_option('--with-rst2man', action='store_true', 543 default=None, help='Use rst2man to generate man page') 544 545 docs_group.add_option('--without-rst2man', action='store_false', 546 dest='with_rst2man', help=optparse.SUPPRESS_HELP) 547 548 docs_group.add_option('--with-doxygen', action='store_true', 549 default=False, help='Use Doxygen') 550 551 docs_group.add_option('--without-doxygen', action='store_false', 552 dest='with_doxygen', help=optparse.SUPPRESS_HELP) 553 554 mods_group = optparse.OptionGroup(parser, 'Module selection') 555 556 mods_group.add_option('--module-policy', dest='module_policy', 557 help="module policy file (see src/build-data/policy)", 558 metavar='POL', default=None) 559 560 mods_group.add_option('--enable-modules', dest='enabled_modules', 561 metavar='MODS', action='append', 562 help='enable specific modules') 563 mods_group.add_option('--disable-modules', dest='disabled_modules', 564 metavar='MODS', action='append', 565 help='disable specific modules') 566 mods_group.add_option('--no-autoload', action='store_true', default=False, 567 help=optparse.SUPPRESS_HELP) 568 mods_group.add_option('--minimized-build', action='store_true', dest='no_autoload', 569 help='minimize build') 570 571 # Should be derived from info.txt but this runs too early 572 third_party = ['boost', 'bzip2', 'lzma', 'openssl', 'commoncrypto', 'sqlite3', 'zlib', 'tpm'] 573 574 for mod in third_party: 575 mods_group.add_option('--with-%s' % (mod), 576 help=('use %s' % (mod)) if mod in third_party else optparse.SUPPRESS_HELP, 577 action='append_const', 578 const=mod, 579 dest='enabled_modules') 580 581 mods_group.add_option('--without-%s' % (mod), 582 help=optparse.SUPPRESS_HELP, 583 action='append_const', 584 const=mod, 585 dest='disabled_modules') 586 587 mods_group.add_option('--with-everything', help=optparse.SUPPRESS_HELP, 588 action='store_true', default=False) 589 590 install_group = optparse.OptionGroup(parser, 'Installation options') 591 592 install_group.add_option('--program-suffix', metavar='SUFFIX', 593 help='append string to program names') 594 install_group.add_option('--library-suffix', metavar='SUFFIX', default='', 595 help='append string to library names') 596 597 install_group.add_option('--prefix', metavar='DIR', 598 help='set the install prefix') 599 install_group.add_option('--docdir', metavar='DIR', 600 help='set the doc install dir') 601 install_group.add_option('--bindir', metavar='DIR', 602 help='set the binary install dir') 603 install_group.add_option('--libdir', metavar='DIR', 604 help='set the library install dir') 605 install_group.add_option('--mandir', metavar='DIR', 606 help='set the install dir for man pages') 607 install_group.add_option('--includedir', metavar='DIR', 608 help='set the include file install dir') 609 610 info_group = optparse.OptionGroup(parser, 'Informational') 611 612 info_group.add_option('--list-modules', dest='list_modules', 613 action='store_true', 614 help='list available modules and exit') 615 616 info_group.add_option('--list-os-features', dest='list_os_features', 617 action='store_true', 618 help='list available OS features and exit') 619 620 parser.add_option_group(target_group) 621 parser.add_option_group(build_group) 622 parser.add_option_group(docs_group) 623 parser.add_option_group(mods_group) 624 parser.add_option_group(install_group) 625 parser.add_option_group(info_group) 626 627 # These exist only for autoconf compatibility (requested by zw for mtn) 628 compat_with_autoconf_options = [ 629 'datadir', 630 'datarootdir', 631 'dvidir', 632 'exec-prefix', 633 'htmldir', 634 'infodir', 635 'libexecdir', 636 'localedir', 637 'localstatedir', 638 'oldincludedir', 639 'pdfdir', 640 'psdir', 641 'sbindir', 642 'sharedstatedir', 643 'sysconfdir' 644 ] 645 646 for opt in compat_with_autoconf_options: 647 parser.add_option('--' + opt, help=optparse.SUPPRESS_HELP) 648 649 (options, args) = parser.parse_args(args) 650 651 if args != []: 652 raise UserError('Unhandled option(s): ' + ' '.join(args)) 653 654 if options.with_endian not in [None, 'little', 'big']: 655 raise UserError('Bad value to --with-endian "%s"' % (options.with_endian)) 656 657 if options.debug_mode: 658 options.no_optimizations = True 659 options.with_debug_info = True 660 661 if options.with_coverage: 662 options.with_coverage_info = True 663 options.no_optimizations = True 664 665 def parse_multiple_enable(modules): 666 if modules is None: 667 return [] 668 669 return sorted({m for m in flatten([s.split(',') for s in modules]) if m != ''}) 670 671 options.enabled_modules = parse_multiple_enable(options.enabled_modules) 672 options.disabled_modules = parse_multiple_enable(options.disabled_modules) 673 674 options.with_os_features = parse_multiple_enable(options.with_os_features) 675 options.without_os_features = parse_multiple_enable(options.without_os_features) 676 677 options.disable_intrinsics = parse_multiple_enable(options.disable_intrinsics) 678 679 return options 680 681def take_options_from_env(options): 682 # Take some values from environment, if not set on command line 683 684 def update_from_env(val, var, name): 685 if val is None: 686 val = os.getenv(var) 687 if val is not None: 688 logging.info('Implicit --%s=%s due to environment variable %s', name, val, var) 689 690 return val 691 692 if os.getenv('CXX') and options.compiler_binary is None and options.compiler is not None: 693 logging.info('CXX environment variable is set which will override compiler path') 694 695 options.ar_command = update_from_env(options.ar_command, 'AR', 'ar-command') 696 options.ar_options = update_from_env(options.ar_options, 'AR_OPTIONS', 'ar-options') 697 options.compiler_binary = update_from_env(options.compiler_binary, 'CXX', 'cc-bin') 698 options.cxxflags = update_from_env(options.cxxflags, 'CXXFLAGS', 'cxxflags') 699 options.ldflags = update_from_env(options.ldflags, 'LDFLAGS', 'ldflags') 700 701class LexResult(object): 702 pass 703 704 705class LexerError(InternalError): 706 def __init__(self, msg, lexfile, line): 707 super(LexerError, self).__init__(msg) 708 self.msg = msg 709 self.lexfile = lexfile 710 self.line = line 711 712 def __str__(self): 713 return '%s at %s:%d' % (self.msg, self.lexfile, self.line) 714 715def parse_lex_dict(as_list, map_name, infofile): 716 if len(as_list) % 3 != 0: 717 raise InternalError("Lex dictionary has invalid format (input not divisible by 3): %s" % as_list) 718 719 result = {} 720 for key, sep, value in [as_list[3*i:3*i+3] for i in range(0, len(as_list)//3)]: 721 if sep != '->': 722 raise InternalError("Map %s in %s has invalid format" % (map_name, infofile)) 723 if key in result: 724 raise InternalError("Duplicate map entry %s in map %s file %s" % (key, map_name, infofile)) 725 result[key] = value 726 return result 727 728def lex_me_harder(infofile, allowed_groups, allowed_maps, name_val_pairs): 729 """ 730 Generic lexer function for info.txt and src/build-data files 731 """ 732 out = LexResult() 733 734 # Format as a nameable Python variable 735 def py_var(group): 736 return group.replace(':', '_') 737 738 lexer = shlex.shlex(open(infofile), infofile, posix=True) 739 lexer.wordchars += '=:.<>/,-!?+*' # handle various funky chars in info.txt 740 741 groups = allowed_groups + allowed_maps 742 for group in groups: 743 out.__dict__[py_var(group)] = [] 744 for (key, val) in name_val_pairs.items(): 745 out.__dict__[key] = val 746 747 def lexed_tokens(): # Convert to an iterator 748 while True: 749 token = lexer.get_token() 750 if token != lexer.eof: 751 yield token 752 else: 753 return 754 755 for token in lexed_tokens(): 756 match = re.match('<(.*)>', token) 757 758 # Check for a grouping 759 if match is not None: 760 group = match.group(1) 761 762 if group not in groups: 763 raise LexerError('Unknown group "%s"' % (group), 764 infofile, lexer.lineno) 765 766 end_marker = '</' + group + '>' 767 768 token = lexer.get_token() 769 while token != end_marker: 770 out.__dict__[py_var(group)].append(token) 771 token = lexer.get_token() 772 if token is None: 773 raise LexerError('Group "%s" not terminated' % (group), 774 infofile, lexer.lineno) 775 776 elif token in name_val_pairs.keys(): 777 if isinstance(out.__dict__[token], list): 778 out.__dict__[token].append(lexer.get_token()) 779 else: 780 out.__dict__[token] = lexer.get_token() 781 782 else: # No match -> error 783 raise LexerError('Bad token "%s"' % (token), infofile, lexer.lineno) 784 785 for group in allowed_maps: 786 out.__dict__[group] = parse_lex_dict(out.__dict__[group], group, infofile) 787 788 return out 789 790class InfoObject(object): 791 def __init__(self, infofile): 792 """ 793 Constructor sets members `infofile`, `lives_in`, `parent_module` and `basename` 794 """ 795 796 self.infofile = infofile 797 (dirname, basename) = os.path.split(infofile) 798 self.lives_in = dirname 799 if basename == 'info.txt': 800 (obj_dir, self.basename) = os.path.split(dirname) 801 if os.access(os.path.join(obj_dir, 'info.txt'), os.R_OK): 802 self.parent_module = os.path.basename(obj_dir) 803 else: 804 self.parent_module = None 805 else: 806 self.basename = basename.replace('.txt', '') 807 808 809class ModuleInfo(InfoObject): 810 """ 811 Represents the information about a particular module 812 """ 813 814 def __init__(self, infofile): 815 # pylint: disable=too-many-statements 816 super(ModuleInfo, self).__init__(infofile) 817 lex = lex_me_harder( 818 infofile, 819 ['header:internal', 'header:public', 'header:external', 'requires', 820 'os_features', 'arch', 'isa', 'cc', 'comment', 'warning'], 821 ['defines', 'libs', 'frameworks'], 822 { 823 'load_on': 'auto', 824 'endian': 'any', 825 }) 826 827 def check_header_duplicates(header_list_public, header_list_internal): 828 pub_header = set(header_list_public) 829 int_header = set(header_list_internal) 830 if not pub_header.isdisjoint(int_header): 831 logging.error("Module %s header contains same header in public and internal sections" % self.infofile) 832 833 check_header_duplicates(lex.header_public, lex.header_internal) 834 835 all_source_files = [] 836 all_header_files = [] 837 838 for fspath in os.listdir(self.lives_in): 839 if fspath.endswith('.cpp'): 840 all_source_files.append(fspath) 841 elif fspath.endswith('.h'): 842 all_header_files.append(fspath) 843 844 self.source = all_source_files 845 846 # If not entry for the headers, all are assumed public 847 if lex.header_internal == [] and lex.header_public == []: 848 self.header_public = list(all_header_files) 849 self.header_internal = [] 850 else: 851 self.header_public = lex.header_public 852 self.header_internal = lex.header_internal 853 self.header_external = lex.header_external 854 855 def convert_lib_list(libs): 856 out = {} 857 for (os_name, lib_list) in libs.items(): 858 out[os_name] = lib_list.split(',') 859 return out 860 861 def combine_lines(c): 862 return ' '.join(c) if c else None 863 864 # Convert remaining lex result to members 865 self.arch = lex.arch 866 self.cc = lex.cc 867 self.comment = combine_lines(lex.comment) 868 self._defines = lex.defines 869 self._validate_defines_content(self._defines) 870 self.frameworks = convert_lib_list(lex.frameworks) 871 self.libs = convert_lib_list(lex.libs) 872 self.load_on = lex.load_on 873 self.isa = lex.isa 874 self.os_features = lex.os_features 875 self.requires = lex.requires 876 self.warning = combine_lines(lex.warning) 877 self.endian = lex.endian 878 879 # Modify members 880 self.source = [normalize_source_path(os.path.join(self.lives_in, s)) for s in self.source] 881 self.header_internal = [os.path.join(self.lives_in, s) for s in self.header_internal] 882 self.header_public = [os.path.join(self.lives_in, s) for s in self.header_public] 883 self.header_external = [os.path.join(self.lives_in, s) for s in self.header_external] 884 885 # Filesystem read access check 886 for src in self.source + self.header_internal + self.header_public + self.header_external: 887 if not os.access(src, os.R_OK): 888 logging.error("Missing file %s in %s" % (src, infofile)) 889 890 # Check for duplicates 891 def intersect_check(type_a, list_a, type_b, list_b): 892 intersection = set.intersection(set(list_a), set(list_b)) 893 if intersection: 894 logging.error('Headers %s marked both %s and %s' % (' '.join(intersection), type_a, type_b)) 895 896 intersect_check('public', self.header_public, 'internal', self.header_internal) 897 intersect_check('public', self.header_public, 'external', self.header_external) 898 intersect_check('external', self.header_external, 'internal', self.header_internal) 899 900 @staticmethod 901 def _validate_defines_content(defines): 902 for key, value in defines.items(): 903 if not re.match('^[0-9A-Za-z_]{3,30}$', key): 904 raise InternalError('Module defines key has invalid format: "%s"' % key) 905 if not re.match('^20[0-9]{6}$', value): 906 raise InternalError('Module defines value has invalid format: "%s"' % value) 907 908 def cross_check(self, arch_info, cc_info, all_os_features, all_isa_extn): 909 910 for feat in set(flatten([o.split(',') for o in self.os_features])): 911 if feat not in all_os_features: 912 logging.error("Module %s uses an OS feature (%s) which no OS supports", self.infofile, feat) 913 914 for supp_cc in self.cc: 915 if supp_cc not in cc_info: 916 colon_idx = supp_cc.find(':') 917 # a versioned compiler dependency 918 if colon_idx > 0 and supp_cc[0:colon_idx] in cc_info: 919 pass 920 else: 921 raise InternalError('Module %s mentions unknown compiler %s' % (self.infofile, supp_cc)) 922 923 for supp_arch in self.arch: 924 if supp_arch not in arch_info: 925 raise InternalError('Module %s mentions unknown arch %s' % (self.infofile, supp_arch)) 926 927 def known_isa(isa): 928 if isa in all_isa_extn: 929 return True 930 931 compound_isa = isa.split(':') 932 if len(compound_isa) == 2 and compound_isa[0] in arch_info and compound_isa[1] in all_isa_extn: 933 return True 934 return False 935 936 for isa in self.isa: 937 if not known_isa(isa): 938 raise InternalError('Module %s uses unknown ISA extension %s' % (self.infofile, isa)) 939 940 def sources(self): 941 return self.source 942 943 def public_headers(self): 944 return self.header_public 945 946 def internal_headers(self): 947 return self.header_internal 948 949 def external_headers(self): 950 return self.header_external 951 952 def isas_needed(self, arch): 953 isas = [] 954 955 for isa in self.isa: 956 if isa.find(':') == -1: 957 isas.append(isa) 958 elif isa.startswith(arch + ':'): 959 isas.append(isa[len(arch)+1:]) 960 961 return isas 962 963 def defines(self): 964 return [(key + ' ' + value) for key, value in self._defines.items()] 965 966 def compatible_cpu(self, archinfo, options): 967 arch_name = archinfo.basename 968 cpu_name = options.cpu 969 970 if self.endian != 'any': 971 if self.endian != options.with_endian: 972 return False 973 974 for isa in self.isa: 975 if isa.find(':') > 0: 976 (arch, isa) = isa.split(':') 977 978 if arch != arch_name: 979 continue 980 981 if isa in options.disable_intrinsics: 982 return False # explicitly disabled 983 984 if isa not in archinfo.isa_extensions: 985 return False 986 987 if self.arch != []: 988 if arch_name not in self.arch and cpu_name not in self.arch: 989 return False 990 991 return True 992 993 def compatible_os(self, os_data, options): 994 if not self.os_features: 995 return True 996 997 def has_all(needed, provided): 998 for n in needed: 999 if n not in provided: 1000 return False 1001 return True 1002 1003 provided_features = os_data.enabled_features(options) 1004 1005 for feature_set in self.os_features: 1006 if has_all(feature_set.split(','), provided_features): 1007 return True 1008 1009 return False 1010 1011 def compatible_compiler(self, ccinfo, cc_min_version, arch): 1012 # Check if this compiler supports the flags we need 1013 def supported_isa_flags(ccinfo, arch): 1014 for isa in self.isa: 1015 if ccinfo.isa_flags_for(isa, arch) is None: 1016 return False 1017 return True 1018 1019 # Check if module gives explicit compiler dependencies 1020 def supported_compiler(ccinfo, cc_min_version): 1021 if self.cc == []: 1022 # no compiler restriction 1023 return True 1024 1025 if ccinfo.basename in self.cc: 1026 # compiler is supported, independent of version 1027 return True 1028 1029 # Maybe a versioned compiler dep 1030 for cc in self.cc: 1031 try: 1032 name, version = cc.split(":") 1033 if name == ccinfo.basename: 1034 min_cc_version = [int(v) for v in version.split('.')] 1035 cur_cc_version = [int(v) for v in cc_min_version.split('.')] 1036 # With lists of ints, this does what we want 1037 return cur_cc_version >= min_cc_version 1038 except ValueError: 1039 # No version part specified 1040 pass 1041 1042 return False # compiler not listed 1043 1044 return supported_isa_flags(ccinfo, arch) and supported_compiler(ccinfo, cc_min_version) 1045 1046 def dependencies(self, osinfo): 1047 # base is an implicit dep for all submodules 1048 deps = ['base'] 1049 if self.parent_module is not None: 1050 deps.append(self.parent_module) 1051 1052 for req in self.requires: 1053 if req.find('?') != -1: 1054 (cond, dep) = req.split('?') 1055 if osinfo is None or cond in osinfo.target_features: 1056 deps.append(dep) 1057 else: 1058 deps.append(req) 1059 1060 return deps 1061 1062 def dependencies_exist(self, modules): 1063 """ 1064 Ensure that all dependencies of this module actually exist, warning 1065 about any that do not 1066 """ 1067 1068 missing = [s for s in self.dependencies(None) if s not in modules] 1069 1070 if missing: 1071 logging.error("Module '%s', dep of '%s', does not exist" % ( 1072 missing, self.basename)) 1073 1074 1075class ModulePolicyInfo(InfoObject): 1076 def __init__(self, infofile): 1077 super(ModulePolicyInfo, self).__init__(infofile) 1078 lex = lex_me_harder( 1079 infofile, 1080 ['required', 'if_available', 'prohibited'], 1081 [], 1082 {}) 1083 1084 self.if_available = lex.if_available 1085 self.required = lex.required 1086 self.prohibited = lex.prohibited 1087 1088 def cross_check(self, modules): 1089 def check(tp, lst): 1090 for mod in lst: 1091 if mod not in modules: 1092 logging.error("Module policy %s includes non-existent module %s in <%s>" % ( 1093 self.infofile, mod, tp)) 1094 1095 check('required', self.required) 1096 check('if_available', self.if_available) 1097 check('prohibited', self.prohibited) 1098 1099 1100class ArchInfo(InfoObject): 1101 def __init__(self, infofile): 1102 super(ArchInfo, self).__init__(infofile) 1103 lex = lex_me_harder( 1104 infofile, 1105 ['aliases', 'isa_extensions'], 1106 [], 1107 { 1108 'endian': None, 1109 'family': None, 1110 'wordsize': 32 1111 }) 1112 1113 self.aliases = lex.aliases 1114 self.endian = lex.endian 1115 self.family = lex.family 1116 self.isa_extensions = lex.isa_extensions 1117 self.wordsize = int(lex.wordsize) 1118 1119 if self.wordsize not in [32, 64]: 1120 logging.error('Unexpected wordsize %d for arch %s', self.wordsize, infofile) 1121 1122 alphanumeric = re.compile('^[a-z0-9]+$') 1123 for isa in self.isa_extensions: 1124 if alphanumeric.match(isa) is None: 1125 logging.error('Invalid name for ISA extension "%s"', isa) 1126 1127 def supported_isa_extensions(self, cc, options): 1128 isas = [] 1129 1130 for isa in self.isa_extensions: 1131 if isa not in options.disable_intrinsics: 1132 if cc.isa_flags_for(isa, self.basename) is not None: 1133 isas.append(isa) 1134 1135 return sorted(isas) 1136 1137 1138class CompilerInfo(InfoObject): # pylint: disable=too-many-instance-attributes 1139 def __init__(self, infofile): 1140 super(CompilerInfo, self).__init__(infofile) 1141 lex = lex_me_harder( 1142 infofile, 1143 [], 1144 ['cpu_flags', 'cpu_flags_no_debug', 'so_link_commands', 'binary_link_commands', 1145 'mach_abi_linking', 'isa_flags', 'sanitizers', 'lib_flags'], 1146 { 1147 'binary_name': None, 1148 'linker_name': None, 1149 'macro_name': None, 1150 'output_to_object': '-o ', 1151 'output_to_exe': '-o ', 1152 'add_include_dir_option': '-I', 1153 'add_lib_dir_option': '-L', 1154 'add_compile_definition_option': '-D', 1155 'add_sysroot_option': '', 1156 'add_lib_option': '-l%s', 1157 'add_framework_option': '-framework ', 1158 'preproc_flags': '-E', 1159 'compile_flags': '-c', 1160 'debug_info_flags': '-g', 1161 'optimization_flags': '', 1162 'size_optimization_flags': '', 1163 'sanitizer_optimization_flags': '', 1164 'coverage_flags': '', 1165 'stack_protector_flags': '', 1166 'shared_flags': '', 1167 'lang_flags': '', 1168 'warning_flags': '', 1169 'maintainer_warning_flags': '', 1170 'visibility_build_flags': '', 1171 'visibility_attribute': '', 1172 'ar_command': '', 1173 'ar_options': '', 1174 'ar_output_to': '', 1175 'werror_flags': '', 1176 }) 1177 1178 self.add_framework_option = lex.add_framework_option 1179 self.add_include_dir_option = lex.add_include_dir_option 1180 self.add_lib_dir_option = lex.add_lib_dir_option 1181 self.add_lib_option = lex.add_lib_option 1182 self.add_compile_definition_option = lex.add_compile_definition_option 1183 self.add_sysroot_option = lex.add_sysroot_option 1184 self.ar_command = lex.ar_command 1185 self.ar_options = lex.ar_options 1186 self.ar_output_to = lex.ar_output_to 1187 self.binary_link_commands = lex.binary_link_commands 1188 self.binary_name = lex.binary_name 1189 self.cpu_flags = lex.cpu_flags 1190 self.cpu_flags_no_debug = lex.cpu_flags_no_debug 1191 self.compile_flags = lex.compile_flags 1192 self.coverage_flags = lex.coverage_flags 1193 self.debug_info_flags = lex.debug_info_flags 1194 self.isa_flags = lex.isa_flags 1195 self.lang_flags = lex.lang_flags 1196 self.lib_flags = lex.lib_flags 1197 self.linker_name = lex.linker_name 1198 self.mach_abi_linking = lex.mach_abi_linking 1199 self.macro_name = lex.macro_name 1200 self.maintainer_warning_flags = lex.maintainer_warning_flags 1201 self.optimization_flags = lex.optimization_flags 1202 self.output_to_exe = lex.output_to_exe 1203 self.output_to_object = lex.output_to_object 1204 self.preproc_flags = lex.preproc_flags 1205 self.sanitizers = lex.sanitizers 1206 self.sanitizer_types = [] 1207 self.sanitizer_optimization_flags = lex.sanitizer_optimization_flags 1208 self.shared_flags = lex.shared_flags 1209 self.size_optimization_flags = lex.size_optimization_flags 1210 self.so_link_commands = lex.so_link_commands 1211 self.stack_protector_flags = lex.stack_protector_flags 1212 self.visibility_attribute = lex.visibility_attribute 1213 self.visibility_build_flags = lex.visibility_build_flags 1214 self.warning_flags = lex.warning_flags 1215 self.werror_flags = lex.werror_flags 1216 1217 def cross_check(self, os_info, arch_info, all_isas): 1218 1219 for isa in self.isa_flags: 1220 if ":" in isa: 1221 (arch, isa) = isa.split(":") 1222 if isa not in all_isas: 1223 raise InternalError('Compiler %s has flags for unknown ISA %s' % (self.infofile, isa)) 1224 if arch not in arch_info: 1225 raise InternalError('Compiler %s has flags for unknown arch/ISA %s:%s' % (self.infofile, arch, isa)) 1226 1227 for os_name in self.binary_link_commands: 1228 if os_name in ["default", "default-debug"]: 1229 continue 1230 if os_name not in os_info: 1231 raise InternalError("Compiler %s has binary_link_command for unknown OS %s" % (self.infofile, os_name)) 1232 1233 for os_name in self.so_link_commands: 1234 if os_name in ["default", "default-debug"]: 1235 continue 1236 if os_name not in os_info: 1237 raise InternalError("Compiler %s has so_link_command for unknown OS %s" % (self.infofile, os_name)) 1238 1239 def isa_flags_for(self, isa, arch): 1240 if isa.find(':') > 0: 1241 (isa_arch, isa) = isa.split(':') 1242 if isa_arch != arch: 1243 return '' 1244 if isa in self.isa_flags: 1245 return self.isa_flags[isa] 1246 1247 if isa in self.isa_flags: 1248 return self.isa_flags[isa] 1249 arch_isa = '%s:%s' % (arch, isa) 1250 if arch_isa in self.isa_flags: 1251 return self.isa_flags[arch_isa] 1252 1253 return None 1254 1255 def get_isa_specific_flags(self, isas, arch, options): 1256 flags = set() 1257 1258 def simd32_impl(): 1259 for simd_isa in ['sse2', 'altivec', 'neon']: 1260 if simd_isa in arch.isa_extensions and \ 1261 simd_isa not in options.disable_intrinsics and \ 1262 self.isa_flags_for(simd_isa, arch.basename): 1263 return simd_isa 1264 return None 1265 1266 for isa in isas: 1267 1268 if isa == 'simd': 1269 isa = simd32_impl() 1270 1271 if isa is None: 1272 continue 1273 1274 flagset = self.isa_flags_for(isa, arch.basename) 1275 if flagset is None: 1276 raise UserError('Compiler %s does not support %s' % (self.basename, isa)) 1277 flags.add(flagset) 1278 1279 return " ".join(sorted(flags)) 1280 1281 def gen_lib_flags(self, options, variables): 1282 """ 1283 Return any flags specific to building the library 1284 (vs the cli or tests) 1285 """ 1286 1287 def flag_builder(): 1288 if options.build_shared_lib: 1289 yield self.shared_flags 1290 yield self.visibility_build_flags 1291 1292 if 'debug' in self.lib_flags and options.with_debug_info: 1293 yield process_template_string(self.lib_flags['debug'], variables, self.infofile) 1294 1295 1296 return ' '.join(list(flag_builder())) 1297 1298 def gen_visibility_attribute(self, options): 1299 if options.build_shared_lib: 1300 return self.visibility_attribute 1301 return '' 1302 1303 def mach_abi_link_flags(self, options, debug_mode=None): 1304 #pylint: disable=too-many-branches 1305 1306 """ 1307 Return the machine specific ABI flags 1308 """ 1309 1310 if debug_mode is None: 1311 debug_mode = options.debug_mode 1312 1313 def mach_abi_groups(): 1314 1315 yield 'all' 1316 1317 if options.msvc_runtime is None: 1318 if debug_mode: 1319 yield 'rt-debug' 1320 else: 1321 yield 'rt' 1322 1323 for all_except in [s for s in self.mach_abi_linking.keys() if s.startswith('all!')]: 1324 exceptions = all_except[4:].split(',') 1325 if options.os not in exceptions and options.arch not in exceptions: 1326 yield all_except 1327 1328 yield options.os 1329 yield options.cpu 1330 1331 abi_link = set() 1332 for what in mach_abi_groups(): 1333 if what in self.mach_abi_linking: 1334 flag = self.mach_abi_linking.get(what) 1335 if flag is not None and flag != '' and flag not in abi_link: 1336 abi_link.add(flag) 1337 1338 if options.msvc_runtime: 1339 abi_link.add("/" + options.msvc_runtime) 1340 1341 if options.with_stack_protector and self.stack_protector_flags != '': 1342 abi_link.add(self.stack_protector_flags) 1343 1344 if options.with_coverage_info: 1345 if self.coverage_flags == '': 1346 raise UserError('No coverage handling for %s' % (self.basename)) 1347 abi_link.add(self.coverage_flags) 1348 1349 if options.with_sanitizers or options.enable_sanitizers != '': 1350 if not self.sanitizers: 1351 raise UserError('No sanitizer handling for %s' % (self.basename)) 1352 1353 default_san = self.sanitizers['default'].split(',') 1354 1355 if options.enable_sanitizers: 1356 san = options.enable_sanitizers.split(',') 1357 else: 1358 san = default_san 1359 1360 for s in san: 1361 if s not in self.sanitizers: 1362 raise UserError('No flags defined for sanitizer %s in %s' % (s, self.basename)) 1363 1364 if s == 'default': 1365 abi_link.update([self.sanitizers[x] for x in default_san]) 1366 else: 1367 abi_link.add(self.sanitizers[s]) 1368 1369 self.sanitizer_types = san 1370 1371 if options.with_openmp: 1372 if 'openmp' not in self.mach_abi_linking: 1373 raise UserError('No support for OpenMP for %s' % (self.basename)) 1374 abi_link.add(self.mach_abi_linking['openmp']) 1375 1376 abi_flags = ' '.join(sorted(abi_link)) 1377 1378 if options.cc_abi_flags != '': 1379 abi_flags += ' ' + options.cc_abi_flags 1380 1381 return abi_flags 1382 1383 def cc_warning_flags(self, options): 1384 def gen_flags(): 1385 yield self.warning_flags 1386 if options.werror_mode or options.maintainer_mode: 1387 yield self.werror_flags 1388 if options.maintainer_mode: 1389 yield self.maintainer_warning_flags 1390 1391 return (' '.join(gen_flags())).strip() 1392 1393 def cc_lang_flags(self): 1394 return self.lang_flags 1395 1396 def cc_compile_flags(self, options, with_debug_info=None, enable_optimizations=None): 1397 #pylint: disable=too-many-branches 1398 1399 def gen_flags(with_debug_info, enable_optimizations): 1400 1401 sanitizers_enabled = options.with_sanitizers or (len(options.enable_sanitizers) > 0) 1402 1403 if with_debug_info is None: 1404 with_debug_info = options.with_debug_info 1405 if enable_optimizations is None: 1406 enable_optimizations = not options.no_optimizations 1407 1408 if with_debug_info: 1409 yield self.debug_info_flags 1410 1411 if enable_optimizations: 1412 if options.optimize_for_size: 1413 if self.size_optimization_flags != '': 1414 yield self.size_optimization_flags 1415 else: 1416 logging.warning("No size optimization flags set for current compiler") 1417 yield self.optimization_flags 1418 elif sanitizers_enabled and self.sanitizer_optimization_flags != '': 1419 yield self.sanitizer_optimization_flags 1420 else: 1421 yield self.optimization_flags 1422 1423 if options.arch in self.cpu_flags: 1424 yield self.cpu_flags[options.arch] 1425 1426 if options.arch in self.cpu_flags_no_debug: 1427 1428 # Only enable these if no debug/sanitizer options enabled 1429 1430 if not (options.debug_mode or sanitizers_enabled): 1431 yield self.cpu_flags_no_debug[options.arch] 1432 1433 for flag in options.extra_cxxflags: 1434 yield flag 1435 1436 for definition in options.define_build_macro: 1437 yield self.add_compile_definition_option + definition 1438 1439 return (' '.join(gen_flags(with_debug_info, enable_optimizations))).strip() 1440 1441 @staticmethod 1442 def _so_link_search(osname, debug_info): 1443 so_link_typ = [osname, 'default'] 1444 if debug_info: 1445 so_link_typ = [l + '-debug' for l in so_link_typ] + so_link_typ 1446 return so_link_typ 1447 1448 def so_link_command_for(self, osname, options): 1449 """ 1450 Return the command needed to link a shared object 1451 """ 1452 1453 for s in self._so_link_search(osname, options.with_debug_info): 1454 if s in self.so_link_commands: 1455 return self.so_link_commands[s] 1456 1457 raise InternalError( 1458 "No shared library link command found for target '%s' in compiler settings '%s'" % 1459 (osname, self.infofile)) 1460 1461 def binary_link_command_for(self, osname, options): 1462 """ 1463 Return the command needed to link an app/test object 1464 """ 1465 1466 for s in self._so_link_search(osname, options.with_debug_info): 1467 if s in self.binary_link_commands: 1468 return self.binary_link_commands[s] 1469 1470 return '$(LINKER)' 1471 1472class OsInfo(InfoObject): # pylint: disable=too-many-instance-attributes 1473 def __init__(self, infofile): 1474 super(OsInfo, self).__init__(infofile) 1475 lex = lex_me_harder( 1476 infofile, 1477 ['aliases', 'target_features', 'feature_macros'], 1478 [], 1479 { 1480 'program_suffix': '', 1481 'obj_suffix': 'o', 1482 'soname_suffix': '', 1483 'soname_pattern_patch': '', 1484 'soname_pattern_abi': '', 1485 'soname_pattern_base': '', 1486 'static_suffix': 'a', 1487 'ar_command': 'ar', 1488 'ar_options': '', 1489 'ar_output_to': '', 1490 'install_root': '/usr/local', 1491 'header_dir': 'include', 1492 'bin_dir': 'bin', 1493 'lib_dir': 'lib', 1494 'doc_dir': 'share/doc', 1495 'man_dir': 'share/man', 1496 'use_stack_protector': 'true', 1497 'cli_exe_name': 'botan', 1498 'lib_prefix': 'lib', 1499 'library_name': 'botan{suffix}-{major}', 1500 'shared_lib_symlinks': 'yes', 1501 'default_compiler': 'gcc', 1502 'uses_pkg_config': 'yes', 1503 }) 1504 1505 if lex.ar_command == 'ar' and lex.ar_options == '': 1506 lex.ar_options = 'crs' 1507 1508 if lex.soname_pattern_base: 1509 self.soname_pattern_base = lex.soname_pattern_base 1510 if lex.soname_pattern_patch == '' and lex.soname_pattern_abi == '': 1511 self.soname_pattern_patch = lex.soname_pattern_base 1512 self.soname_pattern_abi = lex.soname_pattern_base 1513 elif lex.soname_pattern_patch != '' and lex.soname_pattern_abi != '': 1514 self.soname_pattern_patch = lex.soname_pattern_patch 1515 self.soname_pattern_abi = lex.soname_pattern_abi 1516 else: 1517 # base set, only one of patch/abi set 1518 raise InternalError("Invalid soname_patterns in %s" % (self.infofile)) 1519 else: 1520 if lex.soname_suffix: 1521 self.soname_pattern_base = "libbotan{lib_suffix}-{version_major}.%s" % (lex.soname_suffix) 1522 self.soname_pattern_abi = self.soname_pattern_base + ".{abi_rev}" 1523 self.soname_pattern_patch = self.soname_pattern_abi + ".{version_minor}.{version_patch}" 1524 else: 1525 # Could not calculate soname_pattern_* 1526 # This happens for OSs without shared library support (e.g. nacl, mingw, includeos, cygwin) 1527 self.soname_pattern_base = None 1528 self.soname_pattern_abi = None 1529 self.soname_pattern_patch = None 1530 1531 self._aliases = lex.aliases 1532 self.ar_command = lex.ar_command 1533 self.ar_options = lex.ar_options 1534 self.bin_dir = lex.bin_dir 1535 self.cli_exe_name = lex.cli_exe_name 1536 self.doc_dir = lex.doc_dir 1537 self.header_dir = lex.header_dir 1538 self.install_root = lex.install_root 1539 self.lib_dir = lex.lib_dir 1540 self.lib_prefix = lex.lib_prefix 1541 self.library_name = lex.library_name 1542 self.man_dir = lex.man_dir 1543 self.obj_suffix = lex.obj_suffix 1544 self.program_suffix = lex.program_suffix 1545 self.static_suffix = lex.static_suffix 1546 self.target_features = lex.target_features 1547 self.use_stack_protector = (lex.use_stack_protector == "true") 1548 self.shared_lib_uses_symlinks = (lex.shared_lib_symlinks == 'yes') 1549 self.default_compiler = lex.default_compiler 1550 self.uses_pkg_config = (lex.uses_pkg_config == 'yes') 1551 self.feature_macros = lex.feature_macros 1552 1553 def matches_name(self, nm): 1554 if nm in self._aliases: 1555 return True 1556 1557 for alias in self._aliases: 1558 if re.match(alias, nm): 1559 return True 1560 return False 1561 1562 def building_shared_supported(self): 1563 return self.soname_pattern_base is not None 1564 1565 def enabled_features(self, options): 1566 feats = [] 1567 for feat in self.target_features: 1568 if feat not in options.without_os_features: 1569 feats.append(feat) 1570 for feat in options.with_os_features: 1571 if feat not in self.target_features: 1572 feats.append(feat) 1573 1574 return sorted(feats) 1575 1576 def macros(self, cc): 1577 value = [cc.add_compile_definition_option + define 1578 for define in self.feature_macros] 1579 1580 return ' '.join(value) 1581 1582def fixup_proc_name(proc): 1583 proc = proc.lower().replace(' ', '') 1584 for junk in ['(tm)', '(r)']: 1585 proc = proc.replace(junk, '') 1586 return proc 1587 1588def canon_processor(archinfo, proc): 1589 proc = fixup_proc_name(proc) 1590 1591 # First, try to search for an exact match 1592 for ainfo in archinfo.values(): 1593 if ainfo.basename == proc or proc in ainfo.aliases: 1594 return ainfo.basename 1595 1596 return None 1597 1598def system_cpu_info(): 1599 1600 cpu_info = [] 1601 1602 if platform.machine() != '': 1603 cpu_info.append(platform.machine()) 1604 1605 if platform.processor() != '': 1606 cpu_info.append(platform.processor()) 1607 1608 if 'uname' in os.__dict__: 1609 cpu_info.append(os.uname()[4]) 1610 1611 return cpu_info 1612 1613def guess_processor(archinfo): 1614 for info_part in system_cpu_info(): 1615 if info_part: 1616 match = canon_processor(archinfo, info_part) 1617 if match is not None: 1618 logging.debug("Matched '%s' to processor '%s'" % (info_part, match)) 1619 return match, info_part 1620 else: 1621 logging.debug("Failed to deduce CPU from '%s'" % info_part) 1622 1623 raise UserError('Could not determine target CPU; set with --cpu') 1624 1625 1626def read_textfile(filepath): 1627 """ 1628 Read a whole file into memory as a string 1629 """ 1630 if filepath is None: 1631 return '' 1632 1633 with open(filepath) as f: 1634 return ''.join(f.readlines()) 1635 1636 1637def process_template_string(template_text, variables, template_source): 1638 # pylint: disable=too-many-branches,too-many-statements 1639 1640 """ 1641 Perform template substitution 1642 1643 The template language supports (un-nested) conditionals. 1644 """ 1645 class SimpleTemplate(object): 1646 1647 def __init__(self, vals): 1648 self.vals = vals 1649 self.value_pattern = re.compile(r'%{([a-z][a-z_0-9\|]+)}') 1650 self.cond_pattern = re.compile('%{(if|unless) ([a-z][a-z_0-9]+)}') 1651 self.for_pattern = re.compile('(.*)%{for ([a-z][a-z_0-9]+)}') 1652 self.join_pattern = re.compile('(.*)%{join ([a-z][a-z_0-9]+)}') 1653 1654 def substitute(self, template): 1655 # pylint: disable=too-many-locals 1656 def insert_value(match): 1657 v = match.group(1) 1658 if v in self.vals: 1659 return str(self.vals.get(v)) 1660 if v.endswith('|upper'): 1661 v = v.replace('|upper', '') 1662 if v in self.vals: 1663 return str(self.vals.get(v)).upper() 1664 1665 raise KeyError(v) 1666 1667 lines = template.splitlines() 1668 1669 output = "" 1670 idx = 0 1671 1672 while idx < len(lines): 1673 cond_match = self.cond_pattern.match(lines[idx]) 1674 join_match = self.join_pattern.match(lines[idx]) 1675 for_match = self.for_pattern.match(lines[idx]) 1676 1677 if cond_match: 1678 cond_type = cond_match.group(1) 1679 cond_var = cond_match.group(2) 1680 1681 include_cond = False 1682 1683 if cond_type == 'if' and cond_var in self.vals and self.vals.get(cond_var): 1684 include_cond = True 1685 elif cond_type == 'unless' and (cond_var not in self.vals or (not self.vals.get(cond_var))): 1686 include_cond = True 1687 1688 idx += 1 1689 while idx < len(lines): 1690 if lines[idx] == '%{endif}': 1691 break 1692 if include_cond: 1693 output += lines[idx] + "\n" 1694 idx += 1 1695 elif join_match: 1696 join_var = join_match.group(2) 1697 join_str = ' ' 1698 join_line = '%%{join %s}' % (join_var) 1699 output += lines[idx].replace(join_line, join_str.join(self.vals[join_var])) + "\n" 1700 elif for_match: 1701 for_prefix = for_match.group(1) 1702 output += for_prefix 1703 for_var = for_match.group(2) 1704 1705 if for_var not in self.vals: 1706 raise InternalError("Unknown for loop iteration variable '%s'" % (for_var)) 1707 1708 var = self.vals[for_var] 1709 if not isinstance(var, list): 1710 raise InternalError("For loop iteration variable '%s' is not a list" % (for_var)) 1711 idx += 1 1712 1713 for_body = "" 1714 while idx < len(lines): 1715 if lines[idx] == '%{endfor}': 1716 break 1717 for_body += lines[idx] + "\n" 1718 idx += 1 1719 1720 for v in var: 1721 if isinstance(v, dict): 1722 for_val = for_body 1723 for ik, iv in v.items(): 1724 for_val = for_val.replace('%{' + ik + '}', iv) 1725 output += for_val + "\n" 1726 else: 1727 output += for_body.replace('%{i}', v).replace('%{i|upper}', v.upper()) 1728 output += "\n" 1729 else: 1730 output += lines[idx] + "\n" 1731 idx += 1 1732 1733 return self.value_pattern.sub(insert_value, output) + '\n' 1734 1735 try: 1736 return SimpleTemplate(variables).substitute(template_text) 1737 except KeyError as e: 1738 logging.error('Unbound var %s in template %s' % (e, template_source)) 1739 except Exception as e: # pylint: disable=broad-except 1740 logging.error('Exception %s during template processing file %s' % (e, template_source)) 1741 1742def process_template(template_file, variables): 1743 return process_template_string(read_textfile(template_file), variables, template_file) 1744 1745def yield_objectfile_list(sources, obj_dir, obj_suffix, options): 1746 obj_suffix = '.' + obj_suffix 1747 1748 for src in sources: 1749 (directory, filename) = os.path.split(os.path.normpath(src)) 1750 parts = directory.split(os.sep) 1751 1752 if 'src' in parts: 1753 parts = parts[parts.index('src')+2:] 1754 elif options.amalgamation and filename.find(options.name_amalgamation) != -1: 1755 parts = [] 1756 else: 1757 raise InternalError("Unexpected file '%s/%s'" % (directory, filename)) 1758 1759 if parts != []: 1760 # Handle src/X/X.cpp -> X.o 1761 if filename == parts[-1] + '.cpp': 1762 name = '_'.join(parts) + '.cpp' 1763 else: 1764 name = '_'.join(parts) + '_' + filename 1765 1766 def fixup_obj_name(name): 1767 def remove_dups(parts): 1768 last = None 1769 for part in parts: 1770 if last is None or part != last: 1771 last = part 1772 yield part 1773 1774 return '_'.join(remove_dups(name.split('_'))) 1775 1776 name = fixup_obj_name(name) 1777 else: 1778 name = filename 1779 1780 name = name.replace('.cpp', obj_suffix) 1781 yield os.path.join(obj_dir, name) 1782 1783def generate_build_info(build_paths, modules, cc, arch, osinfo, options): 1784 # pylint: disable=too-many-locals 1785 1786 # first create a map of src_file->owning module 1787 1788 module_that_owns = {} 1789 1790 for mod in modules: 1791 for src in mod.sources(): 1792 module_that_owns[src] = mod 1793 1794 def _isa_specific_flags(src): 1795 if os.path.basename(src) == 'test_simd.cpp': 1796 return cc.get_isa_specific_flags(['simd'], arch, options) 1797 1798 if src in module_that_owns: 1799 module = module_that_owns[src] 1800 isas = module.isas_needed(arch.basename) 1801 if 'simd' in module.dependencies(osinfo): 1802 isas.append('simd') 1803 1804 return cc.get_isa_specific_flags(isas, arch, options) 1805 1806 return '' 1807 1808 def _build_info(sources, objects, target_type): 1809 output = [] 1810 for (obj_file, src) in zip(objects, sources): 1811 info = { 1812 'src': src, 1813 'obj': obj_file, 1814 'isa_flags': _isa_specific_flags(src) 1815 } 1816 1817 if target_type == 'fuzzer': 1818 fuzz_basename = os.path.basename(obj_file).replace('.' + osinfo.obj_suffix, '') 1819 info['exe'] = os.path.join(build_paths.fuzzer_output_dir, fuzz_basename) 1820 1821 output.append(info) 1822 1823 return output 1824 1825 out = {} 1826 1827 targets = ['lib', 'cli', 'test', 'fuzzer'] 1828 1829 out['isa_build_info'] = [] 1830 1831 fuzzer_bin = [] 1832 for t in targets: 1833 src_list, src_dir = build_paths.src_info(t) 1834 1835 src_key = '%s_srcs' % (t) 1836 obj_key = '%s_objs' % (t) 1837 build_key = '%s_build_info' % (t) 1838 1839 objects = [] 1840 build_info = [] 1841 1842 if src_list is not None: 1843 src_list.sort() 1844 objects = list(yield_objectfile_list(src_list, src_dir, osinfo.obj_suffix, options)) 1845 build_info = _build_info(src_list, objects, t) 1846 1847 for b in build_info: 1848 if b['isa_flags'] != '': 1849 out['isa_build_info'].append(b) 1850 1851 if t == 'fuzzer': 1852 fuzzer_bin = [b['exe'] for b in build_info] 1853 1854 out[src_key] = src_list if src_list else [] 1855 out[obj_key] = objects 1856 out[build_key] = build_info 1857 1858 out['fuzzer_bin'] = ' '.join(fuzzer_bin) 1859 out['cli_headers'] = build_paths.cli_headers 1860 1861 return out 1862 1863def create_template_vars(source_paths, build_paths, options, modules, cc, arch, osinfo): 1864 #pylint: disable=too-many-locals,too-many-branches,too-many-statements 1865 1866 """ 1867 Create the template variables needed to process the makefile, build.h, etc 1868 """ 1869 1870 def external_link_cmd(): 1871 return ' '.join([cc.add_lib_dir_option + libdir for libdir in options.with_external_libdir]) 1872 1873 def adjust_library_name(info_txt_libname): 1874 """ 1875 Apply custom library name mappings where necessary 1876 """ 1877 1878 # potentially map boost library names to the associated name provided 1879 # via ./configure.py --boost-library-name <build/platform specific name> 1880 # 1881 # We assume that info.txt contains the library name's "stem", i.e. 1882 # 'boost_system'. While the user-provided (actual) library will contain 1883 # the same stem plus a set of prefixes and/or suffixes, e.g. 1884 # libboost_system-vc140-mt-x64-1_69.lib. We use the stem for selecting 1885 # the correct user-provided library name override. 1886 if options.boost_libnames and 'boost_' in info_txt_libname: 1887 adjusted_libnames = [chosen_libname for chosen_libname in options.boost_libnames \ 1888 if info_txt_libname in chosen_libname] 1889 1890 if len(adjusted_libnames) > 1: 1891 logging.warning('Ambiguous boost library names: %s' % ', '.join(adjusted_libnames)) 1892 if len(adjusted_libnames) == 1: 1893 logging.debug('Replacing boost library name %s -> %s' % (info_txt_libname, adjusted_libnames[0])) 1894 return adjusted_libnames[0] 1895 1896 return info_txt_libname 1897 1898 def link_to(module_member_name): 1899 """ 1900 Figure out what external libraries/frameworks are needed based on selected modules 1901 """ 1902 if module_member_name not in ['libs', 'frameworks']: 1903 raise InternalError("Invalid argument") 1904 1905 libs = set() 1906 for module in modules: 1907 for (osname, module_link_to) in getattr(module, module_member_name).items(): 1908 if osname in ['all', osinfo.basename]: 1909 libs |= set(module_link_to) 1910 else: 1911 match = re.match('^all!(.*)', osname) 1912 if match is not None: 1913 exceptions = match.group(1).split(',') 1914 if osinfo.basename not in exceptions: 1915 libs |= set(module_link_to) 1916 1917 return sorted([adjust_library_name(lib) for lib in libs]) 1918 1919 def choose_mp_bits(): 1920 mp_bits = arch.wordsize # allow command line override? 1921 logging.debug('Using MP bits %d' % (mp_bits)) 1922 return mp_bits 1923 1924 def innosetup_arch(os_name, arch): 1925 if os_name == 'windows': 1926 inno_arch = {'x86_32': '', 1927 'x86_64': 'x64', 1928 'ia64': 'ia64'} 1929 if arch in inno_arch: 1930 return inno_arch[arch] 1931 else: 1932 logging.warning('Unknown arch %s in innosetup_arch' % (arch)) 1933 return None 1934 1935 def configure_command_line(): 1936 # Cut absolute path from main executable (e.g. configure.py or python interpreter) 1937 # to get the same result when configuring the same thing on different machines 1938 main_executable = os.path.basename(sys.argv[0]) 1939 return ' '.join([main_executable] + sys.argv[1:]) 1940 1941 def cmake_escape(s): 1942 return s.replace('(', '\\(').replace(')', '\\)') 1943 1944 def sysroot_option(): 1945 if options.with_sysroot_dir == '': 1946 return '' 1947 if cc.add_sysroot_option == '': 1948 logging.error("This compiler doesn't support --sysroot option") 1949 return cc.add_sysroot_option + options.with_sysroot_dir 1950 1951 def ar_command(): 1952 if options.ar_command: 1953 return options.ar_command 1954 1955 if cc.ar_command: 1956 if cc.ar_command == cc.binary_name: 1957 return options.compiler_binary or cc.binary_name 1958 else: 1959 return cc.ar_command 1960 1961 return osinfo.ar_command 1962 1963 build_dir = options.with_build_dir or os.path.curdir 1964 program_suffix = options.program_suffix or osinfo.program_suffix 1965 1966 def join_with_build_dir(path): 1967 # For some unknown reason MinGW doesn't like ./foo 1968 if build_dir == os.path.curdir and options.os == 'mingw': 1969 return path 1970 return os.path.join(build_dir, path) 1971 1972 def all_targets(options): 1973 yield 'libs' 1974 if 'cli' in options.build_targets: 1975 yield 'cli' 1976 if 'tests' in options.build_targets: 1977 yield 'tests' 1978 if options.build_fuzzers: 1979 yield 'fuzzers' 1980 if 'bogo_shim' in options.build_targets: 1981 yield 'bogo_shim' 1982 if options.with_documentation: 1983 yield 'docs' 1984 1985 def install_targets(options): 1986 yield 'libs' 1987 if 'cli' in options.build_targets: 1988 yield 'cli' 1989 if options.with_documentation: 1990 yield 'docs' 1991 1992 def absolute_install_dir(p): 1993 if os.path.isabs(p): 1994 return p 1995 return os.path.join(options.prefix or osinfo.install_root, p) 1996 1997 def choose_python_exe(): 1998 exe = sys.executable 1999 2000 if options.os == 'mingw': # mingw doesn't handle the backslashes in the absolute path well 2001 return exe.replace('\\', '/') 2002 2003 return exe 2004 2005 def choose_cxx_exe(): 2006 cxx = options.compiler_binary or cc.binary_name 2007 2008 if options.compiler_cache is None: 2009 return cxx 2010 else: 2011 return '%s %s' % (options.compiler_cache, cxx) 2012 2013 def extra_libs(libs, cc): 2014 if libs is None: 2015 return '' 2016 2017 return ' '.join([(cc.add_lib_option % lib) for lib in libs.split(',') if lib != '']) 2018 2019 variables = { 2020 'version_major': Version.major(), 2021 'version_minor': Version.minor(), 2022 'version_patch': Version.patch(), 2023 'version_suffix': Version.suffix(), 2024 'version_vc_rev': 'unknown' if options.no_store_vc_rev else Version.vc_rev(), 2025 'abi_rev': Version.so_rev(), 2026 2027 'version': Version.as_string(), 2028 'release_type': Version.release_type(), 2029 'version_datestamp': Version.datestamp(), 2030 2031 'distribution_info': options.distribution_info, 2032 2033 'macos_so_compat_ver': '%s.%s.0' % (Version.packed(), Version.so_rev()), 2034 'macos_so_current_ver': '%s.%s.%s' % (Version.packed(), Version.so_rev(), Version.patch()), 2035 2036 'all_targets': ' '.join(all_targets(options)), 2037 'install_targets': ' '.join(install_targets(options)), 2038 2039 'public_headers': sorted([os.path.basename(h) for h in build_paths.public_headers]), 2040 'internal_headers': sorted([os.path.basename(h) for h in build_paths.internal_headers]), 2041 'external_headers': sorted([os.path.basename(h) for h in build_paths.external_headers]), 2042 2043 'abs_root_dir': os.path.dirname(os.path.realpath(__file__)), 2044 2045 'base_dir': source_paths.base_dir, 2046 'src_dir': source_paths.src_dir, 2047 'test_data_dir': source_paths.test_data_dir, 2048 'doc_dir': source_paths.doc_dir, 2049 'scripts_dir': normalize_source_path(source_paths.scripts_dir), 2050 'python_dir': source_paths.python_dir, 2051 2052 'cli_exe_name': osinfo.cli_exe_name + program_suffix, 2053 'cli_exe': join_with_build_dir(osinfo.cli_exe_name + program_suffix), 2054 'build_cli_exe': bool('cli' in options.build_targets), 2055 'test_exe': join_with_build_dir('botan-test' + program_suffix), 2056 2057 'lib_prefix': osinfo.lib_prefix, 2058 'static_suffix': osinfo.static_suffix, 2059 'lib_suffix': options.library_suffix, 2060 'libname': osinfo.library_name.format(major=Version.major(), 2061 minor=Version.minor(), 2062 suffix=options.library_suffix), 2063 2064 'command_line': configure_command_line(), 2065 'local_config': read_textfile(options.local_config), 2066 2067 'program_suffix': program_suffix, 2068 2069 'prefix': options.prefix or osinfo.install_root, 2070 'bindir': absolute_install_dir(options.bindir or osinfo.bin_dir), 2071 'libdir': absolute_install_dir(options.libdir or osinfo.lib_dir), 2072 'mandir': options.mandir or osinfo.man_dir, 2073 'includedir': options.includedir or osinfo.header_dir, 2074 'docdir': options.docdir or osinfo.doc_dir, 2075 2076 'with_documentation': options.with_documentation, 2077 'with_sphinx': options.with_sphinx, 2078 'with_pdf': options.with_pdf, 2079 'with_rst2man': options.with_rst2man, 2080 'sphinx_config_dir': source_paths.sphinx_config_dir, 2081 'with_doxygen': options.with_doxygen, 2082 'maintainer_mode': options.maintainer_mode, 2083 2084 'out_dir': build_dir, 2085 'build_dir': build_paths.build_dir, 2086 2087 'doc_stamp_file': os.path.join(build_paths.build_dir, 'doc.stamp'), 2088 'makefile_path': os.path.join(build_paths.build_dir, '..', 'Makefile'), 2089 2090 'build_static_lib': options.build_static_lib, 2091 'build_shared_lib': options.build_shared_lib, 2092 2093 'build_fuzzers': options.build_fuzzers, 2094 2095 'build_coverage' : options.with_coverage_info or options.with_coverage, 2096 2097 'symlink_shared_lib': options.build_shared_lib and osinfo.shared_lib_uses_symlinks, 2098 2099 'libobj_dir': build_paths.libobj_dir, 2100 'cliobj_dir': build_paths.cliobj_dir, 2101 'testobj_dir': build_paths.testobj_dir, 2102 'fuzzobj_dir': build_paths.fuzzobj_dir, 2103 2104 'fuzzer_output_dir': build_paths.fuzzer_output_dir if build_paths.fuzzer_output_dir else '', 2105 'doc_output_dir': build_paths.doc_output_dir, 2106 'handbook_output_dir': build_paths.handbook_output_dir, 2107 'doc_output_dir_doxygen': build_paths.doc_output_dir_doxygen, 2108 2109 'compiler_include_dirs': '%s %s' % (build_paths.include_dir, build_paths.external_include_dir), 2110 2111 'os': options.os, 2112 'arch': options.arch, 2113 'compiler': options.compiler, 2114 'cpu_family': arch.family, 2115 'endian': options.with_endian, 2116 'cpu_is_64bit': arch.wordsize == 64, 2117 2118 'bakefile_arch': 'x86' if options.arch == 'x86_32' else 'x86_64', 2119 2120 'innosetup_arch': innosetup_arch(options.os, options.arch), 2121 2122 'mp_bits': choose_mp_bits(), 2123 2124 'python_exe': choose_python_exe(), 2125 'python_version': options.python_version, 2126 'install_python_module': not options.no_install_python_module, 2127 2128 'cxx': choose_cxx_exe(), 2129 'cxx_abi_flags': cc.mach_abi_link_flags(options), 2130 'linker': cc.linker_name or '$(CXX)', 2131 'make_supports_phony': osinfo.basename != 'windows', 2132 2133 'sanitizer_types' : sorted(cc.sanitizer_types), 2134 2135 'cc_compile_opt_flags': cc.cc_compile_flags(options, False, True), 2136 'cc_compile_debug_flags': cc.cc_compile_flags(options, True, False), 2137 2138 # These are for CMake 2139 'cxx_abi_opt_flags': cc.mach_abi_link_flags(options, False), 2140 'cxx_abi_debug_flags': cc.mach_abi_link_flags(options, True), 2141 2142 'dash_o': cc.output_to_object, 2143 'dash_c': cc.compile_flags, 2144 2145 'cc_lang_flags': cc.cc_lang_flags(), 2146 'os_feature_macros': osinfo.macros(cc), 2147 'cc_sysroot': sysroot_option(), 2148 'cc_compile_flags': options.cxxflags or cc.cc_compile_flags(options), 2149 'ldflags': options.ldflags or '', 2150 'extra_libs': extra_libs(options.extra_libs, cc), 2151 'cc_warning_flags': cc.cc_warning_flags(options), 2152 'output_to_exe': cc.output_to_exe, 2153 'cc_macro': cc.macro_name, 2154 2155 'visibility_attribute': cc.gen_visibility_attribute(options), 2156 2157 'lib_link_cmd': cc.so_link_command_for(osinfo.basename, options), 2158 'exe_link_cmd': cc.binary_link_command_for(osinfo.basename, options), 2159 'external_link_cmd': external_link_cmd(), 2160 2161 'ar_command': ar_command(), 2162 'ar_options': options.ar_options or cc.ar_options or osinfo.ar_options, 2163 'ar_output_to': cc.ar_output_to, 2164 2165 'link_to': ' '.join( 2166 [(cc.add_lib_option % lib) for lib in link_to('libs')] + 2167 [cc.add_framework_option + fw for fw in link_to('frameworks')] 2168 ), 2169 2170 'cmake_link_to': ' '.join( 2171 link_to('libs') + 2172 [('"' + cc.add_framework_option + fw + '"') for fw in link_to('frameworks')] 2173 ), 2174 2175 'fuzzer_lib': (cc.add_lib_option % options.fuzzer_lib) if options.fuzzer_lib else '', 2176 'libs_used': [lib.replace('.lib', '') for lib in link_to('libs')], 2177 2178 'include_paths': build_paths.format_include_paths(cc, options.with_external_includedir), 2179 'module_defines': sorted(flatten([m.defines() for m in modules])), 2180 2181 'build_bogo_shim': bool('bogo_shim' in options.build_targets), 2182 'bogo_shim_src': os.path.join(source_paths.src_dir, 'bogo_shim', 'bogo_shim.cpp'), 2183 2184 'os_features': osinfo.enabled_features(options), 2185 'os_name': osinfo.basename, 2186 'cpu_features': arch.supported_isa_extensions(cc, options), 2187 'system_cert_bundle': options.system_cert_bundle, 2188 2189 'fuzzer_mode': options.unsafe_fuzzer_mode, 2190 'fuzzer_type': options.build_fuzzers.upper() if options.build_fuzzers else '', 2191 2192 'with_valgrind': options.with_valgrind, 2193 'with_openmp': options.with_openmp, 2194 'with_debug_asserts': options.with_debug_asserts, 2195 'test_mode': options.test_mode, 2196 'optimize_for_size': options.optimize_for_size, 2197 2198 'mod_list': sorted([m.basename for m in modules]) 2199 } 2200 2201 variables['installed_include_dir'] = os.path.join( 2202 variables['prefix'], 2203 variables['includedir'], 2204 'botan-%d' % (Version.major()), 'botan') 2205 2206 if cc.basename == 'msvc' and variables['cxx_abi_flags'] != '': 2207 # MSVC linker doesn't support/need the ABI options, 2208 # just transfer them over to just the compiler invocations 2209 variables['cc_compile_flags'] = '%s %s' % (variables['cxx_abi_flags'], variables['cc_compile_flags']) 2210 variables['cxx_abi_flags'] = '' 2211 2212 variables['lib_flags'] = cc.gen_lib_flags(options, variables) 2213 variables['cmake_lib_flags'] = cmake_escape(variables['lib_flags']) 2214 2215 if options.with_pkg_config: 2216 variables['botan_pkgconfig'] = os.path.join(build_paths.build_dir, 'botan-%d.pc' % (Version.major())) 2217 2218 # The name is always set because Windows build needs it 2219 variables['static_lib_name'] = '%s%s.%s' % (variables['lib_prefix'], variables['libname'], 2220 variables['static_suffix']) 2221 2222 if options.build_shared_lib: 2223 if osinfo.soname_pattern_base is not None: 2224 variables['soname_base'] = osinfo.soname_pattern_base.format(**variables) 2225 variables['shared_lib_name'] = variables['soname_base'] 2226 2227 if osinfo.soname_pattern_abi is not None: 2228 variables['soname_abi'] = osinfo.soname_pattern_abi.format(**variables) 2229 variables['shared_lib_name'] = variables['soname_abi'] 2230 2231 if osinfo.soname_pattern_patch is not None: 2232 variables['soname_patch'] = osinfo.soname_pattern_patch.format(**variables) 2233 2234 variables['lib_link_cmd'] = variables['lib_link_cmd'].format(**variables) 2235 2236 lib_targets = [] 2237 if options.build_static_lib: 2238 lib_targets.append('static_lib_name') 2239 if options.build_shared_lib: 2240 lib_targets.append('shared_lib_name') 2241 2242 variables['library_targets'] = ' '.join([join_with_build_dir(variables[t]) for t in lib_targets]) 2243 2244 if options.os == 'llvm' or options.compiler == 'msvc': 2245 # llvm-link and msvc require just naming the file directly 2246 variables['link_to_botan'] = os.path.join(build_dir, variables['static_lib_name']) 2247 else: 2248 variables['link_to_botan'] = '%s%s %s' % (cc.add_lib_dir_option, build_dir, 2249 (cc.add_lib_option % variables['libname'])) 2250 2251 return variables 2252 2253class ModulesChooser(object): 2254 """ 2255 Determine which modules to load based on options, target, etc 2256 """ 2257 2258 def __init__(self, modules, module_policy, archinfo, osinfo, ccinfo, cc_min_version, options): 2259 self._modules = modules 2260 self._module_policy = module_policy 2261 self._archinfo = archinfo 2262 self._osinfo = osinfo 2263 self._ccinfo = ccinfo 2264 self._cc_min_version = cc_min_version 2265 self._options = options 2266 2267 self._maybe_dep = set() 2268 self._to_load = set() 2269 # string to set mapping with reasons as key and modules as value 2270 self._not_using_because = collections.defaultdict(set) 2271 2272 ModulesChooser._validate_dependencies_exist(self._modules) 2273 ModulesChooser._validate_user_selection( 2274 self._modules, self._options.enabled_modules, self._options.disabled_modules) 2275 2276 def _check_usable(self, module, modname): 2277 if not module.compatible_cpu(self._archinfo, self._options): 2278 self._not_using_because['incompatible CPU'].add(modname) 2279 return False 2280 elif not module.compatible_os(self._osinfo, self._options): 2281 self._not_using_because['incompatible OS'].add(modname) 2282 return False 2283 elif not module.compatible_compiler(self._ccinfo, self._cc_min_version, self._archinfo.basename): 2284 self._not_using_because['incompatible compiler'].add(modname) 2285 return False 2286 return True 2287 2288 @staticmethod 2289 def _display_module_information_unused(skipped_modules): 2290 for reason in sorted(skipped_modules.keys()): 2291 disabled_mods = sorted(skipped_modules[reason]) 2292 if disabled_mods: 2293 logging.info('Skipping (%s): %s' % (reason, ' '.join(disabled_mods))) 2294 2295 @staticmethod 2296 def _display_module_information_to_load(all_modules, modules_to_load): 2297 sorted_modules_to_load = sorted(modules_to_load) 2298 2299 for modname in sorted_modules_to_load: 2300 if all_modules[modname].comment: 2301 logging.info('%s: %s' % (modname, all_modules[modname].comment)) 2302 if all_modules[modname].warning: 2303 logging.warning('%s: %s' % (modname, all_modules[modname].warning)) 2304 if all_modules[modname].load_on == 'vendor': 2305 logging.info('Enabling use of external dependency %s' % modname) 2306 2307 if sorted_modules_to_load: 2308 logging.info('Loading modules: %s', ' '.join(sorted_modules_to_load)) 2309 else: 2310 logging.error('This configuration disables every submodule and is invalid') 2311 2312 @staticmethod 2313 def _validate_state(used_modules, unused_modules): 2314 for reason, unused_for_reason in unused_modules.items(): 2315 intersection = unused_for_reason & used_modules 2316 if intersection: 2317 raise InternalError( 2318 "Disabled modules (%s) and modules to load have common elements: %s" 2319 % (reason, intersection)) 2320 2321 @staticmethod 2322 def _validate_dependencies_exist(modules): 2323 for module in modules.values(): 2324 module.dependencies_exist(modules) 2325 2326 @staticmethod 2327 def _validate_user_selection(modules, enabled_modules, disabled_modules): 2328 for modname in enabled_modules: 2329 if modname not in modules: 2330 logging.error("Module not found: %s" % modname) 2331 2332 for modname in disabled_modules: 2333 if modname not in modules: 2334 logging.warning("Disabled module not found: %s" % modname) 2335 2336 def _handle_by_module_policy(self, modname, usable): 2337 if self._module_policy is not None: 2338 if modname in self._module_policy.required: 2339 if not usable: 2340 logging.error('Module policy requires module %s not usable on this platform' % (modname)) 2341 elif modname in self._options.disabled_modules: 2342 logging.error('Module %s was disabled but is required by policy' % (modname)) 2343 self._to_load.add(modname) 2344 return True 2345 elif modname in self._module_policy.if_available: 2346 if modname in self._options.disabled_modules: 2347 self._not_using_because['disabled by user'].add(modname) 2348 elif usable: 2349 logging.debug('Enabling optional module %s' % (modname)) 2350 self._to_load.add(modname) 2351 return True 2352 elif modname in self._module_policy.prohibited: 2353 if modname in self._options.enabled_modules: 2354 logging.error('Module %s was requested but is prohibited by policy' % (modname)) 2355 self._not_using_because['prohibited by module policy'].add(modname) 2356 return True 2357 2358 return False 2359 2360 @staticmethod 2361 def resolve_dependencies(available_modules, dependency_table, module, loaded_modules=None): 2362 """ 2363 Parameters 2364 - available_modules: modules to choose from. Constant. 2365 - dependency_table: module to dependencies map. Constant. 2366 - module: name of the module to resolve dependencies. Constant. 2367 - loaded_modules: modules already loaded. Defensive copy in order to not change value for caller. 2368 """ 2369 if loaded_modules is None: 2370 loaded_modules = set([]) 2371 else: 2372 loaded_modules = copy.copy(loaded_modules) 2373 2374 if module not in available_modules: 2375 return False, None 2376 2377 loaded_modules.add(module) 2378 for dependency in dependency_table[module]: 2379 dependency_choices = set(dependency.split('|')) 2380 2381 dependency_met = False 2382 2383 if not set(dependency_choices).isdisjoint(loaded_modules): 2384 dependency_met = True 2385 else: 2386 possible_mods = dependency_choices.intersection(available_modules) 2387 2388 for mod in possible_mods: 2389 ok, dependency_modules = ModulesChooser.resolve_dependencies( 2390 available_modules, dependency_table, mod, loaded_modules) 2391 if ok: 2392 dependency_met = True 2393 loaded_modules.add(mod) 2394 loaded_modules.update(dependency_modules) 2395 break 2396 2397 if not dependency_met: 2398 return False, None 2399 2400 return True, loaded_modules 2401 2402 def _modules_dependency_table(self): 2403 out = {} 2404 for modname in self._modules: 2405 out[modname] = self._modules[modname].dependencies(self._osinfo) 2406 return out 2407 2408 def _resolve_dependencies_for_all_modules(self): 2409 available_modules = set(self._to_load) | set(self._maybe_dep) 2410 dependency_table = self._modules_dependency_table() 2411 2412 successfully_loaded = set() 2413 2414 for modname in self._to_load: 2415 # This will try to recursively load all dependencies of modname 2416 ok, modules = self.resolve_dependencies(available_modules, dependency_table, modname) 2417 if ok: 2418 successfully_loaded.add(modname) 2419 successfully_loaded.update(modules) 2420 else: 2421 # Skip this module 2422 pass 2423 2424 self._not_using_because['dependency failure'].update(self._to_load - successfully_loaded) 2425 self._to_load = successfully_loaded 2426 self._maybe_dep -= successfully_loaded 2427 2428 def _handle_by_load_on(self, module): # pylint: disable=too-many-branches 2429 modname = module.basename 2430 if module.load_on == 'never': 2431 self._not_using_because['disabled as buggy'].add(modname) 2432 elif module.load_on == 'request': 2433 if self._options.with_everything: 2434 self._to_load.add(modname) 2435 else: 2436 self._not_using_because['by request only'].add(modname) 2437 elif module.load_on == 'vendor': 2438 if self._options.with_everything: 2439 self._to_load.add(modname) 2440 else: 2441 self._not_using_because['requires external dependency'].add(modname) 2442 elif module.load_on == 'dep': 2443 self._maybe_dep.add(modname) 2444 2445 elif module.load_on == 'always': 2446 self._to_load.add(modname) 2447 2448 elif module.load_on == 'auto': 2449 if self._options.no_autoload or self._module_policy is not None: 2450 self._maybe_dep.add(modname) 2451 else: 2452 self._to_load.add(modname) 2453 else: 2454 logging.error('Unknown load_on %s in %s' % ( 2455 module.load_on, modname)) 2456 2457 def choose(self): 2458 for (modname, module) in self._modules.items(): 2459 usable = self._check_usable(module, modname) 2460 2461 module_handled = self._handle_by_module_policy(modname, usable) 2462 if module_handled: 2463 continue 2464 2465 if modname in self._options.disabled_modules: 2466 self._not_using_because['disabled by user'].add(modname) 2467 elif usable: 2468 if modname in self._options.enabled_modules: 2469 self._to_load.add(modname) # trust the user 2470 else: 2471 self._handle_by_load_on(module) 2472 2473 if 'compression' in self._to_load: 2474 # Confirm that we have at least one compression library enabled 2475 # Otherwise we leave a lot of useless support code compiled in, plus a 2476 # make_compressor call that always fails 2477 if 'zlib' not in self._to_load and 'bzip2' not in self._to_load and 'lzma' not in self._to_load: 2478 self._to_load.remove('compression') 2479 self._not_using_because['no enabled compression schemes'].add('compression') 2480 2481 self._resolve_dependencies_for_all_modules() 2482 2483 for not_a_dep in self._maybe_dep: 2484 self._not_using_because['not requested'].add(not_a_dep) 2485 2486 ModulesChooser._validate_state(self._to_load, self._not_using_because) 2487 ModulesChooser._display_module_information_unused(self._not_using_because) 2488 ModulesChooser._display_module_information_to_load(self._modules, self._to_load) 2489 2490 return self._to_load 2491 2492def choose_link_method(options): 2493 """ 2494 Choose the link method based on system availability and user request 2495 """ 2496 2497 req = options.link_method 2498 2499 def useable_methods(): 2500 2501 # Symbolic link support on Windows was introduced in Windows 6.0 (Vista) 2502 # and Python 3.2. Furthermore, the SeCreateSymbolicLinkPrivilege is 2503 # required in order to successfully create symlinks. So only try to use 2504 # symlinks on Windows if explicitly requested. 2505 2506 # MinGW declares itself as 'Windows' 2507 host_is_windows = python_platform_identifier() in ['windows', 'cygwin'] 2508 2509 if 'symlink' in os.__dict__: 2510 if host_is_windows: 2511 if req == 'symlink': 2512 yield 'symlink' 2513 else: 2514 yield 'symlink' 2515 2516 if 'link' in os.__dict__: 2517 yield 'hardlink' 2518 2519 yield 'copy' 2520 2521 for method in useable_methods(): 2522 if req is None or req == method: 2523 logging.info('Using %s to link files into build dir ' \ 2524 '(use --link-method to change)' % (method)) 2525 return method 2526 2527 logging.warning('Could not use link method "%s", will copy instead' % (req)) 2528 return 'copy' 2529 2530def portable_symlink(file_path, target_dir, method): 2531 """ 2532 Copy or link the file, depending on what the platform offers 2533 """ 2534 2535 if not os.access(file_path, os.R_OK): 2536 logging.warning('Missing file %s' % (file_path)) 2537 return 2538 2539 if method == 'symlink': 2540 rel_file_path = os.path.relpath(file_path, start=target_dir) 2541 os.symlink(rel_file_path, os.path.join(target_dir, os.path.basename(file_path))) 2542 elif method == 'hardlink': 2543 os.link(file_path, os.path.join(target_dir, os.path.basename(file_path))) 2544 elif method == 'copy': 2545 shutil.copy(file_path, target_dir) 2546 else: 2547 raise UserError('Unknown link method %s' % (method)) 2548 2549 2550class AmalgamationHelper(object): 2551 # All include types may have trailing comment like e.g. '#include <vector> // IWYU pragma: export' 2552 _any_include = re.compile(r'#include <(.*)>') 2553 _botan_include = re.compile(r'#include <botan/(.*)>') 2554 2555 # Only matches at the beginning of the line. By convention, this means that the include 2556 # is not wrapped by condition macros 2557 _unconditional_any_include = re.compile(r'^#include <(.*)>') 2558 # stddef.h is included in ffi.h 2559 _unconditional_std_include = re.compile(r'^#include <([^/\.]+|stddef.h)>') 2560 2561 @staticmethod 2562 def is_any_include(cpp_source_line): 2563 match = AmalgamationHelper._any_include.search(cpp_source_line) 2564 if match: 2565 return match.group(1) 2566 else: 2567 return None 2568 2569 @staticmethod 2570 def is_botan_include(cpp_source_line): 2571 match = AmalgamationHelper._botan_include.search(cpp_source_line) 2572 if match: 2573 return match.group(1) 2574 else: 2575 return None 2576 2577 @staticmethod 2578 def is_unconditional_any_include(cpp_source_line): 2579 match = AmalgamationHelper._unconditional_any_include.search(cpp_source_line) 2580 if match: 2581 return match.group(1) 2582 else: 2583 return None 2584 2585 @staticmethod 2586 def is_unconditional_std_include(cpp_source_line): 2587 match = AmalgamationHelper._unconditional_std_include.search(cpp_source_line) 2588 if match: 2589 return match.group(1) 2590 else: 2591 return None 2592 2593 @staticmethod 2594 def write_banner(fd): 2595 fd.write("""/* 2596* Botan %s Amalgamation 2597* (C) 1999-2020 The Botan Authors 2598* 2599* Botan is released under the Simplified BSD License (see license.txt) 2600*/ 2601""" % (Version.as_string())) 2602 2603 2604class AmalgamationHeader(object): 2605 def __init__(self, input_filepaths): 2606 2607 self.included_already = set() 2608 self.all_std_includes = set() 2609 2610 self.file_contents = {} 2611 for filepath in sorted(input_filepaths): 2612 try: 2613 contents = AmalgamationGenerator.read_header(filepath) 2614 self.file_contents[os.path.basename(filepath)] = contents 2615 except IOError as e: 2616 logging.error('Error processing file %s for amalgamation: %s' % (filepath, e)) 2617 2618 self.contents = '' 2619 for name in sorted(self.file_contents): 2620 self.contents += ''.join(list(self.header_contents(name))) 2621 2622 self.header_includes = '' 2623 for std_header in sorted(self.all_std_includes): 2624 self.header_includes += '#include <%s>\n' % (std_header) 2625 self.header_includes += '\n' 2626 2627 def header_contents(self, name): 2628 name = name.replace('internal/', '') 2629 2630 if name in self.included_already: 2631 return 2632 2633 self.included_already.add(name) 2634 2635 if name not in self.file_contents: 2636 return 2637 2638 depr_marker = 'BOTAN_DEPRECATED_HEADER(%s)\n' % (name) 2639 if depr_marker in self.file_contents[name]: 2640 logging.debug("Ignoring deprecated header %s", name) 2641 return 2642 2643 for line in self.file_contents[name]: 2644 header = AmalgamationHelper.is_botan_include(line) 2645 if header: 2646 for c in self.header_contents(header): 2647 yield c 2648 else: 2649 std_header = AmalgamationHelper.is_unconditional_std_include(line) 2650 2651 if std_header: 2652 self.all_std_includes.add(std_header) 2653 else: 2654 yield line 2655 2656 def write_to_file(self, filepath, include_guard): 2657 with open(filepath, 'w') as f: 2658 AmalgamationHelper.write_banner(f) 2659 f.write("\n#ifndef %s\n#define %s\n\n" % (include_guard, include_guard)) 2660 f.write(self.header_includes) 2661 f.write(self.contents) 2662 f.write("\n#endif // %s\n" % (include_guard)) 2663 2664 2665class AmalgamationGenerator(object): 2666 _header_guard_pattern = re.compile(r'^#define BOTAN_.*_H_\s*$') 2667 _header_endif_pattern = re.compile(r'^#endif.*$') 2668 2669 @staticmethod 2670 def read_header(filepath): 2671 encoding_kwords = {} 2672 if sys.version_info[0] == 3: 2673 encoding_kwords['encoding'] = 'utf8' 2674 with open(filepath, **encoding_kwords) as f: 2675 raw_content = f.readlines() 2676 return AmalgamationGenerator.strip_header_goop(filepath, raw_content) 2677 2678 @staticmethod 2679 def strip_header_goop(header_name, header_lines): 2680 lines = copy.copy(header_lines) # defensive copy 2681 2682 start_header_guard_index = None 2683 for index, line in enumerate(lines): 2684 if AmalgamationGenerator._header_guard_pattern.match(line): 2685 start_header_guard_index = index 2686 break 2687 if start_header_guard_index is None: 2688 raise InternalError("No header guard start found in " + header_name) 2689 2690 end_header_guard_index = None 2691 for index, line in enumerate(lines): 2692 if AmalgamationGenerator._header_endif_pattern.match(line): 2693 end_header_guard_index = index # override with last found 2694 if end_header_guard_index is None: 2695 raise InternalError("No header guard end found in " + header_name) 2696 2697 lines = lines[start_header_guard_index+1 : end_header_guard_index] 2698 2699 # Strip leading and trailing empty lines 2700 while lines[0].strip() == "": 2701 lines = lines[1:] 2702 while lines[-1].strip() == "": 2703 lines = lines[0:-1] 2704 2705 return lines 2706 2707 def __init__(self, prefix, build_paths, modules, options): 2708 self._filename_prefix = prefix 2709 self._build_paths = build_paths 2710 self._modules = modules 2711 self._options = options 2712 2713 def generate(self): 2714 encoding_kwords = {} 2715 if sys.version_info[0] == 3: 2716 encoding_kwords['encoding'] = 'utf8' 2717 2718 pub_header_amalag = AmalgamationHeader(self._build_paths.public_headers) 2719 amalgamation_header_fsname = '%s.h' % (self._filename_prefix) 2720 logging.info('Writing amalgamation header to %s' % (amalgamation_header_fsname)) 2721 pub_header_amalag.write_to_file(amalgamation_header_fsname, "BOTAN_AMALGAMATION_H_") 2722 2723 internal_headers_list = [] 2724 2725 for hdr in self._build_paths.internal_headers: 2726 internal_headers_list.append(hdr) 2727 2728 # file descriptors for all `amalgamation_sources` 2729 amalgamation_fsname = '%s.cpp' % (self._filename_prefix) 2730 logging.info('Writing amalgamation source to %s' % (amalgamation_fsname)) 2731 2732 amalgamation_file = open(amalgamation_fsname, 'w', **encoding_kwords) 2733 2734 AmalgamationHelper.write_banner(amalgamation_file) 2735 amalgamation_file.write('\n#include "%s"\n\n' % (amalgamation_header_fsname)) 2736 2737 internal_headers = AmalgamationHeader(internal_headers_list) 2738 amalgamation_file.write(internal_headers.header_includes) 2739 amalgamation_file.write(internal_headers.contents) 2740 2741 unconditional_headers = set([]) 2742 2743 for mod in sorted(self._modules, key=lambda module: module.basename): 2744 for src in sorted(mod.source): 2745 with open(src, 'r', **encoding_kwords) as f: 2746 for line in f: 2747 if AmalgamationHelper.is_botan_include(line): 2748 # Botan headers are inlined in amalgamation headers 2749 continue 2750 2751 if AmalgamationHelper.is_any_include(line) in unconditional_headers: 2752 # This include (conditional or unconditional) was unconditionally added before 2753 continue 2754 2755 amalgamation_file.write(line) 2756 unconditional_header = AmalgamationHelper.is_unconditional_any_include(line) 2757 if unconditional_header: 2758 unconditional_headers.add(unconditional_header) 2759 2760 amalgamation_file.close() 2761 2762 return ([amalgamation_fsname], [amalgamation_header_fsname]) 2763 2764 2765def have_program(program): 2766 """ 2767 Test for the existence of a program 2768 """ 2769 2770 def exe_test(path, program): 2771 exe_file = os.path.join(path, program) 2772 2773 if os.path.exists(exe_file) and os.access(exe_file, os.X_OK): 2774 logging.debug('Found program %s in %s' % (program, path)) 2775 return True 2776 else: 2777 return False 2778 2779 exe_suffixes = ['', '.exe'] 2780 2781 for path in os.environ['PATH'].split(os.pathsep): 2782 for suffix in exe_suffixes: 2783 if exe_test(path, program + suffix): 2784 return True 2785 2786 logging.debug('Program %s not found' % (program)) 2787 return False 2788 2789 2790class BotanConfigureLogHandler(logging.StreamHandler, object): 2791 def emit(self, record): 2792 # Do the default stuff first 2793 super(BotanConfigureLogHandler, self).emit(record) 2794 # Exit script if and ERROR or worse occurred 2795 if record.levelno >= logging.ERROR: 2796 sys.exit(1) 2797 2798 2799def setup_logging(options): 2800 if options.verbose: 2801 log_level = logging.DEBUG 2802 elif options.quiet: 2803 log_level = logging.WARNING 2804 else: 2805 log_level = logging.INFO 2806 2807 lh = BotanConfigureLogHandler(sys.stdout) 2808 lh.setFormatter(logging.Formatter('%(levelname) 7s: %(message)s')) 2809 logging.getLogger().addHandler(lh) 2810 logging.getLogger().setLevel(log_level) 2811 2812 2813def load_info_files(search_dir, descr, filename_matcher, class_t): 2814 info = {} 2815 2816 def filename_matches(filename): 2817 if isinstance(filename_matcher, str): 2818 return filename == filename_matcher 2819 else: 2820 return filename_matcher.match(filename) is not None 2821 2822 for (dirpath, _, filenames) in os.walk(search_dir): 2823 for filename in filenames: 2824 filepath = os.path.join(dirpath, filename) 2825 if filename_matches(filename): 2826 info_obj = class_t(filepath) 2827 info[info_obj.basename] = info_obj 2828 2829 if info: 2830 infotxt_basenames = ' '.join(sorted(info.keys())) 2831 logging.debug('Loaded %d %s files: %s' % (len(info), descr, infotxt_basenames)) 2832 else: 2833 logging.warning('Failed to load any %s files' % (descr)) 2834 2835 return info 2836 2837 2838def load_build_data_info_files(source_paths, descr, subdir, class_t): 2839 matcher = re.compile(r'[_a-z0-9]+\.txt$') 2840 return load_info_files(os.path.join(source_paths.build_data_dir, subdir), descr, matcher, class_t) 2841 2842 2843# Workaround for Windows systems where antivirus is enabled GH #353 2844def robust_rmtree(path, max_retries=5): 2845 for _ in range(max_retries): 2846 try: 2847 shutil.rmtree(path) 2848 return 2849 except OSError: 2850 time.sleep(0.1) 2851 2852 # Final attempt, pass any exceptions up to caller. 2853 shutil.rmtree(path) 2854 2855 2856# Workaround for Windows systems where antivirus is enabled GH #353 2857def robust_makedirs(directory, max_retries=5): 2858 for _ in range(max_retries): 2859 try: 2860 os.makedirs(directory) 2861 return 2862 except OSError as e: 2863 if e.errno == errno.EEXIST: 2864 raise 2865 2866 time.sleep(0.1) 2867 2868 # Final attempt, pass any exceptions up to caller. 2869 os.makedirs(directory) 2870 2871def python_platform_identifier(): 2872 system_from_python = platform.system().lower() 2873 if re.match('^cygwin_.*', system_from_python): 2874 return 'cygwin' 2875 else: 2876 return system_from_python 2877 2878# This is for otions that have --with-XYZ and --without-XYZ. If user does not 2879# set any of those, we choose a default here. 2880# Mutates `options` 2881def set_defaults_for_unset_options(options, info_arch, info_cc, info_os): # pylint: disable=too-many-branches 2882 if options.os is None: 2883 options.os = python_platform_identifier() 2884 logging.info('Guessing target OS is %s (use --os to set)' % (options.os)) 2885 2886 if options.os not in info_os: 2887 def find_canonical_os_name(os_name_variant): 2888 for (canonical_os_name, os_info) in info_os.items(): 2889 if os_info.matches_name(os_name_variant): 2890 return canonical_os_name 2891 return os_name_variant # not found 2892 options.os = find_canonical_os_name(options.os) 2893 2894 def deduce_compiler_type_from_cc_bin(cc_bin): 2895 if cc_bin.find('clang') != -1 or cc_bin in ['emcc', 'em++']: 2896 return 'clang' 2897 if cc_bin.find('-g++') != -1 or cc_bin.find('g++') != -1: 2898 return 'gcc' 2899 return None 2900 2901 if options.compiler is None and options.compiler_binary is not None: 2902 options.compiler = deduce_compiler_type_from_cc_bin(options.compiler_binary) 2903 2904 if options.compiler is None: 2905 logging.error("Could not figure out what compiler type '%s' is, use --cc to set" % ( 2906 options.compiler_binary)) 2907 2908 if options.compiler is None and options.os in info_os: 2909 options.compiler = info_os[options.os].default_compiler 2910 2911 if not have_program(info_cc[options.compiler].binary_name): 2912 logging.error("Default compiler for system is %s but could not find binary '%s'; use --cc to set" % ( 2913 options.compiler, info_cc[options.compiler].binary_name)) 2914 2915 logging.info('Guessing to use compiler %s (use --cc or CXX to set)' % (options.compiler)) 2916 2917 if options.cpu is None: 2918 (arch, cpu) = guess_processor(info_arch) 2919 options.arch = arch 2920 options.cpu = cpu 2921 logging.info('Guessing target processor is a %s (use --cpu to set)' % (options.arch)) 2922 2923 # OpenBSD uses an old binutils that does not support AVX2 2924 if options.os == 'openbsd': 2925 del info_cc['gcc'].isa_flags['avx2'] 2926 2927 if options.with_documentation is True: 2928 if options.with_sphinx is None and have_program('sphinx-build'): 2929 logging.info('Found sphinx-build (use --without-sphinx to disable)') 2930 options.with_sphinx = True 2931 if options.with_rst2man is None and have_program('rst2man'): 2932 logging.info('Found rst2man (use --without-rst2man to disable)') 2933 options.with_rst2man = True 2934 2935 if options.with_pkg_config is None and options.os in info_os: 2936 options.with_pkg_config = info_os[options.os].uses_pkg_config 2937 2938 if options.system_cert_bundle is None: 2939 default_paths = [ 2940 '/etc/ssl/certs/ca-certificates.crt', # Ubuntu, Debian, Arch, Gentoo 2941 '/etc/pki/tls/certs/ca-bundle.crt', # RHEL 2942 '/etc/ssl/ca-bundle.pem', # SuSE 2943 '/etc/ssl/cert.pem', # OpenBSD, FreeBSD, Alpine 2944 '/etc/certs/ca-certificates.crt', # Solaris 2945 ] 2946 2947 for path in default_paths: 2948 if os.access(path, os.R_OK): 2949 logging.info('Using %s as system certificate store', path) 2950 options.system_cert_bundle = path 2951 break 2952 else: 2953 if not os.access(options.system_cert_bundle, os.R_OK): 2954 logging.warning('Provided system cert bundle path %s not found, ignoring', options.system_cert_bundle) 2955 options.system_cert_bundle = None 2956 2957# Mutates `options` 2958def canonicalize_options(options, info_os, info_arch): 2959 # pylint: disable=too-many-branches 2960 2961 # canonical ARCH/CPU 2962 options.arch = canon_processor(info_arch, options.cpu) 2963 if options.arch is None: 2964 raise UserError('Unknown or unidentifiable processor "%s"' % (options.cpu)) 2965 2966 if options.cpu != options.arch: 2967 logging.info('Canonicalized CPU target %s to %s', options.cpu, options.arch) 2968 2969 # select and sanity check build targets 2970 def canonicalize_build_targets(options): 2971 # --build-targets was not provided: build default targets 2972 if options.build_targets is None: 2973 return ["cli", "tests"] 2974 2975 # flatten the list of multiple --build-targets="" and comma separation 2976 build_targets = [t.strip().lower() for ts in options.build_targets for t in ts.split(",")] 2977 2978 # validate that all requested build targets are available 2979 for build_target in build_targets: 2980 if build_target not in ACCEPTABLE_BUILD_TARGETS: 2981 raise UserError("unknown build target: %s" % build_target) 2982 2983 # building the shared lib desired and without contradiction? 2984 if options.build_shared_lib is None: 2985 options.build_shared_lib = "shared" in build_targets 2986 elif bool(options.build_shared_lib) != bool("shared" in build_targets): 2987 raise UserError("inconsistent usage of --enable/disable-shared-library and --build-targets") 2988 2989 # building the static lib desired and without contradiction? 2990 if options.build_static_lib is None: 2991 options.build_static_lib = "static" in build_targets 2992 elif bool(options.build_static_lib) != bool("static" in build_targets): 2993 raise UserError("inconsistent usage of --enable/disable-static-library and --build-targets") 2994 2995 return build_targets 2996 2997 options.build_targets = canonicalize_build_targets(options) 2998 2999 shared_libs_supported = options.os in info_os and info_os[options.os].building_shared_supported() 3000 3001 if not shared_libs_supported: 3002 if options.build_shared_lib is True: 3003 logging.warning('Shared libs not supported on %s, disabling shared lib support' % (options.os)) 3004 options.build_shared_lib = False 3005 elif options.build_shared_lib is None: 3006 logging.info('Shared libs not supported on %s, disabling shared lib support' % (options.os)) 3007 3008 if options.os == 'windows' and options.build_shared_lib is None and options.build_static_lib is None: 3009 options.build_shared_lib = True 3010 3011 if options.with_stack_protector is None: 3012 if options.os in info_os: 3013 options.with_stack_protector = info_os[options.os].use_stack_protector 3014 3015 if options.build_shared_lib is None: 3016 if options.os == 'windows' and options.build_static_lib: 3017 pass 3018 else: 3019 options.build_shared_lib = shared_libs_supported 3020 3021 if options.build_static_lib is None: 3022 if options.os == 'windows' and options.build_shared_lib: 3023 pass 3024 else: 3025 options.build_static_lib = True 3026 3027 # Set default fuzzing lib 3028 if options.build_fuzzers == 'libfuzzer' and options.fuzzer_lib is None: 3029 options.fuzzer_lib = 'Fuzzer' 3030 3031 if options.ldflags is not None: 3032 extra_libs = [] 3033 link_to_lib = re.compile('^-l(.*)') 3034 for flag in options.ldflags.split(' '): 3035 match = link_to_lib.match(flag) 3036 if match: 3037 extra_libs.append(match.group(1)) 3038 3039 options.extra_libs += ','.join(extra_libs) 3040 3041# Checks user options for consistency 3042# This method DOES NOT change options on behalf of the user but explains 3043# why the given configuration does not work. 3044def validate_options(options, info_os, info_cc, available_module_policies): 3045 # pylint: disable=too-many-branches,too-many-statements 3046 3047 if options.name_amalgamation != 'botan_all': 3048 if options.name_amalgamation == '': 3049 raise UserError('Amalgamation basename must be non-empty') 3050 3051 acceptable_name_re = re.compile('^[a-zA-Z0-9_]+$') 3052 if acceptable_name_re.match(options.name_amalgamation) is None: 3053 raise UserError("Amalgamation basename must match [a-zA-Z0-9_]+") 3054 3055 if options.os == "java": 3056 raise UserError("Jython detected: need --os and --cpu to set target") 3057 3058 if options.os not in info_os: 3059 raise UserError('Unknown OS "%s"; available options: %s' % ( 3060 options.os, ' '.join(sorted(info_os.keys())))) 3061 3062 if options.compiler not in info_cc: 3063 raise UserError('Unknown compiler "%s"; available options: %s' % ( 3064 options.compiler, ' '.join(sorted(info_cc.keys())))) 3065 3066 if options.cc_min_version is not None and not re.match(r'^[0-9]+\.[0-9]+$', options.cc_min_version): 3067 raise UserError("--cc-min-version must have the format MAJOR.MINOR") 3068 3069 if options.module_policy and options.module_policy not in available_module_policies: 3070 raise UserError("Unknown module set %s" % options.module_policy) 3071 3072 if options.cpu == 'llvm' or options.os in ['llvm', 'emscripten']: 3073 if options.compiler != 'clang': 3074 raise UserError('LLVM target requires using Clang') 3075 3076 if options.cpu != 'llvm': 3077 raise UserError('LLVM target requires CPU target set to LLVM bitcode (llvm)') 3078 3079 if options.os not in ['llvm', 'emscripten']: 3080 raise UserError('Target OS is not an LLVM bitcode target') 3081 3082 if options.build_fuzzers is not None: 3083 if options.build_fuzzers not in ['libfuzzer', 'afl', 'klee', 'test']: 3084 raise UserError('Bad value to --build-fuzzers') 3085 3086 if options.build_fuzzers == 'klee' and options.os != 'llvm': 3087 raise UserError('Building for KLEE requires targeting LLVM') 3088 3089 if options.build_static_lib is False and options.build_shared_lib is False: 3090 raise UserError('With both --disable-static-library and --disable-shared-library, nothing to do') 3091 3092 if options.os == 'windows' and options.build_static_lib is True and options.build_shared_lib is True: 3093 raise UserError('On Windows only one of static lib and DLL can be selected') 3094 3095 if options.with_documentation is False: 3096 if options.with_doxygen: 3097 raise UserError('Using --with-doxygen plus --without-documentation makes no sense') 3098 if options.with_sphinx: 3099 raise UserError('Using --with-sphinx plus --without-documentation makes no sense') 3100 if options.with_pdf: 3101 raise UserError('Using --with-pdf plus --without-documentation makes no sense') 3102 3103 if options.with_pdf and not options.with_sphinx: 3104 raise UserError('Option --with-pdf requires --with-sphinx') 3105 3106 if options.with_bakefile: 3107 if options.os != 'windows' or options.compiler != 'msvc' or options.build_shared_lib is False: 3108 raise UserError("Building via bakefile is only supported for MSVC DLL build") 3109 3110 if options.arch not in ['x86_64', 'x86_32']: 3111 raise UserError("Bakefile only supports x86 targets") 3112 3113 # Warnings 3114 if options.os == 'windows' and options.compiler != 'msvc': 3115 logging.warning('The windows target is oriented towards MSVC; maybe you want --os=cygwin or --os=mingw') 3116 3117 if options.msvc_runtime: 3118 if options.compiler != 'msvc': 3119 raise UserError("Makes no sense to specify MSVC runtime for %s" % (options.compiler)) 3120 3121 if options.msvc_runtime not in ['MT', 'MD', 'MTd', 'MDd']: 3122 logging.warning("MSVC runtime option '%s' not known", (options.msvc_runtime)) 3123 3124def run_compiler_preproc(options, ccinfo, source_file, default_return, extra_flags=None): 3125 if extra_flags is None: 3126 extra_flags = [] 3127 3128 cc_bin = options.compiler_binary or ccinfo.binary_name 3129 3130 cmd = cc_bin.split(' ') + ccinfo.preproc_flags.split(' ') + extra_flags + [source_file] 3131 3132 try: 3133 logging.debug("Running '%s'", ' '.join(cmd)) 3134 stdout, _ = subprocess.Popen( 3135 cmd, 3136 stdout=subprocess.PIPE, 3137 stderr=subprocess.PIPE, 3138 universal_newlines=True).communicate() 3139 cc_output = stdout 3140 except OSError as e: 3141 logging.warning('Could not execute %s: %s' % (cmd, e)) 3142 return default_return 3143 3144 def cleanup_output(output): 3145 return ('\n'.join([l for l in output.splitlines() if l.startswith('#') is False])).strip() 3146 3147 return cleanup_output(cc_output) 3148 3149def calculate_cc_min_version(options, ccinfo, source_paths): 3150 version_patterns = { 3151 'msvc': r'^ *MSVC ([0-9]{2})([0-9]{2})$', 3152 'gcc': r'^ *GCC ([0-9]+) ([0-9]+)$', 3153 'clang': r'^ *CLANG ([0-9]+) ([0-9]+)$', 3154 'xlc': r'^ *XLC ([0-9]+) ([0-9]+)$', 3155 } 3156 3157 unknown_pattern = r'UNKNOWN 0 0' 3158 3159 if ccinfo.basename not in version_patterns: 3160 logging.info("No compiler version detection available for %s" % (ccinfo.basename)) 3161 return "0.0" 3162 3163 detect_version_source = os.path.join(source_paths.build_data_dir, "detect_version.cpp") 3164 3165 cc_output = run_compiler_preproc(options, ccinfo, detect_version_source, "0.0") 3166 3167 if re.search(unknown_pattern, cc_output) is not None: 3168 logging.warning('Failed to get version for %s from macro check' % (ccinfo.basename)) 3169 return "0.0" 3170 3171 match = re.search(version_patterns[ccinfo.basename], cc_output, flags=re.MULTILINE) 3172 if match is None: 3173 logging.warning("Tried to get %s version, but output '%s' does not match expected version format" % ( 3174 ccinfo.basename, cc_output)) 3175 return "0.0" 3176 3177 major_version = int(match.group(1), 0) 3178 minor_version = int(match.group(2), 0) 3179 cc_version = "%d.%d" % (major_version, minor_version) 3180 logging.info('Auto-detected compiler version %s' % (cc_version)) 3181 3182 return cc_version 3183 3184def check_compiler_arch(options, ccinfo, archinfo, source_paths): 3185 detect_version_source = os.path.join(source_paths.build_data_dir, 'detect_arch.cpp') 3186 3187 abi_flags = ccinfo.mach_abi_link_flags(options).split(' ') 3188 cc_output = run_compiler_preproc(options, ccinfo, detect_version_source, 'UNKNOWN', abi_flags).lower() 3189 3190 if cc_output == '': 3191 cc_output = run_compiler_preproc(options, ccinfo, detect_version_source, 'UNKNOWN').lower() 3192 3193 if cc_output == 'unknown': 3194 logging.warning('Unable to detect target architecture via compiler macro checks') 3195 return None 3196 3197 if cc_output not in archinfo: 3198 # Should not happen 3199 logging.warning("Error detecting compiler target arch: '%s'", cc_output) 3200 return None 3201 3202 logging.info('Auto-detected compiler arch %s' % (cc_output)) 3203 return cc_output 3204 3205def do_io_for_build(cc, arch, osinfo, using_mods, build_paths, source_paths, template_vars, options): 3206 # pylint: disable=too-many-locals,too-many-branches,too-many-statements 3207 3208 try: 3209 robust_rmtree(build_paths.build_dir) 3210 except OSError as e: 3211 if e.errno != errno.ENOENT: 3212 logging.error('Problem while removing build dir: %s' % (e)) 3213 3214 for build_dir in build_paths.build_dirs(): 3215 try: 3216 robust_makedirs(build_dir) 3217 except OSError as e: 3218 if e.errno != errno.EEXIST: 3219 logging.error('Error while creating "%s": %s' % (build_dir, e)) 3220 3221 def write_template(sink, template): 3222 with open(sink, 'w') as f: 3223 f.write(process_template(template, template_vars)) 3224 3225 def in_build_dir(p): 3226 return os.path.join(build_paths.build_dir, p) 3227 def in_build_data(p): 3228 return os.path.join(source_paths.build_data_dir, p) 3229 3230 write_template(in_build_dir('build.h'), in_build_data('buildh.in')) 3231 write_template(in_build_dir('botan.doxy'), in_build_data('botan.doxy.in')) 3232 3233 if 'botan_pkgconfig' in template_vars: 3234 write_template(template_vars['botan_pkgconfig'], in_build_data('botan.pc.in')) 3235 3236 if options.os == 'windows': 3237 write_template(in_build_dir('botan.iss'), in_build_data('innosetup.in')) 3238 3239 link_method = choose_link_method(options) 3240 3241 def link_headers(headers, visibility, directory): 3242 logging.debug('Linking %d %s header files in %s' % (len(headers), visibility, directory)) 3243 3244 for header_file in headers: 3245 try: 3246 portable_symlink(header_file, directory, link_method) 3247 except OSError as e: 3248 if e.errno != errno.EEXIST: 3249 raise UserError('Error linking %s into %s: %s' % (header_file, directory, e)) 3250 3251 link_headers(build_paths.public_headers, 'public', 3252 build_paths.botan_include_dir) 3253 3254 link_headers(build_paths.internal_headers, 'internal', 3255 build_paths.internal_include_dir) 3256 3257 link_headers(build_paths.external_headers, 'external', 3258 build_paths.external_include_dir) 3259 3260 if options.amalgamation: 3261 (amalg_cpp_files, amalg_headers) = AmalgamationGenerator( 3262 options.name_amalgamation, build_paths, using_mods, options).generate() 3263 build_paths.lib_sources = amalg_cpp_files 3264 template_vars['generated_files'] = ' '.join(amalg_cpp_files + amalg_headers) 3265 3266 # Inserting an amalgamation generated using DLL visibility flags into a 3267 # binary project will either cause errors (on Windows) or unnecessary overhead. 3268 # Provide a hint 3269 if options.build_shared_lib: 3270 logging.warning('Unless you are building a DLL or .so from the amalgamation, use --disable-shared as well') 3271 3272 template_vars.update(generate_build_info(build_paths, using_mods, cc, arch, osinfo, options)) 3273 3274 with open(os.path.join(build_paths.build_dir, 'build_config.json'), 'w') as f: 3275 json.dump(template_vars, f, sort_keys=True, indent=2) 3276 3277 if options.with_cmake: 3278 logging.warning("CMake build is only for development: use make for production builds") 3279 cmake_template = os.path.join(source_paths.build_data_dir, 'cmake.in') 3280 write_template('CMakeLists.txt', cmake_template) 3281 elif options.with_bakefile: 3282 logging.warning("Bakefile build is only for development: use make for production builds") 3283 bakefile_template = os.path.join(source_paths.build_data_dir, 'bakefile.in') 3284 write_template('botan.bkl', bakefile_template) 3285 else: 3286 makefile_template = os.path.join(source_paths.build_data_dir, 'makefile.in') 3287 write_template(template_vars['makefile_path'], makefile_template) 3288 3289 if options.with_rst2man: 3290 rst2man_file = os.path.join(build_paths.build_dir, 'botan.rst') 3291 cli_doc = os.path.join(source_paths.doc_dir, 'cli.rst') 3292 3293 cli_doc_contents = open(cli_doc).readlines() 3294 3295 while cli_doc_contents[0] != "\n": 3296 cli_doc_contents.pop(0) 3297 3298 rst2man_header = """ 3299botan 3300============================= 3301 3302:Subtitle: Botan command line util 3303:Manual section: 1 3304 3305 """.strip() 3306 3307 with open(rst2man_file, 'w') as f: 3308 f.write(rst2man_header) 3309 f.write("\n") 3310 for line in cli_doc_contents: 3311 f.write(line) 3312 3313 logging.info('Botan %s (revision %s) (%s %s) build setup is complete' % ( 3314 Version.as_string(), 3315 Version.vc_rev(), 3316 Version.release_type(), 3317 ('dated %d' % (Version.datestamp())) if Version.datestamp() != 0 else 'undated')) 3318 3319 if options.unsafe_fuzzer_mode: 3320 logging.warning("The fuzzer mode flag is labeled unsafe for a reason, this version is for testing only") 3321 3322def list_os_features(all_os_features, info_os): 3323 for feat in all_os_features: 3324 os_with_feat = [o for o in info_os.keys() if feat in info_os[o].target_features] 3325 os_without_feat = [o for o in info_os.keys() if feat not in info_os[o].target_features] 3326 3327 if len(os_with_feat) < len(os_without_feat): 3328 print("%s: %s" % (feat, ' '.join(sorted(os_with_feat)))) 3329 else: 3330 print("%s: %s" % (feat, '!' + ' !'.join(sorted(os_without_feat)))) 3331 return 0 3332 3333 3334def main(argv): 3335 """ 3336 Main driver 3337 """ 3338 3339 # pylint: disable=too-many-locals,too-many-statements 3340 3341 options = process_command_line(argv[1:]) 3342 3343 setup_logging(options) 3344 3345 source_paths = SourcePaths(os.path.dirname(argv[0])) 3346 3347 info_modules = load_info_files(source_paths.lib_dir, 'Modules', "info.txt", ModuleInfo) 3348 3349 if options.list_modules: 3350 for mod in sorted(info_modules.keys()): 3351 print(mod) 3352 return 0 3353 3354 info_arch = load_build_data_info_files(source_paths, 'CPU info', 'arch', ArchInfo) 3355 info_os = load_build_data_info_files(source_paths, 'OS info', 'os', OsInfo) 3356 info_cc = load_build_data_info_files(source_paths, 'compiler info', 'cc', CompilerInfo) 3357 info_module_policies = load_build_data_info_files(source_paths, 'module policy', 'policy', ModulePolicyInfo) 3358 3359 all_os_features = sorted(set(flatten([o.target_features for o in info_os.values()]))) 3360 all_defined_isas = set(flatten([a.isa_extensions for a in info_arch.values()])) 3361 3362 if options.list_os_features: 3363 return list_os_features(all_os_features, info_os) 3364 3365 for mod in info_modules.values(): 3366 mod.cross_check(info_arch, info_cc, all_os_features, all_defined_isas) 3367 3368 for cc in info_cc.values(): 3369 cc.cross_check(info_os, info_arch, all_defined_isas) 3370 3371 for policy in info_module_policies.values(): 3372 policy.cross_check(info_modules) 3373 3374 logging.info('%s invoked with options "%s"', argv[0], ' '.join(argv[1:])) 3375 logging.info('Configuring to build Botan %s (revision %s)' % ( 3376 Version.as_string(), Version.vc_rev())) 3377 logging.info('Running under %s', sys.version.replace('\n', '')) 3378 3379 take_options_from_env(options) 3380 3381 logging.info('Autodetected platform information: OS="%s" machine="%s" proc="%s"', 3382 platform.system(), platform.machine(), platform.processor()) 3383 3384 logging.debug('Known CPU names: ' + ' '.join( 3385 sorted(flatten([[ainfo.basename] + ainfo.aliases for ainfo in info_arch.values()])))) 3386 3387 set_defaults_for_unset_options(options, info_arch, info_cc, info_os) 3388 canonicalize_options(options, info_os, info_arch) 3389 validate_options(options, info_os, info_cc, info_module_policies) 3390 3391 cc = info_cc[options.compiler] 3392 arch = info_arch[options.arch] 3393 osinfo = info_os[options.os] 3394 module_policy = info_module_policies[options.module_policy] if options.module_policy else None 3395 3396 if options.enable_cc_tests: 3397 cc_min_version = options.cc_min_version or calculate_cc_min_version(options, cc, source_paths) 3398 cc_arch = check_compiler_arch(options, cc, info_arch, source_paths) 3399 3400 if options.arch != 'generic': 3401 if cc_arch is not None and cc_arch != options.arch: 3402 logging.error("Configured target is %s but compiler probe indicates %s", options.arch, cc_arch) 3403 else: 3404 cc_min_version = options.cc_min_version or "0.0" 3405 3406 logging.info('Target is %s:%s-%s-%s' % ( 3407 options.compiler, cc_min_version, options.os, options.arch)) 3408 3409 def choose_endian(arch_info, options): 3410 if options.with_endian is not None: 3411 return options.with_endian 3412 3413 if options.cpu.endswith('eb') or options.cpu.endswith('be'): 3414 return 'big' 3415 elif options.cpu.endswith('el') or options.cpu.endswith('le'): 3416 return 'little' 3417 3418 if arch_info.endian: 3419 logging.info('Assuming target %s is %s endian', arch_info.basename, arch_info.endian) 3420 return arch_info.endian 3421 3422 options.with_endian = choose_endian(arch, options) 3423 3424 chooser = ModulesChooser(info_modules, module_policy, arch, osinfo, cc, cc_min_version, options) 3425 loaded_module_names = chooser.choose() 3426 using_mods = [info_modules[modname] for modname in loaded_module_names] 3427 3428 build_paths = BuildPaths(source_paths, options, using_mods) 3429 build_paths.public_headers.append(os.path.join(build_paths.build_dir, 'build.h')) 3430 3431 template_vars = create_template_vars(source_paths, build_paths, options, using_mods, cc, arch, osinfo) 3432 3433 # Now we start writing to disk 3434 do_io_for_build(cc, arch, osinfo, using_mods, build_paths, source_paths, template_vars, options) 3435 3436 return 0 3437 3438if __name__ == '__main__': 3439 try: 3440 sys.exit(main(argv=sys.argv)) 3441 except UserError as e: 3442 logging.debug(traceback.format_exc()) 3443 logging.error(e) 3444 except Exception as e: # pylint: disable=broad-except 3445 # error() will stop script, so wrap all information into one call 3446 logging.error("""%s 3447An internal error occurred. 3448 3449Don't panic, this is probably not your fault! Please open an issue 3450with the entire output at https://github.com/randombit/botan 3451 3452You'll meet friendly people happy to help!""" % traceback.format_exc()) 3453 3454 sys.exit(0) 3455