1# Copyright (c) 2012 Google Inc. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import errno 6import filecmp 7import os.path 8import re 9import tempfile 10import sys 11import subprocess 12 13try: 14 from collections.abc import MutableSet 15except ImportError: 16 from collections import MutableSet 17 18PY3 = bytes != str 19 20 21# A minimal memoizing decorator. It'll blow up if the args aren't immutable, 22# among other "problems". 23class memoize(object): 24 def __init__(self, func): 25 self.func = func 26 self.cache = {} 27 def __call__(self, *args): 28 try: 29 return self.cache[args] 30 except KeyError: 31 result = self.func(*args) 32 self.cache[args] = result 33 return result 34 35 36class GypError(Exception): 37 """Error class representing an error, which is to be presented 38 to the user. The main entry point will catch and display this. 39 """ 40 pass 41 42 43def ExceptionAppend(e, msg): 44 """Append a message to the given exception's message.""" 45 if not e.args: 46 e.args = (msg,) 47 elif len(e.args) == 1: 48 e.args = (str(e.args[0]) + ' ' + msg,) 49 else: 50 e.args = (str(e.args[0]) + ' ' + msg,) + e.args[1:] 51 52 53def FindQualifiedTargets(target, qualified_list): 54 """ 55 Given a list of qualified targets, return the qualified targets for the 56 specified |target|. 57 """ 58 return [t for t in qualified_list if ParseQualifiedTarget(t)[1] == target] 59 60 61def ParseQualifiedTarget(target): 62 # Splits a qualified target into a build file, target name and toolset. 63 64 # NOTE: rsplit is used to disambiguate the Windows drive letter separator. 65 target_split = target.rsplit(':', 1) 66 if len(target_split) == 2: 67 [build_file, target] = target_split 68 else: 69 build_file = None 70 71 target_split = target.rsplit('#', 1) 72 if len(target_split) == 2: 73 [target, toolset] = target_split 74 else: 75 toolset = None 76 77 return [build_file, target, toolset] 78 79 80def ResolveTarget(build_file, target, toolset): 81 # This function resolves a target into a canonical form: 82 # - a fully defined build file, either absolute or relative to the current 83 # directory 84 # - a target name 85 # - a toolset 86 # 87 # build_file is the file relative to which 'target' is defined. 88 # target is the qualified target. 89 # toolset is the default toolset for that target. 90 [parsed_build_file, target, parsed_toolset] = ParseQualifiedTarget(target) 91 92 if parsed_build_file: 93 if build_file: 94 # If a relative path, parsed_build_file is relative to the directory 95 # containing build_file. If build_file is not in the current directory, 96 # parsed_build_file is not a usable path as-is. Resolve it by 97 # interpreting it as relative to build_file. If parsed_build_file is 98 # absolute, it is usable as a path regardless of the current directory, 99 # and os.path.join will return it as-is. 100 build_file = os.path.normpath(os.path.join(os.path.dirname(build_file), 101 parsed_build_file)) 102 # Further (to handle cases like ../cwd), make it relative to cwd) 103 if not os.path.isabs(build_file): 104 build_file = RelativePath(build_file, '.') 105 else: 106 build_file = parsed_build_file 107 108 if parsed_toolset: 109 toolset = parsed_toolset 110 111 return [build_file, target, toolset] 112 113 114def BuildFile(fully_qualified_target): 115 # Extracts the build file from the fully qualified target. 116 return ParseQualifiedTarget(fully_qualified_target)[0] 117 118 119def GetEnvironFallback(var_list, default): 120 """Look up a key in the environment, with fallback to secondary keys 121 and finally falling back to a default value.""" 122 for var in var_list: 123 if var in os.environ: 124 return os.environ[var] 125 return default 126 127 128def QualifiedTarget(build_file, target, toolset): 129 # "Qualified" means the file that a target was defined in and the target 130 # name, separated by a colon, suffixed by a # and the toolset name: 131 # /path/to/file.gyp:target_name#toolset 132 fully_qualified = build_file + ':' + target 133 if toolset: 134 fully_qualified = fully_qualified + '#' + toolset 135 return fully_qualified 136 137 138@memoize 139def RelativePath(path, relative_to, follow_path_symlink=True): 140 # Assuming both |path| and |relative_to| are relative to the current 141 # directory, returns a relative path that identifies path relative to 142 # relative_to. 143 # If |follow_symlink_path| is true (default) and |path| is a symlink, then 144 # this method returns a path to the real file represented by |path|. If it is 145 # false, this method returns a path to the symlink. If |path| is not a 146 # symlink, this option has no effect. 147 148 # Convert to normalized (and therefore absolute paths). 149 if follow_path_symlink: 150 path = os.path.realpath(path) 151 else: 152 path = os.path.abspath(path) 153 relative_to = os.path.realpath(relative_to) 154 155 # On Windows, we can't create a relative path to a different drive, so just 156 # use the absolute path. 157 if sys.platform == 'win32': 158 if (os.path.splitdrive(path)[0].lower() != 159 os.path.splitdrive(relative_to)[0].lower()): 160 return path 161 162 # Split the paths into components. 163 path_split = path.split(os.path.sep) 164 relative_to_split = relative_to.split(os.path.sep) 165 166 # Determine how much of the prefix the two paths share. 167 prefix_len = len(os.path.commonprefix([path_split, relative_to_split])) 168 169 # Put enough ".." components to back up out of relative_to to the common 170 # prefix, and then append the part of path_split after the common prefix. 171 relative_split = [os.path.pardir] * (len(relative_to_split) - prefix_len) + \ 172 path_split[prefix_len:] 173 174 if len(relative_split) == 0: 175 # The paths were the same. 176 return '' 177 178 # Turn it back into a string and we're done. 179 return os.path.join(*relative_split) 180 181 182@memoize 183def InvertRelativePath(path, toplevel_dir=None): 184 """Given a path like foo/bar that is relative to toplevel_dir, return 185 the inverse relative path back to the toplevel_dir. 186 187 E.g. os.path.normpath(os.path.join(path, InvertRelativePath(path))) 188 should always produce the empty string, unless the path contains symlinks. 189 """ 190 if not path: 191 return path 192 toplevel_dir = '.' if toplevel_dir is None else toplevel_dir 193 return RelativePath(toplevel_dir, os.path.join(toplevel_dir, path)) 194 195 196def FixIfRelativePath(path, relative_to): 197 # Like RelativePath but returns |path| unchanged if it is absolute. 198 if os.path.isabs(path): 199 return path 200 return RelativePath(path, relative_to) 201 202 203def UnrelativePath(path, relative_to): 204 # Assuming that |relative_to| is relative to the current directory, and |path| 205 # is a path relative to the dirname of |relative_to|, returns a path that 206 # identifies |path| relative to the current directory. 207 rel_dir = os.path.dirname(relative_to) 208 return os.path.normpath(os.path.join(rel_dir, path)) 209 210 211# re objects used by EncodePOSIXShellArgument. See IEEE 1003.1 XCU.2.2 at 212# http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_02 213# and the documentation for various shells. 214 215# _quote is a pattern that should match any argument that needs to be quoted 216# with double-quotes by EncodePOSIXShellArgument. It matches the following 217# characters appearing anywhere in an argument: 218# \t, \n, space parameter separators 219# # comments 220# $ expansions (quoted to always expand within one argument) 221# % called out by IEEE 1003.1 XCU.2.2 222# & job control 223# ' quoting 224# (, ) subshell execution 225# *, ?, [ pathname expansion 226# ; command delimiter 227# <, >, | redirection 228# = assignment 229# {, } brace expansion (bash) 230# ~ tilde expansion 231# It also matches the empty string, because "" (or '') is the only way to 232# represent an empty string literal argument to a POSIX shell. 233# 234# This does not match the characters in _escape, because those need to be 235# backslash-escaped regardless of whether they appear in a double-quoted 236# string. 237_quote = re.compile('[\t\n #$%&\'()*;<=>?[{|}~]|^$') 238 239# _escape is a pattern that should match any character that needs to be 240# escaped with a backslash, whether or not the argument matched the _quote 241# pattern. _escape is used with re.sub to backslash anything in _escape's 242# first match group, hence the (parentheses) in the regular expression. 243# 244# _escape matches the following characters appearing anywhere in an argument: 245# " to prevent POSIX shells from interpreting this character for quoting 246# \ to prevent POSIX shells from interpreting this character for escaping 247# ` to prevent POSIX shells from interpreting this character for command 248# substitution 249# Missing from this list is $, because the desired behavior of 250# EncodePOSIXShellArgument is to permit parameter (variable) expansion. 251# 252# Also missing from this list is !, which bash will interpret as the history 253# expansion character when history is enabled. bash does not enable history 254# by default in non-interactive shells, so this is not thought to be a problem. 255# ! was omitted from this list because bash interprets "\!" as a literal string 256# including the backslash character (avoiding history expansion but retaining 257# the backslash), which would not be correct for argument encoding. Handling 258# this case properly would also be problematic because bash allows the history 259# character to be changed with the histchars shell variable. Fortunately, 260# as history is not enabled in non-interactive shells and 261# EncodePOSIXShellArgument is only expected to encode for non-interactive 262# shells, there is no room for error here by ignoring !. 263_escape = re.compile(r'(["\\`])') 264 265def EncodePOSIXShellArgument(argument): 266 """Encodes |argument| suitably for consumption by POSIX shells. 267 268 argument may be quoted and escaped as necessary to ensure that POSIX shells 269 treat the returned value as a literal representing the argument passed to 270 this function. Parameter (variable) expansions beginning with $ are allowed 271 to remain intact without escaping the $, to allow the argument to contain 272 references to variables to be expanded by the shell. 273 """ 274 275 if not isinstance(argument, str): 276 argument = str(argument) 277 278 if _quote.search(argument): 279 quote = '"' 280 else: 281 quote = '' 282 283 encoded = quote + re.sub(_escape, r'\\\1', argument) + quote 284 285 return encoded 286 287 288def EncodePOSIXShellList(list): 289 """Encodes |list| suitably for consumption by POSIX shells. 290 291 Returns EncodePOSIXShellArgument for each item in list, and joins them 292 together using the space character as an argument separator. 293 """ 294 295 encoded_arguments = [] 296 for argument in list: 297 encoded_arguments.append(EncodePOSIXShellArgument(argument)) 298 return ' '.join(encoded_arguments) 299 300 301def DeepDependencyTargets(target_dicts, roots): 302 """Returns the recursive list of target dependencies.""" 303 dependencies = set() 304 pending = set(roots) 305 while pending: 306 # Pluck out one. 307 r = pending.pop() 308 # Skip if visited already. 309 if r in dependencies: 310 continue 311 # Add it. 312 dependencies.add(r) 313 # Add its children. 314 spec = target_dicts[r] 315 pending.update(set(spec.get('dependencies', []))) 316 pending.update(set(spec.get('dependencies_original', []))) 317 return list(dependencies - set(roots)) 318 319 320def BuildFileTargets(target_list, build_file): 321 """From a target_list, returns the subset from the specified build_file. 322 """ 323 return [p for p in target_list if BuildFile(p) == build_file] 324 325 326def AllTargets(target_list, target_dicts, build_file): 327 """Returns all targets (direct and dependencies) for the specified build_file. 328 """ 329 bftargets = BuildFileTargets(target_list, build_file) 330 deptargets = DeepDependencyTargets(target_dicts, bftargets) 331 return bftargets + deptargets 332 333 334def WriteOnDiff(filename): 335 """Write to a file only if the new contents differ. 336 337 Arguments: 338 filename: name of the file to potentially write to. 339 Returns: 340 A file like object which will write to temporary file and only overwrite 341 the target if it differs (on close). 342 """ 343 344 class Writer(object): 345 """Wrapper around file which only covers the target if it differs.""" 346 def __init__(self): 347 # On Cygwin remove the "dir" argument because `C:` prefixed paths are treated as relative, 348 # consequently ending up with current dir "/cygdrive/c/..." being prefixed to those, which was 349 # obviously a non-existent path, for example: "/cygdrive/c/<some folder>/C:\<my win style abs path>". 350 # See https://docs.python.org/2/library/tempfile.html#tempfile.mkstemp for more details 351 base_temp_dir = "" if IsCygwin() else os.path.dirname(filename) 352 # Pick temporary file. 353 tmp_fd, self.tmp_path = tempfile.mkstemp( 354 suffix='.tmp', 355 prefix=os.path.split(filename)[1] + '.gyp.', 356 dir=base_temp_dir) 357 try: 358 self.tmp_file = os.fdopen(tmp_fd, 'wb') 359 except Exception: 360 # Don't leave turds behind. 361 os.unlink(self.tmp_path) 362 raise 363 364 def __getattr__(self, attrname): 365 # Delegate everything else to self.tmp_file 366 return getattr(self.tmp_file, attrname) 367 368 def close(self): 369 try: 370 # Close tmp file. 371 self.tmp_file.close() 372 # Determine if different. 373 same = False 374 try: 375 same = filecmp.cmp(self.tmp_path, filename, False) 376 except OSError as e: 377 if e.errno != errno.ENOENT: 378 raise 379 380 if same: 381 # The new file is identical to the old one, just get rid of the new 382 # one. 383 os.unlink(self.tmp_path) 384 else: 385 # The new file is different from the old one, or there is no old one. 386 # Rename the new file to the permanent name. 387 # 388 # tempfile.mkstemp uses an overly restrictive mode, resulting in a 389 # file that can only be read by the owner, regardless of the umask. 390 # There's no reason to not respect the umask here, which means that 391 # an extra hoop is required to fetch it and reset the new file's mode. 392 # 393 # No way to get the umask without setting a new one? Set a safe one 394 # and then set it back to the old value. 395 umask = os.umask(0o77) 396 os.umask(umask) 397 os.chmod(self.tmp_path, 0o666 & ~umask) 398 if sys.platform == 'win32' and os.path.exists(filename): 399 # NOTE: on windows (but not cygwin) rename will not replace an 400 # existing file, so it must be preceded with a remove. Sadly there 401 # is no way to make the switch atomic. 402 os.remove(filename) 403 os.rename(self.tmp_path, filename) 404 except Exception: 405 # Don't leave turds behind. 406 os.unlink(self.tmp_path) 407 raise 408 409 def write(self, s): 410 self.tmp_file.write(s.encode('utf-8')) 411 412 return Writer() 413 414 415def EnsureDirExists(path): 416 """Make sure the directory for |path| exists.""" 417 try: 418 os.makedirs(os.path.dirname(path)) 419 except OSError: 420 pass 421 422 423def GetFlavor(params): 424 """Returns |params.flavor| if it's set, the system's default flavor else.""" 425 flavors = { 426 'cygwin': 'win', 427 'win32': 'win', 428 'darwin': 'mac', 429 } 430 431 if 'flavor' in params: 432 return params['flavor'] 433 if sys.platform in flavors: 434 return flavors[sys.platform] 435 if sys.platform.startswith('sunos'): 436 return 'solaris' 437 if sys.platform.startswith('freebsd'): 438 return 'freebsd' 439 if sys.platform.startswith('openbsd'): 440 return 'openbsd' 441 if sys.platform.startswith('netbsd'): 442 return 'netbsd' 443 if sys.platform.startswith('aix'): 444 return 'aix' 445 if sys.platform.startswith('zos'): 446 return 'zos' 447 if sys.platform.startswith('os390'): 448 return 'zos' 449 450 return 'linux' 451 452 453def CopyTool(flavor, out_path): 454 """Finds (flock|mac|win)_tool.gyp in the gyp directory and copies it 455 to |out_path|.""" 456 # aix and solaris just need flock emulation. mac and win use more complicated 457 # support scripts. 458 prefix = { 459 'aix': 'flock', 460 'solaris': 'flock', 461 'mac': 'mac', 462 'win': 'win' 463 }.get(flavor, None) 464 if not prefix: 465 return 466 467 # Slurp input file. 468 source_path = os.path.join( 469 os.path.dirname(os.path.abspath(__file__)), '%s_tool.py' % prefix) 470 with open(source_path) as source_file: 471 source = source_file.readlines() 472 473 # Add header and write it out. 474 tool_path = os.path.join(out_path, 'gyp-%s-tool' % prefix) 475 with open(tool_path, 'w') as tool_file: 476 tool_file.write( 477 ''.join([source[0], '# Generated by gyp. Do not edit.\n'] + source[1:])) 478 479 # Make file executable. 480 os.chmod(tool_path, 0o755) 481 482 483# From Alex Martelli, 484# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52560 485# ASPN: Python Cookbook: Remove duplicates from a sequence 486# First comment, dated 2001/10/13. 487# (Also in the printed Python Cookbook.) 488 489def uniquer(seq, idfun=None): 490 if idfun is None: 491 idfun = lambda x: x 492 seen = {} 493 result = [] 494 for item in seq: 495 marker = idfun(item) 496 if marker in seen: continue 497 seen[marker] = 1 498 result.append(item) 499 return result 500 501 502# Based on http://code.activestate.com/recipes/576694/. 503class OrderedSet(MutableSet): 504 def __init__(self, iterable=None): 505 self.end = end = [] 506 end += [None, end, end] # sentinel node for doubly linked list 507 self.map = {} # key --> [key, prev, next] 508 if iterable is not None: 509 self |= iterable 510 511 def __len__(self): 512 return len(self.map) 513 514 def __contains__(self, key): 515 return key in self.map 516 517 def add(self, key): 518 if key not in self.map: 519 end = self.end 520 curr = end[1] 521 curr[2] = end[1] = self.map[key] = [key, curr, end] 522 523 def discard(self, key): 524 if key in self.map: 525 key, prev_item, next_item = self.map.pop(key) 526 prev_item[2] = next_item 527 next_item[1] = prev_item 528 529 def __iter__(self): 530 end = self.end 531 curr = end[2] 532 while curr is not end: 533 yield curr[0] 534 curr = curr[2] 535 536 def __reversed__(self): 537 end = self.end 538 curr = end[1] 539 while curr is not end: 540 yield curr[0] 541 curr = curr[1] 542 543 # The second argument is an addition that causes a pylint warning. 544 def pop(self, last=True): # pylint: disable=W0221 545 if not self: 546 raise KeyError('set is empty') 547 key = self.end[1][0] if last else self.end[2][0] 548 self.discard(key) 549 return key 550 551 def __repr__(self): 552 if not self: 553 return '%s()' % (self.__class__.__name__,) 554 return '%s(%r)' % (self.__class__.__name__, list(self)) 555 556 def __eq__(self, other): 557 if isinstance(other, OrderedSet): 558 return len(self) == len(other) and list(self) == list(other) 559 return set(self) == set(other) 560 561 # Extensions to the recipe. 562 def update(self, iterable): 563 for i in iterable: 564 if i not in self: 565 self.add(i) 566 567 568class CycleError(Exception): 569 """An exception raised when an unexpected cycle is detected.""" 570 def __init__(self, nodes): 571 self.nodes = nodes 572 def __str__(self): 573 return 'CycleError: cycle involving: ' + str(self.nodes) 574 575 576def TopologicallySorted(graph, get_edges): 577 r"""Topologically sort based on a user provided edge definition. 578 579 Args: 580 graph: A list of node names. 581 get_edges: A function mapping from node name to a hashable collection 582 of node names which this node has outgoing edges to. 583 Returns: 584 A list containing all of the node in graph in topological order. 585 It is assumed that calling get_edges once for each node and caching is 586 cheaper than repeatedly calling get_edges. 587 Raises: 588 CycleError in the event of a cycle. 589 Example: 590 graph = {'a': '$(b) $(c)', 'b': 'hi', 'c': '$(b)'} 591 def GetEdges(node): 592 return re.findall(r'\$\(([^))]\)', graph[node]) 593 print TopologicallySorted(graph.keys(), GetEdges) 594 ==> 595 ['a', 'c', b'] 596 """ 597 get_edges = memoize(get_edges) 598 visited = set() 599 visiting = set() 600 ordered_nodes = [] 601 def Visit(node): 602 if node in visiting: 603 raise CycleError(visiting) 604 if node in visited: 605 return 606 visited.add(node) 607 visiting.add(node) 608 for neighbor in get_edges(node): 609 Visit(neighbor) 610 visiting.remove(node) 611 ordered_nodes.insert(0, node) 612 for node in sorted(graph): 613 Visit(node) 614 return ordered_nodes 615 616def CrossCompileRequested(): 617 # TODO: figure out how to not build extra host objects in the 618 # non-cross-compile case when this is enabled, and enable unconditionally. 619 return (os.environ.get('GYP_CROSSCOMPILE') or 620 os.environ.get('AR_host') or 621 os.environ.get('CC_host') or 622 os.environ.get('CXX_host') or 623 os.environ.get('AR_target') or 624 os.environ.get('CC_target') or 625 os.environ.get('CXX_target')) 626 627def IsCygwin(): 628 try: 629 out = subprocess.Popen("uname", 630 stdout=subprocess.PIPE, 631 stderr=subprocess.STDOUT) 632 stdout, stderr = out.communicate() 633 if PY3: 634 stdout = stdout.decode("utf-8") 635 return "CYGWIN" in str(stdout) 636 except Exception: 637 return False 638 639