1#!/usr/bin/env python 2# 3# Copyright 2006 The Closure Library Authors. All Rights Reserved. 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS-IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17 18"""Calculates JavaScript dependencies without requiring Google's build system. 19 20This tool is deprecated and is provided for legacy users. 21See build/closurebuilder.py and build/depswriter.py for the current tools. 22 23It iterates over a number of search paths and builds a dependency tree. With 24the inputs provided, it walks the dependency tree and outputs all the files 25required for compilation. 26""" 27 28 29 30 31 32try: 33 import distutils.version 34except ImportError: 35 # distutils is not available in all environments 36 distutils = None 37 38import logging 39import optparse 40import os 41import re 42import subprocess 43import sys 44 45 46_BASE_REGEX_STRING = '^\s*goog\.%s\(\s*[\'"](.+)[\'"]\s*\)' 47req_regex = re.compile(_BASE_REGEX_STRING % 'require') 48prov_regex = re.compile(_BASE_REGEX_STRING % 'provide') 49ns_regex = re.compile('^ns:((\w+\.)*(\w+))$') 50version_regex = re.compile('[\.0-9]+') 51 52 53def IsValidFile(ref): 54 """Returns true if the provided reference is a file and exists.""" 55 return os.path.isfile(ref) 56 57 58def IsJsFile(ref): 59 """Returns true if the provided reference is a Javascript file.""" 60 return ref.endswith('.js') 61 62 63def IsNamespace(ref): 64 """Returns true if the provided reference is a namespace.""" 65 return re.match(ns_regex, ref) is not None 66 67 68def IsDirectory(ref): 69 """Returns true if the provided reference is a directory.""" 70 return os.path.isdir(ref) 71 72 73def ExpandDirectories(refs): 74 """Expands any directory references into inputs. 75 76 Description: 77 Looks for any directories in the provided references. Found directories 78 are recursively searched for .js files, which are then added to the result 79 list. 80 81 Args: 82 refs: a list of references such as files, directories, and namespaces 83 84 Returns: 85 A list of references with directories removed and replaced by any 86 .js files that are found in them. Also, the paths will be normalized. 87 """ 88 result = [] 89 for ref in refs: 90 if IsDirectory(ref): 91 # Disable 'Unused variable' for subdirs 92 # pylint: disable=unused-variable 93 for (directory, subdirs, filenames) in os.walk(ref): 94 for filename in filenames: 95 if IsJsFile(filename): 96 result.append(os.path.join(directory, filename)) 97 else: 98 result.append(ref) 99 return map(os.path.normpath, result) 100 101 102class DependencyInfo(object): 103 """Represents a dependency that is used to build and walk a tree.""" 104 105 def __init__(self, filename): 106 self.filename = filename 107 self.provides = [] 108 self.requires = [] 109 110 def __str__(self): 111 return '%s Provides: %s Requires: %s' % (self.filename, 112 repr(self.provides), 113 repr(self.requires)) 114 115 116def BuildDependenciesFromFiles(files): 117 """Build a list of dependencies from a list of files. 118 119 Description: 120 Takes a list of files, extracts their provides and requires, and builds 121 out a list of dependency objects. 122 123 Args: 124 files: a list of files to be parsed for goog.provides and goog.requires. 125 126 Returns: 127 A list of dependency objects, one for each file in the files argument. 128 """ 129 result = [] 130 filenames = set() 131 for filename in files: 132 if filename in filenames: 133 continue 134 135 # Python 3 requires the file encoding to be specified 136 if (sys.version_info[0] < 3): 137 file_handle = open(filename, 'r') 138 else: 139 file_handle = open(filename, 'r', encoding='utf8') 140 141 try: 142 dep = CreateDependencyInfo(filename, file_handle) 143 result.append(dep) 144 finally: 145 file_handle.close() 146 147 filenames.add(filename) 148 149 return result 150 151 152def CreateDependencyInfo(filename, source): 153 """Create dependency info. 154 155 Args: 156 filename: Filename for source. 157 source: File-like object containing source. 158 159 Returns: 160 A DependencyInfo object with provides and requires filled. 161 """ 162 dep = DependencyInfo(filename) 163 for line in source: 164 if re.match(req_regex, line): 165 dep.requires.append(re.search(req_regex, line).group(1)) 166 if re.match(prov_regex, line): 167 dep.provides.append(re.search(prov_regex, line).group(1)) 168 return dep 169 170 171def BuildDependencyHashFromDependencies(deps): 172 """Builds a hash for searching dependencies by the namespaces they provide. 173 174 Description: 175 Dependency objects can provide multiple namespaces. This method enumerates 176 the provides of each dependency and adds them to a hash that can be used 177 to easily resolve a given dependency by a namespace it provides. 178 179 Args: 180 deps: a list of dependency objects used to build the hash. 181 182 Raises: 183 Exception: If a multiple files try to provide the same namepace. 184 185 Returns: 186 A hash table { namespace: dependency } that can be used to resolve a 187 dependency by a namespace it provides. 188 """ 189 dep_hash = {} 190 for dep in deps: 191 for provide in dep.provides: 192 if provide in dep_hash: 193 raise Exception('Duplicate provide (%s) in (%s, %s)' % ( 194 provide, 195 dep_hash[provide].filename, 196 dep.filename)) 197 dep_hash[provide] = dep 198 return dep_hash 199 200 201def CalculateDependencies(paths, inputs): 202 """Calculates the dependencies for given inputs. 203 204 Description: 205 This method takes a list of paths (files, directories) and builds a 206 searchable data structure based on the namespaces that each .js file 207 provides. It then parses through each input, resolving dependencies 208 against this data structure. The final output is a list of files, 209 including the inputs, that represent all of the code that is needed to 210 compile the given inputs. 211 212 Args: 213 paths: the references (files, directories) that are used to build the 214 dependency hash. 215 inputs: the inputs (files, directories, namespaces) that have dependencies 216 that need to be calculated. 217 218 Raises: 219 Exception: if a provided input is invalid. 220 221 Returns: 222 A list of all files, including inputs, that are needed to compile the given 223 inputs. 224 """ 225 deps = BuildDependenciesFromFiles(paths + inputs) 226 search_hash = BuildDependencyHashFromDependencies(deps) 227 result_list = [] 228 seen_list = [] 229 for input_file in inputs: 230 if IsNamespace(input_file): 231 namespace = re.search(ns_regex, input_file).group(1) 232 if namespace not in search_hash: 233 raise Exception('Invalid namespace (%s)' % namespace) 234 input_file = search_hash[namespace].filename 235 if not IsValidFile(input_file) or not IsJsFile(input_file): 236 raise Exception('Invalid file (%s)' % input_file) 237 seen_list.append(input_file) 238 file_handle = open(input_file, 'r') 239 try: 240 for line in file_handle: 241 if re.match(req_regex, line): 242 require = re.search(req_regex, line).group(1) 243 ResolveDependencies(require, search_hash, result_list, seen_list) 244 finally: 245 file_handle.close() 246 result_list.append(input_file) 247 248 # All files depend on base.js, so put it first. 249 base_js_path = FindClosureBasePath(paths) 250 if base_js_path: 251 result_list.insert(0, base_js_path) 252 else: 253 logging.warning('Closure Library base.js not found.') 254 255 return result_list 256 257 258def FindClosureBasePath(paths): 259 """Given a list of file paths, return Closure base.js path, if any. 260 261 Args: 262 paths: A list of paths. 263 264 Returns: 265 The path to Closure's base.js file including filename, if found. 266 """ 267 268 for path in paths: 269 pathname, filename = os.path.split(path) 270 271 if filename == 'base.js': 272 f = open(path) 273 274 is_base = False 275 276 # Sanity check that this is the Closure base file. Check that this 277 # is where goog is defined. This is determined by the @provideGoog 278 # flag. 279 for line in f: 280 if '@provideGoog' in line: 281 is_base = True 282 break 283 284 f.close() 285 286 if is_base: 287 return path 288 289def ResolveDependencies(require, search_hash, result_list, seen_list): 290 """Takes a given requirement and resolves all of the dependencies for it. 291 292 Description: 293 A given requirement may require other dependencies. This method 294 recursively resolves all dependencies for the given requirement. 295 296 Raises: 297 Exception: when require does not exist in the search_hash. 298 299 Args: 300 require: the namespace to resolve dependencies for. 301 search_hash: the data structure used for resolving dependencies. 302 result_list: a list of filenames that have been calculated as dependencies. 303 This variable is the output for this function. 304 seen_list: a list of filenames that have been 'seen'. This is required 305 for the dependency->dependant ordering. 306 """ 307 if require not in search_hash: 308 raise Exception('Missing provider for (%s)' % require) 309 310 dep = search_hash[require] 311 if not dep.filename in seen_list: 312 seen_list.append(dep.filename) 313 for sub_require in dep.requires: 314 ResolveDependencies(sub_require, search_hash, result_list, seen_list) 315 result_list.append(dep.filename) 316 317 318def GetDepsLine(dep, base_path): 319 """Returns a JS string for a dependency statement in the deps.js file. 320 321 Args: 322 dep: The dependency that we're printing. 323 base_path: The path to Closure's base.js including filename. 324 """ 325 return 'goog.addDependency("%s", %s, %s);' % ( 326 GetRelpath(dep.filename, base_path), dep.provides, dep.requires) 327 328 329def GetRelpath(path, start): 330 """Return a relative path to |path| from |start|.""" 331 # NOTE: Python 2.6 provides os.path.relpath, which has almost the same 332 # functionality as this function. Since we want to support 2.4, we have 333 # to implement it manually. :( 334 path_list = os.path.abspath(os.path.normpath(path)).split(os.sep) 335 start_list = os.path.abspath( 336 os.path.normpath(os.path.dirname(start))).split(os.sep) 337 338 common_prefix_count = 0 339 for i in range(0, min(len(path_list), len(start_list))): 340 if path_list[i] != start_list[i]: 341 break 342 common_prefix_count += 1 343 344 # Always use forward slashes, because this will get expanded to a url, 345 # not a file path. 346 return '/'.join(['..'] * (len(start_list) - common_prefix_count) + 347 path_list[common_prefix_count:]) 348 349 350def PrintLine(msg, out): 351 out.write(msg) 352 out.write('\n') 353 354 355def PrintDeps(source_paths, deps, out): 356 """Print out a deps.js file from a list of source paths. 357 358 Args: 359 source_paths: Paths that we should generate dependency info for. 360 deps: Paths that provide dependency info. Their dependency info should 361 not appear in the deps file. 362 out: The output file. 363 364 Returns: 365 True on success, false if it was unable to find the base path 366 to generate deps relative to. 367 """ 368 base_path = FindClosureBasePath(source_paths + deps) 369 if not base_path: 370 return False 371 372 PrintLine('// This file was autogenerated by calcdeps.py', out) 373 excludesSet = set(deps) 374 375 for dep in BuildDependenciesFromFiles(source_paths + deps): 376 if not dep.filename in excludesSet: 377 PrintLine(GetDepsLine(dep, base_path), out) 378 379 return True 380 381 382def PrintScript(source_paths, out): 383 for index, dep in enumerate(source_paths): 384 PrintLine('// Input %d' % index, out) 385 f = open(dep, 'r') 386 PrintLine(f.read(), out) 387 f.close() 388 389 390def GetJavaVersion(): 391 """Returns the string for the current version of Java installed.""" 392 proc = subprocess.Popen(['java', '-version'], stderr=subprocess.PIPE) 393 proc.wait() 394 version_line = proc.stderr.read().splitlines()[0] 395 return version_regex.search(version_line).group() 396 397 398def FilterByExcludes(options, files): 399 """Filters the given files by the exlusions specified at the command line. 400 401 Args: 402 options: The flags to calcdeps. 403 files: The files to filter. 404 Returns: 405 A list of files. 406 """ 407 excludes = [] 408 if options.excludes: 409 excludes = ExpandDirectories(options.excludes) 410 411 excludesSet = set(excludes) 412 return [i for i in files if not i in excludesSet] 413 414 415def GetPathsFromOptions(options): 416 """Generates the path files from flag options. 417 418 Args: 419 options: The flags to calcdeps. 420 Returns: 421 A list of files in the specified paths. (strings). 422 """ 423 424 search_paths = options.paths 425 if not search_paths: 426 search_paths = ['.'] # Add default folder if no path is specified. 427 428 search_paths = ExpandDirectories(search_paths) 429 return FilterByExcludes(options, search_paths) 430 431 432def GetInputsFromOptions(options): 433 """Generates the inputs from flag options. 434 435 Args: 436 options: The flags to calcdeps. 437 Returns: 438 A list of inputs (strings). 439 """ 440 inputs = options.inputs 441 if not inputs: # Parse stdin 442 logging.info('No inputs specified. Reading from stdin...') 443 inputs = filter(None, [line.strip('\n') for line in sys.stdin.readlines()]) 444 445 logging.info('Scanning files...') 446 inputs = ExpandDirectories(inputs) 447 448 return FilterByExcludes(options, inputs) 449 450 451def Compile(compiler_jar_path, source_paths, out, flags=None): 452 """Prepares command-line call to Closure compiler. 453 454 Args: 455 compiler_jar_path: Path to the Closure compiler .jar file. 456 source_paths: Source paths to build, in order. 457 flags: A list of additional flags to pass on to Closure compiler. 458 """ 459 args = ['java', '-jar', compiler_jar_path] 460 for path in source_paths: 461 args += ['--js', path] 462 463 if flags: 464 args += flags 465 466 logging.info('Compiling with the following command: %s', ' '.join(args)) 467 proc = subprocess.Popen(args, stdout=subprocess.PIPE) 468 (stdoutdata, stderrdata) = proc.communicate() 469 if proc.returncode != 0: 470 logging.error('JavaScript compilation failed.') 471 sys.exit(1) 472 else: 473 out.write(stdoutdata) 474 475 476def main(): 477 """The entrypoint for this script.""" 478 479 logging.basicConfig(format='calcdeps.py: %(message)s', level=logging.INFO) 480 481 usage = 'usage: %prog [options] arg' 482 parser = optparse.OptionParser(usage) 483 parser.add_option('-i', 484 '--input', 485 dest='inputs', 486 action='append', 487 help='The inputs to calculate dependencies for. Valid ' 488 'values can be files, directories, or namespaces ' 489 '(ns:goog.net.XhrIo). Only relevant to "list" and ' 490 '"script" output.') 491 parser.add_option('-p', 492 '--path', 493 dest='paths', 494 action='append', 495 help='The paths that should be traversed to build the ' 496 'dependencies.') 497 parser.add_option('-d', 498 '--dep', 499 dest='deps', 500 action='append', 501 help='Directories or files that should be traversed to ' 502 'find required dependencies for the deps file. ' 503 'Does not generate dependency information for names ' 504 'provided by these files. Only useful in "deps" mode.') 505 parser.add_option('-e', 506 '--exclude', 507 dest='excludes', 508 action='append', 509 help='Files or directories to exclude from the --path ' 510 'and --input flags') 511 parser.add_option('-o', 512 '--output_mode', 513 dest='output_mode', 514 action='store', 515 default='list', 516 help='The type of output to generate from this script. ' 517 'Options are "list" for a list of filenames, "script" ' 518 'for a single script containing the contents of all the ' 519 'file, "deps" to generate a deps.js file for all ' 520 'paths, or "compiled" to produce compiled output with ' 521 'the Closure compiler.') 522 parser.add_option('-c', 523 '--compiler_jar', 524 dest='compiler_jar', 525 action='store', 526 help='The location of the Closure compiler .jar file.') 527 parser.add_option('-f', 528 '--compiler_flag', 529 '--compiler_flags', # for backwards compatability 530 dest='compiler_flags', 531 action='append', 532 help='Additional flag to pass to the Closure compiler. ' 533 'May be specified multiple times to pass multiple flags.') 534 parser.add_option('--output_file', 535 dest='output_file', 536 action='store', 537 help=('If specified, write output to this path instead of ' 538 'writing to standard output.')) 539 540 (options, args) = parser.parse_args() 541 542 search_paths = GetPathsFromOptions(options) 543 544 if options.output_file: 545 out = open(options.output_file, 'w') 546 else: 547 out = sys.stdout 548 549 if options.output_mode == 'deps': 550 result = PrintDeps(search_paths, ExpandDirectories(options.deps or []), out) 551 if not result: 552 logging.error('Could not find Closure Library in the specified paths') 553 sys.exit(1) 554 555 return 556 557 inputs = GetInputsFromOptions(options) 558 559 logging.info('Finding Closure dependencies...') 560 deps = CalculateDependencies(search_paths, inputs) 561 output_mode = options.output_mode 562 563 if output_mode == 'script': 564 PrintScript(deps, out) 565 elif output_mode == 'list': 566 # Just print out a dep per line 567 for dep in deps: 568 PrintLine(dep, out) 569 elif output_mode == 'compiled': 570 # Make sure a .jar is specified. 571 if not options.compiler_jar: 572 logging.error('--compiler_jar flag must be specified if --output is ' 573 '"compiled"') 574 sys.exit(1) 575 576 # User friendly version check. 577 if distutils and not (distutils.version.LooseVersion(GetJavaVersion()) > 578 distutils.version.LooseVersion('1.6')): 579 logging.error('Closure Compiler requires Java 1.6 or higher.') 580 logging.error('Please visit http://www.java.com/getjava') 581 sys.exit(1) 582 583 Compile(options.compiler_jar, deps, out, options.compiler_flags) 584 585 else: 586 logging.error('Invalid value for --output flag.') 587 sys.exit(1) 588 589if __name__ == '__main__': 590 main() 591