1#!/usr/bin/env python 2# Copyright 2013 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Traverses the source tree, parses all found DEPS files, and constructs 7a dependency rule table to be used by subclasses. 8 9See README.md for the format of the deps file. 10""" 11 12import copy 13import os.path 14import posixpath 15import subprocess 16 17from rules import Rule, Rules 18 19 20# Variable name used in the DEPS file to add or subtract include files from 21# the module-level deps. 22INCLUDE_RULES_VAR_NAME = 'include_rules' 23 24# Variable name used in the DEPS file to add or subtract include files 25# from module-level deps specific to files whose basename (last 26# component of path) matches a given regular expression. 27SPECIFIC_INCLUDE_RULES_VAR_NAME = 'specific_include_rules' 28 29# Optionally present in the DEPS file to list subdirectories which should not 30# be checked. This allows us to skip third party code, for example. 31SKIP_SUBDIRS_VAR_NAME = 'skip_child_includes' 32 33# Optionally discard rules from parent directories, similar to "noparent" in 34# OWNERS files. For example, if //ash/components has "noparent = True" then 35# it will not inherit rules from //ash/DEPS, forcing each //ash/component/foo 36# to declare all its dependencies. 37NOPARENT_VAR_NAME = 'noparent' 38 39 40class DepsBuilderError(Exception): 41 """Base class for exceptions in this module.""" 42 pass 43 44 45def NormalizePath(path): 46 """Returns a path normalized to how we write DEPS rules and compare paths.""" 47 return os.path.normcase(path).replace(os.path.sep, posixpath.sep) 48 49 50def _GitSourceDirectories(base_directory): 51 """Returns set of normalized paths to subdirectories containing sources 52 managed by git.""" 53 base_dir_norm = NormalizePath(base_directory) 54 git_source_directories = set([base_dir_norm]) 55 56 git_cmd = 'git.bat' if os.name == 'nt' else 'git' 57 git_ls_files_cmd = [git_cmd, 'ls-files'] 58 # FIXME: Use a context manager in Python 3.2+ 59 popen = subprocess.Popen(git_ls_files_cmd, 60 stdout=subprocess.PIPE, 61 bufsize=1, # line buffering, since read by line 62 cwd=base_directory) 63 try: 64 try: 65 for line in popen.stdout: 66 dir_path = os.path.join(base_directory, os.path.dirname(line)) 67 dir_path_norm = NormalizePath(dir_path) 68 # Add the directory as well as all the parent directories, 69 # stopping once we reach an already-listed directory. 70 while dir_path_norm not in git_source_directories: 71 git_source_directories.add(dir_path_norm) 72 dir_path_norm = posixpath.dirname(dir_path_norm) 73 finally: 74 popen.stdout.close() 75 finally: 76 popen.wait() 77 78 return git_source_directories 79 80 81class DepsBuilder(object): 82 """Parses include_rules from DEPS files.""" 83 84 def __init__(self, 85 base_directory=None, 86 extra_repos=[], 87 verbose=False, 88 being_tested=False, 89 ignore_temp_rules=False, 90 ignore_specific_rules=False): 91 """Creates a new DepsBuilder. 92 93 Args: 94 base_directory: local path to root of checkout, e.g. C:\chr\src. 95 verbose: Set to True for debug output. 96 being_tested: Set to True to ignore the DEPS file at 97 buildtools/checkdeps/DEPS. 98 ignore_temp_rules: Ignore rules that start with Rule.TEMP_ALLOW ("!"). 99 """ 100 base_directory = (base_directory or 101 os.path.join(os.path.dirname(__file__), 102 os.path.pardir, os.path.pardir)) 103 self.base_directory = os.path.abspath(base_directory) # Local absolute path 104 self.extra_repos = extra_repos 105 self.verbose = verbose 106 self._under_test = being_tested 107 self._ignore_temp_rules = ignore_temp_rules 108 self._ignore_specific_rules = ignore_specific_rules 109 self._git_source_directories = None 110 111 if os.path.exists(os.path.join(base_directory, '.git')): 112 self.is_git = True 113 elif os.path.exists(os.path.join(base_directory, '.svn')): 114 self.is_git = False 115 else: 116 raise DepsBuilderError("%s is not a repository root" % base_directory) 117 118 # Map of normalized directory paths to rules to use for those 119 # directories, or None for directories that should be skipped. 120 # Normalized is: absolute, lowercase, / for separator. 121 self.directory_rules = {} 122 self._ApplyDirectoryRulesAndSkipSubdirs(Rules(), self.base_directory) 123 124 def _ApplyRules(self, existing_rules, includes, specific_includes, 125 cur_dir_norm): 126 """Applies the given include rules, returning the new rules. 127 128 Args: 129 existing_rules: A set of existing rules that will be combined. 130 include: The list of rules from the "include_rules" section of DEPS. 131 specific_includes: E.g. {'.*_unittest\.cc': ['+foo', '-blat']} rules 132 from the "specific_include_rules" section of DEPS. 133 cur_dir_norm: The current directory, normalized path. We will create an 134 implicit rule that allows inclusion from this directory. 135 136 Returns: A new set of rules combining the existing_rules with the other 137 arguments. 138 """ 139 rules = copy.deepcopy(existing_rules) 140 141 # First apply the implicit "allow" rule for the current directory. 142 base_dir_norm = NormalizePath(self.base_directory) 143 if not cur_dir_norm.startswith(base_dir_norm): 144 raise Exception( 145 'Internal error: base directory is not at the beginning for\n' 146 ' %s and base dir\n' 147 ' %s' % (cur_dir_norm, base_dir_norm)) 148 relative_dir = posixpath.relpath(cur_dir_norm, base_dir_norm) 149 150 # Make the help string a little more meaningful. 151 source = relative_dir or 'top level' 152 rules.AddRule('+' + relative_dir, 153 relative_dir, 154 'Default rule for ' + source) 155 156 def ApplyOneRule(rule_str, dependee_regexp=None): 157 """Deduces a sensible description for the rule being added, and 158 adds the rule with its description to |rules|. 159 160 If we are ignoring temporary rules, this function does nothing 161 for rules beginning with the Rule.TEMP_ALLOW character. 162 """ 163 if self._ignore_temp_rules and rule_str.startswith(Rule.TEMP_ALLOW): 164 return 165 166 rule_block_name = 'include_rules' 167 if dependee_regexp: 168 rule_block_name = 'specific_include_rules' 169 if relative_dir: 170 rule_description = relative_dir + "'s %s" % rule_block_name 171 else: 172 rule_description = 'the top level %s' % rule_block_name 173 rules.AddRule(rule_str, relative_dir, rule_description, dependee_regexp) 174 175 # Apply the additional explicit rules. 176 for rule_str in includes: 177 ApplyOneRule(rule_str) 178 179 # Finally, apply the specific rules. 180 if self._ignore_specific_rules: 181 return rules 182 183 for regexp, specific_rules in specific_includes.iteritems(): 184 for rule_str in specific_rules: 185 ApplyOneRule(rule_str, regexp) 186 187 return rules 188 189 def _ApplyDirectoryRules(self, existing_rules, dir_path_local_abs): 190 """Combines rules from the existing rules and the new directory. 191 192 Any directory can contain a DEPS file. Top-level DEPS files can contain 193 module dependencies which are used by gclient. We use these, along with 194 additional include rules and implicit rules for the given directory, to 195 come up with a combined set of rules to apply for the directory. 196 197 Args: 198 existing_rules: The rules for the parent directory. We'll add-on to these. 199 dir_path_local_abs: The directory path that the DEPS file may live in (if 200 it exists). This will also be used to generate the 201 implicit rules. This is a local path. 202 203 Returns: A 2-tuple of: 204 (1) the combined set of rules to apply to the sub-tree, 205 (2) a list of all subdirectories that should NOT be checked, as specified 206 in the DEPS file (if any). 207 Subdirectories are single words, hence no OS dependence. 208 """ 209 dir_path_norm = NormalizePath(dir_path_local_abs) 210 211 # Check the DEPS file in this directory. 212 if self.verbose: 213 print 'Applying rules from', dir_path_local_abs 214 def FromImpl(*_): 215 pass # NOP function so "From" doesn't fail. 216 217 def FileImpl(_): 218 pass # NOP function so "File" doesn't fail. 219 220 class _VarImpl: 221 def __init__(self, local_scope): 222 self._local_scope = local_scope 223 224 def Lookup(self, var_name): 225 """Implements the Var syntax.""" 226 try: 227 return self._local_scope['vars'][var_name] 228 except KeyError: 229 raise Exception('Var is not defined: %s' % var_name) 230 231 local_scope = {} 232 global_scope = { 233 'File': FileImpl, 234 'From': FromImpl, 235 'Var': _VarImpl(local_scope).Lookup, 236 'Str': str, 237 } 238 deps_file_path = os.path.join(dir_path_local_abs, 'DEPS') 239 240 # The second conditional here is to disregard the 241 # buildtools/checkdeps/DEPS file while running tests. This DEPS file 242 # has a skip_child_includes for 'testdata' which is necessary for 243 # running production tests, since there are intentional DEPS 244 # violations under the testdata directory. On the other hand when 245 # running tests, we absolutely need to verify the contents of that 246 # directory to trigger those intended violations and see that they 247 # are handled correctly. 248 if os.path.isfile(deps_file_path) and not ( 249 self._under_test and 250 os.path.basename(dir_path_local_abs) == 'checkdeps'): 251 execfile(deps_file_path, global_scope, local_scope) 252 elif self.verbose: 253 print ' No deps file found in', dir_path_local_abs 254 255 # Even if a DEPS file does not exist we still invoke ApplyRules 256 # to apply the implicit "allow" rule for the current directory 257 include_rules = local_scope.get(INCLUDE_RULES_VAR_NAME, []) 258 specific_include_rules = local_scope.get(SPECIFIC_INCLUDE_RULES_VAR_NAME, 259 {}) 260 skip_subdirs = local_scope.get(SKIP_SUBDIRS_VAR_NAME, []) 261 noparent = local_scope.get(NOPARENT_VAR_NAME, False) 262 if noparent: 263 parent_rules = Rules() 264 else: 265 parent_rules = existing_rules 266 267 return (self._ApplyRules(parent_rules, include_rules, 268 specific_include_rules, dir_path_norm), 269 skip_subdirs) 270 271 def _ApplyDirectoryRulesAndSkipSubdirs(self, parent_rules, 272 dir_path_local_abs): 273 """Given |parent_rules| and a subdirectory |dir_path_local_abs| of the 274 directory that owns the |parent_rules|, add |dir_path_local_abs|'s rules to 275 |self.directory_rules|, and add None entries for any of its 276 subdirectories that should be skipped. 277 """ 278 directory_rules, excluded_subdirs = self._ApplyDirectoryRules( 279 parent_rules, dir_path_local_abs) 280 dir_path_norm = NormalizePath(dir_path_local_abs) 281 self.directory_rules[dir_path_norm] = directory_rules 282 for subdir in excluded_subdirs: 283 subdir_path_norm = posixpath.join(dir_path_norm, subdir) 284 self.directory_rules[subdir_path_norm] = None 285 286 def GetAllRulesAndFiles(self, dir_name=None): 287 """Yields (rules, filenames) for each repository directory with DEPS rules. 288 289 This walks the directory tree while staying in the repository. Specify 290 |dir_name| to walk just one directory and its children; omit |dir_name| to 291 walk the entire repository. 292 293 Yields: 294 Two-element (rules, filenames) tuples. |rules| is a rules.Rules object 295 for a directory, and |filenames| is a list of the absolute local paths 296 of all files in that directory. 297 """ 298 if self.is_git and self._git_source_directories is None: 299 self._git_source_directories = _GitSourceDirectories(self.base_directory) 300 for repo in self.extra_repos: 301 repo_path = os.path.join(self.base_directory, repo) 302 self._git_source_directories.update(_GitSourceDirectories(repo_path)) 303 304 # Collect a list of all files and directories to check. 305 files_to_check = [] 306 if dir_name and not os.path.isabs(dir_name): 307 dir_name = os.path.join(self.base_directory, dir_name) 308 dirs_to_check = [dir_name or self.base_directory] 309 while dirs_to_check: 310 current_dir = dirs_to_check.pop() 311 312 # Check that this directory is part of the source repository. This 313 # prevents us from descending into third-party code or directories 314 # generated by the build system. 315 if self.is_git: 316 if NormalizePath(current_dir) not in self._git_source_directories: 317 continue 318 elif not os.path.exists(os.path.join(current_dir, '.svn')): 319 continue 320 321 current_dir_rules = self.GetDirectoryRules(current_dir) 322 323 if not current_dir_rules: 324 continue # Handle the 'skip_child_includes' case. 325 326 current_dir_contents = sorted(os.listdir(current_dir)) 327 file_names = [] 328 sub_dirs = [] 329 for file_name in current_dir_contents: 330 full_name = os.path.join(current_dir, file_name) 331 if os.path.isdir(full_name): 332 sub_dirs.append(full_name) 333 else: 334 file_names.append(full_name) 335 dirs_to_check.extend(reversed(sub_dirs)) 336 337 yield (current_dir_rules, file_names) 338 339 def GetDirectoryRules(self, dir_path_local): 340 """Returns a Rules object to use for the given directory, or None 341 if the given directory should be skipped. 342 343 Also modifies |self.directory_rules| to store the Rules. 344 This takes care of first building rules for parent directories (up to 345 |self.base_directory|) if needed, which may add rules for skipped 346 subdirectories. 347 348 Args: 349 dir_path_local: A local path to the directory you want rules for. 350 Can be relative and unnormalized. It is the caller's responsibility 351 to ensure that this is part of the repository rooted at 352 |self.base_directory|. 353 """ 354 if os.path.isabs(dir_path_local): 355 dir_path_local_abs = dir_path_local 356 else: 357 dir_path_local_abs = os.path.join(self.base_directory, dir_path_local) 358 dir_path_norm = NormalizePath(dir_path_local_abs) 359 360 if dir_path_norm in self.directory_rules: 361 return self.directory_rules[dir_path_norm] 362 363 parent_dir_local_abs = os.path.dirname(dir_path_local_abs) 364 parent_rules = self.GetDirectoryRules(parent_dir_local_abs) 365 # We need to check for an entry for our dir_path again, since 366 # GetDirectoryRules can modify entries for subdirectories, namely setting 367 # to None if they should be skipped, via _ApplyDirectoryRulesAndSkipSubdirs. 368 # For example, if dir_path == 'A/B/C' and A/B/DEPS specifies that the C 369 # subdirectory be skipped, GetDirectoryRules('A/B') will fill in the entry 370 # for 'A/B/C' as None. 371 if dir_path_norm in self.directory_rules: 372 return self.directory_rules[dir_path_norm] 373 374 if parent_rules: 375 self._ApplyDirectoryRulesAndSkipSubdirs(parent_rules, dir_path_local_abs) 376 else: 377 # If the parent directory should be skipped, then the current 378 # directory should also be skipped. 379 self.directory_rules[dir_path_norm] = None 380 return self.directory_rules[dir_path_norm] 381