1<?php 2 3/** 4 * Build maps of libphutil libraries. libphutil uses the library map to locate 5 * and load classes and functions in the library. 6 * 7 * @task map Mapping libphutil Libraries 8 * @task path Path Management 9 * @task symbol Symbol Analysis and Caching 10 * @task source Source Management 11 */ 12final class PhutilLibraryMapBuilder extends Phobject { 13 14 private $root; 15 private $quiet = true; 16 private $subprocessLimit = 8; 17 18 private $fileSymbolMap; 19 private $librarySymbolMap; 20 21 const LIBRARY_MAP_VERSION_KEY = '__library_version__'; 22 const LIBRARY_MAP_VERSION = 2; 23 24 const SYMBOL_CACHE_VERSION_KEY = '__symbol_cache_version__'; 25 const SYMBOL_CACHE_VERSION = 11; 26 27 28/* -( Mapping libphutil Libraries )---------------------------------------- */ 29 30 /** 31 * Create a new map builder for a library. 32 * 33 * @param string Path to the library root. 34 * 35 * @task map 36 */ 37 public function __construct($root) { 38 $this->root = $root; 39 } 40 41 /** 42 * Control status output. Use `--quiet` to set this. 43 * 44 * @param bool If true, don't show status output. 45 * @return this 46 * 47 * @task map 48 */ 49 public function setQuiet($quiet) { 50 $this->quiet = $quiet; 51 return $this; 52 } 53 54 /** 55 * Control subprocess parallelism limit. Use `--limit` to set this. 56 * 57 * @param int Maximum number of subprocesses to run in parallel. 58 * @return this 59 * 60 * @task map 61 */ 62 public function setSubprocessLimit($limit) { 63 $this->subprocessLimit = $limit; 64 return $this; 65 } 66 67 /** 68 * Get the map of symbols in this library, analyzing the library to build it 69 * if necessary. 70 * 71 * @return map<string, wild> Information about symbols in this library. 72 * 73 * @task map 74 */ 75 public function buildMap() { 76 if ($this->librarySymbolMap === null) { 77 $this->analyzeLibrary(); 78 } 79 return $this->librarySymbolMap; 80 } 81 82 83 /** 84 * Get the map of files in this library, analyzing the library to build it 85 * if necessary. 86 * 87 * Returns a map of file paths to information about symbols used and defined 88 * in the file. 89 * 90 * @return map<string, wild> Information about files in this library. 91 * 92 * @task map 93 */ 94 public function buildFileSymbolMap() { 95 if ($this->fileSymbolMap === null) { 96 $this->analyzeLibrary(); 97 } 98 return $this->fileSymbolMap; 99 } 100 101 /** 102 * Build and update the library map. 103 * 104 * @return void 105 * 106 * @task map 107 */ 108 public function buildAndWriteMap() { 109 $library_map = $this->buildMap(); 110 111 $this->log(pht('Writing map...')); 112 $this->writeLibraryMap($library_map); 113 } 114 115 /** 116 * Write a status message to the user, if not running in quiet mode. 117 * 118 * @param string Message to write. 119 * @return this 120 * 121 * @task map 122 */ 123 private function log($message) { 124 if (!$this->quiet) { 125 @fwrite(STDERR, "%s\n", $message); 126 } 127 return $this; 128 } 129 130 131/* -( Path Management )---------------------------------------------------- */ 132 133 /** 134 * Get the path to some file in the library. 135 * 136 * @param string A library-relative path. If omitted, returns the library 137 * root path. 138 * @return string An absolute path. 139 * 140 * @task path 141 */ 142 private function getPath($path = '') { 143 return $this->root.'/'.$path; 144 } 145 146 /** 147 * Get the path to the symbol cache file. 148 * 149 * @return string Absolute path to symbol cache. 150 * 151 * @task path 152 */ 153 private function getPathForSymbolCache() { 154 return $this->getPath('.phutil_module_cache'); 155 } 156 157 /** 158 * Get the path to the map file. 159 * 160 * @return string Absolute path to the library map. 161 * 162 * @task path 163 */ 164 private function getPathForLibraryMap() { 165 return $this->getPath('__phutil_library_map__.php'); 166 } 167 168 /** 169 * Get the path to the library init file. 170 * 171 * @return string Absolute path to the library init file 172 * 173 * @task path 174 */ 175 private function getPathForLibraryInit() { 176 return $this->getPath('__phutil_library_init__.php'); 177 } 178 179 180/* -( Symbol Analysis and Caching )---------------------------------------- */ 181 182 /** 183 * Load the library symbol cache, if it exists and is readable and valid. 184 * 185 * @return dict Map of content hashes to cache of output from 186 * `extract-symbols.php`. 187 * 188 * @task symbol 189 */ 190 private function loadSymbolCache() { 191 $cache_file = $this->getPathForSymbolCache(); 192 193 try { 194 $cache = Filesystem::readFile($cache_file); 195 } catch (Exception $ex) { 196 $cache = null; 197 } 198 199 $symbol_cache = array(); 200 if ($cache) { 201 try { 202 $symbol_cache = phutil_json_decode($cache); 203 } catch (PhutilJSONParserException $ex) { 204 $symbol_cache = array(); 205 } 206 } 207 208 $version = idx($symbol_cache, self::SYMBOL_CACHE_VERSION_KEY); 209 if ($version != self::SYMBOL_CACHE_VERSION) { 210 // Throw away caches from a different version of the library. 211 $symbol_cache = array(); 212 } 213 unset($symbol_cache[self::SYMBOL_CACHE_VERSION_KEY]); 214 215 return $symbol_cache; 216 } 217 218 /** 219 * Write a symbol map to disk cache. 220 * 221 * @param dict Symbol map of relative paths to symbols. 222 * @param dict Source map (like @{method:loadSourceFileMap}). 223 * @return void 224 * 225 * @task symbol 226 */ 227 private function writeSymbolCache(array $symbol_map, array $source_map) { 228 $cache_file = $this->getPathForSymbolCache(); 229 230 $cache = array( 231 self::SYMBOL_CACHE_VERSION_KEY => self::SYMBOL_CACHE_VERSION, 232 ); 233 234 foreach ($symbol_map as $file => $symbols) { 235 $cache[$source_map[$file]] = $symbols; 236 } 237 238 $json = json_encode($cache); 239 try { 240 Filesystem::writeFile($cache_file, $json); 241 } catch (FilesystemException $ex) { 242 $this->log(pht('Unable to save the cache!')); 243 } 244 } 245 246 /** 247 * Drop the symbol cache, forcing a clean rebuild. 248 * 249 * @return this 250 * 251 * @task symbol 252 */ 253 public function dropSymbolCache() { 254 $this->log(pht('Dropping symbol cache...')); 255 Filesystem::remove($this->getPathForSymbolCache()); 256 } 257 258 /** 259 * Build a future which returns a `extract-symbols.php` analysis of a source 260 * file. 261 * 262 * @param string Relative path to the source file to analyze. 263 * @return Future Analysis future. 264 * 265 * @task symbol 266 */ 267 private function buildSymbolAnalysisFuture($file) { 268 $absolute_file = $this->getPath($file); 269 $bin = dirname(__FILE__).'/../../support/lib/extract-symbols.php'; 270 271 return new ExecFuture('%%PHP_CMD%% -f %R -- --ugly %R', $bin, $absolute_file); 272 } 273 274 275/* -( Source Management )-------------------------------------------------- */ 276 277 /** 278 * Build a map of all source files in a library to hashes of their content. 279 * Returns an array like this: 280 * 281 * array( 282 * 'src/parser/ExampleParser.php' => '60b725f10c9c85c70d97880dfe8191b3', 283 * // ... 284 * ); 285 * 286 * @return dict Map of library-relative paths to content hashes. 287 * @task source 288 */ 289 private function loadSourceFileMap() { 290 $root = $this->getPath(); 291 292 $init = $this->getPathForLibraryInit(); 293 if (!Filesystem::pathExists($init)) { 294 throw new Exception( 295 pht( 296 "Provided path '%s' is not a %s library.", 297 $root, 298 'phutil')); 299 } 300 301 $files = id(new FileFinder($root)) 302 ->withType('f') 303 ->withSuffix('php') 304 ->excludePath('*/.*') 305 ->setGenerateChecksums(true) 306 ->find(); 307 308 $extensions_dir = 'extensions/'; 309 $extensions_len = strlen($extensions_dir); 310 311 $map = array(); 312 foreach ($files as $file => $hash) { 313 $file = Filesystem::readablePath($file, $root); 314 $file = ltrim($file, '/'); 315 316 if (dirname($file) == '.') { 317 // We don't permit normal source files at the root level, so just ignore 318 // them; they're special library files. 319 continue; 320 } 321 322 // Ignore files in the extensions/ directory. 323 if (!strncmp($file, $extensions_dir, $extensions_len)) { 324 continue; 325 } 326 327 // We include also filename in the hash to handle cases when the file is 328 // moved without modifying its content. 329 $map[$file] = md5($hash.$file); 330 } 331 332 return $map; 333 } 334 335 /** 336 * Convert the symbol analysis of all the source files in the library into 337 * a library map. 338 * 339 * @param dict Symbol analysis of all source files. 340 * @return dict Library map. 341 * @task source 342 */ 343 private function buildLibraryMap(array $symbol_map) { 344 $library_map = array( 345 'class' => array(), 346 'function' => array(), 347 'xmap' => array(), 348 ); 349 350 // Detect duplicate symbols within the library. 351 foreach ($symbol_map as $file => $info) { 352 foreach ($info['have'] as $type => $symbols) { 353 foreach ($symbols as $symbol => $declaration) { 354 $lib_type = ($type == 'interface') ? 'class' : $type; 355 if (!empty($library_map[$lib_type][$symbol])) { 356 $prior = $library_map[$lib_type][$symbol]; 357 throw new Exception( 358 pht( 359 "Definition of %s '%s' in file '%s' duplicates prior ". 360 "definition in file '%s'. You can not declare the ". 361 "same symbol twice.", 362 $type, 363 $symbol, 364 $file, 365 $prior)); 366 } 367 $library_map[$lib_type][$symbol] = $file; 368 } 369 } 370 $library_map['xmap'] += $info['xmap']; 371 } 372 373 // Simplify the common case (one parent) to make the file a little easier 374 // to deal with. 375 foreach ($library_map['xmap'] as $class => $extends) { 376 if (count($extends) == 1) { 377 $library_map['xmap'][$class] = reset($extends); 378 } 379 } 380 381 // Sort the map so it is relatively stable across changes. 382 foreach ($library_map as $key => $symbols) { 383 ksort($symbols); 384 $library_map[$key] = $symbols; 385 } 386 ksort($library_map); 387 388 return $library_map; 389 } 390 391 /** 392 * Write a finalized library map. 393 * 394 * @param dict Library map structure to write. 395 * @return void 396 * 397 * @task source 398 */ 399 private function writeLibraryMap(array $library_map) { 400 $map_file = $this->getPathForLibraryMap(); 401 $version = self::LIBRARY_MAP_VERSION; 402 403 $library_map = array( 404 self::LIBRARY_MAP_VERSION_KEY => $version, 405 ) + $library_map; 406 407 $library_map = phutil_var_export($library_map); 408 $at = '@'; 409 410 $source_file = <<<EOPHP 411<?php 412 413/** 414 * This file is automatically generated. Use 'arc liberate' to rebuild it. 415 * 416 * {$at}generated 417 * {$at}phutil-library-version {$version} 418 */ 419phutil_register_library_map({$library_map}); 420 421EOPHP; 422 423 Filesystem::writeFile($map_file, $source_file); 424 } 425 426 427 /** 428 * Analyze the library, generating the file and symbol maps. 429 * 430 * @return void 431 */ 432 private function analyzeLibrary() { 433 // Identify all the ".php" source files in the library. 434 $this->log(pht('Finding source files...')); 435 $source_map = $this->loadSourceFileMap(); 436 $this->log( 437 pht('Found %s files.', new PhutilNumber(count($source_map)))); 438 439 // Load the symbol cache with existing parsed symbols. This allows us 440 // to remap libraries quickly by analyzing only changed files. 441 $this->log(pht('Loading symbol cache...')); 442 $symbol_cache = $this->loadSymbolCache(); 443 444 // If the XHPAST binary is not up-to-date, build it now. Otherwise, 445 // `extract-symbols.php` will attempt to build the binary and will fail 446 // miserably because it will be trying to build the same file multiple 447 // times in parallel. 448 if (!PhutilXHPASTBinary::isAvailable()) { 449 PhutilXHPASTBinary::build(); 450 } 451 452 // Build out the symbol analysis for all the files in the library. For 453 // each file, check if it's in cache. If we miss in the cache, do a fresh 454 // analysis. 455 $symbol_map = array(); 456 $futures = array(); 457 foreach ($source_map as $file => $hash) { 458 if (!empty($symbol_cache[$hash])) { 459 $symbol_map[$file] = $symbol_cache[$hash]; 460 continue; 461 } 462 $futures[$file] = $this->buildSymbolAnalysisFuture($file); 463 } 464 $this->log( 465 pht('Found %s files in cache.', new PhutilNumber(count($symbol_map)))); 466 467 // Run the analyzer on any files which need analysis. 468 if ($futures) { 469 $limit = $this->subprocessLimit; 470 471 $this->log( 472 pht( 473 'Analyzing %s file(s) with %s subprocess(es)...', 474 phutil_count($futures), 475 new PhutilNumber($limit))); 476 477 $progress = new PhutilConsoleProgressBar(); 478 if ($this->quiet) { 479 $progress->setQuiet(true); 480 } 481 $progress->setTotal(count($futures)); 482 483 $futures = id(new FutureIterator($futures)) 484 ->limit($limit); 485 foreach ($futures as $file => $future) { 486 $result = $future->resolveJSON(); 487 if (empty($result['error'])) { 488 $symbol_map[$file] = $result; 489 } else { 490 $progress->done(false); 491 throw new XHPASTSyntaxErrorException( 492 $result['line'], 493 $file.': '.$result['error']); 494 } 495 $progress->update(1); 496 } 497 $progress->done(); 498 } 499 500 $this->fileSymbolMap = $symbol_map; 501 502 // We're done building the cache, so write it out immediately. Note that 503 // we've only retained entries for files we found, so this implicitly cleans 504 // out old cache entries. 505 $this->writeSymbolCache($symbol_map, $source_map); 506 507 // Our map is up to date, so either show it on stdout or write it to disk. 508 $this->log(pht('Building library map...')); 509 510 $this->librarySymbolMap = $this->buildLibraryMap($symbol_map); 511 } 512 513 514} 515