1<?php
2
3/**
4 * Build maps of libphutil libraries. libphutil uses the library map to locate
5 * and load classes and functions in the library.
6 *
7 * @task map      Mapping libphutil Libraries
8 * @task path     Path Management
9 * @task symbol   Symbol Analysis and Caching
10 * @task source   Source Management
11 */
12final class PhutilLibraryMapBuilder extends Phobject {
13
14  private $root;
15  private $quiet = true;
16  private $subprocessLimit = 8;
17
18  private $fileSymbolMap;
19  private $librarySymbolMap;
20
21  const LIBRARY_MAP_VERSION_KEY   = '__library_version__';
22  const LIBRARY_MAP_VERSION       = 2;
23
24  const SYMBOL_CACHE_VERSION_KEY  = '__symbol_cache_version__';
25  const SYMBOL_CACHE_VERSION      = 11;
26
27
28/* -(  Mapping libphutil Libraries  )---------------------------------------- */
29
30  /**
31   * Create a new map builder for a library.
32   *
33   * @param string Path to the library root.
34   *
35   * @task map
36   */
37  public function __construct($root) {
38    $this->root = $root;
39  }
40
41  /**
42   * Control status output. Use `--quiet` to set this.
43   *
44   * @param  bool  If true, don't show status output.
45   * @return this
46   *
47   * @task map
48   */
49  public function setQuiet($quiet) {
50    $this->quiet = $quiet;
51    return $this;
52  }
53
54  /**
55   * Control subprocess parallelism limit. Use `--limit` to set this.
56   *
57   * @param  int   Maximum number of subprocesses to run in parallel.
58   * @return this
59   *
60   * @task map
61   */
62  public function setSubprocessLimit($limit) {
63    $this->subprocessLimit = $limit;
64    return $this;
65  }
66
67  /**
68   * Get the map of symbols in this library, analyzing the library to build it
69   * if necessary.
70   *
71   * @return map<string, wild> Information about symbols in this library.
72   *
73   * @task map
74   */
75  public function buildMap() {
76    if ($this->librarySymbolMap === null) {
77      $this->analyzeLibrary();
78    }
79    return $this->librarySymbolMap;
80  }
81
82
83  /**
84   * Get the map of files in this library, analyzing the library to build it
85   * if necessary.
86   *
87   * Returns a map of file paths to information about symbols used and defined
88   * in the file.
89   *
90   * @return map<string, wild> Information about files in this library.
91   *
92   * @task map
93   */
94  public function buildFileSymbolMap() {
95    if ($this->fileSymbolMap === null) {
96      $this->analyzeLibrary();
97    }
98    return $this->fileSymbolMap;
99  }
100
101  /**
102   * Build and update the library map.
103   *
104   * @return void
105   *
106   * @task map
107   */
108  public function buildAndWriteMap() {
109    $library_map = $this->buildMap();
110
111    $this->log(pht('Writing map...'));
112    $this->writeLibraryMap($library_map);
113  }
114
115  /**
116   * Write a status message to the user, if not running in quiet mode.
117   *
118   * @param  string  Message to write.
119   * @return this
120   *
121   * @task map
122   */
123  private function log($message) {
124    if (!$this->quiet) {
125      @fwrite(STDERR, "%s\n", $message);
126    }
127    return $this;
128  }
129
130
131/* -(  Path Management  )---------------------------------------------------- */
132
133  /**
134   * Get the path to some file in the library.
135   *
136   * @param  string  A library-relative path. If omitted, returns the library
137   *                 root path.
138   * @return string  An absolute path.
139   *
140   * @task path
141   */
142  private function getPath($path = '') {
143    return $this->root.'/'.$path;
144  }
145
146  /**
147   * Get the path to the symbol cache file.
148   *
149   * @return string Absolute path to symbol cache.
150   *
151   * @task path
152   */
153  private function getPathForSymbolCache() {
154    return $this->getPath('.phutil_module_cache');
155  }
156
157  /**
158   * Get the path to the map file.
159   *
160   * @return string Absolute path to the library map.
161   *
162   * @task path
163   */
164  private function getPathForLibraryMap() {
165    return $this->getPath('__phutil_library_map__.php');
166  }
167
168  /**
169   * Get the path to the library init file.
170   *
171   * @return string Absolute path to the library init file
172   *
173   * @task path
174   */
175  private function getPathForLibraryInit() {
176    return $this->getPath('__phutil_library_init__.php');
177  }
178
179
180/* -(  Symbol Analysis and Caching  )---------------------------------------- */
181
182  /**
183   * Load the library symbol cache, if it exists and is readable and valid.
184   *
185   * @return dict  Map of content hashes to cache of output from
186   *               `extract-symbols.php`.
187   *
188   * @task symbol
189   */
190  private function loadSymbolCache() {
191    $cache_file = $this->getPathForSymbolCache();
192
193    try {
194      $cache = Filesystem::readFile($cache_file);
195    } catch (Exception $ex) {
196      $cache = null;
197    }
198
199    $symbol_cache = array();
200    if ($cache) {
201      try {
202        $symbol_cache = phutil_json_decode($cache);
203      } catch (PhutilJSONParserException $ex) {
204        $symbol_cache = array();
205      }
206    }
207
208    $version = idx($symbol_cache, self::SYMBOL_CACHE_VERSION_KEY);
209    if ($version != self::SYMBOL_CACHE_VERSION) {
210      // Throw away caches from a different version of the library.
211      $symbol_cache = array();
212    }
213    unset($symbol_cache[self::SYMBOL_CACHE_VERSION_KEY]);
214
215    return $symbol_cache;
216  }
217
218  /**
219   * Write a symbol map to disk cache.
220   *
221   * @param  dict  Symbol map of relative paths to symbols.
222   * @param  dict  Source map (like @{method:loadSourceFileMap}).
223   * @return void
224   *
225   * @task symbol
226   */
227  private function writeSymbolCache(array $symbol_map, array $source_map) {
228    $cache_file = $this->getPathForSymbolCache();
229
230    $cache = array(
231      self::SYMBOL_CACHE_VERSION_KEY => self::SYMBOL_CACHE_VERSION,
232    );
233
234    foreach ($symbol_map as $file => $symbols) {
235      $cache[$source_map[$file]] = $symbols;
236    }
237
238    $json = json_encode($cache);
239    try {
240      Filesystem::writeFile($cache_file, $json);
241    } catch (FilesystemException $ex) {
242      $this->log(pht('Unable to save the cache!'));
243    }
244  }
245
246  /**
247   * Drop the symbol cache, forcing a clean rebuild.
248   *
249   * @return this
250   *
251   * @task symbol
252   */
253  public function dropSymbolCache() {
254    $this->log(pht('Dropping symbol cache...'));
255    Filesystem::remove($this->getPathForSymbolCache());
256  }
257
258  /**
259   * Build a future which returns a `extract-symbols.php` analysis of a source
260   * file.
261   *
262   * @param  string  Relative path to the source file to analyze.
263   * @return Future  Analysis future.
264   *
265   * @task symbol
266   */
267  private function buildSymbolAnalysisFuture($file) {
268    $absolute_file = $this->getPath($file);
269    $bin = dirname(__FILE__).'/../../support/lib/extract-symbols.php';
270
271    return new ExecFuture('%%PHP_CMD%% -f %R -- --ugly %R', $bin, $absolute_file);
272  }
273
274
275/* -(  Source Management  )-------------------------------------------------- */
276
277  /**
278   * Build a map of all source files in a library to hashes of their content.
279   * Returns an array like this:
280   *
281   *   array(
282   *     'src/parser/ExampleParser.php' => '60b725f10c9c85c70d97880dfe8191b3',
283   *     // ...
284   *   );
285   *
286   * @return dict  Map of library-relative paths to content hashes.
287   * @task source
288   */
289  private function loadSourceFileMap() {
290    $root = $this->getPath();
291
292    $init = $this->getPathForLibraryInit();
293    if (!Filesystem::pathExists($init)) {
294      throw new Exception(
295        pht(
296          "Provided path '%s' is not a %s library.",
297          $root,
298          'phutil'));
299    }
300
301    $files = id(new FileFinder($root))
302      ->withType('f')
303      ->withSuffix('php')
304      ->excludePath('*/.*')
305      ->setGenerateChecksums(true)
306      ->find();
307
308    $extensions_dir = 'extensions/';
309    $extensions_len = strlen($extensions_dir);
310
311    $map = array();
312    foreach ($files as $file => $hash) {
313      $file = Filesystem::readablePath($file, $root);
314      $file = ltrim($file, '/');
315
316      if (dirname($file) == '.') {
317        // We don't permit normal source files at the root level, so just ignore
318        // them; they're special library files.
319        continue;
320      }
321
322      // Ignore files in the extensions/ directory.
323      if (!strncmp($file, $extensions_dir, $extensions_len)) {
324        continue;
325      }
326
327      // We include also filename in the hash to handle cases when the file is
328      // moved without modifying its content.
329      $map[$file] = md5($hash.$file);
330    }
331
332    return $map;
333  }
334
335  /**
336   * Convert the symbol analysis of all the source files in the library into
337   * a library map.
338   *
339   * @param   dict  Symbol analysis of all source files.
340   * @return  dict  Library map.
341   * @task source
342   */
343  private function buildLibraryMap(array $symbol_map) {
344    $library_map = array(
345      'class'     => array(),
346      'function'  => array(),
347      'xmap'      => array(),
348    );
349
350    // Detect duplicate symbols within the library.
351    foreach ($symbol_map as $file => $info) {
352      foreach ($info['have'] as $type => $symbols) {
353        foreach ($symbols as $symbol => $declaration) {
354          $lib_type = ($type == 'interface') ? 'class' : $type;
355          if (!empty($library_map[$lib_type][$symbol])) {
356            $prior = $library_map[$lib_type][$symbol];
357            throw new Exception(
358              pht(
359                "Definition of %s '%s' in file '%s' duplicates prior ".
360                "definition in file '%s'. You can not declare the ".
361                "same symbol twice.",
362                $type,
363                $symbol,
364                $file,
365                $prior));
366          }
367          $library_map[$lib_type][$symbol] = $file;
368        }
369      }
370      $library_map['xmap'] += $info['xmap'];
371    }
372
373    // Simplify the common case (one parent) to make the file a little easier
374    // to deal with.
375    foreach ($library_map['xmap'] as $class => $extends) {
376      if (count($extends) == 1) {
377        $library_map['xmap'][$class] = reset($extends);
378      }
379    }
380
381    // Sort the map so it is relatively stable across changes.
382    foreach ($library_map as $key => $symbols) {
383      ksort($symbols);
384      $library_map[$key] = $symbols;
385    }
386    ksort($library_map);
387
388    return $library_map;
389  }
390
391  /**
392   * Write a finalized library map.
393   *
394   * @param  dict Library map structure to write.
395   * @return void
396   *
397   * @task source
398   */
399  private function writeLibraryMap(array $library_map) {
400    $map_file = $this->getPathForLibraryMap();
401    $version = self::LIBRARY_MAP_VERSION;
402
403    $library_map = array(
404      self::LIBRARY_MAP_VERSION_KEY => $version,
405    ) + $library_map;
406
407    $library_map = phutil_var_export($library_map);
408    $at = '@';
409
410    $source_file = <<<EOPHP
411<?php
412
413/**
414 * This file is automatically generated. Use 'arc liberate' to rebuild it.
415 *
416 * {$at}generated
417 * {$at}phutil-library-version {$version}
418 */
419phutil_register_library_map({$library_map});
420
421EOPHP;
422
423    Filesystem::writeFile($map_file, $source_file);
424  }
425
426
427  /**
428   * Analyze the library, generating the file and symbol maps.
429   *
430   * @return void
431   */
432  private function analyzeLibrary() {
433    // Identify all the ".php" source files in the library.
434    $this->log(pht('Finding source files...'));
435    $source_map = $this->loadSourceFileMap();
436    $this->log(
437      pht('Found %s files.', new PhutilNumber(count($source_map))));
438
439    // Load the symbol cache with existing parsed symbols. This allows us
440    // to remap libraries quickly by analyzing only changed files.
441    $this->log(pht('Loading symbol cache...'));
442    $symbol_cache = $this->loadSymbolCache();
443
444    // If the XHPAST binary is not up-to-date, build it now. Otherwise,
445    // `extract-symbols.php` will attempt to build the binary and will fail
446    // miserably because it will be trying to build the same file multiple
447    // times in parallel.
448    if (!PhutilXHPASTBinary::isAvailable()) {
449      PhutilXHPASTBinary::build();
450    }
451
452    // Build out the symbol analysis for all the files in the library. For
453    // each file, check if it's in cache. If we miss in the cache, do a fresh
454    // analysis.
455    $symbol_map = array();
456    $futures = array();
457    foreach ($source_map as $file => $hash) {
458      if (!empty($symbol_cache[$hash])) {
459        $symbol_map[$file] = $symbol_cache[$hash];
460        continue;
461      }
462      $futures[$file] = $this->buildSymbolAnalysisFuture($file);
463    }
464    $this->log(
465      pht('Found %s files in cache.', new PhutilNumber(count($symbol_map))));
466
467    // Run the analyzer on any files which need analysis.
468    if ($futures) {
469      $limit = $this->subprocessLimit;
470
471      $this->log(
472        pht(
473          'Analyzing %s file(s) with %s subprocess(es)...',
474          phutil_count($futures),
475          new PhutilNumber($limit)));
476
477      $progress = new PhutilConsoleProgressBar();
478      if ($this->quiet) {
479        $progress->setQuiet(true);
480      }
481      $progress->setTotal(count($futures));
482
483      $futures = id(new FutureIterator($futures))
484        ->limit($limit);
485      foreach ($futures as $file => $future) {
486        $result = $future->resolveJSON();
487        if (empty($result['error'])) {
488          $symbol_map[$file] = $result;
489        } else {
490          $progress->done(false);
491          throw new XHPASTSyntaxErrorException(
492            $result['line'],
493            $file.': '.$result['error']);
494        }
495        $progress->update(1);
496      }
497      $progress->done();
498    }
499
500    $this->fileSymbolMap = $symbol_map;
501
502    // We're done building the cache, so write it out immediately. Note that
503    // we've only retained entries for files we found, so this implicitly cleans
504    // out old cache entries.
505    $this->writeSymbolCache($symbol_map, $source_map);
506
507    // Our map is up to date, so either show it on stdout or write it to disk.
508    $this->log(pht('Building library map...'));
509
510    $this->librarySymbolMap = $this->buildLibraryMap($symbol_map);
511  }
512
513
514}
515