1#!/usr/bin/env python3 2# Copyright 2020 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5"""Command-line tool to run jdeps and process its output into a JSON file.""" 6 7import argparse 8import functools 9import math 10import multiprocessing 11import pathlib 12import os 13 14from typing import List, Tuple 15 16import class_dependency 17import git_utils 18import package_dependency 19import serialization 20import subprocess_utils 21 22DEFAULT_ROOT_TARGET = 'chrome/android:monochrome_public_bundle' 23 24 25def class_is_interesting(name: str): 26 """Checks if a jdeps class is a class we are actually interested in.""" 27 if name.startswith('org.chromium.'): 28 return True 29 return False 30 31 32# pylint: disable=useless-object-inheritance 33class JavaClassJdepsParser(object): 34 """A parser for jdeps class-level dependency output.""" 35 def __init__(self): # pylint: disable=missing-function-docstring 36 self._graph = class_dependency.JavaClassDependencyGraph() 37 38 @property 39 def graph(self): 40 """The dependency graph of the jdeps output. 41 42 Initialized as empty and updated using parse_raw_jdeps_output. 43 """ 44 return self._graph 45 46 def parse_raw_jdeps_output(self, build_target: str, jdeps_output: str): 47 """Parses the entirety of the jdeps output.""" 48 for line in jdeps_output.split('\n'): 49 self.parse_line(build_target, line) 50 51 def parse_line(self, build_target: str, line: str): 52 """Parses a line of jdeps output. 53 54 The assumed format of the line starts with 'name_1 -> name_2'. 55 """ 56 parsed = line.split() 57 if len(parsed) <= 3: 58 return 59 if parsed[2] == 'not' and parsed[3] == 'found': 60 return 61 if parsed[1] != '->': 62 return 63 64 dep_from = parsed[0] 65 dep_to = parsed[2] 66 if not class_is_interesting(dep_from): 67 return 68 69 key_from, nested_from = class_dependency.split_nested_class_from_key( 70 dep_from) 71 from_node: class_dependency.JavaClass = self._graph.add_node_if_new( 72 key_from) 73 from_node.add_build_target(build_target) 74 75 if not class_is_interesting(dep_to): 76 return 77 78 key_to, nested_to = class_dependency.split_nested_class_from_key( 79 dep_to) 80 81 self._graph.add_node_if_new(key_to) 82 if key_from != key_to: # Skip self-edges (class-nested dependency) 83 self._graph.add_edge_if_new(key_from, key_to) 84 if nested_from is not None: 85 from_node.add_nested_class(nested_from) 86 if nested_to is not None: 87 from_node.add_nested_class(nested_to) 88 89 90def _run_jdeps(jdeps_path: str, filepath: pathlib.Path) -> str: 91 """Runs jdeps on the given filepath and returns the output.""" 92 print(f'Running jdeps and parsing output for {filepath}') 93 return subprocess_utils.run_command( 94 [jdeps_path, '-R', '-verbose:class', filepath]) 95 96 97def _run_gn_desc_list_dependencies(build_output_dir: str, target: str, 98 gn_path: str) -> str: 99 """Runs gn desc to list all jars that a target depends on. 100 101 This includes direct and indirect dependencies.""" 102 return subprocess_utils.run_command( 103 [gn_path, 'desc', '--all', build_output_dir, target, 'deps']) 104 105 106JarTargetList = List[Tuple[str, pathlib.Path]] 107 108 109def list_original_targets_and_jars(gn_desc_output: str, build_output_dir: str, 110 cr_position: int) -> JarTargetList: 111 """Parses gn desc output to list original java targets and output jar paths. 112 113 Returns a list of tuples (build_target: str, jar_path: str), where: 114 - build_target is the original java dependency target in the form 115 "//path/to:target" 116 - jar_path is the path to the built jar in the build_output_dir, 117 including the path to the output dir 118 """ 119 jar_tuples: JarTargetList = [] 120 for build_target_line in gn_desc_output.split('\n'): 121 if not build_target_line.endswith('__compile_java'): 122 continue 123 build_target = build_target_line.strip() 124 original_build_target = build_target.replace('__compile_java', '') 125 jar_path = _get_jar_path_for_target(build_output_dir, build_target, 126 cr_position) 127 jar_tuples.append((original_build_target, jar_path)) 128 return jar_tuples 129 130 131def _get_jar_path_for_target(build_output_dir: str, build_target: str, 132 cr_position: int) -> str: 133 if cr_position == 0: # Not running on main branch, use current convention. 134 subdirectory = 'obj' 135 elif cr_position < 761560: # crrev.com/c/2161205 136 subdirectory = 'gen' 137 else: 138 subdirectory = 'obj' 139 """Calculates the output location of a jar for a java build target.""" 140 target_path, target_name = build_target.split(':') 141 assert target_path.startswith('//'), \ 142 f'Build target should start with "//" but is: "{build_target}"' 143 jar_dir = target_path[len('//'):] 144 jar_name = target_name.replace('__compile_java', '.javac.jar') 145 return pathlib.Path(build_output_dir) / subdirectory / jar_dir / jar_name 146 147 148def main(): 149 """Runs jdeps on all JARs a build target depends on. 150 151 Creates a JSON file from the jdeps output.""" 152 arg_parser = argparse.ArgumentParser( 153 description='Runs jdeps (dependency analysis tool) on all JARs a root ' 154 'build target depends on and writes the resulting dependency graph ' 155 'into a JSON file. The default root build target is ' 156 'chrome/android:monochrome_public_bundle.') 157 required_arg_group = arg_parser.add_argument_group('required arguments') 158 required_arg_group.add_argument('-C', 159 '--build_output_dir', 160 required=True, 161 help='Build output directory.') 162 required_arg_group.add_argument( 163 '-o', 164 '--output', 165 required=True, 166 help='Path to the file to write JSON output to. Will be created ' 167 'if it does not yet exist and overwrite existing ' 168 'content if it does.') 169 arg_parser.add_argument('-t', 170 '--target', 171 default=DEFAULT_ROOT_TARGET, 172 help='Root build target.') 173 arg_parser.add_argument('-d', 174 '--checkout-dir', 175 help='Path to the chromium checkout directory.') 176 arg_parser.add_argument('-j', 177 '--jdeps-path', 178 help='Path to the jdeps executable.') 179 arg_parser.add_argument('-g', 180 '--gn-path', 181 default='gn', 182 help='Path to the gn executable.') 183 arguments = arg_parser.parse_args() 184 185 if arguments.checkout_dir: 186 src_path = pathlib.Path(arguments.checkout_dir) 187 else: 188 src_path = pathlib.Path(__file__).resolve().parents[3] 189 190 if arguments.jdeps_path: 191 jdeps_path = pathlib.Path(arguments.jdeps_path) 192 else: 193 jdeps_path = src_path.joinpath('third_party/jdk/current/bin/jdeps') 194 195 # gn and git must be run from inside the git checkout. 196 os.chdir(src_path) 197 198 cr_position_str = git_utils.get_last_commit_cr_position() 199 cr_position = int(cr_position_str) if cr_position_str else 0 200 201 print('Getting list of dependency jars...') 202 gn_desc_output = _run_gn_desc_list_dependencies(arguments.build_output_dir, 203 arguments.target, 204 arguments.gn_path) 205 target_jars: JarTargetList = list_original_targets_and_jars( 206 gn_desc_output, arguments.build_output_dir, cr_position) 207 208 print('Running jdeps...') 209 # jdeps already has some parallelism 210 jdeps_process_number = math.ceil(multiprocessing.cpu_count() / 2) 211 with multiprocessing.Pool(jdeps_process_number) as pool: 212 jar_paths = [target_jar for _, target_jar in target_jars] 213 jdeps_outputs = pool.map(functools.partial(_run_jdeps, jdeps_path), 214 jar_paths) 215 216 print('Parsing jdeps output...') 217 jdeps_parser = JavaClassJdepsParser() 218 for raw_jdeps_output, (build_target, _) in zip(jdeps_outputs, target_jars): 219 jdeps_parser.parse_raw_jdeps_output(build_target, raw_jdeps_output) 220 221 class_graph = jdeps_parser.graph 222 print(f'Parsed class-level dependency graph, ' 223 f'got {class_graph.num_nodes} nodes ' 224 f'and {class_graph.num_edges} edges.') 225 226 package_graph = package_dependency.JavaPackageDependencyGraph(class_graph) 227 print(f'Created package-level dependency graph, ' 228 f'got {package_graph.num_nodes} nodes ' 229 f'and {package_graph.num_edges} edges.') 230 231 print(f'Dumping JSON representation to {arguments.output}.') 232 serialization.dump_class_and_package_graphs_to_file( 233 class_graph, package_graph, arguments.output) 234 235 236if __name__ == '__main__': 237 main() 238