1#!/usr/bin/env python2.7 2# Copyright 2018 The Chromium Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Tool for doing Java refactors over native methods. 7 8Converts 9(a) non-static natives to static natives using @JCaller 10e.g. 11class A { 12 native void nativeFoo(int a); 13 14 void bar() { 15 nativeFoo(5); 16 } 17} 18-> 19import .....JCaller 20class A { 21 static native void nativeFoo(@JCaller caller, int a); 22 23 void bar() { 24 nativeFoo(A.this, 5); 25 } 26} 27 28(b) static natives to new mockable static natives. 29e.g. 30class A { 31 static native void nativeFoo(@JCaller caller, int a); 32 33 void bar() { 34 nativeFoo(5); 35 } 36} 37-> 38import .....JCaller 39class A { 40 void bar() { 41 AJni.get().foo(A.this, 5); 42 } 43 44 @NativeMethods 45 interface Natives { 46 static native void foo(@JCaller caller, int a); 47 } 48} 49 50Created for large refactors to @NativeMethods. 51Note: This tool does most of the heavy lifting in the conversion but 52there are some things that are difficult to implement with regex and 53infrequent enough that they can be done by hand. 54 55These exceptions are: 561) public native methods calls used in other classes are not refactored. 572) native methods inherited from a super class are not refactored 583) Non-static methods are always assumed to be called by the class instance 59instead of by another class using that object. 60""" 61 62from __future__ import print_function 63 64import argparse 65import sys 66import string 67import re 68import os 69import pickle 70 71import jni_generator 72 73_JNI_INTERFACE_TEMPLATES = string.Template(""" 74 @NativeMethods 75 interface ${INTERFACE_NAME} {${METHODS} 76 } 77""") 78 79_JNI_METHOD_DECL = string.Template(""" 80 ${RETURN_TYPE} ${NAME}($PARAMS);""") 81 82_COMMENT_REGEX_STRING = r'(?:(?:(?:\/\*[^\/]*\*\/)+|(?:\/\/[^\n]*?\n))+\s*)*' 83 84_NATIVES_REGEX = re.compile( 85 r'(?P<comments>' + _COMMENT_REGEX_STRING + ')' 86 r'(?P<annotations>(@NativeClassQualifiedName' 87 r'\(\"(?P<native_class_name>.*?)\"\)\s+)?' 88 r'(@NativeCall(\(\"(?P<java_class_name>.*?)\"\))\s+)?)' 89 r'(?P<qualifiers>\w+\s\w+|\w+|\s+)\s+static\s*native ' 90 r'(?P<return_type>\S*) ' 91 r'(?P<name>native\w+)\((?P<params>.*?)\);\n', re.DOTALL) 92 93_NON_STATIC_NATIVES_REGEX = re.compile( 94 r'(?P<comments>' + _COMMENT_REGEX_STRING + ')' 95 r'(?P<annotations>(@NativeClassQualifiedName' 96 r'\(\"(?P<native_class_name>.*?)\"\)\s+)?' 97 r'(@NativeCall(\(\"(?P<java_class_name>.*?)\"\))\s+)?)' 98 r'(?P<qualifiers>\w+\s\w+|\w+|\s+)\s*native ' 99 r'(?P<return_type>\S*) ' 100 r'(?P<name>native\w+)\((?P<params>.*?)\);\n', re.DOTALL) 101_NATIVE_PTR_REGEX = re.compile(r'\s*long native.*') 102 103JNI_IMPORT_STRING = 'import org.chromium.base.annotations.NativeMethods;' 104IMPORT_REGEX = re.compile(r'^import .*?;', re.MULTILINE) 105 106PICKLE_LOCATION = './jni_ref_pickle' 107 108 109def build_method_declaration(return_type, name, params, annotations, comments): 110 out = _JNI_METHOD_DECL.substitute({ 111 'RETURN_TYPE': return_type, 112 'NAME': name, 113 'PARAMS': params 114 }) 115 if annotations: 116 out = '\n' + annotations + out 117 if comments: 118 out = '\n' + comments + out 119 if annotations or comments: 120 out += '\n' 121 return out 122 123 124def add_chromium_import_to_java_file(contents, import_string): 125 # Just in cases there are no imports default to after the package statement. 126 import_insert = contents.find(';') + 1 127 128 # Insert in alphabetical order into org.chromium. This assumes 129 # that all files will contain some org.chromium import. 130 for match in IMPORT_REGEX.finditer(contents): 131 import_name = match.group() 132 133 if not 'import org.chromium' in import_name: 134 continue 135 if import_name > import_insert: 136 import_insert = match.start() 137 break 138 else: 139 import_insert = match.end() + 1 140 141 return "%s%s\n%s" % (contents[:import_insert], import_string, 142 contents[import_insert:]) 143 144 145def convert_nonstatic_to_static(java_file_name, 146 skip_caller=False, 147 dry=False, 148 verbose=True): 149 if java_file_name is None: 150 return 151 if not os.path.isfile(java_file_name): 152 if verbose: 153 print('%s does not exist', java_file_name) 154 return 155 156 with open(java_file_name, 'r') as f: 157 contents = f.read() 158 159 no_comment_content = jni_generator.RemoveComments(contents) 160 parsed_natives = jni_generator.ExtractNatives(no_comment_content, 'long') 161 non_static_natives = [n for n in parsed_natives if not n.static] 162 if not non_static_natives: 163 if verbose: 164 print('no natives found') 165 return 166 167 class_name = jni_generator.ExtractFullyQualifiedJavaClassName( 168 java_file_name, no_comment_content).split('/')[-1] 169 170 # For fixing call sites. 171 replace_patterns = [] 172 should_append_comma = [] 173 should_prepend_comma = [] 174 175 new_contents = contents 176 177 # 1. Change non-static -> static. 178 insertion_offset = 0 179 180 matches = [] 181 for match in _NON_STATIC_NATIVES_REGEX.finditer(contents): 182 if not 'static' in match.group('qualifiers'): 183 matches.append(match) 184 # Insert static as a keyword. 185 qual_end = match.end('qualifiers') + insertion_offset 186 insert_str = ' static ' 187 new_contents = new_contents[:qual_end] + insert_str + new_contents[ 188 qual_end:] 189 insertion_offset += len(insert_str) 190 191 if skip_caller: 192 continue 193 194 # Insert an object param. 195 insert_str = '%s caller' % class_name 196 # No params. 197 if not match.group('params'): 198 start = insertion_offset + match.start('params') 199 append_comma = False 200 prepend_comma = False 201 202 # One or more params. 203 else: 204 # Has mNativePtr. 205 if _NATIVE_PTR_REGEX.match(match.group('params')): 206 # Only 1 param, append to end of params. 207 if match.group('params').count(',') == 0: 208 start = insertion_offset + match.end('params') 209 append_comma = False 210 prepend_comma = True 211 # Multiple params, insert after first param. 212 else: 213 comma_pos = match.group('params').find(',') 214 start = insertion_offset + match.start('params') + comma_pos + 1 215 append_comma = True 216 prepend_comma = False 217 else: 218 # No mNativePtr, insert as first param. 219 start = insertion_offset + match.start('params') 220 append_comma = True 221 prepend_comma = False 222 223 if prepend_comma: 224 insert_str = ', ' + insert_str 225 if append_comma: 226 insert_str = insert_str + ', ' 227 new_contents = new_contents[:start] + insert_str + new_contents[start:] 228 229 # Match lines that don't have a native keyword. 230 native_match = r'\((\s*?(([ms]Native\w+)|([ms]\w+Android(Ptr)?)),?)?)' 231 replace_patterns.append(r'(^\s*' + match.group('name') + native_match) 232 replace_patterns.append(r'(return ' + match.group('name') + native_match) 233 replace_patterns.append(r'([\:\)\(\+\*\?\&\|,\.\-\=\!\/][ \t]*' + 234 match.group('name') + native_match) 235 236 should_append_comma.extend([append_comma] * 3) 237 should_prepend_comma.extend([prepend_comma] * 3) 238 insertion_offset += len(insert_str) 239 240 assert len(matches) == len(non_static_natives), ('Regex missed a native ' 241 'method that was found by ' 242 'the jni_generator.') 243 244 # 2. Add a this param to all calls. 245 for i, r in enumerate(replace_patterns): 246 prepend_comma = ', ' if should_prepend_comma[i] else '' 247 append_comma = ', ' if should_append_comma[i] else '' 248 repl_str = '\g<1>' + prepend_comma + ' %s.this' + append_comma 249 new_contents = re.sub( 250 r, repl_str % class_name, new_contents, flags=re.MULTILINE) 251 252 if dry: 253 print(new_contents) 254 else: 255 with open(java_file_name, 'w') as f: 256 f.write(new_contents) 257 258 259def filter_files_with_natives(files, verbose=True): 260 filtered = [] 261 i = 1 262 for java_file_name in files: 263 if not os.path.isfile(java_file_name): 264 print('does not exist') 265 return 266 if verbose: 267 print('Processing %s/%s - %s ' % (i, len(files), java_file_name)) 268 with open(java_file_name, 'r') as f: 269 contents = f.read() 270 no_comment_content = jni_generator.RemoveComments(contents) 271 natives = jni_generator.ExtractNatives(no_comment_content, 'long') 272 273 if len(natives) > 1: 274 filtered.append(java_file_name) 275 i += 1 276 return filtered 277 278 279def convert_file_to_proxy_natives(java_file_name, dry=False, verbose=True): 280 if not os.path.isfile(java_file_name): 281 if verbose: 282 print('%s does not exist', java_file_name) 283 return 284 285 with open(java_file_name, 'r') as f: 286 contents = f.read() 287 288 no_comment_content = jni_generator.RemoveComments(contents) 289 natives = jni_generator.ExtractNatives(no_comment_content, 'long') 290 291 static_natives = [n for n in natives if n.static] 292 if not static_natives: 293 if verbose: 294 print('%s has no static natives.', java_file_name) 295 return 296 297 contents = add_chromium_import_to_java_file(contents, JNI_IMPORT_STRING) 298 299 # Extract comments and annotations above native methods. 300 native_map = {} 301 for itr in re.finditer(_NATIVES_REGEX, contents): 302 n_dict = {} 303 n_dict['annotations'] = itr.group('annotations').strip() 304 n_dict['comments'] = itr.group('comments').strip() 305 n_dict['params'] = itr.group('params').strip() 306 native_map[itr.group('name')] = n_dict 307 308 # Using static natives here ensures all the methods that are picked up by 309 # the JNI generator are also caught by our own regex. 310 methods = [] 311 for n in static_natives: 312 new_name = n.name[0].lower() + n.name[1:] 313 n_dict = native_map['native' + n.name] 314 params = n_dict['params'] 315 comments = n_dict['comments'] 316 annotations = n_dict['annotations'] 317 methods.append( 318 build_method_declaration(n.return_type, new_name, params, annotations, 319 comments)) 320 321 fully_qualified_class = jni_generator.ExtractFullyQualifiedJavaClassName( 322 java_file_name, contents) 323 class_name = fully_qualified_class.split('/')[-1] 324 jni_class_name = class_name + 'Jni' 325 326 # Remove all old declarations. 327 for n in static_natives: 328 pattern = _NATIVES_REGEX 329 contents = re.sub(pattern, '', contents) 330 331 # Replace occurences with new signature. 332 for n in static_natives: 333 # Okay not to match first (. 334 # Since even if natives share a prefix, the replacement is the same. 335 # E.g. if nativeFoo() and nativeFooBar() are both statics 336 # and in nativeFooBar() we replace nativeFoo -> AJni.get().foo 337 # the result is the same as replacing nativeFooBar() -> AJni.get().fooBar 338 pattern = r'native%s' % n.name 339 lower_name = n.name[0].lower() + n.name[1:] 340 contents = re.sub(pattern, '%s.get().%s' % (jni_class_name, lower_name), 341 contents) 342 343 # Build and insert the @NativeMethods interface. 344 interface = _JNI_INTERFACE_TEMPLATES.substitute({ 345 'INTERFACE_NAME': 'Natives', 346 'METHODS': ''.join(methods) 347 }) 348 349 # Insert the interface at the bottom of the top level class. 350 # Most of the time this will be before the last }. 351 insertion_point = contents.rfind('}') 352 contents = contents[:insertion_point] + '\n' + interface + contents[ 353 insertion_point:] 354 355 if not dry: 356 with open(java_file_name, 'w') as f: 357 f.write(contents) 358 else: 359 print(contents) 360 return contents 361 362 363def main(argv): 364 arg_parser = argparse.ArgumentParser() 365 366 mutually_ex_group = arg_parser.add_mutually_exclusive_group() 367 368 mutually_ex_group.add_argument( 369 '-R', 370 '--recursive', 371 action='store_true', 372 help='Run recursively over all java files ' 373 'descendants of the current directory.', 374 default=False) 375 mutually_ex_group.add_argument( 376 '--read_cache', 377 help='Reads paths to refactor from pickled file %s.' % PICKLE_LOCATION, 378 action='store_true', 379 default=False) 380 mutually_ex_group.add_argument( 381 '--source', help='Path to refactor single source file.', default=None) 382 383 arg_parser.add_argument( 384 '--cache', 385 action='store_true', 386 help='Finds all java files with native functions recursively from ' 387 'the current directory, then pickles and saves them to %s and then' 388 'exits.' % PICKLE_LOCATION, 389 default=False) 390 arg_parser.add_argument( 391 '--dry_run', 392 default=False, 393 action='store_true', 394 help='Print refactor output to console instead ' 395 'of replacing the contents of files.') 396 arg_parser.add_argument( 397 '--nonstatic', 398 default=False, 399 action='store_true', 400 help='If true converts native nonstatic methods to static methods' 401 ' instead of converting static methods to new jni.') 402 arg_parser.add_argument( 403 '--verbose', default=False, action='store_true', help='') 404 arg_parser.add_argument( 405 '--ignored-paths', 406 action='append', 407 help='Paths to ignore during conversion.') 408 arg_parser.add_argument( 409 '--skip-caller-arg', 410 default=False, 411 action='store_true', 412 help='Do not add the "this" param when converting non-static methods.') 413 414 args = arg_parser.parse_args() 415 416 java_file_paths = [] 417 418 if args.source: 419 java_file_paths = [args.source] 420 elif args.read_cache: 421 print('Reading paths from ' + PICKLE_LOCATION) 422 with open(PICKLE_LOCATION, 'r') as file: 423 java_file_paths = pickle.load(file) 424 print('Found %s java paths.' % len(java_file_paths)) 425 elif args.recursive: 426 for root, _, files in os.walk(os.path.abspath('.')): 427 java_file_paths.extend( 428 ['%s/%s' % (root, f) for f in files if f.endswith('.java')]) 429 430 else: 431 # Get all java files in current dir. 432 java_file_paths = filter(lambda x: x.endswith('.java'), 433 map(os.path.abspath, os.listdir('.'))) 434 435 if args.ignored_paths: 436 java_file_paths = [ 437 path for path in java_file_paths 438 if all(p not in path for p in args.ignored_paths) 439 ] 440 441 if args.cache: 442 with open(PICKLE_LOCATION, 'w') as file: 443 pickle.dump(filter_files_with_natives(java_file_paths), file) 444 print('Java files with proxy natives written to ' + PICKLE_LOCATION) 445 446 i = 1 447 for f in java_file_paths: 448 print(f) 449 if args.nonstatic: 450 convert_nonstatic_to_static( 451 f, 452 skip_caller=args.skip_caller_arg, 453 dry=args.dry_run, 454 verbose=args.verbose) 455 else: 456 convert_file_to_proxy_natives(f, dry=args.dry_run, verbose=args.verbose) 457 print('Done converting %s/%s' % (i, len(java_file_paths))) 458 i += 1 459 460 print('Done please run git cl format.') 461 462 463if __name__ == '__main__': 464 sys.exit(main(sys.argv)) 465