1#!/usr/bin/env python2.7
2# Copyright 2018 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Tool for doing Java refactors over native methods.
7
8Converts
9(a) non-static natives to static natives using @JCaller
10e.g.
11class A {
12  native void nativeFoo(int a);
13
14  void bar() {
15    nativeFoo(5);
16  }
17}
18->
19import .....JCaller
20class A {
21  static native void nativeFoo(@JCaller caller, int a);
22
23  void bar() {
24    nativeFoo(A.this, 5);
25  }
26}
27
28(b) static natives to new mockable static natives.
29e.g.
30class A {
31  static native void nativeFoo(@JCaller caller, int a);
32
33  void bar() {
34    nativeFoo(5);
35  }
36}
37->
38import .....JCaller
39class A {
40  void bar() {
41    AJni.get().foo(A.this, 5);
42  }
43
44  @NativeMethods
45  interface Natives {
46    static native void foo(@JCaller caller, int a);
47  }
48}
49
50Created for large refactors to @NativeMethods.
51Note: This tool does most of the heavy lifting in the conversion but
52there are some things that are difficult to implement with regex and
53infrequent enough that they can be done by hand.
54
55These exceptions are:
561) public native methods calls used in other classes are not refactored.
572) native methods inherited from a super class are not refactored
583) Non-static methods are always assumed to be called by the class instance
59instead of by another class using that object.
60"""
61
62from __future__ import print_function
63
64import argparse
65import sys
66import string
67import re
68import os
69import pickle
70
71import jni_generator
72
73_JNI_INTERFACE_TEMPLATES = string.Template("""
74    @NativeMethods
75    interface ${INTERFACE_NAME} {${METHODS}
76    }
77""")
78
79_JNI_METHOD_DECL = string.Template("""
80        ${RETURN_TYPE} ${NAME}($PARAMS);""")
81
82_COMMENT_REGEX_STRING = r'(?:(?:(?:\/\*[^\/]*\*\/)+|(?:\/\/[^\n]*?\n))+\s*)*'
83
84_NATIVES_REGEX = re.compile(
85    r'(?P<comments>' + _COMMENT_REGEX_STRING + ')'
86    r'(?P<annotations>(@NativeClassQualifiedName'
87    r'\(\"(?P<native_class_name>.*?)\"\)\s+)?'
88    r'(@NativeCall(\(\"(?P<java_class_name>.*?)\"\))\s+)?)'
89    r'(?P<qualifiers>\w+\s\w+|\w+|\s+)\s+static\s*native '
90    r'(?P<return_type>\S*) '
91    r'(?P<name>native\w+)\((?P<params>.*?)\);\n', re.DOTALL)
92
93_NON_STATIC_NATIVES_REGEX = re.compile(
94    r'(?P<comments>' + _COMMENT_REGEX_STRING + ')'
95    r'(?P<annotations>(@NativeClassQualifiedName'
96    r'\(\"(?P<native_class_name>.*?)\"\)\s+)?'
97    r'(@NativeCall(\(\"(?P<java_class_name>.*?)\"\))\s+)?)'
98    r'(?P<qualifiers>\w+\s\w+|\w+|\s+)\s*native '
99    r'(?P<return_type>\S*) '
100    r'(?P<name>native\w+)\((?P<params>.*?)\);\n', re.DOTALL)
101_NATIVE_PTR_REGEX = re.compile(r'\s*long native.*')
102
103JNI_IMPORT_STRING = 'import org.chromium.base.annotations.NativeMethods;'
104IMPORT_REGEX = re.compile(r'^import .*?;', re.MULTILINE)
105
106PICKLE_LOCATION = './jni_ref_pickle'
107
108
109def build_method_declaration(return_type, name, params, annotations, comments):
110  out = _JNI_METHOD_DECL.substitute({
111      'RETURN_TYPE': return_type,
112      'NAME': name,
113      'PARAMS': params
114  })
115  if annotations:
116    out = '\n' + annotations + out
117  if comments:
118    out = '\n' + comments + out
119  if annotations or comments:
120    out += '\n'
121  return out
122
123
124def add_chromium_import_to_java_file(contents, import_string):
125  # Just in cases there are no imports default to after the package statement.
126  import_insert = contents.find(';') + 1
127
128  # Insert in alphabetical order into org.chromium. This assumes
129  # that all files will contain some org.chromium import.
130  for match in IMPORT_REGEX.finditer(contents):
131    import_name = match.group()
132
133    if not 'import org.chromium' in import_name:
134      continue
135    if import_name > import_insert:
136      import_insert = match.start()
137      break
138    else:
139      import_insert = match.end() + 1
140
141  return "%s%s\n%s" % (contents[:import_insert], import_string,
142                       contents[import_insert:])
143
144
145def convert_nonstatic_to_static(java_file_name,
146                                skip_caller=False,
147                                dry=False,
148                                verbose=True):
149  if java_file_name is None:
150    return
151  if not os.path.isfile(java_file_name):
152    if verbose:
153      print('%s does not exist', java_file_name)
154    return
155
156  with open(java_file_name, 'r') as f:
157    contents = f.read()
158
159  no_comment_content = jni_generator.RemoveComments(contents)
160  parsed_natives = jni_generator.ExtractNatives(no_comment_content, 'long')
161  non_static_natives = [n for n in parsed_natives if not n.static]
162  if not non_static_natives:
163    if verbose:
164      print('no natives found')
165    return
166
167  class_name = jni_generator.ExtractFullyQualifiedJavaClassName(
168      java_file_name, no_comment_content).split('/')[-1]
169
170  # For fixing call sites.
171  replace_patterns = []
172  should_append_comma = []
173  should_prepend_comma = []
174
175  new_contents = contents
176
177  # 1. Change non-static -> static.
178  insertion_offset = 0
179
180  matches = []
181  for match in _NON_STATIC_NATIVES_REGEX.finditer(contents):
182    if not 'static' in match.group('qualifiers'):
183      matches.append(match)
184      # Insert static as a keyword.
185      qual_end = match.end('qualifiers') + insertion_offset
186      insert_str = ' static '
187      new_contents = new_contents[:qual_end] + insert_str + new_contents[
188          qual_end:]
189      insertion_offset += len(insert_str)
190
191      if skip_caller:
192        continue
193
194      # Insert an object param.
195      insert_str = '%s caller' % class_name
196      # No params.
197      if not match.group('params'):
198        start = insertion_offset + match.start('params')
199        append_comma = False
200        prepend_comma = False
201
202      # One or more params.
203      else:
204        # Has mNativePtr.
205        if _NATIVE_PTR_REGEX.match(match.group('params')):
206          # Only 1 param, append to end of params.
207          if match.group('params').count(',') == 0:
208            start = insertion_offset + match.end('params')
209            append_comma = False
210            prepend_comma = True
211          # Multiple params, insert after first param.
212          else:
213            comma_pos = match.group('params').find(',')
214            start = insertion_offset + match.start('params') + comma_pos + 1
215            append_comma = True
216            prepend_comma = False
217        else:
218          # No mNativePtr, insert as first param.
219          start = insertion_offset + match.start('params')
220          append_comma = True
221          prepend_comma = False
222
223      if prepend_comma:
224        insert_str = ', ' + insert_str
225      if append_comma:
226        insert_str = insert_str + ', '
227      new_contents = new_contents[:start] + insert_str + new_contents[start:]
228
229      # Match lines that don't have a native keyword.
230      native_match = r'\((\s*?(([ms]Native\w+)|([ms]\w+Android(Ptr)?)),?)?)'
231      replace_patterns.append(r'(^\s*' + match.group('name') + native_match)
232      replace_patterns.append(r'(return ' + match.group('name') + native_match)
233      replace_patterns.append(r'([\:\)\(\+\*\?\&\|,\.\-\=\!\/][ \t]*' +
234                              match.group('name') + native_match)
235
236      should_append_comma.extend([append_comma] * 3)
237      should_prepend_comma.extend([prepend_comma] * 3)
238      insertion_offset += len(insert_str)
239
240  assert len(matches) == len(non_static_natives), ('Regex missed a native '
241                                                   'method that was found by '
242                                                   'the jni_generator.')
243
244  # 2. Add a this param to all calls.
245  for i, r in enumerate(replace_patterns):
246    prepend_comma = ', ' if should_prepend_comma[i] else ''
247    append_comma = ', ' if should_append_comma[i] else ''
248    repl_str = '\g<1>' + prepend_comma + ' %s.this' + append_comma
249    new_contents = re.sub(
250        r, repl_str % class_name, new_contents, flags=re.MULTILINE)
251
252  if dry:
253    print(new_contents)
254  else:
255    with open(java_file_name, 'w') as f:
256      f.write(new_contents)
257
258
259def filter_files_with_natives(files, verbose=True):
260  filtered = []
261  i = 1
262  for java_file_name in files:
263    if not os.path.isfile(java_file_name):
264      print('does not exist')
265      return
266    if verbose:
267      print('Processing %s/%s - %s ' % (i, len(files), java_file_name))
268    with open(java_file_name, 'r') as f:
269      contents = f.read()
270    no_comment_content = jni_generator.RemoveComments(contents)
271    natives = jni_generator.ExtractNatives(no_comment_content, 'long')
272
273    if len(natives) > 1:
274      filtered.append(java_file_name)
275    i += 1
276  return filtered
277
278
279def convert_file_to_proxy_natives(java_file_name, dry=False, verbose=True):
280  if not os.path.isfile(java_file_name):
281    if verbose:
282      print('%s does not exist', java_file_name)
283    return
284
285  with open(java_file_name, 'r') as f:
286    contents = f.read()
287
288  no_comment_content = jni_generator.RemoveComments(contents)
289  natives = jni_generator.ExtractNatives(no_comment_content, 'long')
290
291  static_natives = [n for n in natives if n.static]
292  if not static_natives:
293    if verbose:
294      print('%s has no static natives.', java_file_name)
295    return
296
297  contents = add_chromium_import_to_java_file(contents, JNI_IMPORT_STRING)
298
299  # Extract comments and annotations above native methods.
300  native_map = {}
301  for itr in re.finditer(_NATIVES_REGEX, contents):
302    n_dict = {}
303    n_dict['annotations'] = itr.group('annotations').strip()
304    n_dict['comments'] = itr.group('comments').strip()
305    n_dict['params'] = itr.group('params').strip()
306    native_map[itr.group('name')] = n_dict
307
308  # Using static natives here ensures all the methods that are picked up by
309  # the JNI generator are also caught by our own regex.
310  methods = []
311  for n in static_natives:
312    new_name = n.name[0].lower() + n.name[1:]
313    n_dict = native_map['native' + n.name]
314    params = n_dict['params']
315    comments = n_dict['comments']
316    annotations = n_dict['annotations']
317    methods.append(
318        build_method_declaration(n.return_type, new_name, params, annotations,
319                                 comments))
320
321  fully_qualified_class = jni_generator.ExtractFullyQualifiedJavaClassName(
322      java_file_name, contents)
323  class_name = fully_qualified_class.split('/')[-1]
324  jni_class_name = class_name + 'Jni'
325
326  # Remove all old declarations.
327  for n in static_natives:
328    pattern = _NATIVES_REGEX
329    contents = re.sub(pattern, '', contents)
330
331  # Replace occurences with new signature.
332  for n in static_natives:
333    # Okay not to match first (.
334    # Since even if natives share a prefix, the replacement is the same.
335    # E.g. if nativeFoo() and nativeFooBar() are both statics
336    # and in nativeFooBar() we replace nativeFoo -> AJni.get().foo
337    # the result is the same as replacing nativeFooBar() -> AJni.get().fooBar
338    pattern = r'native%s' % n.name
339    lower_name = n.name[0].lower() + n.name[1:]
340    contents = re.sub(pattern, '%s.get().%s' % (jni_class_name, lower_name),
341                      contents)
342
343  # Build and insert the @NativeMethods interface.
344  interface = _JNI_INTERFACE_TEMPLATES.substitute({
345      'INTERFACE_NAME': 'Natives',
346      'METHODS': ''.join(methods)
347  })
348
349  # Insert the interface at the bottom of the top level class.
350  # Most of the time this will be before the last }.
351  insertion_point = contents.rfind('}')
352  contents = contents[:insertion_point] + '\n' + interface + contents[
353      insertion_point:]
354
355  if not dry:
356    with open(java_file_name, 'w') as f:
357      f.write(contents)
358  else:
359    print(contents)
360  return contents
361
362
363def main(argv):
364  arg_parser = argparse.ArgumentParser()
365
366  mutually_ex_group = arg_parser.add_mutually_exclusive_group()
367
368  mutually_ex_group.add_argument(
369      '-R',
370      '--recursive',
371      action='store_true',
372      help='Run recursively over all java files '
373      'descendants of the current directory.',
374      default=False)
375  mutually_ex_group.add_argument(
376      '--read_cache',
377      help='Reads paths to refactor from pickled file %s.' % PICKLE_LOCATION,
378      action='store_true',
379      default=False)
380  mutually_ex_group.add_argument(
381      '--source', help='Path to refactor single source file.', default=None)
382
383  arg_parser.add_argument(
384      '--cache',
385      action='store_true',
386      help='Finds all java files with native functions recursively from '
387      'the current directory, then pickles and saves them to %s and then'
388      'exits.' % PICKLE_LOCATION,
389      default=False)
390  arg_parser.add_argument(
391      '--dry_run',
392      default=False,
393      action='store_true',
394      help='Print refactor output to console instead '
395      'of replacing the contents of files.')
396  arg_parser.add_argument(
397      '--nonstatic',
398      default=False,
399      action='store_true',
400      help='If true converts native nonstatic methods to static methods'
401      ' instead of converting static methods to new jni.')
402  arg_parser.add_argument(
403      '--verbose', default=False, action='store_true', help='')
404  arg_parser.add_argument(
405      '--ignored-paths',
406      action='append',
407      help='Paths to ignore during conversion.')
408  arg_parser.add_argument(
409      '--skip-caller-arg',
410      default=False,
411      action='store_true',
412      help='Do not add the "this" param when converting non-static methods.')
413
414  args = arg_parser.parse_args()
415
416  java_file_paths = []
417
418  if args.source:
419    java_file_paths = [args.source]
420  elif args.read_cache:
421    print('Reading paths from ' + PICKLE_LOCATION)
422    with open(PICKLE_LOCATION, 'r') as file:
423      java_file_paths = pickle.load(file)
424      print('Found %s java paths.' % len(java_file_paths))
425  elif args.recursive:
426    for root, _, files in os.walk(os.path.abspath('.')):
427      java_file_paths.extend(
428          ['%s/%s' % (root, f) for f in files if f.endswith('.java')])
429
430  else:
431    # Get all java files in current dir.
432    java_file_paths = filter(lambda x: x.endswith('.java'),
433                             map(os.path.abspath, os.listdir('.')))
434
435  if args.ignored_paths:
436    java_file_paths = [
437        path for path in java_file_paths
438        if all(p not in path for p in args.ignored_paths)
439    ]
440
441  if args.cache:
442    with open(PICKLE_LOCATION, 'w') as file:
443      pickle.dump(filter_files_with_natives(java_file_paths), file)
444      print('Java files with proxy natives written to ' + PICKLE_LOCATION)
445
446  i = 1
447  for f in java_file_paths:
448    print(f)
449    if args.nonstatic:
450      convert_nonstatic_to_static(
451          f,
452          skip_caller=args.skip_caller_arg,
453          dry=args.dry_run,
454          verbose=args.verbose)
455    else:
456      convert_file_to_proxy_natives(f, dry=args.dry_run, verbose=args.verbose)
457    print('Done converting %s/%s' % (i, len(java_file_paths)))
458    i += 1
459
460  print('Done please run git cl format.')
461
462
463if __name__ == '__main__':
464  sys.exit(main(sys.argv))
465