Android: passes a list of qualified JNI parameters as a param to the generator.

base/android/jni_generator/jni_generator.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-# TODO (qinmin): Need to refactor this file as base should not know about
-# higher level concepts. Currently this file has knowledge about higher level
-# java classes.
-
 """Extracts native methods from a Java file and generates the JNI bindings.
 If you change this, please run and update the tests."""
 
 import collections
 import optparse
 import os
 import re
 import string
 from string import Template
 import subprocess
 import sys
 import textwrap
 import zipfile
 
-UNKNOWN_JAVA_TYPE_PREFIX = 'UNKNOWN_JAVA_TYPE: '
-
 
 class ParseError(Exception):
   """Exception thrown when we can't parse the input file."""
 
   def __init__(self, description, *context_lines):
     Exception.__init__(self)
     self.description = description
     self.context_lines = context_lines
 
   def __str__(self):
@@ skipping 70 matching lines @@
   elif java_type in java_type_map:
     return java_type_map[java_type]
   elif java_type.endswith('[]'):
     if java_type[:-2] in java_pod_type_map:
       return java_pod_type_map[java_type[:-2]] + 'Array'
     return 'jobjectArray'
   else:
     return 'jobject'
 
 
-def JavaParamToJni(param):
-  """Converts a java param into a JNI signature type."""
-  pod_param_map = {
-      'int': 'I',
-      'boolean': 'Z',
-      'long': 'J',
-      'double': 'D',
-      'float': 'F',
-      'byte': 'B',
-      'void': 'V',
-  }
-  object_param_list = [
-      'Ljava/lang/Boolean',
-      'Ljava/lang/Integer',
-      'Ljava/lang/Long',
-      'Ljava/lang/Object',
-      'Ljava/lang/String',
-      'Ljava/util/ArrayList',
-      'Ljava/util/HashMap',
-      'Ljava/util/List',
-      'Landroid/content/Context',
-      'Landroid/graphics/Bitmap',
-      'Landroid/graphics/Canvas',
-      'Landroid/graphics/Rect',
-      'Landroid/graphics/RectF',
-      'Landroid/graphics/Matrix',
-      'Landroid/graphics/Point',
-      'Landroid/graphics/SurfaceTexture$OnFrameAvailableListener',
-      'Landroid/media/MediaPlayer',
-      'Landroid/os/Message',
-      'Landroid/view/KeyEvent',
-      'Landroid/view/Surface',
-      'Landroid/view/View',
-      'Landroid/webkit/ValueCallback',
-      'Ljava/io/InputStream',
-      'Ljava/nio/ByteBuffer',
-      'Ljava/util/Vector',
-  ]
-  app_param_list = [
-      'Landroid/graphics/SurfaceTexture',
-      'Lcom/google/android/apps/chrome/ChromeContextMenuInfo',
-      'Lcom/google/android/apps/chrome/ChromeWindow',
-      'Lcom/google/android/apps/chrome/GoogleLocationSettingsHelperImpl',
-      'Lcom/google/android/apps/chrome/OmniboxSuggestion',
-      'Lcom/google/android/apps/chrome/PageInfoViewer',
-      'Lcom/google/android/apps/chrome/Tab',
-      'Lcom/google/android/apps/chrome/infobar/AutoLogin',
-      'Lcom/google/android/apps/chrome/infobar/InfoBarContainer',
-      'Lcom/google/android/apps/chrome/infobar/InfoBarContainer$NativeInfoBar',
-      ('Lcom/google/android/apps/chrome/preferences/ChromeNativePreferences$'
-       'PasswordListObserver'),
-      'Lorg/chromium/android_webview/AwContents',
-      'Lorg/chromium/android_webview/AwContentsClient',
-      'Lorg/chromium/android_webview/AwHttpAuthHandler',
-      'Lorg/chromium/android_webview/AwContentsIoThreadClient',
-      'Lorg/chromium/android_webview/AwWebContentsDelegate',
-      'Lorg/chromium/android_webview/InterceptedRequestData',
-      'Lorg/chromium/android_webview/JsPromptResultReceiver',
-      'Lorg/chromium/android_webview/JsResultHandler',
-      'Lorg/chromium/android_webview/JsResultReceiver',
-      'Lorg/chromium/base/SystemMessageHandler',
-      'Lorg/chromium/chrome/browser/autofill/AutofillExternalDelegate',
-      'Lorg/chromium/chrome/browser/autofill/AutofillSuggestion',
-      'Lorg/chromium/chrome/browser/ChromeBrowserProvider$BookmarkNode',
-      'Lorg/chromium/chrome/browser/ChromeHttpAuthHandler',
-      'Lorg/chromium/chrome/browser/ChromeWebContentsDelegateAndroid',
-      'Lorg/chromium/chrome/browser/FindMatchRectsDetails',
-      'Lorg/chromium/chrome/browser/FindNotificationDetails',
-      'Lorg/chromium/chrome/browser/GoogleLocationSettingsHelper',
-      'Lorg/chromium/chrome/browser/GoogleLocationSettingsHelperStub',
-      'Lorg/chromium/chrome/browser/JavascriptAppModalDialog',
-      'Lorg/chromium/chrome/browser/ProcessUtils',
-      ('Lorg/chromium/chrome/browser/component/navigation_interception/'
-       'InterceptNavigationDelegate'),
-      ('Lorg/chromium/chrome/browser/component/web_contents_delegate_android/'
-       'WebContentsDelegateAndroid'),
-      'Lorg/chromium/chrome/browser/database/SQLiteCursor',
-      'Lorg/chromium/content/app/SandboxedProcessService',
-      'Lorg/chromium/content/browser/ContainerViewDelegate',
-      'Lorg/chromium/content/browser/ContentVideoView',
-      'Lorg/chromium/content/browser/ContentViewCore',
-      'Lorg/chromium/content/browser/DeviceOrientation',
-      'Lorg/chromium/content/browser/JavaInputStream',
-      'Lorg/chromium/content/browser/LocationProvider',
-      'Lorg/chromium/content/browser/SandboxedProcessArgs',
-      'Lorg/chromium/content/browser/SandboxedProcessConnection',
-      'Lorg/chromium/content/browser/TouchPoint',
-      'Lorg/chromium/content/browser/WaitableNativeEvent',
-      'Lorg/chromium/content/browser/WebContentsObserverAndroid',
-      'Lorg/chromium/content/common/DeviceInfo',
-      'Lorg/chromium/content/common/SurfaceTextureListener',
-      'Lorg/chromium/media/MediaPlayerListener',
-      'Lorg/chromium/net/NetworkChangeNotifier',
-      'Lorg/chromium/net/ProxyChangeListener',
-      'Lorg/chromium/ui/gfx/NativeWindow',
-      'Lorg/chromium/ui/SelectFileDialog',
-  ]
-  if param == 'byte[][]':
-    return '[[B'
-  prefix = ''
-  # Array?
-  if param[-2:] == '[]':
-    prefix = '['
-    param = param[:-2]
-  # Generic?
-  if '<' in param:
-    param = param[:param.index('<')]
-  if param in pod_param_map:
-    return prefix + pod_param_map[param]
-  if '/' in param:
-    # Coming from javap, use the fully qualified param directly.
-    return 'L' + param + ';'
-  for qualified_name in object_param_list + app_param_list:
-    if (qualified_name.endswith('/' + param) or
-        qualified_name.endswith('$' + param.replace('.', '$')) or
-        qualified_name == 'L' + param):
-      return prefix + qualified_name + ';'
-  else:
-    return UNKNOWN_JAVA_TYPE_PREFIX + prefix + param + ';'
+class JniParams(object):
+  _UNKNOWN_JAVA_TYPE_PREFIX = 'UNKNOWN_JAVA_TYPE: '
+  _external_param_files = set()
+  _external_param_list = []

[Nico, 2012/10/31 17:41:28]

Meh, that's still global state. Why not make this a real object that gets passed
to where the static methods get called?

[bulach, 2012/10/31 17:53:10]

I agree, but that's a much more complex change :)
ok to do separately? this patch as it is moves towards that anyways..

 
+  @staticmethod
+  def ReadExternalParamList(external_param_files):
+    if not external_param_files:
+      return
+    assert not JniParams._external_param_files
+    JniParams._external_param_files = set(external_param_files)
+    for external_param_file in JniParams._external_param_files:
+      with file(external_param_file, 'r') as f:
+        contents = f.read()
+        JniParams._external_param_list += [x.strip()
+                                           for x in contents.splitlines()
+                                           if x and not x.startswith('#')]
 
-def JniSignature(params, returns, wrap):
-  """Returns the JNI signature for the given datatypes."""
-  items = ['(']
-  items += [JavaParamToJni(param.datatype) for param in params]
-  items += [')']
-  items += [JavaParamToJni(returns)]
-  if wrap:
-    return '\n' + '\n'.join(['"' + item + '"' for item in items])
-  else:
-    return '"' + ''.join(items) + '"'
+  @staticmethod
+  def JavaToJni(param):
+    """Converts a java param into a JNI signature type."""
+    pod_param_map = {
+        'int': 'I',
+        'boolean': 'Z',
+        'long': 'J',
+        'double': 'D',
+        'float': 'F',
+        'byte': 'B',
+        'void': 'V',
+    }
+    object_param_list = [
+        'Ljava/lang/Boolean',
+        'Ljava/lang/Integer',
+        'Ljava/lang/Long',
+        'Ljava/lang/Object',
+        'Ljava/lang/String',
+        'Ljava/util/ArrayList',
+        'Ljava/util/HashMap',
+        'Ljava/util/List',
+        'Landroid/content/Context',
+        'Landroid/graphics/Bitmap',
+        'Landroid/graphics/Canvas',
+        'Landroid/graphics/Rect',
+        'Landroid/graphics/RectF',
+        'Landroid/graphics/Matrix',
+        'Landroid/graphics/Point',
+        'Landroid/graphics/SurfaceTexture',
+        'Landroid/graphics/SurfaceTexture$OnFrameAvailableListener',
+        'Landroid/media/MediaPlayer',
+        'Landroid/os/Message',
+        'Landroid/view/KeyEvent',
+        'Landroid/view/Surface',
+        'Landroid/view/View',
+        'Landroid/webkit/ValueCallback',
+        'Ljava/io/InputStream',
+        'Ljava/nio/ByteBuffer',
+        'Ljava/util/Vector',
+    ]
+    if param == 'byte[][]':
+      return '[[B'
+    prefix = ''
+    # Array?
+    if param[-2:] == '[]':
+      prefix = '['
+      param = param[:-2]
+    # Generic?
+    if '<' in param:
+      param = param[:param.index('<')]
+    if param in pod_param_map:
+      return prefix + pod_param_map[param]
+    if '/' in param:
+      # Coming from javap, use the fully qualified param directly.
+      return 'L' + param + ';'
+    for qualified_name in object_param_list + JniParams._external_param_list:
+      if (qualified_name.endswith('/' + param) or
+          qualified_name.endswith('$' + param.replace('.', '$')) or
+          qualified_name == 'L' + param):
+        return prefix + qualified_name + ';'
+    else:
+      return JniParams._UNKNOWN_JAVA_TYPE_PREFIX + prefix + param + ';'
 
+  @staticmethod
+  def Signature(params, returns, wrap):
+    """Returns the JNI signature for the given datatypes."""
+    items = ['(']
+    items += [JniParams.JavaToJni(param.datatype) for param in params]
+    items += [')']
+    items += [JniParams.JavaToJni(returns)]
+    if wrap:
+      return '\n' + '\n'.join(['"' + item + '"' for item in items])
+    else:
+      return '"' + ''.join(items) + '"'
 
-def ParseParams(params):
-  """Parses the params into a list of Param objects."""
-  if not params:
-    return []
-  ret = []
-  for p in [p.strip() for p in params.split(',')]:
-    items = p.split(' ')
-    if 'final' in items:
-      items.remove('final')
-    param = Param(
-        datatype=items[0],
-        name=(items[1] if len(items) > 1 else 'p%s' % len(ret)),
-    )
-    ret += [param]
-  return ret
+  @staticmethod
+  def Parse(params):
+    """Parses the params into a list of Param objects."""
+    if not params:
+      return []
+    ret = []
+    for p in [p.strip() for p in params.split(',')]:
+      items = p.split(' ')
+      if 'final' in items:
+        items.remove('final')
+      param = Param(
+          datatype=items[0],
+          name=(items[1] if len(items) > 1 else 'p%s' % len(ret)),
+      )
+      ret += [param]
+    return ret
 
+  @staticmethod
+  def CheckUnknownDatatypes(fully_qualified_class, items):
+    unknown_datatypes = JniParams._GetUnknownDatatypes(items)
+    if unknown_datatypes:
+      msg = ('There are a few unknown datatypes in %s' %
+             fully_qualified_class)
+      msg += '\nPlease, edit %s' % str(JniParams._external_param_files)
+      msg += '\nand add the JNI type(s):\n'
+      for unknown_datatype in unknown_datatypes:
+        msg += '\n%s in methods:\n' % unknown_datatype
+        msg += '\n '.join(unknown_datatypes[unknown_datatype])
+      raise SyntaxError(msg)
 
-def GetUnknownDatatypes(items):
-  """Returns a list containing the unknown datatypes."""
-  unknown_types = {}
-  for item in items:
-    all_datatypes = ([JavaParamToJni(param.datatype)
-                      for param in item.params] +
-                     [JavaParamToJni(item.return_type)])
-    for d in all_datatypes:
-      if d.startswith(UNKNOWN_JAVA_TYPE_PREFIX):
-        unknown_types[d] = (unknown_types.get(d, []) +
-                            [item.name or 'Unable to parse'])
-  return unknown_types
+  @staticmethod
+  def _GetUnknownDatatypes(items):
+    """Returns a list containing the unknown datatypes."""
+    unknown_types = {}
+    for item in items:
+      all_datatypes = ([JniParams.JavaToJni(param.datatype)
+                        for param in item.params] +
+                       [JniParams.JavaToJni(item.return_type)])
+      for d in all_datatypes:
+        if d.startswith(JniParams._UNKNOWN_JAVA_TYPE_PREFIX):
+          unknown_types[d] = (unknown_types.get(d, []) +
+                              [item.name or 'Unable to parse'])
+    return unknown_types
 
 
 def ExtractJNINamespace(contents):
   re_jni_namespace = re.compile('.*?@JNINamespace\("(.*?)"\)')
   m = re.findall(re_jni_namespace, contents)
   if not m:
     return ''
   return m[0]
 
 
@@ skipping 16 matching lines @@
                          '(?P<qualifiers>\w+\s\w+|\w+|\s+)\s*?native '
                          '(?P<return>\S*?) '
                          '(?P<name>\w+?)\((?P<params>.*?)\);')
   for match in re.finditer(re_native, contents):
     native = NativeMethod(
         static='static' in match.group('qualifiers'),
         java_class_name=match.group('java_class_name'),
         native_class_name=match.group('native_class_name'),
         return_type=match.group('return'),
         name=match.group('name').replace('native', ''),
-        params=ParseParams(match.group('params')))
+        params=JniParams.Parse(match.group('params')))
     natives += [native]
   return natives
 
 
 def GetStaticCastForReturnType(return_type):
   if return_type == 'String':
     return 'jstring'
   return None
 
 
@@ skipping 41 matching lines @@
   Args:
      name: string.
      params: list of Param.
      return_type: string.
 
   Returns:
       A mangled name.
   """
   mangled_items = []
   for datatype in [return_type] + [x.datatype for x in params]:
-    mangled_items += [GetMangledParam(JavaParamToJni(datatype))]
+    mangled_items += [GetMangledParam(JniParams.JavaToJni(datatype))]
   mangled_name = name + '_'.join(mangled_items)
   assert re.match(r'[0-9a-zA-Z_]+', mangled_name)
   return mangled_name
 
 
 def MangleCalledByNatives(called_by_natives):
   """Mangles all the overloads from the call_by_natives list."""
   method_counts = collections.defaultdict(
       lambda: collections.defaultdict(lambda: 0))
   for called_by_native in called_by_natives:
@@ skipping 40 matching lines @@
   """
   called_by_natives = []
   for match in re.finditer(RE_CALLED_BY_NATIVE, contents):
     called_by_natives += [CalledByNative(
         system_class=False,
         unchecked='Unchecked' in match.group('Unchecked'),
         static='static' in match.group('prefix'),
         java_class_name=match.group('annotation') or '',
         return_type=match.group('return_type'),
         name=match.group('name'),
-        params=ParseParams(match.group('params')))]
+        params=JniParams.Parse(match.group('params')))]
   # Check for any @CalledByNative occurrences that weren't matched.
   unmatched_lines = re.sub(RE_CALLED_BY_NATIVE, '', contents).split('\n')
   for line1, line2 in zip(unmatched_lines, unmatched_lines[1:]):
     if '@CalledByNative' in line1:
       raise ParseError('could not parse @CalledByNative method signature',
                        line1, line2)
   return MangleCalledByNatives(called_by_natives)
 
 
 class JNIFromJavaP(object):
@@ skipping 15 matching lines @@
       match = re.match(re_method, content)
       if not match:
         continue
       self.called_by_natives += [CalledByNative(
           system_class=True,
           unchecked=False,
           static='static' in match.group('prefix'),
           java_class_name='',
           return_type=match.group('return_type'),
           name=match.group('name'),
-          params=ParseParams(match.group('params').replace('.', '/')))]
+          params=JniParams.Parse(match.group('params').replace('.', '/')))]
     re_constructor = re.compile('.*? public ' +
                                 self.fully_qualified_class.replace('/', '.') +
                                 '\((?P<params>.*?)\)')
     for content in contents[2:]:
       match = re.match(re_constructor, content)
       if not match:
         continue
       self.called_by_natives += [CalledByNative(
           system_class=True,
           unchecked=False,
           static=False,
           java_class_name='',
           return_type=self.fully_qualified_class,
           name='Constructor',
-          params=ParseParams(match.group('params').replace('.', '/')),
+          params=JniParams.Parse(match.group('params').replace('.', '/')),
           is_constructor=True)]
     self.called_by_natives = MangleCalledByNatives(self.called_by_natives)
     self.inl_header_file_generator = InlHeaderFileGenerator(
         self.namespace, self.fully_qualified_class, [], self.called_by_natives)
 
   def GetContent(self):
     return self.inl_header_file_generator.GetContent()
 
   @staticmethod
   def CreateFromClass(class_file, namespace):
@@ skipping 55 matching lines @@
   """Generates an inline header file for JNI integration."""
 
   def __init__(self, namespace, fully_qualified_class, natives,
                called_by_natives):
     self.namespace = namespace
     self.fully_qualified_class = fully_qualified_class
     self.class_name = self.fully_qualified_class.split('/')[-1]
     self.natives = natives
     self.called_by_natives = called_by_natives
     self.header_guard = fully_qualified_class.replace('/', '_') + '_JNI'
-    unknown_datatypes = GetUnknownDatatypes(self.natives +
-                                            self.called_by_natives)
-    if unknown_datatypes:
-      msg = ('There are a few unknown datatypes in %s' %
-             self.fully_qualified_class)
-      msg += '\nPlease, edit %s' % sys.argv[0]
-      msg += '\nand add the java type to JavaParamToJni()\n'
-      for unknown_datatype in unknown_datatypes:
-        msg += '\n%s in methods:\n' % unknown_datatype
-        msg += '\n '.join(unknown_datatypes[unknown_datatype])
-      raise SyntaxError(msg)
+    JniParams.CheckUnknownDatatypes(self.fully_qualified_class,
+                                    self.natives + self.called_by_natives)
 
   def GetContent(self):
     """Returns the content of the JNI binding file."""
     template = Template("""\
 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 
 // This file is autogenerated by
@@ skipping 273 matching lines @@
       values['FUNCTION_HEADER'] = (
           function_header_with_unused_template.substitute(values))
     else:
       values['FUNCTION_HEADER'] = function_header_template.substitute(values)
     return template.substitute(values)
 
   def GetKMethodArrayEntry(self, native):
     template = Template("""\
     { "native${NAME}", ${JNI_SIGNATURE}, reinterpret_cast<void*>(${NAME}) },""")
     values = {'NAME': native.name,
-              'JNI_SIGNATURE': JniSignature(native.params, native.return_type,
-                                            True)}
+              'JNI_SIGNATURE': JniParams.Signature(native.params,
+                                                   native.return_type,
+                                                   True)}
     return template.substitute(values)
 
   def GetUniqueClasses(self, origin):
     ret = {self.class_name: self.fully_qualified_class}
     for entry in origin:
       class_name = self.class_name
       jni_class_path = self.fully_qualified_class
       if entry.java_class_name:
         class_name = entry.java_class_name
         jni_class_path = self.fully_qualified_class + '$' + class_name
@@ skipping 50 matching lines @@
     jni_name = called_by_native.name
     jni_return_type = called_by_native.return_type
     if called_by_native.is_constructor:
       jni_name = '<init>'
       jni_return_type = 'void'
     values = {
         'JAVA_CLASS': called_by_native.java_class_name or self.class_name,
         'JNI_NAME': jni_name,
         'METHOD_ID_VAR_NAME': called_by_native.method_id_var_name,
         'STATIC': 'STATIC' if called_by_native.static else 'INSTANCE',
-        'JNI_SIGNATURE': JniSignature(called_by_native.params,
-                                      jni_return_type,
-                                      True)
+        'JNI_SIGNATURE': JniParams.Signature(called_by_native.params,
+                                             jni_return_type,
+                                             True)
     }
     return template.substitute(values)
 
 
 def WrapOutput(output):
   ret = []
   for line in output.splitlines():
     # Do not wrap lines under 80 characters or preprocessor directives.
     if len(line) < 80 or line.lstrip()[:1] == '#':
       stripped = line.rstrip()
@@ skipping 74 matching lines @@
   option_parser.add_option('-n', dest='namespace',
                            help='Uses as a namespace in the generated header,'
                            ' instead of the javap class name.')
   option_parser.add_option('--input_file',
                            help='Single input file name. The output file name '
                            'will be derived from it. Must be used with '
                            '--output_dir.')
   option_parser.add_option('--output_dir',
                            help='The output directory. Must be used with '
                            '--input')
+  option_parser.add_option('--external_param_list',
+                           help='A file name containing a list with extra '
+                           'fully-qualified param names. Can be used multiple '
+                           'times.',
+                           action='append')
   options, args = option_parser.parse_args(argv)
+  JniParams.ReadExternalParamList(options.external_param_list)
   if options.jar_file:
     input_file = ExtractJarInputFile(options.jar_file, options.input_file,
                                      options.output_dir)
   else:
     input_file = options.input_file
   output_file = None
   if options.output_dir:
     root_name = os.path.splitext(os.path.basename(input_file))[0]
     output_file = os.path.join(options.output_dir, root_name) + '_jni.h'
   GenerateJNIHeader(input_file, output_file, options.namespace)
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))