Speed up fremontcut/dartdomgenerator

lib/html/scripts/databasebuilder.py

 #!/usr/bin/python
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import copy
 import database
 import idlparser
 import logging
+import multiprocessing
 import os
 import os.path
 import re
 
 from idlnode import *
 
 _logger = logging.getLogger('databasebuilder')
 
 # Used in source annotations to specify the parent interface declaring
 # a displaced declaration. The 'via' attribute specifies the parent interface
@@ skipping 36 matching lines @@
     self.source_attributes = source_attributes
     self.idl_syntax = idl_syntax
     self.idl_defines = idl_defines
     self.type_rename_map = type_rename_map
     self.rename_operation_arguments_on_merge = \
         rename_operation_arguments_on_merge
     self.add_new_interfaces = add_new_interfaces
     self.obsolete_old_declarations = obsolete_old_declarations
 
 
+def _load_idl_file(file_name, import_options, result_queue):
+  """Loads an IDL file into memory"""
+  idl_parser = idlparser.IDLParser(import_options.idl_syntax)
+
+  try:
+    f = open(file_name, 'r')
+    content = f.read()
+    f.close()
+
+    idl_ast = idl_parser.parse(content,

[Anton Muhin, 2012/09/26 06:37:29]

nit: unless it's against the style, maybe idl_parser.parse(\ncontent, defines=)
as you're touching this code anyway?

[vsm, 2012/09/26 16:22:10]

Done.

+      defines=import_options.idl_defines)
+    result = IDLFile(idl_ast, file_name)
+    result_queue.put(result)
+  except SyntaxError, e:
+    raise RuntimeError('Failed to load file %s: %s' % (file_name, e))

[Anton Muhin, 2012/09/26 06:37:29]

in multiprocessing, what would be result of throwing an exception from child
process?

[vsm, 2012/09/26 16:22:10]

It'll quite the process, but the code below would hang waiting for the
result_queue.  I've modified to make the code more robust and tested with some
IDL errors to make sure they're propagated correctly.

On 2012/09/26 06:37:29, Anton Muhin wrote:

+
 class DatabaseBuilder(object):
   def __init__(self, database):
     """DatabaseBuilder is used for importing and merging interfaces into
     the Database"""
     self._database = database
     self._imported_interfaces = []
     self._impl_stmts = []
 
-  def _load_idl_file(self, file_name, import_options):
-    """Loads an IDL file intor memory"""
-    idl_parser = idlparser.IDLParser(import_options.idl_syntax)
-
-    try:
-      f = open(file_name, 'r')
-      content = f.read()
-      f.close()
-
-      idl_ast = idl_parser.parse(content,
-        defines=import_options.idl_defines)
-      return IDLFile(idl_ast, file_name)
-    except SyntaxError, e:
-      raise RuntimeError('Failed to load file %s: %s' % (file_name, e))
-
   def _resolve_type_defs(self, idl_file):
     type_def_map = {}
     # build map
     for type_def in idl_file.all(IDLTypeDef):
       if type_def.type.id != type_def.id: # sanity check
         type_def_map[type_def.id] = type_def.type.id
     # use the map
     for type_node in idl_file.all(IDLType):
       while type_node.id in type_def_map:
         type_node.id = type_def_map[type_node.id]
 
   def _strip_ext_attributes(self, idl_file):
     """Strips unuseful extended attributes."""
     for ext_attrs in idl_file.all(IDLExtAttrs):
       # TODO: Decide which attributes are uninteresting.
       pass
 
   def _rename_types(self, idl_file, import_options):
     """Rename interface and type names with names provided in the
     options. Also clears scopes from scoped names"""
 
     def rename(name):
       name_parts = name.split('::')
       name = name_parts[-1]
       if name in import_options.type_rename_map:
         name = import_options.type_rename_map[name]
       return name
 
     def rename_node(idl_node):
-      idl_node.id = rename(idl_node.id)
+      new_name = rename(idl_node.id)

[Anton Muhin, 2012/09/26 06:37:29]

why this change?  does that belong to this CL?

[vsm, 2012/09/26 16:22:10]

It turns out that writing a database to file and reading it back in was not
idempotent.  I'm trying to fix part of that here.  Before, these js names were
not renamed properly, but if the database was written and read, they'd be
properly reset to idl_name.id so it worked anyway.

On 2012/09/26 06:37:29, Anton Muhin wrote:

[Anton Muhin, 2012/09/26 17:53:09]

Weird.  Might be worth splitting those change if things will go weird on
windows.

Up to you to decide though.

On 2012/09/26 16:22:10, vsm wrote:

+      if new_name != idl_node.id:
+        idl_node.id = new_name
+        if isinstance(idl_node, IDLInterface):
+          idl_node.doc_js_name = new_name
+          idl_node.javascript_binding_name = new_name
+          for member in idl_node.operations:
+            member.doc_js_interface_name = new_name
+          for member in idl_node.attributes:
+            member.doc_js_interface_name = new_name
+          for member in idl_node.constants:
+            member.doc_js_interface_name = new_name
 
     def rename_ext_attrs(ext_attrs_node):
       for type_valued_attribute_name in ['Supplemental']:
         if type_valued_attribute_name in ext_attrs_node:
           value = ext_attrs_node[type_valued_attribute_name]
           if isinstance(value, str):
             ext_attrs_node[type_valued_attribute_name] = rename(value)
 
     map(rename_node, idl_file.all(IDLInterface))
     map(rename_node, idl_file.all(IDLType))
@@ skipping 186 matching lines @@
         not new_interface.is_supplemental):
       old_interface.annotations[source] = new_interface.annotations[source]
       changed = True
 
     def merge_list(what):
       old_list = old_interface.__dict__[what]
       new_list = new_interface.__dict__[what]
 
       if what != 'parents' and old_interface.id != new_interface.id:
         for node in new_list:
+          node.doc_js_interface_name = old_interface.id

[Anton Muhin, 2012/09/26 06:37:29]

ditto

[vsm, 2012/09/26 16:22:10]

Same issue as above.

On 2012/09/26 06:37:29, Anton Muhin wrote:

           node.ext_attrs['ImplementedBy'] = new_interface.id
 
       changed = self._merge_nodes(old_list, new_list, import_options)
 
       # Delete list items with zero remaining annotations.
       if changed and import_options.obsolete_old_declarations:
 
         def has_annotations(idl_node):
           return len(idl_node.annotations)
 
@@ skipping 79 matching lines @@
         else:
           raise Exception("Supplemental target '%s' not found", target)
 
     # Step 4: Resolve 'implements' statements
     for impl_stmt, import_options in self._impl_stmts:
       self._merge_impl_stmt(impl_stmt, import_options)
 
     self._impl_stmts = []
     self._imported_interfaces = []
 
-  def import_idl_file(self, file_path,
+  def import_idl_files(self, file_paths, import_options):
+    # Parse the IDL files in parallel.
+    result_queue = multiprocessing.Queue()
+    jobs = [ multiprocessing.Process(target=_load_idl_file,
+                                     args=(file_path, import_options,
+                                           result_queue))
+             for file_path in file_paths ]
+    for job in jobs:
+      job.start()
+    for job in jobs:
+      result = result_queue.get()

[Anton Muhin, 2012/09/26 06:37:29]

same question as above: if some of child processes die for any reason, will it
throw an exception or we'll block forever?

[vsm, 2012/09/26 16:22:10]

PTAL.  I've tested both bad IDL and timeout cases (by setting the timeout to 0).
 It quits and cleans up dead processes in both cases.

On 2012/09/26 06:37:29, Anton Muhin wrote:

+      self._process_idl_file(result, import_options)
+
+  def _process_idl_file(self, idl_file,

[Anton Muhin, 2012/09/26 06:37:29]

should it be a method on its own or it might be a closure?  And if it can be a
closure, is it required for import_options to be an optional argument?

[vsm, 2012/09/26 16:22:10]

I've dropped the optional argument.  It's not called anywhere else, but I think
the import function above is a little more readable if I take this out.

[Anton Muhin, 2012/09/26 17:57:26]

I'd rather make it a closure of the only caller, but it's your call

On 2012/09/26 16:22:10, vsm wrote:

             import_options=DatabaseBuilderOptions()):
-    """Parses, loads into memory and cleans up and IDL file"""
-    idl_file = self._load_idl_file(file_path, import_options)
-
     self._strip_ext_attributes(idl_file)
     self._resolve_type_defs(idl_file)
     self._rename_types(idl_file, import_options)
 
     def enabled(idl_node):
       return self._is_node_enabled(idl_node, import_options.idl_defines)
 
     for module in idl_file.modules:
       for interface in module.interfaces:
         if not self._is_node_enabled(interface, import_options.idl_defines):
-          _logger.info('skipping interface %s/%s (source=%s file=%s)'
-            % (module.id, interface.id, import_options.source,
-               file_path))
+          _logger.info('skipping interface %s/%s (source=%s)'
+            % (module.id, interface.id, import_options.source))
           continue
 
-        _logger.info('importing interface %s/%s (source=%s file=%s)'
-          % (module.id, interface.id, import_options.source,
-             file_path))
+        _logger.info('importing interface %s/%s (source=%s)'
+          % (module.id, interface.id, import_options.source))
         interface.attributes = filter(enabled, interface.attributes)
         interface.operations = filter(enabled, interface.operations)
         self._imported_interfaces.append((interface, module.id, import_options))
 
       for implStmt in module.implementsStatements:
         self._impl_stmts.append((implStmt, import_options))
 
   def _is_node_enabled(self, node, idl_defines):
     if not 'Conditional' in node.ext_attrs:
       return True
@@ skipping 93 matching lines @@
       # TODO(antonm): Ideally we'd like to have pristine copy of WebKit IDLs and fetch
       # this information directly from it.  Unfortunately right now database is massaged
       # a lot so it's difficult to maintain necessary information on DOMWindow itself.
       interface = self._database.GetInterface(options.type_rename_map.get(type, type))
       if 'V8EnabledPerContext' in attr.ext_attrs:
         interface.ext_attrs['synthesizedV8EnabledPerContext'] = \
             attr.ext_attrs['V8EnabledPerContext']
       if 'V8EnabledAtRuntime' in attr.ext_attrs:
         interface.ext_attrs['synthesizedV8EnabledAtRuntime'] = \
             attr.ext_attrs['V8EnabledAtRuntime'] or attr.id