Speed up fremontcut/dartdomgenerator

lib/html/scripts/databasebuilder.py

Index: lib/html/scripts/databasebuilder.py
diff --git a/lib/html/scripts/databasebuilder.py b/lib/html/scripts/databasebuilder.py
index 9c4726e5a08c106e31bed6168d184746805d882a..48b0101ce95c3d9ad419a5d70f7347a0327ba6e2 100755
--- a/lib/html/scripts/databasebuilder.py
+++ b/lib/html/scripts/databasebuilder.py
@@ -7,6 +7,7 @@ import copy
 import database
 import idlparser
 import logging
+import multiprocessing
 import os
 import os.path
 import re
@@ -63,6 +64,22 @@ class DatabaseBuilderOptions(object):
     self.obsolete_old_declarations = obsolete_old_declarations
 
 
+def _load_idl_file(file_name, import_options, result_queue):
+  """Loads an IDL file into memory"""
+  idl_parser = idlparser.IDLParser(import_options.idl_syntax)
+
+  try:
+    f = open(file_name, 'r')
+    content = f.read()
+    f.close()
+
+    idl_ast = idl_parser.parse(content,

[Anton Muhin, 2012/09/26 06:37:29]

nit: unless it's against the style, maybe idl_parser.parse(\ncontent, defines=)
as you're touching this code anyway?

[vsm, 2012/09/26 16:22:10]

Done.

+      defines=import_options.idl_defines)
+    result = IDLFile(idl_ast, file_name)
+    result_queue.put(result)
+  except SyntaxError, e:
+    raise RuntimeError('Failed to load file %s: %s' % (file_name, e))

[Anton Muhin, 2012/09/26 06:37:29]

in multiprocessing, what would be result of throwing an exception from child
process?

[vsm, 2012/09/26 16:22:10]

It'll quite the process, but the code below would hang waiting for the
result_queue.  I've modified to make the code more robust and tested with some
IDL errors to make sure they're propagated correctly.

On 2012/09/26 06:37:29, Anton Muhin wrote:

+
 class DatabaseBuilder(object):
   def __init__(self, database):
     """DatabaseBuilder is used for importing and merging interfaces into
@@ -71,21 +88,6 @@ class DatabaseBuilder(object):
     self._imported_interfaces = []
     self._impl_stmts = []
 
-  def _load_idl_file(self, file_name, import_options):
-    """Loads an IDL file intor memory"""
-    idl_parser = idlparser.IDLParser(import_options.idl_syntax)
-
-    try:
-      f = open(file_name, 'r')
-      content = f.read()
-      f.close()
-
-      idl_ast = idl_parser.parse(content,
-        defines=import_options.idl_defines)
-      return IDLFile(idl_ast, file_name)
-    except SyntaxError, e:
-      raise RuntimeError('Failed to load file %s: %s' % (file_name, e))
-
   def _resolve_type_defs(self, idl_file):
     type_def_map = {}
     # build map
@@ -115,7 +117,18 @@ class DatabaseBuilder(object):
       return name
 
     def rename_node(idl_node):
-      idl_node.id = rename(idl_node.id)
+      new_name = rename(idl_node.id)

[Anton Muhin, 2012/09/26 06:37:29]

why this change?  does that belong to this CL?

[vsm, 2012/09/26 16:22:10]

It turns out that writing a database to file and reading it back in was not
idempotent.  I'm trying to fix part of that here.  Before, these js names were
not renamed properly, but if the database was written and read, they'd be
properly reset to idl_name.id so it worked anyway.

On 2012/09/26 06:37:29, Anton Muhin wrote:

[Anton Muhin, 2012/09/26 17:53:09]

Weird.  Might be worth splitting those change if things will go weird on
windows.

Up to you to decide though.

On 2012/09/26 16:22:10, vsm wrote:

+      if new_name != idl_node.id:
+        idl_node.id = new_name
+        if isinstance(idl_node, IDLInterface):
+          idl_node.doc_js_name = new_name
+          idl_node.javascript_binding_name = new_name
+          for member in idl_node.operations:
+            member.doc_js_interface_name = new_name
+          for member in idl_node.attributes:
+            member.doc_js_interface_name = new_name
+          for member in idl_node.constants:
+            member.doc_js_interface_name = new_name
 
     def rename_ext_attrs(ext_attrs_node):
       for type_valued_attribute_name in ['Supplemental']:
@@ -322,6 +335,7 @@ class DatabaseBuilder(object):
 
       if what != 'parents' and old_interface.id != new_interface.id:
         for node in new_list:
+          node.doc_js_interface_name = old_interface.id

[Anton Muhin, 2012/09/26 06:37:29]

ditto

[vsm, 2012/09/26 16:22:10]

Same issue as above.

On 2012/09/26 06:37:29, Anton Muhin wrote:

           node.ext_attrs['ImplementedBy'] = new_interface.id
 
       changed = self._merge_nodes(old_list, new_list, import_options)
@@ -421,11 +435,21 @@ class DatabaseBuilder(object):
     self._impl_stmts = []
     self._imported_interfaces = []
 
-  def import_idl_file(self, file_path,
+  def import_idl_files(self, file_paths, import_options):
+    # Parse the IDL files in parallel.
+    result_queue = multiprocessing.Queue()
+    jobs = [ multiprocessing.Process(target=_load_idl_file,
+                                     args=(file_path, import_options,
+                                           result_queue))
+             for file_path in file_paths ]
+    for job in jobs:
+      job.start()
+    for job in jobs:
+      result = result_queue.get()

[Anton Muhin, 2012/09/26 06:37:29]

same question as above: if some of child processes die for any reason, will it
throw an exception or we'll block forever?

[vsm, 2012/09/26 16:22:10]

PTAL.  I've tested both bad IDL and timeout cases (by setting the timeout to 0).
 It quits and cleans up dead processes in both cases.

On 2012/09/26 06:37:29, Anton Muhin wrote:

+      self._process_idl_file(result, import_options)
+
+  def _process_idl_file(self, idl_file,

[Anton Muhin, 2012/09/26 06:37:29]

should it be a method on its own or it might be a closure?  And if it can be a
closure, is it required for import_options to be an optional argument?

[vsm, 2012/09/26 16:22:10]

I've dropped the optional argument.  It's not called anywhere else, but I think
the import function above is a little more readable if I take this out.

[Anton Muhin, 2012/09/26 17:57:26]

I'd rather make it a closure of the only caller, but it's your call

On 2012/09/26 16:22:10, vsm wrote:

             import_options=DatabaseBuilderOptions()):
-    """Parses, loads into memory and cleans up and IDL file"""
-    idl_file = self._load_idl_file(file_path, import_options)
-
     self._strip_ext_attributes(idl_file)
     self._resolve_type_defs(idl_file)
     self._rename_types(idl_file, import_options)
@@ -436,14 +460,12 @@ class DatabaseBuilder(object):
     for module in idl_file.modules:
       for interface in module.interfaces:
         if not self._is_node_enabled(interface, import_options.idl_defines):
-          _logger.info('skipping interface %s/%s (source=%s file=%s)'
-            % (module.id, interface.id, import_options.source,
-               file_path))
+          _logger.info('skipping interface %s/%s (source=%s)'
+            % (module.id, interface.id, import_options.source))
           continue
 
-        _logger.info('importing interface %s/%s (source=%s file=%s)'
-          % (module.id, interface.id, import_options.source,
-             file_path))
+        _logger.info('importing interface %s/%s (source=%s)'
+          % (module.id, interface.id, import_options.source))
         interface.attributes = filter(enabled, interface.attributes)
         interface.operations = filter(enabled, interface.operations)
         self._imported_interfaces.append((interface, module.id, import_options))