Overhaul JSON Schema Compiler to support a number of features required to

tools/json_schema_compiler/h_generator.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from code import Code
-from model import PropertyType
+from model import PropertyType, Type
 import cpp_util
 import schema_util
 
 class HGenerator(object):
   """A .h generator for a namespace.
   """
   def __init__(self, namespace, cpp_type_generator):
-    self._cpp_type_generator = cpp_type_generator
+    self._type_helper = cpp_type_generator
     self._namespace = namespace
     self._target_namespace = (
-        self._cpp_type_generator.GetCppNamespaceName(self._namespace))
+        self._type_helper.GetCppNamespaceName(self._namespace))
 
   def Generate(self):
     """Generates a Code object with the .h for a single namespace.
     """
     c = Code()
     (c.Append(cpp_util.CHROMIUM_LICENSE)
       .Append()
       .Append(cpp_util.GENERATED_FILE_MESSAGE % self._namespace.source_file)
       .Append()
     )
 
     ifndef_name = cpp_util.GenerateIfndefName(self._namespace.source_file_dir,
                                               self._target_namespace)
     (c.Append('#ifndef %s' % ifndef_name)
       .Append('#define %s' % ifndef_name)
       .Append()
+      .Append('#include <map>')
       .Append('#include <string>')
       .Append('#include <vector>')
       .Append()
       .Append('#include "base/basictypes.h"')
       .Append('#include "base/logging.h"')
       .Append('#include "base/memory/linked_ptr.h"')
       .Append('#include "base/memory/scoped_ptr.h"')
       .Append('#include "base/values.h"')
-      .Append('#include "tools/json_schema_compiler/any.h"')
       .Append()
     )
 
-    c.Concat(self._cpp_type_generator.GetRootNamespaceStart())
+    c.Concat(self._type_helper.GetRootNamespaceStart())
     # TODO(calamity): These forward declarations should be #includes to allow
     # $ref types from other files to be used as required params. This requires
     # some detangling of windows and tabs which will currently lead to circular
     # #includes.
     forward_declarations = (
-        self._cpp_type_generator.GenerateForwardDeclarations())
+        self._type_helper.GenerateForwardDeclarations())
     if not forward_declarations.IsEmpty():
       (c.Append()
-        .Concat(forward_declarations)
-        .Append()
+        .Cblock(forward_declarations)
       )
 
-    c.Concat(self._cpp_type_generator.GetNamespaceStart())
+    c.Concat(self._type_helper.GetNamespaceStart())
     c.Append()
     if self._namespace.properties:
       (c.Append('//')
         .Append('// Properties')
         .Append('//')
         .Append()
       )
       for property in self._namespace.properties.values():
-        property_code = self._cpp_type_generator.GeneratePropertyValues(
+        property_code = self._type_helper.GeneratePropertyValues(
             property,
             'extern const %(type)s %(name)s;')
         if property_code:
-          c.Concat(property_code).Append()
+          c.Cblock(property_code)
     if self._namespace.types:
       (c.Append('//')
         .Append('// Types')
         .Append('//')
         .Append()
+        .Cblock(self._GenerateTypes(self._FieldDependencyOrder(),
+                                    is_toplevel=True,
+                                    generate_typedefs=True))
       )
-      for type_ in self._FieldDependencyOrder():
-        (c.Concat(self._GenerateType(type_))
-          .Append()
-        )
     if self._namespace.functions:
       (c.Append('//')
         .Append('// Functions')
         .Append('//')
         .Append()
       )
       for function in self._namespace.functions.values():
-        (c.Concat(self._GenerateFunction(function))
-          .Append()
-        )
+        c.Cblock(self._GenerateFunction(function))
     if self._namespace.events:
       (c.Append('//')
         .Append('// Events')
         .Append('//')
         .Append()
       )
       for event in self._namespace.events.values():
-        (c.Concat(self._GenerateEvent(event))
-          .Append()
-        )
-    (c.Concat(self._cpp_type_generator.GetNamespaceEnd())
-      .Concat(self._cpp_type_generator.GetRootNamespaceEnd())
-      .Append()
+        c.Cblock(self._GenerateEvent(event))
+    (c.Concat(self._type_helper.GetNamespaceEnd())
+      .Cblock(self._type_helper.GetRootNamespaceEnd())
       .Append('#endif  // %s' % ifndef_name)
       .Append()
     )
     return c
 
   def _FieldDependencyOrder(self):
     """Generates the list of types in the current namespace in an order in which
     depended-upon types appear before types which depend on them.
     """
     dependency_order = []
 
     def ExpandType(path, type_):
       if type_ in path:
         raise ValueError("Illegal circular dependency via cycle " +
                          ", ".join(map(lambda x: x.name, path + [type_])))
       for prop in type_.properties.values():
         if (prop.type_ == PropertyType.REF and
             schema_util.GetNamespace(prop.ref_type) == self._namespace.name):
           ExpandType(path + [type_], self._namespace.types[prop.ref_type])
       if not type_ in dependency_order:
         dependency_order.append(type_)
 
     for type_ in self._namespace.types.values():
       ExpandType([], type_)
     return dependency_order
 
-  def _GenerateEnumDeclaration(self, enum_name, prop, values):
-    """Generate the declaration of a C++ enum for the given property and
-    values.
+  def _GenerateEnumDeclaration(self, enum_name, type_):
+    """Generate the declaration of a C++ enum.
     """
     c = Code()
     c.Sblock('enum %s {' % enum_name)
-    c.Append(self._cpp_type_generator.GetEnumNoneValue(prop) + ',')
-    for value in values:
-      c.Append(self._cpp_type_generator.GetEnumValue(prop, value) + ',')
-    (c.Eblock('};')
-      .Append()
-    )
-    return c
+    c.Append(self._type_helper.GetEnumNoneValue(type_) + ',')
+    for value in type_.enum_values:
+      c.Append(self._type_helper.GetEnumValue(type_, value) + ',')
+    return c.Eblock('};')
 
   def _GenerateFields(self, props):
     """Generates the field declarations when declaring a type.
     """
     c = Code()
-    # Generate the enums needed for any fields with "choices"
+    needs_blank_line = False
     for prop in props:
-      if prop.type_ == PropertyType.CHOICES:
-        enum_name = self._cpp_type_generator.GetChoicesEnumType(prop)
-        c.Append('%s %s_type;' % (enum_name, prop.unix_name))
+      if needs_blank_line:
         c.Append()
-
-    for prop in self._cpp_type_generator.ExpandParams(props):
+      needs_blank_line = True
       if prop.description:
         c.Comment(prop.description)
+      # ANY is a base::Value which is abstract and cannot be a direct member, so
+      # we always need to wrap it in a scoped_ptr.
+      is_optional = (prop.optional or

[Yoyo Zhou, 2013/01/15 01:49:25]

Is is_optional still an appropriate name, or is it better to say that it needs
to be in a scoped_ptr?

[not at google - send to devlin, 2013/01/15 21:47:27]

Done.

+                     prop.type_.property_type == PropertyType.ANY)
       (c.Append('%s %s;' % (
-           self._cpp_type_generator.GetCompiledType(prop, wrap_optional=True),
+           self._type_helper.GetCppType(prop.type_, is_optional=is_optional),
            prop.unix_name))
-        .Append()
       )
     return c
 
-  def _GenerateType(self, type_):
+  def _GenerateType(self, type_, is_toplevel=False, generate_typedefs=False):
     """Generates a struct for a type.

[Yoyo Zhou, 2013/01/15 01:49:25]

Please document the optional arguments.

[not at google - send to devlin, 2013/01/15 21:47:27]

Done.

     """
     classname = cpp_util.Classname(schema_util.StripSchemaNamespace(type_.name))
     c = Code()
 
     if type_.functions:
-      c.Sblock('namespace %(classname)s {')
+      # Wrap functions within types in the type's namespace.
+      (c.Append('namespace %s {' % classname)
+        .Append()
+      )
       for function in type_.functions.values():
-        (c.Concat(self._GenerateFunction(function))
-          .Append()
+        c.Cblock(self._GenerateFunction(function))
+      c.Append('}  // namespace %s' % classname)
+    elif type_.property_type == PropertyType.ARRAY:
+      if generate_typedefs and type_.description:
+        c.Comment(type_.description)
+      c.Cblock(self._GenerateType(type_.item_type))
+      if generate_typedefs:
+        (c.Append('typedef std::vector<%s > %s;' % (
+                       self._type_helper.GetCppType(type_.item_type),
+                       classname))
         )
-      c.Eblock('}')
-    elif type_.type_ == PropertyType.ARRAY:
-      if type_.description:
-        c.Comment(type_.description)
-      c.Append('typedef std::vector<%(item_type)s> %(classname)s;')
-      c.Substitute({'classname': classname, 'item_type':
-          self._cpp_type_generator.GetCompiledType(type_.item_type,
-                                                   wrap_optional=True)})
-    elif type_.type_ == PropertyType.STRING:
-      if type_.description:
-        c.Comment(type_.description)
-      c.Append('typedef std::string %(classname)s;')
-    elif type_.type_ == PropertyType.ENUM:
+    elif type_.property_type == PropertyType.STRING:
+      if generate_typedefs:
+        if type_.description:
+          c.Comment(type_.description)
+        c.Append('typedef std::string %(classname)s;')
+    elif type_.property_type == PropertyType.ENUM:
       if type_.description:
         c.Comment(type_.description)
       c.Sblock('enum %(classname)s {')
-      c.Append('%s,' % self._cpp_type_generator.GetEnumNoneValue(type_))
+      c.Append('%s,' % self._type_helper.GetEnumNoneValue(type_))
       for value in type_.enum_values:
-        c.Append('%s,' % self._cpp_type_generator.GetEnumValue(type_, value))
+        c.Append('%s,' % self._type_helper.GetEnumValue(type_, value))
+      # Top level enums are in a namespace scope so the methods shouldn't be
+      # static. On the other hand, those declared inline (e.g. in an object) do.
+      maybe_static = '' if is_toplevel else 'static '
       (c.Eblock('};')
         .Append()
-        .Append('scoped_ptr<base::Value> CreateEnumValue(%s %s);' %
-                (classname, classname.lower()))
-        .Append('std::string ToString(%s enum_param);' % classname)
-        .Append('%s From%sString(const std::string& enum_string);' %
-            (classname, classname))
+        .Append('%sstd::string ToString(%s as_enum);' %
+                    (maybe_static, classname))
+        .Append('%s%s Parse%s(const std::string& as_string);' %
+                    (maybe_static, classname, classname))
       )
-    else:
+    elif type_.property_type == PropertyType.OBJECT:
       if type_.description:
         c.Comment(type_.description)
       (c.Sblock('struct %(classname)s {')
+          .Append('%(classname)s();')
           .Append('~%(classname)s();')
+      )
+      if type_.origin.from_json:
+        (c.Append()
+          .Comment('Populates a %s object from a base::Value. Returns'
+                   ' whether |out| was successfully populated.' % classname)
+          .Append('static bool Populate(const base::Value& value, '
+                  '%(classname)s* out);')
+        )
+      if type_.origin.from_client:
+        (c.Append()
+          .Comment('Returns a new base::DictionaryValue representing the'
+                   ' serialized form of this %s object.' % classname)
+          .Append('scoped_ptr<base::DictionaryValue> ToValue() const;')
+        )
+      properties = type_.properties.values()
+      (c.Append()
+        .Cblock(self._GenerateTypes(p.type_ for p in properties))
+        .Cblock(self._GenerateFields(properties)))
+      if type_.additional_properties is not None:
+        (c.Cblock(self._GenerateType(type_.additional_properties))
+          .Append('std::map<std::string, %s> additional_properties;' %
+              self._type_helper.GetCppType(type_.additional_properties,
+                                           is_in_container=True))
+        )
+      (c.Eblock()
+        .Sblock(' private:')
+          .Append('DISALLOW_COPY_AND_ASSIGN(%(classname)s);')
+        .Eblock('};')
+      )
+    elif type_.property_type == PropertyType.CHOICES:
+      if type_.description:
+        c.Comment(type_.description)
+      # Choices are modelled with optional fields for each choice. Exactly one
+      # field of the choice is guaranteed to be set by the compiler.
+      (c.Sblock('struct %(classname)s {')
           .Append('%(classname)s();')
-          .Append()
-          .Concat(self._GeneratePropertyStructures(type_.properties.values()))
-          .Concat(self._GenerateFields(type_.properties.values()))
-      )
-      if type_.from_json:
+          .Append('~%(classname)s();')
+          .Append())
+      c.Cblock(self._GenerateTypes(type_.choices))
+      if type_.origin.from_json:
         (c.Comment('Populates a %s object from a base::Value. Returns'
                    ' whether |out| was successfully populated.' % classname)
           .Append('static bool Populate(const base::Value& value, '
                   '%(classname)s* out);')
           .Append()
         )
-
-      if type_.from_client:
-        (c.Comment('Returns a new base::DictionaryValue representing the'
-                   ' serialized form of this %s object. Passes '
-                   'ownership to caller.' % classname)
-          .Append('scoped_ptr<base::DictionaryValue> ToValue() const;')
+      if type_.origin.from_client:
+        (c.Comment('Returns a new base::Value representing the'
+                   ' serialized form of this %s object.' % classname)
+          .Append('scoped_ptr<base::Value> ToValue() const;')
+          .Append()
         )
-
-      (c.Eblock()
-        .Sblock(' private:')
-          .Concat(self._GeneratePrivatePropertyStructures(
-              type_.properties.values()))
-          .Append()
-          .Append('DISALLOW_COPY_AND_ASSIGN(%(classname)s);')
-        .Eblock('};')
-      )
+      c.Append('// Choices:')
+      for choice_type in type_.choices:
+        c.Append('%s as_%s;' % (
+            self._type_helper.GetCppType(choice_type, is_optional=True),
+            choice_type.unix_name))
+      c.Eblock('};')
     c.Substitute({'classname': classname})
     return c
 
   def _GenerateEvent(self, event):
     """Generates the namespaces for an event.
     """
     c = Code()
-    (c.Sblock('namespace %s {' % cpp_util.Classname(event.name))
-        .Concat(self._GenerateCreateCallbackArguments(event,
-                                                      generate_to_json=True))
-      .Eblock('};')
+    # TODO(kalman): use event.unix_name not Classname.
+    event_namespace = cpp_util.Classname(event.name)
+    (c.Append('namespace %s {' % event_namespace)
+      .Append()
+      .Concat(self._GenerateCreateCallbackArguments(event))
+      .Eblock('}  // namespace %s' % event_namespace)
     )
     return c
 
   def _GenerateFunction(self, function):
     """Generates the namespaces and structs for a function.
     """
     c = Code()
-    (c.Sblock('namespace %s {' % cpp_util.Classname(function.name))
-        .Concat(self._GenerateFunctionParams(function))
-        .Append()
+    # TODO(kalman): Use function.unix_name not Classname here.
+    function_namespace = cpp_util.Classname(function.name)
+    (c.Append('namespace %s {' % function_namespace)
+      .Append()
+      .Cblock(self._GenerateFunctionParams(function))
     )
     if function.callback:
-      (c.Concat(self._GenerateFunctionResults(function.callback))
-        .Append()
-      )
-    c.Eblock('};')
-
+      c.Cblock(self._GenerateFunctionResults(function.callback))
+    c.Append('}  // namespace %s' % function_namespace)
     return c
 
   def _GenerateFunctionParams(self, function):
     """Generates the struct for passing parameters from JSON to a function.
     """
+    if not function.params:
+      return Code()
+
     c = Code()
-
-    if function.params:
-      (c.Sblock('struct Params {')
-          .Concat(self._GeneratePropertyStructures(function.params))
-          .Concat(self._GenerateFields(function.params))
-          .Append('~Params();')
-          .Append()
-          .Append('static scoped_ptr<Params> Create('
-                  'const base::ListValue& args);')
-        .Eblock()
-        .Sblock(' private:')
-          .Append('Params();')
-          .Append()
-          .Append('DISALLOW_COPY_AND_ASSIGN(Params);')
-        .Eblock('};')
-      )
-
+    (c.Sblock('struct Params {')
+      .Append('static scoped_ptr<Params> Create(const base::ListValue& args);')
+      .Append('~Params();')
+      .Append()
+      .Cblock(self._GenerateTypes(p.type_ for p in function.params))
+      .Cblock(self._GenerateFields(function.params))
+      .Eblock()
+      .Sblock(' private:')
+        .Append('Params();')
+        .Append()
+        .Append('DISALLOW_COPY_AND_ASSIGN(Params);')
+      .Eblock('};')
+    )
     return c
 
-  def _GeneratePropertyStructures(self, props):
+  def _GenerateTypes(self, types, is_toplevel=False, generate_typedefs=False):
     """Generate the structures required by a property such as OBJECT classes
     and enums.
     """
     c = Code()
-    for prop in props:
-      if prop.type_ == PropertyType.OBJECT:
-        c.Concat(self._GenerateType(prop))
-        c.Append()
-      elif prop.type_ == PropertyType.ARRAY:
-        c.Concat(self._GeneratePropertyStructures([prop.item_type]))
-        c.Append()
-      elif prop.type_ == PropertyType.CHOICES:
-        c.Concat(self._GenerateEnumDeclaration(
-            self._cpp_type_generator.GetChoicesEnumType(prop),
-            prop,
-            [choice.type_.name for choice in prop.choices.values()]))
-        c.Concat(self._GeneratePropertyStructures(prop.choices.values()))
-      elif prop.type_ == PropertyType.ENUM:
-        enum_name = self._cpp_type_generator.GetCompiledType(prop)
-        c.Concat(self._GenerateEnumDeclaration(
-            enum_name,
-            prop,
-            prop.enum_values))
-        create_enum_value = ('scoped_ptr<base::Value> CreateEnumValue(%s %s);' %
-                            (enum_name, prop.unix_name))
-        enum_to_string = 'std::string ToString(%s enum_param);' % enum_name
-        enum_from_string = ('%s From%sString(const std::string& enum_string);' %
-            (enum_name, enum_name))
-        # If the property is from the UI then we're in a struct so this function
-        # should be static. If it's from the client, then we're just in a
-        # namespace so we can't have the static keyword.
-        if prop.from_json:
-          create_enum_value = 'static %s' % create_enum_value
-          enum_to_string = 'static %s' % enum_to_string
-          enum_from_string = 'static %s' % enum_from_string
-        (c.Append(create_enum_value)
-          .Append(enum_to_string)
-          .Append(enum_from_string))
+    for type_ in types:
+      c.Cblock(self._GenerateType(type_,
+                                  is_toplevel=is_toplevel,
+                                  generate_typedefs=generate_typedefs))
     return c
 
-  def _GeneratePrivatePropertyStructures(self, props):
-    """Generate the private structures required by a property such as OBJECT
-    classes and enums.
-    """
-    c = Code()
-    for prop in props:
-      if prop.type_ == PropertyType.ARRAY:
-        c.Concat(self._GeneratePrivatePropertyStructures([prop.item_type]))
-        c.Append()
-      elif prop.type_ == PropertyType.CHOICES:
-        # We only need GetChoiceValue() if there is a ToValue() method.
-        if prop.from_client:
-          c.Append('scoped_ptr<base::Value> Get%sChoiceValue() const;' % (
-              cpp_util.Classname(prop.name)))
-    return c
-
-  def _GenerateCreateCallbackArguments(self, function, generate_to_json=False):
+  def _GenerateCreateCallbackArguments(self, function):
     """Generates functions for passing paramaters to a callback.

[Yoyo Zhou, 2013/01/15 01:49:25]

typo: parameters

[not at google - send to devlin, 2013/01/15 21:47:27]

Done.

     """
     c = Code()
     params = function.params
-    c.Concat(self._GeneratePropertyStructures(params))
+    c.Cblock(self._GenerateTypes((p.type_ for p in params), is_toplevel=True))
 
-    param_lists = self._cpp_type_generator.GetAllPossibleParameterLists(params)
-    for param_list in param_lists:
-      declaration_list = []
-      for param in param_list:
-        if param.description:
-          c.Comment(param.description)
-        declaration_list.append('const %s' % cpp_util.GetParameterDeclaration(
-            param, self._cpp_type_generator.GetCompiledType(param)))
-      c.Append('scoped_ptr<base::ListValue> Create(%s);' %
-               ', '.join(declaration_list))
-      if generate_to_json:
-        c.Append('std::string ToJson(%s);' % ', '.join(declaration_list))
+    declaration_list = []
+    for param in params:
+      if param.description:
+        c.Comment(param.description)
+      declaration_list.append(cpp_util.GetParameterDeclaration(
+          param, self._type_helper.GetCppType(param.type_)))
+    c.Append('scoped_ptr<base::ListValue> Create(%s);' %
+             ', '.join(declaration_list))
     return c
 
   def _GenerateFunctionResults(self, callback):
     """Generates namespace for passing a function's result back.
     """
     c = Code()
-    (c.Sblock('namespace Results {')
-        .Concat(self._GenerateCreateCallbackArguments(callback))
-      .Eblock('};')
+    (c.Append('namespace Results {')
+      .Append()
+      .Concat(self._GenerateCreateCallbackArguments(callback))
+      .Append('}  // namespace Results')
     )
     return c