Overhaul JSON Schema Compiler to support a number of features required to

tools/json_schema_compiler/cpp_type_generator.py

Index: tools/json_schema_compiler/cpp_type_generator.py
diff --git a/tools/json_schema_compiler/cpp_type_generator.py b/tools/json_schema_compiler/cpp_type_generator.py
index deade36d839e2cbdf7b2d221aa10f9589f7d9cf4..c9ad6d73ca1d5838fd982b79cf482278d076cbac 100644
--- a/tools/json_schema_compiler/cpp_type_generator.py
+++ b/tools/json_schema_compiler/cpp_type_generator.py
@@ -3,8 +3,7 @@
 # found in the LICENSE file.
 
 from code import Code
-from model import PropertyType
-import any_helper
+from model import Namespace, PropertyType, Type
 import cpp_util
 import operator
 import schema_util
@@ -24,42 +23,22 @@ class CppTypeGenerator(object):
     if namespace and cpp_namespace:
       self._namespace = namespace
       self.AddNamespace(namespace, cpp_namespace)
+    else:
+      self._namespace = None
 
   def AddNamespace(self, namespace, cpp_namespace):
     """Maps a model.Namespace to its C++ namespace name. All mappings are
     beneath the root namespace.
     """
-    for type_ in namespace.types:
-      if type_ in self._type_namespaces:
-        raise ValueError('Type %s is declared in both %s and %s' %
-            (type_, namespace.name, self._type_namespaces[type_].name))
-      self._type_namespaces[type_] = namespace
     self._cpp_namespaces[namespace] = cpp_namespace
-
-  def ExpandParams(self, params):
-    """Returns the given parameters with PropertyType.CHOICES parameters
-    expanded so that each choice is a separate parameter.
-    """
-    expanded = []
-    for param in params:
-      if param.type_ == PropertyType.CHOICES:
-        for choice in param.choices.values():
-          expanded.append(choice)
-      else:
-        expanded.append(param)
-    return expanded
-
-  def GetAllPossibleParameterLists(self, params):
-    """Returns all possible parameter lists for the given set of parameters.
-    Every combination of arguments passed to any of the PropertyType.CHOICES
-    parameters will have a corresponding parameter list returned here.
-    """
-    if not params:
-      return [[]]
-    partial_parameter_lists = self.GetAllPossibleParameterLists(params[1:])
-    return [[param] + partial_list
-            for param in self.ExpandParams(params[:1])
-            for partial_list in partial_parameter_lists]
+    for type_name in namespace.types:
+      # Allow $refs to refer to just 'Type' within namespaces. Otherwise they
+      # must be qualified with 'namespace.Type'.
+      type_aliases = ['%s.%s' % (namespace.name, type_name)]
+      if namespace is self._namespace:
+        type_aliases.append(type_name)
+      for alias in type_aliases:
+        self._type_namespaces[alias] = namespace
 
   def GetCppNamespaceName(self, namespace):
     """Gets the mapped C++ namespace name for the given namespace relative to
@@ -95,106 +74,89 @@ class CppTypeGenerator(object):
     return Code().Append('}  // %s' %
         self.GetCppNamespaceName(self._namespace))
 
-  def GetEnumNoneValue(self, prop):
+  def GetEnumNoneValue(self, type_):
     """Gets the enum value in the given model.Property indicating no value has
     been set.
     """
-    return '%s_NONE' % self.GetReferencedProperty(prop).unix_name.upper()
+    return '%s_NONE' % self.FollowRef(type_).unix_name.upper()
 
-  def GetEnumValue(self, prop, enum_value):
+  def GetEnumValue(self, type_, enum_value):
     """Gets the enum value of the given model.Property of the given type.
 
     e.g VAR_STRING
     """
-    return '%s_%s' % (self.GetReferencedProperty(prop).unix_name.upper(),
+    return '%s_%s' % (self.FollowRef(type_).unix_name.upper(),
                       cpp_util.Classname(enum_value.upper()))
 
-  def GetChoicesEnumType(self, prop):
-    """Gets the type of the enum for the given model.Property.
-
-    e.g VarType
-    """
-    return cpp_util.Classname(prop.name) + 'Type'
-
-  def GetType(self, prop, pad_for_generics=False, wrap_optional=False):
-    return self._GetTypeHelper(prop, pad_for_generics, wrap_optional)
-
-  def GetCompiledType(self, prop, pad_for_generics=False, wrap_optional=False):
-    return self._GetTypeHelper(prop, pad_for_generics, wrap_optional,
-                               use_compiled_type=True)
-
-  def _GetTypeHelper(self, prop, pad_for_generics=False, wrap_optional=False,
-                     use_compiled_type=False):
-    """Translates a model.Property into its C++ type.
+  def GetCppType(self, type_, is_optional=False, is_in_container=False):
+    """Translates a model.Property or model.Type into its C++ type.
 
     If REF types from different namespaces are referenced, will resolve
     using self._type_namespaces.
 
-    Use pad_for_generics when using as a generic to avoid operator ambiguity.
+    Use |is_optional| if the type is optional. This will wrap the type in a
+    scoped_ptr if possible (i.e. it is not possible to wrap an enum).

[Yoyo Zhou, 2013/01/15 01:49:25]

nit: don't say i.e. here

[not at google - send to devlin, 2013/01/15 21:47:27]

Done.

 
-    Use wrap_optional to wrap the type in a scoped_ptr<T> if the Property is
-    optional.
-
-    Use use_compiled_type when converting from prop.type_ to prop.compiled_type.
+    Use |is_in_container| if the type is appearing in a collection, e.g. a
+    std::vector or std::map. This will wrap it in the correct type with spacing.
     """
     cpp_type = None
-    type_ = prop.type_ if not use_compiled_type else prop.compiled_type
-
-    if type_ == PropertyType.REF:
-      dependency_namespace = self._ResolveTypeNamespace(prop.ref_type)
+    if type_.property_type == PropertyType.REF:
+      ref = type_.ref_type
+      dependency_namespace = self._ResolveTypeNamespace(ref)
       if not dependency_namespace:
-        raise KeyError('Cannot find referenced type: %s' % prop.ref_type)
+        raise KeyError('Cannot find referenced type: %s' % ref)
       if self._namespace != dependency_namespace:
         cpp_type = '%s::%s' % (self._cpp_namespaces[dependency_namespace],
-            schema_util.StripSchemaNamespace(prop.ref_type))
+                               schema_util.StripSchemaNamespace(ref))
       else:
-        cpp_type = schema_util.StripSchemaNamespace(prop.ref_type)
-    elif type_ == PropertyType.BOOLEAN:
+        cpp_type = schema_util.StripSchemaNamespace(ref)
+    elif type_.property_type == PropertyType.BOOLEAN:
       cpp_type = 'bool'
-    elif type_ == PropertyType.INTEGER:
+    elif type_.property_type == PropertyType.INTEGER:
       cpp_type = 'int'
-    elif type_ == PropertyType.INT64:
+    elif type_.property_type == PropertyType.INT64:
       cpp_type = 'int64'
-    elif type_ == PropertyType.DOUBLE:
+    elif type_.property_type == PropertyType.DOUBLE:
       cpp_type = 'double'
-    elif type_ == PropertyType.STRING:
+    elif type_.property_type == PropertyType.STRING:
       cpp_type = 'std::string'
-    elif type_ == PropertyType.ENUM:
-      cpp_type = cpp_util.Classname(prop.name)
-    elif type_ == PropertyType.ADDITIONAL_PROPERTIES:
-      cpp_type = 'base::DictionaryValue'
-    elif type_ == PropertyType.ANY:
-      cpp_type = any_helper.ANY_CLASS
-    elif type_ == PropertyType.OBJECT:
-      cpp_type = cpp_util.Classname(prop.name)
-    elif type_ == PropertyType.FUNCTION:
+    elif type_.property_type == PropertyType.ENUM:
+      cpp_type = cpp_util.Classname(type_.name)
+    elif type_.property_type == PropertyType.ANY:
+      cpp_type = 'base::Value'
+    elif (type_.property_type == PropertyType.OBJECT or
+          type_.property_type == PropertyType.CHOICES):
+      cpp_type = cpp_util.Classname(type_.name)
+    elif type_.property_type == PropertyType.FUNCTION:
       # Functions come into the json schema compiler as empty objects. We can
       # record these as empty DictionaryValue's so that we know if the function

[Yoyo Zhou, 2013/01/15 01:49:25]

nit: no '

[not at google - send to devlin, 2013/01/15 21:47:27]

Done.

       # was passed in or not.
       cpp_type = 'base::DictionaryValue'
-    elif type_ == PropertyType.ARRAY:
-      item_type = prop.item_type
-      if item_type.type_ == PropertyType.REF:
-        item_type = self.GetReferencedProperty(item_type)
-      if item_type.type_ in (
-          PropertyType.REF, PropertyType.ANY, PropertyType.OBJECT):
-        cpp_type = 'std::vector<linked_ptr<%s> > '
-      else:
-        cpp_type = 'std::vector<%s> '
-      cpp_type = cpp_type % self.GetType(
-          prop.item_type, pad_for_generics=True)
-    elif type_ == PropertyType.BINARY:
+    elif type_.property_type == PropertyType.ARRAY:
+      cpp_type = 'std::vector<%s>' % self.GetCppType(type_.item_type,
+                                                     is_in_container=True)
+    elif type_.property_type == PropertyType.BINARY:
       cpp_type = 'std::string'
     else:
-      raise NotImplementedError(type_)
+      raise NotImplementedError('%s with %s' % (type_, type_.type_))

[Yoyo Zhou, 2013/01/15 01:49:25]

Does this error message make sense?

[not at google - send to devlin, 2013/01/15 21:47:27]

Not really, but it does now!

+
+    # Optional ENUM is represented elsewhere with a _NONE value.
+    if is_optional and is_in_container:
+      raise ValueError('Cannot have optional types within collections')
+    if (is_optional and
+        not self.FollowRef(type_).property_type == PropertyType.ENUM):
+      cpp_type = 'scoped_ptr<%s>' % cpp_util.PadForGenerics(cpp_type)
+    if is_in_container and not self.IsCopyable(type_):
+      cpp_type = 'linked_ptr<%s> ' % cpp_util.PadForGenerics(cpp_type)
 
-    # Enums aren't wrapped because C++ won't allow it. Optional enums have a
-    # NONE value generated instead.
-    if wrap_optional and prop.optional and not self.IsEnumOrEnumRef(prop):
-      cpp_type = 'scoped_ptr<%s> ' % cpp_type
-    if pad_for_generics:
-      return cpp_type
-    return cpp_type.strip()
+    return cpp_type
+
+  def IsCopyable(self, type_):
+    return not (self.FollowRef(type_).property_type in (PropertyType.ANY,
+                                                        PropertyType.ARRAY,
+                                                        PropertyType.OBJECT,
+                                                        PropertyType.CHOICES))
 
   def GenerateForwardDeclarations(self):
     """Returns the forward declarations for self._namespace.
@@ -207,24 +169,17 @@ class CppTypeGenerator(object):
     for namespace in sorted(namespace_type_dependencies.keys(),
                             key=operator.attrgetter('name')):
       c.Append('namespace %s {' % namespace.name)
-      for type_ in sorted(namespace_type_dependencies[namespace],
-                          key=schema_util.StripSchemaNamespace):
-        type_name = schema_util.StripSchemaNamespace(type_)
-        if namespace.types[type_].type_ == PropertyType.STRING:
-          c.Append('typedef std::string %s;' % type_name)
-        elif namespace.types[type_].type_ == PropertyType.ARRAY:
-          c.Append('typedef std::vector<%(item_type)s> %(name)s;')
-          c.Substitute({
-            'name': type_name,
-            'item_type': self.GetType(namespace.types[type_].item_type,
-                                      wrap_optional=True)})
-        # Enums cannot be forward declared.
-        elif namespace.types[type_].type_ != PropertyType.ENUM:
-          c.Append('struct %s;' % type_name)
+      for type_name in sorted(namespace_type_dependencies[namespace],
+                              key=schema_util.StripSchemaNamespace):
+        simple_type_name = schema_util.StripSchemaNamespace(type_name)
+        type_ = namespace.types[simple_type_name];

[Yoyo Zhou, 2013/01/15 01:49:25]

;?

[not at google - send to devlin, 2013/01/15 21:47:27]

Done.

+        # Add more ways to forward declare things as necessary.
+        if type_.property_type == PropertyType.OBJECT:
+          c.Append('struct %s;' % simple_type_name)
       c.Append('}')
     c.Concat(self.GetNamespaceStart())
     for (name, type_) in self._namespace.types.items():
-      if not type_.functions and type_.type_ == PropertyType.OBJECT:
+      if not type_.functions and type_.property_type == PropertyType.OBJECT:
         c.Append('struct %s;' % schema_util.StripSchemaNamespace(name))
     c.Concat(self.GetNamespaceEnd())
     return c
@@ -244,37 +199,28 @@ class CppTypeGenerator(object):
       c.Append('#include "base/json/json_writer.h"')
     return c
 
-  def _ResolveTypeNamespace(self, ref_type):
+  def _ResolveTypeNamespace(self, qualified_name):
     """Resolves a type, which must be explicitly qualified, to its enclosing
     namespace.
     """
-    if ref_type in self._type_namespaces:
-      return self._type_namespaces[ref_type]
-    raise KeyError('Cannot resolve type: %s. Maybe it needs a namespace prefix '
-                   'if it comes from another namespace?' % ref_type)
-    return None
+    return self._type_namespaces.get(qualified_name, None)

[Yoyo Zhou, 2013/01/15 01:49:25]

Shouldn't it still be an error if this would return None?

[not at google - send to devlin, 2013/01/15 21:47:27]

Done.

 
-  def GetReferencedProperty(self, prop):
-    """Returns the property a property of type REF is referring to.
+  def FollowRef(self, type_):
+    """Follows $ref link of types to resolve the concrete type a ref refers to.
 
     If the property passed in is not of type PropertyType.REF, it will be
     returned unchanged.
     """
-    if prop.type_ != PropertyType.REF:
-      return prop
-    return self._ResolveTypeNamespace(prop.ref_type).types.get(prop.ref_type,
-        None)
+    if not type_.property_type == PropertyType.REF:
+      return type_
+    ref = type_.ref_type
 
-  def IsEnumOrEnumRef(self, prop):
-    """Returns true if the property is an ENUM or a reference to an ENUM.
-    """
-    return self.GetReferencedProperty(prop).type_ == PropertyType.ENUM
+    without_namespace = ref
+    if '.' in ref:
+      without_namespace = ref.split('.', 1)[1]
 
-  def IsEnumRef(self, prop):
-    """Returns true if the property is a reference to an ENUM.
-    """
-    return (prop.type_ == PropertyType.REF and
-            self.GetReferencedProperty(prop).type_ == PropertyType.ENUM)
+    # TODO(kalman): Do we need to keep on resolving?
+    return self._ResolveTypeNamespace(ref).types[without_namespace]
 
   def _NamespaceTypeDependencies(self):
     """Returns a dict containing a mapping of model.Namespace to the C++ type
@@ -283,16 +229,16 @@ class CppTypeGenerator(object):
     dependencies = set()
     for function in self._namespace.functions.values():
       for param in function.params:
-        dependencies |= self._PropertyTypeDependencies(param)
+        dependencies |= self._TypeDependencies(param.type_)
       if function.callback:
         for param in function.callback.params:
-          dependencies |= self._PropertyTypeDependencies(param)
+          dependencies |= self._TypeDependencies(param.type_)
     for type_ in self._namespace.types.values():
       for prop in type_.properties.values():
-        dependencies |= self._PropertyTypeDependencies(prop)
+        dependencies |= self._TypeDependencies(prop.type_)
     for event in self._namespace.events.values():
       for param in event.params:
-        dependencies |= self._PropertyTypeDependencies(param)
+        dependencies |= self._TypeDependencies(param.type_)
 
     dependency_namespaces = dict()
     for dependency in dependencies:
@@ -302,18 +248,17 @@ class CppTypeGenerator(object):
         dependency_namespaces[namespace].append(dependency)
     return dependency_namespaces
 
-  def _PropertyTypeDependencies(self, prop):
+  def _TypeDependencies(self, type_):
     """Recursively gets all the type dependencies of a property.
     """
     deps = set()
-    if prop:
-      if prop.type_ == PropertyType.REF:
-        deps.add(prop.ref_type)
-      elif prop.type_ == PropertyType.ARRAY:
-        deps = self._PropertyTypeDependencies(prop.item_type)
-      elif prop.type_ == PropertyType.OBJECT:
-        for p in prop.properties.values():
-          deps |= self._PropertyTypeDependencies(p)
+    if type_.property_type == PropertyType.REF:
+      deps.add(type_.ref_type)
+    elif type_.property_type == PropertyType.ARRAY:
+      deps = self._TypeDependencies(type_.item_type)
+    elif type_.property_type == PropertyType.OBJECT:
+      for p in type_.properties.values():
+        deps |= self._TypeDependencies(p.type_)
     return deps
 
   def GeneratePropertyValues(self, property, line, nodoc=False):
@@ -323,20 +268,19 @@ class CppTypeGenerator(object):
     if not nodoc:
       c.Comment(property.description)
 
-    if property.has_value:
+    if property.value is not None:
       c.Append(line % {
-          "type": self._GetPrimitiveType(property.type_),
+          "type": self.GetCppType(property.type_),
           "name": property.name,
           "value": property.value
         })
     else:
       has_child_code = False
       c.Sblock('namespace %s {' % property.name)
-      for child_property in property.properties.values():
-        child_code = self.GeneratePropertyValues(
-            child_property,
-            line,
-            nodoc=nodoc)
+      for child_property in property.type_.properties.values():
+        child_code = self.GeneratePropertyValues(child_property,
+                                                 line,
+                                                 nodoc=nodoc)
         if child_code:
           has_child_code = True
           c.Concat(child_code)
@@ -344,16 +288,3 @@ class CppTypeGenerator(object):
       if not has_child_code:
         c = None
     return c
-
-  def _GetPrimitiveType(self, type_):
-    """Like |GetType| but only accepts and returns C++ primitive types.
-    """
-    if type_ == PropertyType.BOOLEAN:
-      return 'bool'
-    elif type_ == PropertyType.INTEGER:
-      return 'int'
-    elif type_ == PropertyType.DOUBLE:
-      return 'double'
-    elif type_ == PropertyType.STRING:
-      return 'char*'
-    raise Exception(type_ + ' is not primitive')