[package.proto] convert deps from list to map.

recipe_engine/package.py

 # Copyright 2015 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 import copy
 import difflib
 import json
 import logging
 import operator
 import os
@@ skipping 42 matching lines @@
     return os.path.dirname( # <repo root>
             os.path.dirname( # infra
               os.path.dirname( # config
                 os.path.abspath(recipes_cfg)))) # recipes.cfg
 
 
 class ProtoFile(object):
   """A collection of functions operating on a proto path.
 
   This is an object so that it can be mocked in the tests.
+
+  Proto files read will always be upconverted to the current proto in
+  package.proto, and will be written back in their original format.
   """
+  API_VERSIONS = (1, 2)
+
   def __init__(self, path):
     self._path = path
 
   @property
   def path(self):
     return os.path.realpath(self._path)
 
   def read_raw(self):
     with open(self._path, 'r') as fh:
       return fh.read()
 
   def read(self):
-    text = self.read_raw()
+    obj = json.loads(self.read_raw())
+
+    vers = obj.get('api_version')
+    assert vers in self.API_VERSIONS, (
+      'expected %r to be one of %r' % (vers, self.API_VERSIONS)
+    )
+
+    # upconvert old deps-as-a-list to deps-as-a-dict
+    if 'deps' in obj and vers == 1:
+      obj['deps'] = {d.pop('project_id'): d for d in obj['deps']}
+
     buf = package_pb2.Package()
-    json_format.Parse(text, buf, ignore_unknown_fields=True)
+    json_format.ParseDict(obj, buf, ignore_unknown_fields=True)
     return buf
 
   def to_raw(self, buf):
     obj = json_format.MessageToDict(buf, preserving_proto_field_name=True)
+
+    # downconvert if api_version is 1
+    if buf.deps and buf.api_version < 2:
+      deps = []
+      for pid, d in sorted(obj['deps'].iteritems()):
+        d['project_id'] = pid
+        deps.append(d)
+      obj['deps'] = deps
+
     return json.dumps(obj, indent=2, sort_keys=True).replace(' \n', '\n')
 
   def write(self, buf):
     with open(self._path, 'w') as fh:
       fh.write(self.to_raw(buf))
 
 
 class PackageContext(object):
   """Contains information about where the root package and its dependency
   checkouts live.
@@ skipping 105 matching lines @@
     checkout_dir = self._dep_dir(context)
     self.backend.checkout(
         self.repo, self.revision, checkout_dir, context.allow_fetch)
     cleanup_pyc(checkout_dir)
 
   def repo_root(self, context):
     return os.path.join(self._dep_dir(context), self.path)
 
   def dump(self):
     buf = package_pb2.DepSpec(
-        project_id=self.project_id,
         url=self.repo,
         branch=self.branch,
         revision=self.revision)
     if self.path:
       buf.path_override = self.path
 
     # Only dump repo_type if it's different from default. This preserves
     # compatibility e.g. with recipes.py bootstrap scripts in client repos
     # which may not handle repo_type correctly.
     # TODO(phajdan.jr): programmatically extract the default value.
@@ skipping 104 matching lines @@
     Requires a good checkout."""
     return ProtoFile(InfraRepoConfig().to_recipes_cfg(self.path))
 
   def updates(self, _context, _other_revision=None):
     """Returns (empty) list of potential updates for this spec."""
     return []
 
   def dump(self):
     """Returns the package.proto DepSpec form of this RepoSpec."""
     return package_pb2.DepSpec(
-        project_id=self.project_id,
         url="file://"+self.path)
 
   def __eq__(self, other):
     if not isinstance(other, type(self)):
       return False
     return self.path == other.path
 
 
 class RootRepoSpec(RepoSpec):
   def __init__(self, proto_file):
@@ skipping 111 matching lines @@
         # Prevent rolling backwards.
         more_recent_revision = other_spec.get_more_recent_revision(
             self._context, current_revision, other_spec.revision)
         if more_recent_revision != other_spec.revision:
           return False
 
         self._updates[other_spec.project_id] = other_spec
 
   def get_rolled_spec(self):
     """Returns a PackageSpec with all the deps updates from this roll."""
-    # TODO(phajdan.jr): does this preserve comments? should it?
     new_deps = _updated(
         self._package_spec.deps,
         { project_id: spec for project_id, spec in
           self._updates.iteritems() })
     return PackageSpec(
+        self._package_spec.api_version,
         self._package_spec.project_id,
         self._package_spec.recipes_path,
         new_deps)
 
   def get_commit_infos(self):
     """Returns a mapping project_id -> list of commits from that repo
     that are getting pulled by this roll.
     """
     commit_infos = {}
 
@@ skipping 11 matching lines @@
 
   def get_diff(self):
     """Returns a unified diff between original package spec and one after roll.
     """
     orig = str(self._package_spec.dump()).splitlines()
     new = str(self.get_rolled_spec().dump()).splitlines()
     return '\n'.join(difflib.unified_diff(orig, new, lineterm=''))
 
 
 class PackageSpec(object):
-  API_VERSION = 1
-
-  def __init__(self, project_id, recipes_path, deps):
+  def __init__(self, api_version, project_id, recipes_path, deps):
+    self._api_version = api_version
     self._project_id = project_id
     self._recipes_path = recipes_path
     self._deps = deps
 
   def __repr__(self):
     return 'PackageSpec(%s, %s, %r)' % (self._project_id, self._recipes_path,
                                         self._deps)
 
   @classmethod
   def load_proto(cls, proto_file):
     buf = proto_file.read()
-    assert buf.api_version == cls.API_VERSION
 
-    deps = { str(dep.project_id): cls.spec_for_dep(dep)
-             for dep in buf.deps }
-    return cls(str(buf.project_id), str(buf.recipes_path), deps)
+    deps = { pid: cls.spec_for_dep(pid, dep)
+             for pid, dep in buf.deps.iteritems() }
+    return cls(buf.api_version, str(buf.project_id), str(buf.recipes_path),
+               deps)
 
   @classmethod
-  def spec_for_dep(cls, dep):
+  def spec_for_dep(cls, project_id, dep):
     """Returns a RepoSpec for the given dependency protobuf."""
     url = str(dep.url)
     if url.startswith("file://"):
-      return PathRepoSpec(str(dep.project_id), url[len("file://"):])
+      return PathRepoSpec(str(project_id), url[len("file://"):])
 
     if dep.repo_type in (package_pb2.DepSpec.GIT, package_pb2.DepSpec.GITILES):
       if dep.repo_type == package_pb2.DepSpec.GIT:
         backend = fetch.GitBackend()
       elif dep.repo_type == package_pb2.DepSpec.GITILES:
         backend = fetch.GitilesBackend()
-      return GitRepoSpec(str(dep.project_id),
+      return GitRepoSpec(str(project_id),
                          url,
                          str(dep.branch),
                          str(dep.revision),
                          str(dep.path_override),
                          backend)
 
     assert False, 'Unexpected repo type: %s' % dep
 
   @property
   def project_id(self):
     return self._project_id
 
   @property
   def recipes_path(self):
     return self._recipes_path
 
   @property
   def deps(self):
     return self._deps
 
+  @property
+  def api_version(self):
+    return self._api_version
+
   def dump(self):
     return package_pb2.Package(
-        api_version=self.API_VERSION,
+        api_version=self._api_version,
         project_id=self._project_id,
         recipes_path=self._recipes_path,
-        deps=[ self._deps[dep].dump() for dep in sorted(self._deps.keys()) ])
+        deps={k: v.dump() for k, v in self._deps.iteritems()})
 
   def roll_candidates(self, root_spec, context):
     """Returns list of consistent roll candidates, and rejected roll candidates.
 
     The first one is sorted by score, descending. The more commits are pulled by
     the roll, the higher score.
 
     Second list is included to distinguish between a situation where there are
     no roll candidates from one where there are updates but they're not
     consistent.
@@ skipping 121 matching lines @@
   >>> d = { 'x': 1, 'y': 2 }
   >>> sorted(_updated(d, { 'y': 3, 'z': 4 }).items())
   [('x', 1), ('y', 3), ('z', 4)]
   >>> sorted(d.items())
   [('x', 1), ('y', 2)]
   """
 
   d = copy.copy(d)
   d.update(updates)
   return d