Add step generator protocol, remove annotated_checkout, remove script.

scripts/slave/unittests/recipes_test.py

 #!/usr/bin/python
 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+"""Provides test coverage for individual recipes.
+
+Recipe tests are located in ../recipes_test/*.py.
+
+Each py file's splitext'd name is expected to match a recipe in ../recipes/*.py.
+
+Each test py file contains one or more test functions:
+  * A test function's name ends with '_test' and takes an instance of TestAPI
+    as its only parameter.
+  * The test should return a dictionary with any of the following keys:
+    * factory_properties
+    * build_properties
+    * test_data
+      * test_data's value should be a dictionary in the form of
+        {stepname -> (retcode, json_data)}
+      * Since the test doesn't run any steps, test_data allows you to simulate
+        return values for particular steps.
+
+Once your test methods are set up, run `recipes_test.py --train`. This will
+take your tests and simulate what steps would have run, given the test inputs,
+and will record them as JSON into files of the form:
+  ../recipes_test/<recipe_name>.<test_name>.expected
+
+If those files look right, make sure they get checked in with your changes.
+
+When this file runs as a test (i.e. as `recipes_test.py`), it will re-evaluate
+the recipes using the test function input data and compare the result to the
+values recorded in the .expected files.
+
+Additionally, this test cannot pass unless every recipe in ../recipes has 100%
+code coverage when executed via the tests in ../recipes_test.
+"""
+
 import contextlib
 import json
 import os
 import sys
 import unittest
 
 from glob import glob
 
 import test_env  # pylint: disable=W0611
 
@@ skipping 62 matching lines @@
   for name, value in gvars.iteritems():
     if name.endswith('_test'):
       ret[name[:-len('_test')]] = value
   return ret
 
 
 def execute_test_case(test_fn, recipe_path):
   test_data = test_fn(TestAPI())
   bp = test_data.get('build_properties', {})
   fp = test_data.get('factory_properties', {})
+  td = test_data.get('test_data', {})
   fp['recipe'] = os.path.basename(os.path.splitext(recipe_path)[0])
 
   stream = annotator.StructuredAnnotationStream(stream=open(os.devnull, 'w'))
   with cover():
     with recipe_util.mock_paths():
-      retval = annotated_run.make_steps(stream, bp, fp, True)
-      assert retval.status_code is None
-      return retval.script or retval.steps
+      step_data = annotated_run.run_steps(stream, bp, fp, td).steps_ran.values()
+      return [s.step for s in step_data]
 
 
 def train_from_tests(recipe_path):
   if not has_test(recipe_path):
     print 'FATAL: Recipe %s has NO tests!' % recipe_path
     return False
 
   for path in glob(expected_for(recipe_path, '*')):
     os.unlink(path)
 
   for name, test_fn in exec_test_file(recipe_path).iteritems():
     steps = execute_test_case(test_fn, recipe_path)
     expected_path = expected_for(recipe_path, name)
     print 'Writing', expected_path
     with open(expected_path, 'w') as f:
-      json.dump(steps, f, indent=2, sort_keys=True)
+      f.write('[')
+      first = True
+      for step in steps:
+        f.write(('' if first else ',')+'\n  ')
+        json.dump(step, f, sort_keys=True)

[Mike Stip (use stip instead), 2013/05/18 00:37:22]

might be possible to find a json prettifier...

[iannucci, 2013/05/18 04:00:36]

Maybe. One thing I toyed with is emitting things in the format of:

[
  {'cmd': [...],
   'other_attr': ..,
   'cool_setting': ...
  },
  {
  ...
  }
]

Do you think that would be a significant improvement?

Actually, screw it, that WOULD be a significant improvement :p. Done.

+        first = False
+      f.write('\n]')
 
   return True
 
 
 def load_tests(loader, _standard_tests, _pattern):
   """This method is invoked by unittest.main's automatic testloader."""
   def create_test_class(recipe_path):
     class RecipeTest(unittest.TestCase):
       def testExists(self):
         self.assertTrue(has_test(recipe_path))
@@ skipping 10 matching lines @@
               with open(expected_path, 'r') as f:
                 expected = json.load(f)
               self.assertEqual(steps, expected)
             test_.__name__ += name
             setattr(cls, test_.__name__, test_)
           add_test(test_fn, expected_path)
 
     if has_test(recipe_path):
       RecipeTest.add_test_methods()
 
-    RecipeTest.__name__ += 'for_%s' % os.path.basename(recipe_path)
+    RecipeTest.__name__ += '_for_%s' % (
+      os.path.splitext(os.path.basename(recipe_path))[0])
     return RecipeTest
 
   suite = unittest.TestSuite()
   for test_class in map(create_test_class, loop_over_recipes()):
     suite.addTest(loader.loadTestsFromTestCase(test_class))
   return suite
 
 
 def loop_over_recipes():
   for _name, path in BASE_DIRS.iteritems():
@@ skipping 42 matching lines @@
     total_covered = COVERAGE.report()
     if total_covered != 100.0:
       print 'FATAL: Recipes are not at 100% coverage.'
       retcode = retcode or 2
 
   return retcode
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))