Add step generator protocol, remove annotated_checkout, remove script.

scripts/slave/unittests/recipes_test.py

Index: scripts/slave/unittests/recipes_test.py
diff --git a/scripts/slave/unittests/recipes_test.py b/scripts/slave/unittests/recipes_test.py
index 7508b6e1bd21776481d6fdf445321b49112d061c..c27195719370e423d1549285b1aea56a15a3b74f 100755
--- a/scripts/slave/unittests/recipes_test.py
+++ b/scripts/slave/unittests/recipes_test.py
@@ -3,6 +3,39 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+"""Provides test coverage for individual recipes.
+
+Recipe tests are located in ../recipes_test/*.py.
+
+Each py file's splitext'd name is expected to match a recipe in ../recipes/*.py.
+
+Each test py file contains one or more test functions:
+  * A test function's name ends with '_test' and takes an instance of TestAPI
+    as its only parameter.
+  * The test should return a dictionary with any of the following keys:
+    * factory_properties
+    * build_properties
+    * test_data
+      * test_data's value should be a dictionary in the form of
+        {stepname -> (retcode, json_data)}
+      * Since the test doesn't run any steps, test_data allows you to simulate
+        return values for particular steps.
+
+Once your test methods are set up, run `recipes_test.py --train`. This will
+take your tests and simulate what steps would have run, given the test inputs,
+and will record them as JSON into files of the form:
+  ../recipes_test/<recipe_name>.<test_name>.expected
+
+If those files look right, make sure they get checked in with your changes.
+
+When this file runs as a test (i.e. as `recipes_test.py`), it will re-evaluate
+the recipes using the test function input data and compare the result to the
+values recorded in the .expected files.
+
+Additionally, this test cannot pass unless every recipe in ../recipes has 100%
+code coverage when executed via the tests in ../recipes_test.
+"""
+
 import contextlib
 import json
 import os
@@ -85,14 +118,14 @@ def execute_test_case(test_fn, recipe_path):
   test_data = test_fn(TestAPI())
   bp = test_data.get('build_properties', {})
   fp = test_data.get('factory_properties', {})
+  td = test_data.get('test_data', {})
   fp['recipe'] = os.path.basename(os.path.splitext(recipe_path)[0])
 
   stream = annotator.StructuredAnnotationStream(stream=open(os.devnull, 'w'))
   with cover():
     with recipe_util.mock_paths():
-      retval = annotated_run.make_steps(stream, bp, fp, True)
-      assert retval.status_code is None
-      return retval.script or retval.steps
+      step_data = annotated_run.run_steps(stream, bp, fp, td).steps_ran.values()
+      return [s.step for s in step_data]
 
 
 def train_from_tests(recipe_path):
@@ -108,7 +141,13 @@ def train_from_tests(recipe_path):
     expected_path = expected_for(recipe_path, name)
     print 'Writing', expected_path
     with open(expected_path, 'w') as f:
-      json.dump(steps, f, indent=2, sort_keys=True)
+      f.write('[')
+      first = True
+      for step in steps:
+        f.write(('' if first else ',')+'\n  ')
+        json.dump(step, f, sort_keys=True)

[Mike Stip (use stip instead), 2013/05/18 00:37:22]

might be possible to find a json prettifier...

[iannucci, 2013/05/18 04:00:36]

Maybe. One thing I toyed with is emitting things in the format of:

[
  {'cmd': [...],
   'other_attr': ..,
   'cool_setting': ...
  },
  {
  ...
  }
]

Do you think that would be a significant improvement?

Actually, screw it, that WOULD be a significant improvement :p. Done.

+        first = False
+      f.write('\n]')
 
   return True
 
@@ -139,7 +178,8 @@ def load_tests(loader, _standard_tests, _pattern):
     if has_test(recipe_path):
       RecipeTest.add_test_methods()
 
-    RecipeTest.__name__ += 'for_%s' % os.path.basename(recipe_path)
+    RecipeTest.__name__ += '_for_%s' % (
+      os.path.splitext(os.path.basename(recipe_path))[0])
     return RecipeTest
 
   suite = unittest.TestSuite()