OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/env python |
| 2 # Copyright (c) 2013 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. |
| 5 |
| 6 import contextlib |
| 7 import json |
| 8 import os |
| 9 import sys |
| 10 import unittest |
| 11 |
| 12 from glob import glob |
| 13 |
| 14 import test_env # pylint: disable=W0611 |
| 15 |
| 16 import coverage |
| 17 |
| 18 from common import annotator |
| 19 from slave import annotated_run |
| 20 from slave import recipe_util |
| 21 |
| 22 SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) |
| 23 ROOT_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, os.pardir, os.pardir, |
| 24 os.pardir)) |
| 25 SLAVE_DIR = os.path.join(ROOT_PATH, 'slave', 'fake_slave', 'build') |
| 26 BASE_DIRS = { |
| 27 'Public': os.path.dirname(SCRIPT_PATH) |
| 28 } |
| 29 # TODO(iannucci): Check for duplicate recipe names when we have more than one |
| 30 # base_dir |
| 31 |
| 32 COVERAGE = coverage.coverage( |
| 33 include=[os.path.join(x, 'recipes', '*') for x in BASE_DIRS.values()]) |
| 34 |
| 35 @contextlib.contextmanager |
| 36 def cover(): |
| 37 COVERAGE.start() |
| 38 try: |
| 39 yield |
| 40 finally: |
| 41 COVERAGE.stop() |
| 42 |
| 43 |
| 44 class TestAPI(object): |
| 45 @staticmethod |
| 46 def tryserver_build_properties(**kwargs): |
| 47 ret = { |
| 48 'issue': 12853011, |
| 49 'patchset': 1, |
| 50 'blamelist': ['cool_dev1337@chromium.org'], |
| 51 'rietveld': 'https://chromiumcodereview.appspot.com', |
| 52 } |
| 53 ret.update(kwargs) |
| 54 return ret |
| 55 |
| 56 |
| 57 class DevNull(object): |
| 58 @staticmethod |
| 59 def write(data): |
| 60 pass |
| 61 |
| 62 @staticmethod |
| 63 def flush(): |
| 64 pass |
| 65 |
| 66 |
| 67 def test_path_for_recipe(recipe_path): |
| 68 root = os.path.dirname(os.path.dirname(recipe_path)) |
| 69 return os.path.join(root, 'recipes_test', os.path.basename(recipe_path)) |
| 70 |
| 71 |
| 72 def has_test(recipe_path): |
| 73 return os.path.exists(test_path_for_recipe(recipe_path)) |
| 74 |
| 75 |
| 76 def expected_for(recipe_path, test_name): |
| 77 test_base = os.path.splitext(test_path_for_recipe(recipe_path))[0] |
| 78 return "%s.%s.expected" % (test_base, test_name) |
| 79 |
| 80 |
| 81 def exec_test_file(recipe_path): |
| 82 test_path = test_path_for_recipe(recipe_path) |
| 83 gvars = {} |
| 84 execfile(test_path, gvars) |
| 85 return dict(item for item in gvars.iteritems() if item[0].endswith('_test')) |
| 86 |
| 87 |
| 88 def execute_test_case(test_fn, recipe_path): |
| 89 test_data = test_fn(TestAPI()) |
| 90 bp = test_data.get('build_properties', {}) |
| 91 fp = test_data.get('factory_properties', {}) |
| 92 fp['recipe'] = os.path.basename(os.path.splitext(recipe_path)[0]) |
| 93 |
| 94 stream = annotator.StructuredAnnotationStream(stream=DevNull()) |
| 95 with cover(): |
| 96 with recipe_util.mock_paths(): |
| 97 retval = annotated_run.make_steps(stream, bp, fp, True) |
| 98 assert retval.status_code is None |
| 99 return retval.script or retval.steps |
| 100 |
| 101 |
| 102 def train_from_tests(_base_dir_type, recipe_path): |
| 103 if not has_test(recipe_path): |
| 104 print 'FATAL: Recipe %s has NO tests!' % recipe_path |
| 105 return False |
| 106 |
| 107 for path in glob(expected_for(recipe_path, '*')): |
| 108 os.unlink(path) |
| 109 |
| 110 for name, test_fn in exec_test_file(recipe_path).iteritems(): |
| 111 steps = execute_test_case(test_fn, recipe_path) |
| 112 expected_path = expected_for(recipe_path, name) |
| 113 print 'Writing', expected_path |
| 114 with open(expected_path, 'w') as f: |
| 115 json.dump(steps, f, indent=2, sort_keys=True) |
| 116 |
| 117 return True |
| 118 |
| 119 |
| 120 def NoTestsForRecipe(recipe_path): |
| 121 def test_DoesntExist(self): |
| 122 self.assert_(False, 'No tests exist for %s' % recipe_path) |
| 123 return test_DoesntExist |
| 124 |
| 125 |
| 126 def RecipeTest(name, expected_path, test_fn, recipe_path): |
| 127 def test_(self): |
| 128 steps = execute_test_case(test_fn, recipe_path) |
| 129 # Roundtrip json to get same string encoding as load |
| 130 steps = json.loads(json.dumps(steps)) |
| 131 with open(expected_path, 'r') as f: |
| 132 expected = json.load(f) |
| 133 self.assertEqual(steps, expected) |
| 134 |
| 135 test_.__name__ += name |
| 136 return test_ |
| 137 |
| 138 |
| 139 def add_tests(_base_dir_type, recipe_path, loader, suite): |
| 140 tests = [] |
| 141 if not has_test(recipe_path): |
| 142 tests.append(NoTestsForRecipe(recipe_path)) |
| 143 else: |
| 144 for name, test_fn in exec_test_file(recipe_path).iteritems(): |
| 145 tests.append(RecipeTest(name, expected_for(recipe_path, name), |
| 146 test_fn, recipe_path)) |
| 147 |
| 148 suite.addTest(loader.loadTestsFromTestCase( |
| 149 type('RecipeTest_for_%s' % os.path.basename(recipe_path), |
| 150 (unittest.TestCase,), dict((x.__name__, x) for x in tests)))) |
| 151 return True |
| 152 |
| 153 |
| 154 def load_tests(loader, standard_tests, _pattern): |
| 155 """This method is invoked by unittest.main's automatic testloader when |
| 156 this module is loaded for tests.""" |
| 157 assert not standard_tests.countTestCases() |
| 158 suite = unittest.TestSuite() |
| 159 for result in loop_over_recipes(add_tests, loader, suite): |
| 160 assert result, 'Got bad result: %s' % result |
| 161 return suite |
| 162 |
| 163 |
| 164 def loop_over_recipes(func, *args, **kwargs): |
| 165 for name, path in BASE_DIRS.iteritems(): |
| 166 recipe_dir = os.path.join(path, 'recipes') |
| 167 for root, _dirs, files in os.walk(recipe_dir): |
| 168 for recipe in (f for f in files if f.endswith('.py') and f[0] != '_'): |
| 169 recipe_path = os.path.join(root, recipe) |
| 170 with cover(): |
| 171 # Force this file into coverage, even if there's no test for it. |
| 172 execfile(recipe_path, {}) |
| 173 yield func(name, recipe_path, *args, **kwargs) |
| 174 |
| 175 |
| 176 def main(argv): |
| 177 if not os.path.exists(SLAVE_DIR): |
| 178 os.makedirs(SLAVE_DIR) |
| 179 |
| 180 os.chdir(SLAVE_DIR) |
| 181 |
| 182 training = False |
| 183 is_help = False |
| 184 if '--help' in argv or '-h' in argv: |
| 185 print 'Pass --train to enter training mode.' |
| 186 print |
| 187 is_help = True |
| 188 if '--train' in argv: |
| 189 argv.remove('--train') |
| 190 training = True |
| 191 |
| 192 had_errors = False |
| 193 if training and not is_help: |
| 194 for result in loop_over_recipes(train_from_tests): |
| 195 had_errors = had_errors or result |
| 196 |
| 197 retcode = 1 if had_errors else 0 |
| 198 |
| 199 if not training: |
| 200 try: |
| 201 unittest.main() |
| 202 except SystemExit as e: |
| 203 retcode = e.code or retcode |
| 204 |
| 205 if not is_help: |
| 206 total_covered = COVERAGE.report() |
| 207 if total_covered != 100.0: |
| 208 print 'FATAL: Recipes are not at 100% coverage.' |
| 209 retcode = retcode or 2 |
| 210 |
| 211 return retcode |
| 212 |
| 213 |
| 214 if __name__ == '__main__': |
| 215 sys.exit(main(sys.argv)) |
OLD | NEW |