Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(1)

Side by Side Diff: scripts/slave/unittests/recipes_test.py

Issue 14988009: First cut of testing infrastructure for recipes. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/build
Patch Set: Address comments Created 7 years, 7 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 import contextlib
7 import json
8 import os
9 import sys
10 import unittest
11
12 from glob import glob
13
14 import test_env # pylint: disable=W0611
15
16 import coverage
17
18 from common import annotator
19 from slave import annotated_run
20 from slave import recipe_util
21
22 SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__))
23 ROOT_PATH = os.path.abspath(os.path.join(SCRIPT_PATH, os.pardir, os.pardir,
24 os.pardir))
25 SLAVE_DIR = os.path.join(ROOT_PATH, 'slave', 'fake_slave', 'build')
26 BASE_DIRS = {
27 'Public': os.path.dirname(SCRIPT_PATH)
28 }
29 # TODO(iannucci): Check for duplicate recipe names when we have more than one
30 # base_dir
31
32 COVERAGE = coverage.coverage(
33 include=[os.path.join(x, 'recipes', '*') for x in BASE_DIRS.values()])
34
35 @contextlib.contextmanager
36 def cover():
37 COVERAGE.start()
38 try:
39 yield
40 finally:
41 COVERAGE.stop()
42
43
44 class TestAPI(object):
45 @staticmethod
46 def tryserver_build_properties(**kwargs):
47 ret = {
48 'issue': 12853011,
49 'patchset': 1,
50 'blamelist': ['cool_dev1337@chromium.org'],
51 'rietveld': 'https://chromiumcodereview.appspot.com',
52 }
53 ret.update(kwargs)
54 return ret
55
56
57 class DevNull(object):
58 @staticmethod
59 def write(data):
60 pass
61
62 @staticmethod
63 def flush():
64 pass
65
66
67 def test_path_for_recipe(recipe_path):
68 root = os.path.dirname(os.path.dirname(recipe_path))
69 return os.path.join(root, 'recipes_test', os.path.basename(recipe_path))
70
71
72 def has_test(recipe_path):
73 return os.path.exists(test_path_for_recipe(recipe_path))
74
75
76 def expected_for(recipe_path, test_name):
77 test_base = os.path.splitext(test_path_for_recipe(recipe_path))[0]
78 return "%s.%s.expected" % (test_base, test_name)
79
80
81 def exec_test_file(recipe_path):
82 test_path = test_path_for_recipe(recipe_path)
83 gvars = {}
84 execfile(test_path, gvars)
85 return dict(item for item in gvars.iteritems() if item[0].endswith('_test'))
86
87
88 def execute_test_case(test_fn, recipe_path):
89 test_data = test_fn(TestAPI())
90 bp = test_data.get('build_properties', {})
91 fp = test_data.get('factory_properties', {})
92 fp['recipe'] = os.path.basename(os.path.splitext(recipe_path)[0])
93
94 stream = annotator.StructuredAnnotationStream(stream=DevNull())
95 with cover():
96 with recipe_util.mock_paths():
97 retval = annotated_run.make_steps(stream, bp, fp, True)
98 assert retval.status_code is None
99 return retval.script or retval.steps
100
101
102 def train_from_tests(_base_dir_type, recipe_path):
103 if not has_test(recipe_path):
104 print 'FATAL: Recipe %s has NO tests!' % recipe_path
105 return False
106
107 for path in glob(expected_for(recipe_path, '*')):
108 os.unlink(path)
109
110 for name, test_fn in exec_test_file(recipe_path).iteritems():
111 steps = execute_test_case(test_fn, recipe_path)
112 expected_path = expected_for(recipe_path, name)
113 print 'Writing', expected_path
114 with open(expected_path, 'w') as f:
115 json.dump(steps, f, indent=2, sort_keys=True)
116
117 return True
118
119
120 def NoTestsForRecipe(recipe_path):
121 def test_DoesntExist(self):
122 self.assert_(False, 'No tests exist for %s' % recipe_path)
123 return test_DoesntExist
124
125
126 def RecipeTest(name, expected_path, test_fn, recipe_path):
127 def test_(self):
128 steps = execute_test_case(test_fn, recipe_path)
129 # Roundtrip json to get same string encoding as load
130 steps = json.loads(json.dumps(steps))
131 with open(expected_path, 'r') as f:
132 expected = json.load(f)
133 self.assertEqual(steps, expected)
134
135 test_.__name__ += name
136 return test_
137
138
139 def add_tests(_base_dir_type, recipe_path, loader, suite):
140 tests = []
141 if not has_test(recipe_path):
142 tests.append(NoTestsForRecipe(recipe_path))
143 else:
144 for name, test_fn in exec_test_file(recipe_path).iteritems():
145 tests.append(RecipeTest(name, expected_for(recipe_path, name),
146 test_fn, recipe_path))
147
148 suite.addTest(loader.loadTestsFromTestCase(
149 type('RecipeTest_for_%s' % os.path.basename(recipe_path),
150 (unittest.TestCase,), dict((x.__name__, x) for x in tests))))
151 return True
152
153
154 def load_tests(loader, standard_tests, _pattern):
155 """This method is invoked by unittest.main's automatic testloader when
156 this module is loaded for tests."""
157 assert not standard_tests.countTestCases()
158 suite = unittest.TestSuite()
159 for result in loop_over_recipes(add_tests, loader, suite):
160 assert result, 'Got bad result: %s' % result
161 return suite
162
163
164 def loop_over_recipes(func, *args, **kwargs):
165 for name, path in BASE_DIRS.iteritems():
166 recipe_dir = os.path.join(path, 'recipes')
167 for root, _dirs, files in os.walk(recipe_dir):
168 for recipe in (f for f in files if f.endswith('.py') and f[0] != '_'):
169 recipe_path = os.path.join(root, recipe)
170 with cover():
171 # Force this file into coverage, even if there's no test for it.
172 execfile(recipe_path, {})
173 yield func(name, recipe_path, *args, **kwargs)
174
175
176 def main(argv):
177 if not os.path.exists(SLAVE_DIR):
178 os.makedirs(SLAVE_DIR)
179
180 os.chdir(SLAVE_DIR)
181
182 training = False
183 is_help = False
184 if '--help' in argv or '-h' in argv:
185 print 'Pass --train to enter training mode.'
186 print
187 is_help = True
188 if '--train' in argv:
189 argv.remove('--train')
190 training = True
191
192 had_errors = False
193 if training and not is_help:
194 for result in loop_over_recipes(train_from_tests):
195 had_errors = had_errors or result
196
197 retcode = 1 if had_errors else 0
198
199 if not training:
200 try:
201 unittest.main()
202 except SystemExit as e:
203 retcode = e.code or retcode
204
205 if not is_help:
206 total_covered = COVERAGE.report()
207 if total_covered != 100.0:
208 print 'FATAL: Recipes are not at 100% coverage.'
209 retcode = retcode or 2
210
211 return retcode
212
213
214 if __name__ == '__main__':
215 sys.exit(main(sys.argv))
OLDNEW

Powered by Google App Engine
This is Rietveld 408576698