| OLD | NEW |
| 1 #!/usr/bin/python | 1 # Copyright 2013-2015 The Chromium Authors. All rights reserved. |
| 2 # Copyright 2013 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 5 | 4 |
| 6 """Provides test coverage for common recipe configurations. | 5 """Provides test coverage for common recipe configurations. |
| 7 | 6 |
| 8 recipe config expectations are located in ../recipe_configs_test/*.expected | 7 recipe config expectations are located in ../recipe_configs_test/*.expected |
| 9 | 8 |
| 10 In training mode, this will loop over every config item in ../recipe_configs.py | 9 In training mode, this will loop over every config item in ../recipe_configs.py |
| 11 crossed with every platform, and spit out the as_json() representation to | 10 crossed with every platform, and spit out the as_json() representation to |
| 12 ../recipe_configs_test | 11 ../recipe_configs_test |
| 13 | 12 |
| 14 You must have 100% coverage of ../recipe_configs.py for this test to pass. | 13 You must have 100% coverage of ../recipe_configs.py for this test to pass. |
| 15 """ | 14 """ |
| 16 | 15 |
| 17 import argparse | 16 import argparse |
| 18 import multiprocessing | 17 import multiprocessing |
| 19 import os | 18 import os |
| 20 import sys | 19 import sys |
| 21 import traceback | 20 import traceback |
| 22 from itertools import product, imap | 21 from itertools import product, imap |
| 23 | 22 |
| 24 import test_env # "relative import" pylint: disable=W0403,W0611 | 23 from . import loader |
| 25 | 24 from . import config |
| 26 from slave import recipe_loader | |
| 27 from slave import recipe_util | |
| 28 | 25 |
| 29 import coverage | 26 import coverage |
| 30 | 27 |
| 31 SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) | 28 SCRIPT_PATH = os.path.abspath(os.path.dirname(__file__)) |
| 32 SLAVE_DIR = os.path.abspath(os.path.join(SCRIPT_PATH, os.pardir)) | 29 SLAVE_DIR = os.path.abspath(os.path.join(SCRIPT_PATH, os.pardir)) |
| 33 | 30 |
| 34 COVERAGE = (lambda: coverage.coverage( | 31 UNIVERSE = None |
| 35 include=[os.path.join(x, '*', '*config.py') | 32 COVERAGE = None |
| 36 for x in recipe_util.MODULE_DIRS()], | |
| 37 data_file='.recipe_configs_test_coverage', data_suffix=True))() | |
| 38 | 33 |
| 39 def covered(fn, *args, **kwargs): | 34 def covered(fn, *args, **kwargs): |
| 40 COVERAGE.start() | 35 COVERAGE.start() |
| 41 try: | 36 try: |
| 42 return fn(*args, **kwargs) | 37 return fn(*args, **kwargs) |
| 43 finally: | 38 finally: |
| 44 COVERAGE.stop() | 39 COVERAGE.stop() |
| 45 | 40 |
| 46 UNIVERSE = recipe_loader.RecipeUniverse() | |
| 47 | 41 |
| 48 def load_recipe_modules(): | 42 def load_recipe_modules(): |
| 49 modules = {} | 43 modules = {} |
| 50 for modpath in recipe_loader.loop_over_recipe_modules(): | 44 for modpath in UNIVERSE.loop_over_recipe_modules(): |
| 51 # That's right, we're using the path as the local name! The local | 45 # That's right, we're using the path as the local name! The local |
| 52 # name really could be anything unique, we don't use it. | 46 # name really could be anything unique, we don't use it. |
| 53 modules[modpath] = UNIVERSE.load(recipe_loader.PathDependency( | 47 modules[modpath] = UNIVERSE.load(loader.PathDependency( |
| 54 modpath, local_name=modpath, base_path=os.curdir)) | 48 modpath, local_name=modpath, base_path=os.curdir, universe=UNIVERSE)) |
| 55 return modules | 49 return modules |
| 56 | 50 |
| 57 | 51 |
| 58 RECIPE_MODULES = None | 52 RECIPE_MODULES = None |
| 59 def init_recipe_modules(): | 53 def init_recipe_modules(): |
| 60 global RECIPE_MODULES | 54 global RECIPE_MODULES |
| 61 RECIPE_MODULES = covered(load_recipe_modules) | 55 RECIPE_MODULES = covered(load_recipe_modules) |
| 62 | 56 |
| 63 from slave import recipe_config # pylint: disable=F0401 | |
| 64 | |
| 65 | 57 |
| 66 def evaluate_configurations(args): | 58 def evaluate_configurations(args): |
| 67 mod_id, var_assignments = args | 59 mod_id, var_assignments = args |
| 68 mod = RECIPE_MODULES[mod_id] | 60 mod = RECIPE_MODULES[mod_id] |
| 69 ctx = mod.CONFIG_CTX | 61 ctx = mod.CONFIG_CTX |
| 70 | 62 |
| 71 config_name = None | 63 config_name = None |
| 72 try: | 64 try: |
| 73 make_item = lambda: covered(ctx.CONFIG_SCHEMA, **var_assignments) | 65 make_item = lambda: covered(ctx.CONFIG_SCHEMA, **var_assignments) |
| 74 | 66 |
| 75 # Try ROOT_CONFIG_ITEM first. If it raises BadConf, then we can skip | 67 # Try ROOT_CONFIG_ITEM first. If it raises BadConf, then we can skip |
| 76 # this config. | 68 # this config. |
| 77 root_item = ctx.ROOT_CONFIG_ITEM | 69 root_item = ctx.ROOT_CONFIG_ITEM |
| 78 if root_item: | 70 if root_item: |
| 79 config_name = root_item.__name__ | 71 config_name = root_item.__name__ |
| 80 try: | 72 try: |
| 81 result = covered(root_item, make_item()) | 73 result = covered(root_item, make_item()) |
| 82 if result.complete(): | 74 if result.complete(): |
| 83 covered(result.as_jsonish) | 75 covered(result.as_jsonish) |
| 84 except recipe_config.BadConf, e: | 76 except config.BadConf, e: |
| 85 pass # This is a possibly expected failure mode. | 77 pass # This is a possibly expected failure mode. |
| 86 | 78 |
| 87 for config_name, fn in ctx.CONFIG_ITEMS.iteritems(): | 79 for config_name, fn in ctx.CONFIG_ITEMS.iteritems(): |
| 88 if fn.NO_TEST or fn.IS_ROOT: | 80 if fn.NO_TEST or fn.IS_ROOT: |
| 89 continue | 81 continue |
| 90 try: | 82 try: |
| 91 result = covered(fn, make_item()) | 83 result = covered(fn, make_item()) |
| 92 if result.complete(): | 84 if result.complete(): |
| 93 covered(result.as_jsonish) | 85 covered(result.as_jsonish) |
| 94 except recipe_config.BadConf: | 86 except config.BadConf: |
| 95 pass # This is a possibly expected failure mode. | 87 pass # This is a possibly expected failure mode. |
| 96 return True | 88 return True |
| 97 except Exception as e: | 89 except Exception as e: |
| 98 print ('Caught unknown exception [%s] for config name [%s] for module ' | 90 print ('Caught unknown exception [%s] for config name [%s] for module ' |
| 99 '[%s] with args %s') % (e, config_name, mod_id, var_assignments) | 91 '[%s] with args %s') % (e, config_name, mod_id, var_assignments) |
| 100 traceback.print_exc() | 92 traceback.print_exc() |
| 101 return False | 93 return False |
| 102 | 94 |
| 103 | 95 |
| 104 def multiprocessing_init(): | 96 def multiprocessing_init(): |
| (...skipping 27 matching lines...) Expand all Loading... |
| 132 pool = multiprocessing.Pool(initializer=multiprocessing_init) | 124 pool = multiprocessing.Pool(initializer=multiprocessing_init) |
| 133 try: | 125 try: |
| 134 return pool.map_async(fn, combination_generator).get(999999) | 126 return pool.map_async(fn, combination_generator).get(999999) |
| 135 finally: | 127 finally: |
| 136 # necessary so that the subprocesses will write out their coverage due to | 128 # necessary so that the subprocesses will write out their coverage due to |
| 137 # the hack in multiprocessing_init() | 129 # the hack in multiprocessing_init() |
| 138 pool.close() | 130 pool.close() |
| 139 pool.join() | 131 pool.join() |
| 140 | 132 |
| 141 | 133 |
| 142 def main(): | 134 def main(universe): |
| 135 global UNIVERSE |
| 136 global COVERAGE |
| 137 UNIVERSE = universe |
| 138 COVERAGE = coverage.coverage( |
| 139 include=[os.path.join(x, '*', '*config.py') |
| 140 for x in UNIVERSE.module_dirs], |
| 141 data_file='.recipe_configs_test_coverage', data_suffix=True) |
| 143 COVERAGE.erase() | 142 COVERAGE.erase() |
| 144 init_recipe_modules() | 143 init_recipe_modules() |
| 145 | 144 |
| 146 success = all(coverage_parallel_map(evaluate_configurations)) | 145 success = all(coverage_parallel_map(evaluate_configurations)) |
| 147 | 146 |
| 148 COVERAGE.combine() | 147 COVERAGE.combine() |
| 149 total_covered = COVERAGE.report() | 148 total_covered = COVERAGE.report() |
| 150 all_covered = total_covered == 100.0 | 149 all_covered = total_covered == 100.0 |
| 151 | 150 |
| 152 if not success: | 151 if not success: |
| 153 print 'FATAL: Some recipe configuration(s) failed' | 152 print 'FATAL: Some recipe configuration(s) failed' |
| 154 if not all_covered: | 153 if not all_covered: |
| 155 print 'FATAL: Recipes configs are not at 100% coverage.' | 154 print 'FATAL: Recipes configs are not at 100% coverage.' |
| 156 | 155 |
| 157 return 1 if (not success or not all_covered) else 0 | 156 return 1 if (not success or not all_covered) else 0 |
| 158 | 157 |
| 159 | 158 |
| 160 if __name__ == '__main__': | 159 if __name__ == '__main__': |
| 161 sys.exit(main()) | 160 sys.exit(main()) |
| OLD | NEW |