| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright 2013 The Chromium Authors. All rights reserved. | |
| 2 # Use of this source code is governed by a BSD-style license that can be | |
| 3 # found in the LICENSE file. | |
| 4 | |
| 5 import imp | |
| 6 import inspect | |
| 7 import os | |
| 8 import sys | |
| 9 | |
| 10 from .recipe_util import (ROOT_PATH, RECIPE_DIRS, MODULE_DIRS, | |
| 11 cached_unary, scan_directory) | |
| 12 from .recipe_api import RecipeApi, RecipeApiPlain | |
| 13 from .recipe_config import ConfigContext | |
| 14 from .recipe_config_types import Path, ModuleBasePath, RECIPE_MODULE_PREFIX | |
| 15 from .recipe_test_api import RecipeTestApi, DisabledTestData | |
| 16 | |
| 17 | |
| 18 class NoSuchRecipe(Exception): | |
| 19 """Raised by load_recipe is recipe is not found.""" | |
| 20 | |
| 21 | |
| 22 class RecipeScript(object): | |
| 23 """Holds dict of an evaluated recipe script.""" | |
| 24 | |
| 25 def __init__(self, recipe_dict): | |
| 26 for k, v in recipe_dict.iteritems(): | |
| 27 setattr(self, k, v) | |
| 28 | |
| 29 @classmethod | |
| 30 def from_script_path(cls, script_path, universe): | |
| 31 """Evaluates a script and returns RecipeScript instance.""" | |
| 32 script_vars = {} | |
| 33 execfile(script_path, script_vars) | |
| 34 script_vars['__file__'] = script_path | |
| 35 script_vars['LOADED_DEPS'] = universe.deps_from_mixed( | |
| 36 script_vars.get('DEPS', []), os.path.basename(script_path)) | |
| 37 return cls(script_vars) | |
| 38 | |
| 39 | |
| 40 class Dependency(object): | |
| 41 def load(self, universe): | |
| 42 raise NotImplementedError() | |
| 43 | |
| 44 @property | |
| 45 def local_name(self): | |
| 46 raise NotImplementedError() | |
| 47 | |
| 48 @property | |
| 49 def unique_name(self): | |
| 50 """A unique identifier for the module that this dependency refers to. | |
| 51 This must be generated without loading the module.""" | |
| 52 raise NotImplementedError() | |
| 53 | |
| 54 | |
| 55 class PathDependency(Dependency): | |
| 56 def __init__(self, path, local_name, base_path=None): | |
| 57 self._path = _normalize_path(base_path, path) | |
| 58 self._local_name = local_name | |
| 59 | |
| 60 # We forbid modules from living outside our main paths to keep clients | |
| 61 # from going crazy before we have standardized recipe locations. | |
| 62 mod_dir = os.path.dirname(path) | |
| 63 assert mod_dir in MODULE_DIRS(), ( | |
| 64 'Modules living outside of approved directories are forbidden: ' | |
| 65 '%s is not in %s' % (mod_dir, MODULE_DIRS())) | |
| 66 | |
| 67 def load(self, universe): | |
| 68 return _load_recipe_module_module(self._path, universe) | |
| 69 | |
| 70 @property | |
| 71 def local_name(self): | |
| 72 return self._local_name | |
| 73 | |
| 74 @property | |
| 75 def unique_name(self): | |
| 76 return self._path | |
| 77 | |
| 78 | |
| 79 class NamedDependency(PathDependency): | |
| 80 def __init__(self, name): | |
| 81 for path in MODULE_DIRS(): | |
| 82 mod_path = os.path.join(path, name) | |
| 83 if os.path.exists(os.path.join(mod_path, '__init__.py')): | |
| 84 super(NamedDependency, self).__init__(mod_path, name) | |
| 85 return | |
| 86 raise NoSuchRecipe('Recipe module named %s does not exist' % name) | |
| 87 | |
| 88 | |
| 89 class RecipeUniverse(object): | |
| 90 def __init__(self): | |
| 91 self._loaded = {} | |
| 92 | |
| 93 def load(self, dep): | |
| 94 """Load a Dependency.""" | |
| 95 name = dep.unique_name | |
| 96 if name in self._loaded: | |
| 97 mod = self._loaded[name] | |
| 98 assert mod is not None, ( | |
| 99 'Cyclic dependency when trying to load %s' % name) | |
| 100 return mod | |
| 101 else: | |
| 102 self._loaded[name] = None | |
| 103 mod = dep.load(self) | |
| 104 self._loaded[name] = mod | |
| 105 return mod | |
| 106 | |
| 107 def deps_from_names(self, deps): | |
| 108 """Load dependencies given a list simple module names (old style).""" | |
| 109 return { dep: self.load(NamedDependency(dep)) for dep in deps } | |
| 110 | |
| 111 def deps_from_paths(self, deps, base_path): | |
| 112 """Load dependencies given a dictionary of local names to module paths | |
| 113 (new style).""" | |
| 114 return { name: self.load(PathDependency(path, name, base_path)) | |
| 115 for name, path in deps.iteritems() } | |
| 116 | |
| 117 def deps_from_mixed(self, deps, base_path): | |
| 118 """Load dependencies given either a new style or old style deps spec.""" | |
| 119 if isinstance(deps, (list, tuple)): | |
| 120 return self.deps_from_names(deps) | |
| 121 elif isinstance(deps, dict): | |
| 122 return self.deps_from_paths(deps, base_path) | |
| 123 else: | |
| 124 raise ValueError('%s is not a valid or known deps structure' % deps) | |
| 125 | |
| 126 def load_recipe(self, recipe): | |
| 127 """Given name of a recipe, loads and returns it as RecipeScript instance. | |
| 128 | |
| 129 Args: | |
| 130 recipe (str): name of a recipe, can be in form '<module>:<recipe>'. | |
| 131 | |
| 132 Returns: | |
| 133 RecipeScript instance. | |
| 134 | |
| 135 Raises: | |
| 136 NoSuchRecipe: recipe is not found. | |
| 137 """ | |
| 138 # If the recipe is specified as "module:recipe", then it is an recipe | |
| 139 # contained in a recipe_module as an example. Look for it in the modules | |
| 140 # imported by load_recipe_modules instead of the normal search paths. | |
| 141 if ':' in recipe: | |
| 142 module_name, example = recipe.split(':') | |
| 143 assert example.endswith('example') | |
| 144 for module_dir in MODULE_DIRS(): | |
| 145 for subitem in os.listdir(module_dir): | |
| 146 if module_name == subitem: | |
| 147 return RecipeScript.from_script_path( | |
| 148 os.path.join(module_dir, subitem, 'example.py'), self) | |
| 149 raise NoSuchRecipe(recipe, | |
| 150 'Recipe example %s:%s does not exist' % | |
| 151 (module_name, example)) | |
| 152 else: | |
| 153 for recipe_path in (os.path.join(p, recipe) for p in RECIPE_DIRS()): | |
| 154 if os.path.exists(recipe_path + '.py'): | |
| 155 return RecipeScript.from_script_path(recipe_path + '.py', self) | |
| 156 raise NoSuchRecipe(recipe) | |
| 157 | |
| 158 | |
| 159 | |
| 160 | |
| 161 def _find_and_load_module(fullname, modname, path): | |
| 162 imp.acquire_lock() | |
| 163 try: | |
| 164 if fullname not in sys.modules: | |
| 165 fil = None | |
| 166 try: | |
| 167 fil, pathname, descr = imp.find_module(modname, | |
| 168 [os.path.dirname(path)]) | |
| 169 imp.load_module(fullname, fil, pathname, descr) | |
| 170 finally: | |
| 171 if fil: | |
| 172 fil.close() | |
| 173 return sys.modules[fullname] | |
| 174 finally: | |
| 175 imp.release_lock() | |
| 176 | |
| 177 | |
| 178 def _load_recipe_module_module(path, universe): | |
| 179 modname = os.path.splitext(os.path.basename(path))[0] | |
| 180 fullname = '%s.%s' % (RECIPE_MODULE_PREFIX, modname) | |
| 181 mod = _find_and_load_module(fullname, modname, path) | |
| 182 | |
| 183 # This actually loads the dependencies. | |
| 184 mod.LOADED_DEPS = universe.deps_from_mixed( | |
| 185 getattr(mod, 'DEPS', []), os.path.basename(path)) | |
| 186 | |
| 187 # TODO(luqui): Remove this hack once configs are cleaned. | |
| 188 sys.modules['%s.DEPS' % fullname] = mod.LOADED_DEPS | |
| 189 _recursive_import(path, RECIPE_MODULE_PREFIX) | |
| 190 _patchup_module(modname, mod) | |
| 191 | |
| 192 return mod | |
| 193 | |
| 194 | |
| 195 def _recursive_import(path, prefix): | |
| 196 modname = os.path.splitext(os.path.basename(path))[0] | |
| 197 fullname = '%s.%s' % (prefix, modname) | |
| 198 mod = _find_and_load_module(fullname, modname, path) | |
| 199 if not os.path.isdir(path): | |
| 200 return mod | |
| 201 | |
| 202 for subitem in os.listdir(path): | |
| 203 subpath = os.path.join(path, subitem) | |
| 204 subname = os.path.splitext(subitem)[0] | |
| 205 if os.path.isdir(subpath): | |
| 206 if not os.path.exists(os.path.join(subpath, '__init__.py')): | |
| 207 continue | |
| 208 elif not subpath.endswith('.py') or subitem.startswith('__init__.py'): | |
| 209 continue | |
| 210 | |
| 211 submod = _recursive_import(subpath, fullname) | |
| 212 | |
| 213 if not hasattr(mod, subname): | |
| 214 setattr(mod, subname, submod) | |
| 215 else: | |
| 216 prev = getattr(mod, subname) | |
| 217 assert submod is prev, ( | |
| 218 'Conflicting modules: %s and %s' % (prev, mod)) | |
| 219 | |
| 220 return mod | |
| 221 | |
| 222 | |
| 223 def _patchup_module(name, submod): | |
| 224 """Finds framework related classes and functions in a |submod| and adds | |
| 225 them to |submod| as top level constants with well known names such as | |
| 226 API, CONFIG_CTX and TEST_API. | |
| 227 | |
| 228 |submod| is a recipe module (akin to python package) with submodules such as | |
| 229 'api', 'config', 'test_api'. This function scans through dicts of that | |
| 230 submodules to find subclasses of RecipeApi, RecipeTestApi, etc. | |
| 231 """ | |
| 232 submod.NAME = name | |
| 233 submod.UNIQUE_NAME = name # TODO(luqui): use a luci-config unique name | |
| 234 submod.MODULE_DIRECTORY = Path(ModuleBasePath(submod)) | |
| 235 submod.CONFIG_CTX = getattr(submod, 'CONFIG_CTX', None) | |
| 236 | |
| 237 if hasattr(submod, 'config'): | |
| 238 for v in submod.config.__dict__.itervalues(): | |
| 239 if isinstance(v, ConfigContext): | |
| 240 assert not submod.CONFIG_CTX, ( | |
| 241 'More than one configuration context: %s, %s' % | |
| 242 (submod.config, submod.CONFIG_CTX)) | |
| 243 submod.CONFIG_CTX = v | |
| 244 assert submod.CONFIG_CTX, 'Config file, but no config context?' | |
| 245 | |
| 246 submod.API = getattr(submod, 'API', None) | |
| 247 for v in submod.api.__dict__.itervalues(): | |
| 248 if inspect.isclass(v) and issubclass(v, RecipeApiPlain): | |
| 249 assert not submod.API, ( | |
| 250 '%s has more than one Api subclass: %s, %s' % (name, v, submod.api)) | |
| 251 submod.API = v | |
| 252 assert submod.API, 'Submodule has no api? %s' % (submod) | |
| 253 | |
| 254 submod.TEST_API = getattr(submod, 'TEST_API', None) | |
| 255 if hasattr(submod, 'test_api'): | |
| 256 for v in submod.test_api.__dict__.itervalues(): | |
| 257 if inspect.isclass(v) and issubclass(v, RecipeTestApi): | |
| 258 assert not submod.TEST_API, ( | |
| 259 'More than one TestApi subclass: %s' % submod.api) | |
| 260 submod.TEST_API = v | |
| 261 assert submod.API, ( | |
| 262 'Submodule has test_api.py but no TestApi subclass? %s' | |
| 263 % (submod) | |
| 264 ) | |
| 265 | |
| 266 | |
| 267 class DependencyMapper(object): | |
| 268 """DependencyMapper topologically traverses the dependency DAG beginning at | |
| 269 a module, executing a callback ("instantiator") for each module. | |
| 270 | |
| 271 For example, if the dependency DAG looked like this: | |
| 272 | |
| 273 A | |
| 274 / \ | |
| 275 B C | |
| 276 \ / | |
| 277 D | |
| 278 | |
| 279 (with D depending on B and C, etc.), DependencyMapper(f).instantiate(D) would | |
| 280 construct | |
| 281 | |
| 282 f_A = f(A, {}) | |
| 283 f_B = f(B, { 'A': f_A }) | |
| 284 f_C = f(C, { 'A': f_A }) | |
| 285 f_D = f(D, { 'B': f_B, 'C': f_C }) | |
| 286 | |
| 287 finally returning f_D. instantiate can be called multiple times, which reuses | |
| 288 already-computed results. | |
| 289 """ | |
| 290 | |
| 291 def __init__(self, instantiator): | |
| 292 self._instantiator = instantiator | |
| 293 self._instances = {} | |
| 294 | |
| 295 def instantiate(self, mod): | |
| 296 if mod in self._instances: | |
| 297 return self._instances[mod] | |
| 298 deps_dict = { name: self.instantiate(dep) | |
| 299 for name, dep in mod.LOADED_DEPS.iteritems() } | |
| 300 self._instances[mod] = self._instantiator(mod, deps_dict) | |
| 301 return self._instances[mod] | |
| 302 | |
| 303 | |
| 304 def create_recipe_api(toplevel_deps, engine, test_data=DisabledTestData()): | |
| 305 def instantiator(mod, deps): | |
| 306 # TODO(luqui): test_data will need to use canonical unique names. | |
| 307 mod_api = mod.API(module=mod, engine=engine, | |
| 308 test_data=test_data.get_module_test_data(mod.NAME)) | |
| 309 mod_api.test_api = (getattr(mod, 'TEST_API', None) | |
| 310 or RecipeTestApi)(module=mod) | |
| 311 for k, v in deps.iteritems(): | |
| 312 setattr(mod_api.m, k, v) | |
| 313 setattr(mod_api.test_api.m, k, v.test_api) | |
| 314 return mod_api | |
| 315 | |
| 316 mapper = DependencyMapper(instantiator) | |
| 317 api = RecipeApi(module=None, engine=engine, | |
| 318 test_data=test_data.get_module_test_data(None)) | |
| 319 for k, v in toplevel_deps.iteritems(): | |
| 320 setattr(api, k, mapper.instantiate(v)) | |
| 321 return api | |
| 322 | |
| 323 | |
| 324 def create_test_api(toplevel_deps, universe): | |
| 325 def instantiator(mod, deps): | |
| 326 modapi = (getattr(mod, 'TEST_API', None) or RecipeTestApi)(module=mod) | |
| 327 for k,v in deps.iteritems(): | |
| 328 setattr(modapi.m, k, v) | |
| 329 return modapi | |
| 330 | |
| 331 mapper = DependencyMapper(instantiator) | |
| 332 api = RecipeTestApi(module=None) | |
| 333 for k,v in toplevel_deps.iteritems(): | |
| 334 setattr(api, k, mapper.instantiate(v)) | |
| 335 return api | |
| 336 | |
| 337 | |
| 338 def loop_over_recipe_modules(): | |
| 339 for path in MODULE_DIRS(): | |
| 340 if os.path.isdir(path): | |
| 341 for item in os.listdir(path): | |
| 342 subpath = os.path.join(path, item) | |
| 343 if os.path.isdir(subpath): | |
| 344 yield subpath | |
| 345 | |
| 346 | |
| 347 def loop_over_recipes(): | |
| 348 """Yields pairs (path to recipe, recipe name). | |
| 349 | |
| 350 Enumerates real recipes in recipes/* as well as examples in recipe_modules/*. | |
| 351 """ | |
| 352 for path in RECIPE_DIRS(): | |
| 353 for recipe in scan_directory( | |
| 354 path, lambda f: f.endswith('.py') and f[0] != '_'): | |
| 355 yield recipe, recipe[len(path)+1:-len('.py')] | |
| 356 for path in MODULE_DIRS(): | |
| 357 for recipe in scan_directory( | |
| 358 path, lambda f: f.endswith('example.py')): | |
| 359 module_name = os.path.dirname(recipe)[len(path)+1:] | |
| 360 yield recipe, '%s:example' % module_name | |
| 361 | |
| 362 | |
| 363 def _normalize_path(base_path, path): | |
| 364 if base_path is None or os.path.isabs(path): | |
| 365 return os.path.realpath(path) | |
| 366 else: | |
| 367 return os.path.realpath(os.path.join(base_path, path)) | |
| 368 | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1151423002:
Move recipe engine to third_party/recipe_engine. (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/tools/build