| OLD | NEW |
| 1 # Copyright 2015 The LUCI Authors. All rights reserved. | 1 # Copyright 2015 The LUCI Authors. All rights reserved. |
| 2 # Use of this source code is governed under the Apache License, Version 2.0 | 2 # Use of this source code is governed under the Apache License, Version 2.0 |
| 3 # that can be found in the LICENSE file. | 3 # that can be found in the LICENSE file. |
| 4 | 4 |
| 5 import copy | 5 import copy |
| 6 import difflib | 6 import difflib |
| 7 import logging | 7 import logging |
| 8 import operator | 8 import operator |
| 9 import os | 9 import os |
| 10 import subprocess | 10 import subprocess |
| (...skipping 18 matching lines...) Expand all Loading... |
| 29 | 29 |
| 30 class CyclicDependencyError(Exception): | 30 class CyclicDependencyError(Exception): |
| 31 pass | 31 pass |
| 32 | 32 |
| 33 | 33 |
| 34 def cleanup_pyc(path): | 34 def cleanup_pyc(path): |
| 35 """Removes any .pyc files from |path|'s directory tree. | 35 """Removes any .pyc files from |path|'s directory tree. |
| 36 | 36 |
| 37 This ensures we always use the fresh code. | 37 This ensures we always use the fresh code. |
| 38 """ | 38 """ |
| 39 for root, _dirs, files in os.walk(path): | 39 for root, dirs, files in os.walk(path): |
| 40 for f in files: | 40 for f in files: |
| 41 if f.endswith('.pyc'): | 41 if f.endswith('.pyc'): |
| 42 os.unlink(os.path.join(root, f)) | 42 os.unlink(os.path.join(root, f)) |
| 43 | 43 |
| 44 | 44 |
| 45 class InfraRepoConfig(object): | 45 class InfraRepoConfig(object): |
| 46 def to_recipes_cfg(self, repo_root): | 46 def to_recipes_cfg(self, repo_root): |
| 47 # TODO(luqui): This is not always correct. It can be configured in | 47 # TODO(luqui): This is not always correct. It can be configured in |
| 48 # infra/config:refs.cfg. | 48 # infra/config:refs.cfg. |
| 49 return os.path.join(repo_root, 'infra', 'config', 'recipes.cfg') | 49 return os.path.join(repo_root, 'infra', 'config', 'recipes.cfg') |
| (...skipping 304 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 354 return False | 354 return False |
| 355 return self._proto_file == other._proto_file | 355 return self._proto_file == other._proto_file |
| 356 | 356 |
| 357 | 357 |
| 358 class Package(object): | 358 class Package(object): |
| 359 """Package represents a loaded package, and contains path and dependency | 359 """Package represents a loaded package, and contains path and dependency |
| 360 information. | 360 information. |
| 361 | 361 |
| 362 This is accessed by loader.py through RecipeDeps.get_package. | 362 This is accessed by loader.py through RecipeDeps.get_package. |
| 363 """ | 363 """ |
| 364 def __init__(self, name, repo_spec, deps, repo_root, relative_recipes_dir, | 364 def __init__(self, name, repo_spec, deps, repo_root, relative_recipes_dir): |
| 365 canonical_base_url): | |
| 366 self.name = name | 365 self.name = name |
| 367 self.repo_spec = repo_spec | 366 self.repo_spec = repo_spec |
| 368 self.deps = deps | 367 self.deps = deps |
| 369 self.repo_root = repo_root | 368 self.repo_root = repo_root |
| 370 self.relative_recipes_dir = relative_recipes_dir | 369 self.relative_recipes_dir = relative_recipes_dir |
| 371 self.canonical_base_url = canonical_base_url | |
| 372 | 370 |
| 373 @property | 371 @property |
| 374 def recipes_dir(self): | 372 def recipes_dir(self): |
| 375 return os.path.join(self.repo_root, self.relative_recipes_dir) | 373 return os.path.join(self.repo_root, self.relative_recipes_dir) |
| 376 | 374 |
| 377 @property | 375 @property |
| 378 def recipe_dir(self): | 376 def recipe_dir(self): |
| 379 return os.path.join(self.recipes_dir, 'recipes') | 377 return os.path.join(self.recipes_dir, 'recipes') |
| 380 | 378 |
| 381 @property | 379 @property |
| 382 def module_dir(self): | 380 def module_dir(self): |
| 383 return os.path.join(self.recipes_dir, 'recipe_modules') | 381 return os.path.join(self.recipes_dir, 'recipe_modules') |
| 384 | 382 |
| 385 def find_dep(self, dep_name): | 383 def find_dep(self, dep_name): |
| 386 if dep_name == self.name: | 384 if dep_name == self.name: |
| 387 return self | 385 return self |
| 388 | 386 |
| 389 assert dep_name in self.deps, ( | 387 assert dep_name in self.deps, ( |
| 390 '%s does not exist or is not declared as a dependency of %s' % ( | 388 '%s does not exist or is not declared as a dependency of %s' % ( |
| 391 dep_name, self.name)) | 389 dep_name, self.name)) |
| 392 return self.deps[dep_name] | 390 return self.deps[dep_name] |
| 393 | 391 |
| 394 def module_path(self, module_name): | 392 def module_path(self, module_name): |
| 395 return os.path.join(self.recipes_dir, 'recipe_modules', module_name) | 393 return os.path.join(self.recipes_dir, 'recipe_modules', module_name) |
| 396 | 394 |
| 397 def __repr__(self): | 395 def __repr__(self): |
| 398 return 'Package(%r, %r, %r, %r, %r)' % ( | 396 return 'Package(%r, %r, %r, %r)' % ( |
| 399 self.name, self.repo_spec, self.deps, self.recipe_dir, | 397 self.name, self.repo_spec, self.deps, self.recipe_dir) |
| 400 self.canonical_base_url) | |
| 401 | 398 |
| 402 def __str__(self): | 399 def __str__(self): |
| 403 return 'Package %s, with dependencies %s' % (self.name, self.deps.keys()) | 400 return 'Package %s, with dependencies %s' % (self.name, self.deps.keys()) |
| 404 | 401 |
| 405 | 402 |
| 406 class RollCandidate(object): | 403 class RollCandidate(object): |
| 407 """RollCandidate represents a recipe roll candidate, i.e. updates | 404 """RollCandidate represents a recipe roll candidate, i.e. updates |
| 408 to pinned revisions of recipe dependencies. | 405 to pinned revisions of recipe dependencies. |
| 409 | 406 |
| 410 This is mostly used by recipes.py autoroll command. | 407 This is mostly used by recipes.py autoroll command. |
| (...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 462 def get_rolled_spec(self): | 459 def get_rolled_spec(self): |
| 463 """Returns a PackageSpec with all the deps updates from this roll.""" | 460 """Returns a PackageSpec with all the deps updates from this roll.""" |
| 464 # TODO(phajdan.jr): does this preserve comments? should it? | 461 # TODO(phajdan.jr): does this preserve comments? should it? |
| 465 new_deps = _updated( | 462 new_deps = _updated( |
| 466 self._package_spec.deps, | 463 self._package_spec.deps, |
| 467 { project_id: spec for project_id, spec in | 464 { project_id: spec for project_id, spec in |
| 468 self._updates.iteritems() }) | 465 self._updates.iteritems() }) |
| 469 return PackageSpec( | 466 return PackageSpec( |
| 470 self._package_spec.project_id, | 467 self._package_spec.project_id, |
| 471 self._package_spec.recipes_path, | 468 self._package_spec.recipes_path, |
| 472 new_deps, | 469 new_deps) |
| 473 self._package_spec.canonical_base_url) | |
| 474 | 470 |
| 475 def get_commit_infos(self): | 471 def get_commit_infos(self): |
| 476 """Returns a mapping project_id -> list of commits from that repo | 472 """Returns a mapping project_id -> list of commits from that repo |
| 477 that are getting pulled by this roll. | 473 that are getting pulled by this roll. |
| 478 """ | 474 """ |
| 479 commit_infos = {} | 475 commit_infos = {} |
| 480 | 476 |
| 481 for project_id, update in self._updates.iteritems(): | 477 for project_id, update in self._updates.iteritems(): |
| 482 commit_infos[project_id] = self._package_spec.deps[ | 478 commit_infos[project_id] = self._package_spec.deps[ |
| 483 project_id].commit_infos(self._context, update.revision) | 479 project_id].commit_infos(self._context, update.revision) |
| (...skipping 10 matching lines...) Expand all Loading... |
| 494 """Returns a unified diff between original package spec and one after roll. | 490 """Returns a unified diff between original package spec and one after roll. |
| 495 """ | 491 """ |
| 496 orig = str(self._package_spec.dump()).splitlines() | 492 orig = str(self._package_spec.dump()).splitlines() |
| 497 new = str(self.get_rolled_spec().dump()).splitlines() | 493 new = str(self.get_rolled_spec().dump()).splitlines() |
| 498 return '\n'.join(difflib.unified_diff(orig, new, lineterm='')) | 494 return '\n'.join(difflib.unified_diff(orig, new, lineterm='')) |
| 499 | 495 |
| 500 | 496 |
| 501 class PackageSpec(object): | 497 class PackageSpec(object): |
| 502 API_VERSION = 1 | 498 API_VERSION = 1 |
| 503 | 499 |
| 504 def __init__(self, project_id, recipes_path, deps, canonical_base_url): | 500 def __init__(self, project_id, recipes_path, deps): |
| 505 self._project_id = project_id | 501 self._project_id = project_id |
| 506 self._recipes_path = recipes_path | 502 self._recipes_path = recipes_path |
| 507 self._deps = deps | 503 self._deps = deps |
| 508 self._canonical_base_url = canonical_base_url | |
| 509 | 504 |
| 510 @classmethod | 505 @classmethod |
| 511 def load_proto(cls, proto_file): | 506 def load_proto(cls, proto_file): |
| 512 buf = proto_file.read() | 507 buf = proto_file.read() |
| 513 assert buf.api_version == cls.API_VERSION | 508 assert buf.api_version == cls.API_VERSION |
| 514 | 509 |
| 515 deps = { str(dep.project_id): cls.spec_for_dep(dep) | 510 deps = { str(dep.project_id): cls.spec_for_dep(dep) |
| 516 for dep in buf.deps } | 511 for dep in buf.deps } |
| 517 return cls(str(buf.project_id), str(buf.recipes_path), deps, | 512 return cls(str(buf.project_id), str(buf.recipes_path), deps) |
| 518 buf.canonical_base_url) | |
| 519 | 513 |
| 520 @classmethod | 514 @classmethod |
| 521 def spec_for_dep(cls, dep): | 515 def spec_for_dep(cls, dep): |
| 522 """Returns a RepoSpec for the given dependency protobuf.""" | 516 """Returns a RepoSpec for the given dependency protobuf.""" |
| 523 url = str(dep.url) | 517 url = str(dep.url) |
| 524 if url.startswith("file://"): | 518 if url.startswith("file://"): |
| 525 return PathRepoSpec(str(dep.project_id), url[len("file://"):]) | 519 return PathRepoSpec(str(dep.project_id), url[len("file://"):]) |
| 526 | 520 |
| 527 if dep.repo_type in (package_pb2.DepSpec.GIT, package_pb2.DepSpec.GITILES): | 521 if dep.repo_type in (package_pb2.DepSpec.GIT, package_pb2.DepSpec.GITILES): |
| 528 if dep.repo_type == package_pb2.DepSpec.GIT: | 522 if dep.repo_type == package_pb2.DepSpec.GIT: |
| (...skipping 14 matching lines...) Expand all Loading... |
| 543 return self._project_id | 537 return self._project_id |
| 544 | 538 |
| 545 @property | 539 @property |
| 546 def recipes_path(self): | 540 def recipes_path(self): |
| 547 return self._recipes_path | 541 return self._recipes_path |
| 548 | 542 |
| 549 @property | 543 @property |
| 550 def deps(self): | 544 def deps(self): |
| 551 return self._deps | 545 return self._deps |
| 552 | 546 |
| 553 @property | |
| 554 def canonical_base_url(self): | |
| 555 return self._canonical_base_url | |
| 556 | |
| 557 def dump(self): | 547 def dump(self): |
| 558 return package_pb2.Package( | 548 return package_pb2.Package( |
| 559 api_version=self.API_VERSION, | 549 api_version=self.API_VERSION, |
| 560 project_id=self._project_id, | 550 project_id=self._project_id, |
| 561 recipes_path=self._recipes_path, | 551 recipes_path=self._recipes_path, |
| 562 deps=[ self._deps[dep].dump() for dep in sorted(self._deps.keys()) ], | 552 deps=[ self._deps[dep].dump() for dep in sorted(self._deps.keys()) ]) |
| 563 canonical_base_url=self._canonical_base_url) | |
| 564 | 553 |
| 565 def roll_candidates(self, root_spec, context): | 554 def roll_candidates(self, root_spec, context): |
| 566 """Returns list of consistent roll candidates, and rejected roll candidates. | 555 """Returns list of consistent roll candidates, and rejected roll candidates. |
| 567 | 556 |
| 568 The first one is sorted by score, descending. The more commits are pulled by | 557 The first one is sorted by score, descending. The more commits are pulled by |
| 569 the roll, the higher score. | 558 the roll, the higher score. |
| 570 | 559 |
| 571 Second list is included to distinguish between a situation where there are | 560 Second list is included to distinguish between a situation where there are |
| 572 no roll candidates from one where there are updates but they're not | 561 no roll candidates from one where there are updates but they're not |
| 573 consistent. | 562 consistent. |
| (...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 628 are the override path. | 617 are the override path. |
| 629 """ | 618 """ |
| 630 context = PackageContext.from_proto_file(repo_root, proto_file, allow_fetch, | 619 context = PackageContext.from_proto_file(repo_root, proto_file, allow_fetch, |
| 631 deps_path=deps_path) | 620 deps_path=deps_path) |
| 632 | 621 |
| 633 if overrides: | 622 if overrides: |
| 634 overrides = {project_id: PathRepoSpec(project_id, path) | 623 overrides = {project_id: PathRepoSpec(project_id, path) |
| 635 for project_id, path in overrides.iteritems()} | 624 for project_id, path in overrides.iteritems()} |
| 636 package_deps = cls(context, overrides=overrides) | 625 package_deps = cls(context, overrides=overrides) |
| 637 | 626 |
| 638 package_deps._root_package = package_deps._create_package( | 627 package_deps._root_package = package_deps._create_package(RootRepoSpec(proto
_file)) |
| 639 RootRepoSpec(proto_file)) | |
| 640 | 628 |
| 641 return package_deps | 629 return package_deps |
| 642 | 630 |
| 643 def _create_package(self, repo_spec): | 631 def _create_package(self, repo_spec): |
| 644 repo_spec.checkout(self._context) | 632 repo_spec.checkout(self._context) |
| 645 package_spec = PackageSpec.load_proto(repo_spec.proto_file(self._context)) | 633 package_spec = PackageSpec.load_proto(repo_spec.proto_file(self._context)) |
| 646 return self._create_from_spec(repo_spec, package_spec) | 634 return self._create_from_spec(repo_spec, package_spec) |
| 647 | 635 |
| 648 def _create_from_spec(self, repo_spec, package_spec): | 636 def _create_from_spec(self, repo_spec, package_spec): |
| 649 project_id = package_spec.project_id | 637 project_id = package_spec.project_id |
| (...skipping 12 matching lines...) Expand all Loading... |
| 662 self._packages[project_id] = None | 650 self._packages[project_id] = None |
| 663 | 651 |
| 664 deps = {} | 652 deps = {} |
| 665 for dep, dep_repo in sorted(package_spec.deps.items()): | 653 for dep, dep_repo in sorted(package_spec.deps.items()): |
| 666 dep_repo = self._overrides.get(dep, dep_repo) | 654 dep_repo = self._overrides.get(dep, dep_repo) |
| 667 deps[dep] = self._create_package(dep_repo) | 655 deps[dep] = self._create_package(dep_repo) |
| 668 | 656 |
| 669 package = Package( | 657 package = Package( |
| 670 project_id, repo_spec, deps, | 658 project_id, repo_spec, deps, |
| 671 repo_spec.repo_root(self._context), | 659 repo_spec.repo_root(self._context), |
| 672 package_spec.recipes_path, | 660 package_spec.recipes_path) |
| 673 package_spec.canonical_base_url) | |
| 674 | 661 |
| 675 self._packages[project_id] = package | 662 self._packages[project_id] = package |
| 676 return package | 663 return package |
| 677 | 664 |
| 678 # TODO(luqui): Remove this, so all accesses to packages are done | 665 # TODO(luqui): Remove this, so all accesses to packages are done |
| 679 # via other packages with properly scoped deps. | 666 # via other packages with properly scoped deps. |
| 680 def get_package(self, package_id): | 667 def get_package(self, package_id): |
| 681 return self._packages[package_id] | 668 return self._packages[package_id] |
| 682 | 669 |
| 683 @property | 670 @property |
| (...skipping 12 matching lines...) Expand all Loading... |
| 696 >>> d = { 'x': 1, 'y': 2 } | 683 >>> d = { 'x': 1, 'y': 2 } |
| 697 >>> sorted(_updated(d, { 'y': 3, 'z': 4 }).items()) | 684 >>> sorted(_updated(d, { 'y': 3, 'z': 4 }).items()) |
| 698 [('x', 1), ('y', 3), ('z', 4)] | 685 [('x', 1), ('y', 3), ('z', 4)] |
| 699 >>> sorted(d.items()) | 686 >>> sorted(d.items()) |
| 700 [('x', 1), ('y', 2)] | 687 [('x', 1), ('y', 2)] |
| 701 """ | 688 """ |
| 702 | 689 |
| 703 d = copy.copy(d) | 690 d = copy.copy(d) |
| 704 d.update(updates) | 691 d.update(updates) |
| 705 return d | 692 return d |
| OLD | NEW |