OLD | NEW |
1 # Copyright 2015 The LUCI Authors. All rights reserved. | 1 # Copyright 2015 The LUCI Authors. All rights reserved. |
2 # Use of this source code is governed under the Apache License, Version 2.0 | 2 # Use of this source code is governed under the Apache License, Version 2.0 |
3 # that can be found in the LICENSE file. | 3 # that can be found in the LICENSE file. |
4 | 4 |
5 import copy | 5 import copy |
6 import difflib | 6 import difflib |
7 import logging | 7 import logging |
8 import operator | 8 import operator |
9 import os | 9 import os |
10 import subprocess | 10 import subprocess |
(...skipping 18 matching lines...) Expand all Loading... |
29 | 29 |
30 class CyclicDependencyError(Exception): | 30 class CyclicDependencyError(Exception): |
31 pass | 31 pass |
32 | 32 |
33 | 33 |
34 def cleanup_pyc(path): | 34 def cleanup_pyc(path): |
35 """Removes any .pyc files from |path|'s directory tree. | 35 """Removes any .pyc files from |path|'s directory tree. |
36 | 36 |
37 This ensures we always use the fresh code. | 37 This ensures we always use the fresh code. |
38 """ | 38 """ |
39 for root, _dirs, files in os.walk(path): | 39 for root, dirs, files in os.walk(path): |
40 for f in files: | 40 for f in files: |
41 if f.endswith('.pyc'): | 41 if f.endswith('.pyc'): |
42 os.unlink(os.path.join(root, f)) | 42 os.unlink(os.path.join(root, f)) |
43 | 43 |
44 | 44 |
45 class InfraRepoConfig(object): | 45 class InfraRepoConfig(object): |
46 def to_recipes_cfg(self, repo_root): | 46 def to_recipes_cfg(self, repo_root): |
47 # TODO(luqui): This is not always correct. It can be configured in | 47 # TODO(luqui): This is not always correct. It can be configured in |
48 # infra/config:refs.cfg. | 48 # infra/config:refs.cfg. |
49 return os.path.join(repo_root, 'infra', 'config', 'recipes.cfg') | 49 return os.path.join(repo_root, 'infra', 'config', 'recipes.cfg') |
(...skipping 304 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
354 return False | 354 return False |
355 return self._proto_file == other._proto_file | 355 return self._proto_file == other._proto_file |
356 | 356 |
357 | 357 |
358 class Package(object): | 358 class Package(object): |
359 """Package represents a loaded package, and contains path and dependency | 359 """Package represents a loaded package, and contains path and dependency |
360 information. | 360 information. |
361 | 361 |
362 This is accessed by loader.py through RecipeDeps.get_package. | 362 This is accessed by loader.py through RecipeDeps.get_package. |
363 """ | 363 """ |
364 def __init__(self, name, repo_spec, deps, repo_root, relative_recipes_dir, | 364 def __init__(self, name, repo_spec, deps, repo_root, relative_recipes_dir): |
365 canonical_base_url): | |
366 self.name = name | 365 self.name = name |
367 self.repo_spec = repo_spec | 366 self.repo_spec = repo_spec |
368 self.deps = deps | 367 self.deps = deps |
369 self.repo_root = repo_root | 368 self.repo_root = repo_root |
370 self.relative_recipes_dir = relative_recipes_dir | 369 self.relative_recipes_dir = relative_recipes_dir |
371 self.canonical_base_url = canonical_base_url | |
372 | 370 |
373 @property | 371 @property |
374 def recipes_dir(self): | 372 def recipes_dir(self): |
375 return os.path.join(self.repo_root, self.relative_recipes_dir) | 373 return os.path.join(self.repo_root, self.relative_recipes_dir) |
376 | 374 |
377 @property | 375 @property |
378 def recipe_dir(self): | 376 def recipe_dir(self): |
379 return os.path.join(self.recipes_dir, 'recipes') | 377 return os.path.join(self.recipes_dir, 'recipes') |
380 | 378 |
381 @property | 379 @property |
382 def module_dir(self): | 380 def module_dir(self): |
383 return os.path.join(self.recipes_dir, 'recipe_modules') | 381 return os.path.join(self.recipes_dir, 'recipe_modules') |
384 | 382 |
385 def find_dep(self, dep_name): | 383 def find_dep(self, dep_name): |
386 if dep_name == self.name: | 384 if dep_name == self.name: |
387 return self | 385 return self |
388 | 386 |
389 assert dep_name in self.deps, ( | 387 assert dep_name in self.deps, ( |
390 '%s does not exist or is not declared as a dependency of %s' % ( | 388 '%s does not exist or is not declared as a dependency of %s' % ( |
391 dep_name, self.name)) | 389 dep_name, self.name)) |
392 return self.deps[dep_name] | 390 return self.deps[dep_name] |
393 | 391 |
394 def module_path(self, module_name): | 392 def module_path(self, module_name): |
395 return os.path.join(self.recipes_dir, 'recipe_modules', module_name) | 393 return os.path.join(self.recipes_dir, 'recipe_modules', module_name) |
396 | 394 |
397 def __repr__(self): | 395 def __repr__(self): |
398 return 'Package(%r, %r, %r, %r, %r)' % ( | 396 return 'Package(%r, %r, %r, %r)' % ( |
399 self.name, self.repo_spec, self.deps, self.recipe_dir, | 397 self.name, self.repo_spec, self.deps, self.recipe_dir) |
400 self.canonical_base_url) | |
401 | 398 |
402 def __str__(self): | 399 def __str__(self): |
403 return 'Package %s, with dependencies %s' % (self.name, self.deps.keys()) | 400 return 'Package %s, with dependencies %s' % (self.name, self.deps.keys()) |
404 | 401 |
405 | 402 |
406 class RollCandidate(object): | 403 class RollCandidate(object): |
407 """RollCandidate represents a recipe roll candidate, i.e. updates | 404 """RollCandidate represents a recipe roll candidate, i.e. updates |
408 to pinned revisions of recipe dependencies. | 405 to pinned revisions of recipe dependencies. |
409 | 406 |
410 This is mostly used by recipes.py autoroll command. | 407 This is mostly used by recipes.py autoroll command. |
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
462 def get_rolled_spec(self): | 459 def get_rolled_spec(self): |
463 """Returns a PackageSpec with all the deps updates from this roll.""" | 460 """Returns a PackageSpec with all the deps updates from this roll.""" |
464 # TODO(phajdan.jr): does this preserve comments? should it? | 461 # TODO(phajdan.jr): does this preserve comments? should it? |
465 new_deps = _updated( | 462 new_deps = _updated( |
466 self._package_spec.deps, | 463 self._package_spec.deps, |
467 { project_id: spec for project_id, spec in | 464 { project_id: spec for project_id, spec in |
468 self._updates.iteritems() }) | 465 self._updates.iteritems() }) |
469 return PackageSpec( | 466 return PackageSpec( |
470 self._package_spec.project_id, | 467 self._package_spec.project_id, |
471 self._package_spec.recipes_path, | 468 self._package_spec.recipes_path, |
472 new_deps, | 469 new_deps) |
473 self._package_spec.canonical_base_url) | |
474 | 470 |
475 def get_commit_infos(self): | 471 def get_commit_infos(self): |
476 """Returns a mapping project_id -> list of commits from that repo | 472 """Returns a mapping project_id -> list of commits from that repo |
477 that are getting pulled by this roll. | 473 that are getting pulled by this roll. |
478 """ | 474 """ |
479 commit_infos = {} | 475 commit_infos = {} |
480 | 476 |
481 for project_id, update in self._updates.iteritems(): | 477 for project_id, update in self._updates.iteritems(): |
482 commit_infos[project_id] = self._package_spec.deps[ | 478 commit_infos[project_id] = self._package_spec.deps[ |
483 project_id].commit_infos(self._context, update.revision) | 479 project_id].commit_infos(self._context, update.revision) |
(...skipping 10 matching lines...) Expand all Loading... |
494 """Returns a unified diff between original package spec and one after roll. | 490 """Returns a unified diff between original package spec and one after roll. |
495 """ | 491 """ |
496 orig = str(self._package_spec.dump()).splitlines() | 492 orig = str(self._package_spec.dump()).splitlines() |
497 new = str(self.get_rolled_spec().dump()).splitlines() | 493 new = str(self.get_rolled_spec().dump()).splitlines() |
498 return '\n'.join(difflib.unified_diff(orig, new, lineterm='')) | 494 return '\n'.join(difflib.unified_diff(orig, new, lineterm='')) |
499 | 495 |
500 | 496 |
501 class PackageSpec(object): | 497 class PackageSpec(object): |
502 API_VERSION = 1 | 498 API_VERSION = 1 |
503 | 499 |
504 def __init__(self, project_id, recipes_path, deps, canonical_base_url): | 500 def __init__(self, project_id, recipes_path, deps): |
505 self._project_id = project_id | 501 self._project_id = project_id |
506 self._recipes_path = recipes_path | 502 self._recipes_path = recipes_path |
507 self._deps = deps | 503 self._deps = deps |
508 self._canonical_base_url = canonical_base_url | |
509 | 504 |
510 @classmethod | 505 @classmethod |
511 def load_proto(cls, proto_file): | 506 def load_proto(cls, proto_file): |
512 buf = proto_file.read() | 507 buf = proto_file.read() |
513 assert buf.api_version == cls.API_VERSION | 508 assert buf.api_version == cls.API_VERSION |
514 | 509 |
515 deps = { str(dep.project_id): cls.spec_for_dep(dep) | 510 deps = { str(dep.project_id): cls.spec_for_dep(dep) |
516 for dep in buf.deps } | 511 for dep in buf.deps } |
517 return cls(str(buf.project_id), str(buf.recipes_path), deps, | 512 return cls(str(buf.project_id), str(buf.recipes_path), deps) |
518 buf.canonical_base_url) | |
519 | 513 |
520 @classmethod | 514 @classmethod |
521 def spec_for_dep(cls, dep): | 515 def spec_for_dep(cls, dep): |
522 """Returns a RepoSpec for the given dependency protobuf.""" | 516 """Returns a RepoSpec for the given dependency protobuf.""" |
523 url = str(dep.url) | 517 url = str(dep.url) |
524 if url.startswith("file://"): | 518 if url.startswith("file://"): |
525 return PathRepoSpec(str(dep.project_id), url[len("file://"):]) | 519 return PathRepoSpec(str(dep.project_id), url[len("file://"):]) |
526 | 520 |
527 if dep.repo_type in (package_pb2.DepSpec.GIT, package_pb2.DepSpec.GITILES): | 521 if dep.repo_type in (package_pb2.DepSpec.GIT, package_pb2.DepSpec.GITILES): |
528 if dep.repo_type == package_pb2.DepSpec.GIT: | 522 if dep.repo_type == package_pb2.DepSpec.GIT: |
(...skipping 14 matching lines...) Expand all Loading... |
543 return self._project_id | 537 return self._project_id |
544 | 538 |
545 @property | 539 @property |
546 def recipes_path(self): | 540 def recipes_path(self): |
547 return self._recipes_path | 541 return self._recipes_path |
548 | 542 |
549 @property | 543 @property |
550 def deps(self): | 544 def deps(self): |
551 return self._deps | 545 return self._deps |
552 | 546 |
553 @property | |
554 def canonical_base_url(self): | |
555 return self._canonical_base_url | |
556 | |
557 def dump(self): | 547 def dump(self): |
558 return package_pb2.Package( | 548 return package_pb2.Package( |
559 api_version=self.API_VERSION, | 549 api_version=self.API_VERSION, |
560 project_id=self._project_id, | 550 project_id=self._project_id, |
561 recipes_path=self._recipes_path, | 551 recipes_path=self._recipes_path, |
562 deps=[ self._deps[dep].dump() for dep in sorted(self._deps.keys()) ], | 552 deps=[ self._deps[dep].dump() for dep in sorted(self._deps.keys()) ]) |
563 canonical_base_url=self._canonical_base_url) | |
564 | 553 |
565 def roll_candidates(self, root_spec, context): | 554 def roll_candidates(self, root_spec, context): |
566 """Returns list of consistent roll candidates, and rejected roll candidates. | 555 """Returns list of consistent roll candidates, and rejected roll candidates. |
567 | 556 |
568 The first one is sorted by score, descending. The more commits are pulled by | 557 The first one is sorted by score, descending. The more commits are pulled by |
569 the roll, the higher score. | 558 the roll, the higher score. |
570 | 559 |
571 Second list is included to distinguish between a situation where there are | 560 Second list is included to distinguish between a situation where there are |
572 no roll candidates from one where there are updates but they're not | 561 no roll candidates from one where there are updates but they're not |
573 consistent. | 562 consistent. |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
628 are the override path. | 617 are the override path. |
629 """ | 618 """ |
630 context = PackageContext.from_proto_file(repo_root, proto_file, allow_fetch, | 619 context = PackageContext.from_proto_file(repo_root, proto_file, allow_fetch, |
631 deps_path=deps_path) | 620 deps_path=deps_path) |
632 | 621 |
633 if overrides: | 622 if overrides: |
634 overrides = {project_id: PathRepoSpec(project_id, path) | 623 overrides = {project_id: PathRepoSpec(project_id, path) |
635 for project_id, path in overrides.iteritems()} | 624 for project_id, path in overrides.iteritems()} |
636 package_deps = cls(context, overrides=overrides) | 625 package_deps = cls(context, overrides=overrides) |
637 | 626 |
638 package_deps._root_package = package_deps._create_package( | 627 package_deps._root_package = package_deps._create_package(RootRepoSpec(proto
_file)) |
639 RootRepoSpec(proto_file)) | |
640 | 628 |
641 return package_deps | 629 return package_deps |
642 | 630 |
643 def _create_package(self, repo_spec): | 631 def _create_package(self, repo_spec): |
644 repo_spec.checkout(self._context) | 632 repo_spec.checkout(self._context) |
645 package_spec = PackageSpec.load_proto(repo_spec.proto_file(self._context)) | 633 package_spec = PackageSpec.load_proto(repo_spec.proto_file(self._context)) |
646 return self._create_from_spec(repo_spec, package_spec) | 634 return self._create_from_spec(repo_spec, package_spec) |
647 | 635 |
648 def _create_from_spec(self, repo_spec, package_spec): | 636 def _create_from_spec(self, repo_spec, package_spec): |
649 project_id = package_spec.project_id | 637 project_id = package_spec.project_id |
(...skipping 12 matching lines...) Expand all Loading... |
662 self._packages[project_id] = None | 650 self._packages[project_id] = None |
663 | 651 |
664 deps = {} | 652 deps = {} |
665 for dep, dep_repo in sorted(package_spec.deps.items()): | 653 for dep, dep_repo in sorted(package_spec.deps.items()): |
666 dep_repo = self._overrides.get(dep, dep_repo) | 654 dep_repo = self._overrides.get(dep, dep_repo) |
667 deps[dep] = self._create_package(dep_repo) | 655 deps[dep] = self._create_package(dep_repo) |
668 | 656 |
669 package = Package( | 657 package = Package( |
670 project_id, repo_spec, deps, | 658 project_id, repo_spec, deps, |
671 repo_spec.repo_root(self._context), | 659 repo_spec.repo_root(self._context), |
672 package_spec.recipes_path, | 660 package_spec.recipes_path) |
673 package_spec.canonical_base_url) | |
674 | 661 |
675 self._packages[project_id] = package | 662 self._packages[project_id] = package |
676 return package | 663 return package |
677 | 664 |
678 # TODO(luqui): Remove this, so all accesses to packages are done | 665 # TODO(luqui): Remove this, so all accesses to packages are done |
679 # via other packages with properly scoped deps. | 666 # via other packages with properly scoped deps. |
680 def get_package(self, package_id): | 667 def get_package(self, package_id): |
681 return self._packages[package_id] | 668 return self._packages[package_id] |
682 | 669 |
683 @property | 670 @property |
(...skipping 12 matching lines...) Expand all Loading... |
696 >>> d = { 'x': 1, 'y': 2 } | 683 >>> d = { 'x': 1, 'y': 2 } |
697 >>> sorted(_updated(d, { 'y': 3, 'z': 4 }).items()) | 684 >>> sorted(_updated(d, { 'y': 3, 'z': 4 }).items()) |
698 [('x', 1), ('y', 3), ('z', 4)] | 685 [('x', 1), ('y', 3), ('z', 4)] |
699 >>> sorted(d.items()) | 686 >>> sorted(d.items()) |
700 [('x', 1), ('y', 2)] | 687 [('x', 1), ('y', 2)] |
701 """ | 688 """ |
702 | 689 |
703 d = copy.copy(d) | 690 d = copy.copy(d) |
704 d.update(updates) | 691 d.update(updates) |
705 return d | 692 return d |
OLD | NEW |