Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(202)

Side by Side Diff: recipe_engine/package.py

Issue 2739803002: Revert of [package.proto] add canonical_base_url field. (Closed)
Patch Set: Created 3 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « recipe_engine/package.proto ('k') | recipe_engine/package_pb2.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2015 The LUCI Authors. All rights reserved. 1 # Copyright 2015 The LUCI Authors. All rights reserved.
2 # Use of this source code is governed under the Apache License, Version 2.0 2 # Use of this source code is governed under the Apache License, Version 2.0
3 # that can be found in the LICENSE file. 3 # that can be found in the LICENSE file.
4 4
5 import copy 5 import copy
6 import difflib 6 import difflib
7 import logging 7 import logging
8 import operator 8 import operator
9 import os 9 import os
10 import subprocess 10 import subprocess
(...skipping 18 matching lines...) Expand all
29 29
30 class CyclicDependencyError(Exception): 30 class CyclicDependencyError(Exception):
31 pass 31 pass
32 32
33 33
34 def cleanup_pyc(path): 34 def cleanup_pyc(path):
35 """Removes any .pyc files from |path|'s directory tree. 35 """Removes any .pyc files from |path|'s directory tree.
36 36
37 This ensures we always use the fresh code. 37 This ensures we always use the fresh code.
38 """ 38 """
39 for root, _dirs, files in os.walk(path): 39 for root, dirs, files in os.walk(path):
40 for f in files: 40 for f in files:
41 if f.endswith('.pyc'): 41 if f.endswith('.pyc'):
42 os.unlink(os.path.join(root, f)) 42 os.unlink(os.path.join(root, f))
43 43
44 44
45 class InfraRepoConfig(object): 45 class InfraRepoConfig(object):
46 def to_recipes_cfg(self, repo_root): 46 def to_recipes_cfg(self, repo_root):
47 # TODO(luqui): This is not always correct. It can be configured in 47 # TODO(luqui): This is not always correct. It can be configured in
48 # infra/config:refs.cfg. 48 # infra/config:refs.cfg.
49 return os.path.join(repo_root, 'infra', 'config', 'recipes.cfg') 49 return os.path.join(repo_root, 'infra', 'config', 'recipes.cfg')
(...skipping 304 matching lines...) Expand 10 before | Expand all | Expand 10 after
354 return False 354 return False
355 return self._proto_file == other._proto_file 355 return self._proto_file == other._proto_file
356 356
357 357
358 class Package(object): 358 class Package(object):
359 """Package represents a loaded package, and contains path and dependency 359 """Package represents a loaded package, and contains path and dependency
360 information. 360 information.
361 361
362 This is accessed by loader.py through RecipeDeps.get_package. 362 This is accessed by loader.py through RecipeDeps.get_package.
363 """ 363 """
364 def __init__(self, name, repo_spec, deps, repo_root, relative_recipes_dir, 364 def __init__(self, name, repo_spec, deps, repo_root, relative_recipes_dir):
365 canonical_base_url):
366 self.name = name 365 self.name = name
367 self.repo_spec = repo_spec 366 self.repo_spec = repo_spec
368 self.deps = deps 367 self.deps = deps
369 self.repo_root = repo_root 368 self.repo_root = repo_root
370 self.relative_recipes_dir = relative_recipes_dir 369 self.relative_recipes_dir = relative_recipes_dir
371 self.canonical_base_url = canonical_base_url
372 370
373 @property 371 @property
374 def recipes_dir(self): 372 def recipes_dir(self):
375 return os.path.join(self.repo_root, self.relative_recipes_dir) 373 return os.path.join(self.repo_root, self.relative_recipes_dir)
376 374
377 @property 375 @property
378 def recipe_dir(self): 376 def recipe_dir(self):
379 return os.path.join(self.recipes_dir, 'recipes') 377 return os.path.join(self.recipes_dir, 'recipes')
380 378
381 @property 379 @property
382 def module_dir(self): 380 def module_dir(self):
383 return os.path.join(self.recipes_dir, 'recipe_modules') 381 return os.path.join(self.recipes_dir, 'recipe_modules')
384 382
385 def find_dep(self, dep_name): 383 def find_dep(self, dep_name):
386 if dep_name == self.name: 384 if dep_name == self.name:
387 return self 385 return self
388 386
389 assert dep_name in self.deps, ( 387 assert dep_name in self.deps, (
390 '%s does not exist or is not declared as a dependency of %s' % ( 388 '%s does not exist or is not declared as a dependency of %s' % (
391 dep_name, self.name)) 389 dep_name, self.name))
392 return self.deps[dep_name] 390 return self.deps[dep_name]
393 391
394 def module_path(self, module_name): 392 def module_path(self, module_name):
395 return os.path.join(self.recipes_dir, 'recipe_modules', module_name) 393 return os.path.join(self.recipes_dir, 'recipe_modules', module_name)
396 394
397 def __repr__(self): 395 def __repr__(self):
398 return 'Package(%r, %r, %r, %r, %r)' % ( 396 return 'Package(%r, %r, %r, %r)' % (
399 self.name, self.repo_spec, self.deps, self.recipe_dir, 397 self.name, self.repo_spec, self.deps, self.recipe_dir)
400 self.canonical_base_url)
401 398
402 def __str__(self): 399 def __str__(self):
403 return 'Package %s, with dependencies %s' % (self.name, self.deps.keys()) 400 return 'Package %s, with dependencies %s' % (self.name, self.deps.keys())
404 401
405 402
406 class RollCandidate(object): 403 class RollCandidate(object):
407 """RollCandidate represents a recipe roll candidate, i.e. updates 404 """RollCandidate represents a recipe roll candidate, i.e. updates
408 to pinned revisions of recipe dependencies. 405 to pinned revisions of recipe dependencies.
409 406
410 This is mostly used by recipes.py autoroll command. 407 This is mostly used by recipes.py autoroll command.
(...skipping 51 matching lines...) Expand 10 before | Expand all | Expand 10 after
462 def get_rolled_spec(self): 459 def get_rolled_spec(self):
463 """Returns a PackageSpec with all the deps updates from this roll.""" 460 """Returns a PackageSpec with all the deps updates from this roll."""
464 # TODO(phajdan.jr): does this preserve comments? should it? 461 # TODO(phajdan.jr): does this preserve comments? should it?
465 new_deps = _updated( 462 new_deps = _updated(
466 self._package_spec.deps, 463 self._package_spec.deps,
467 { project_id: spec for project_id, spec in 464 { project_id: spec for project_id, spec in
468 self._updates.iteritems() }) 465 self._updates.iteritems() })
469 return PackageSpec( 466 return PackageSpec(
470 self._package_spec.project_id, 467 self._package_spec.project_id,
471 self._package_spec.recipes_path, 468 self._package_spec.recipes_path,
472 new_deps, 469 new_deps)
473 self._package_spec.canonical_base_url)
474 470
475 def get_commit_infos(self): 471 def get_commit_infos(self):
476 """Returns a mapping project_id -> list of commits from that repo 472 """Returns a mapping project_id -> list of commits from that repo
477 that are getting pulled by this roll. 473 that are getting pulled by this roll.
478 """ 474 """
479 commit_infos = {} 475 commit_infos = {}
480 476
481 for project_id, update in self._updates.iteritems(): 477 for project_id, update in self._updates.iteritems():
482 commit_infos[project_id] = self._package_spec.deps[ 478 commit_infos[project_id] = self._package_spec.deps[
483 project_id].commit_infos(self._context, update.revision) 479 project_id].commit_infos(self._context, update.revision)
(...skipping 10 matching lines...) Expand all
494 """Returns a unified diff between original package spec and one after roll. 490 """Returns a unified diff between original package spec and one after roll.
495 """ 491 """
496 orig = str(self._package_spec.dump()).splitlines() 492 orig = str(self._package_spec.dump()).splitlines()
497 new = str(self.get_rolled_spec().dump()).splitlines() 493 new = str(self.get_rolled_spec().dump()).splitlines()
498 return '\n'.join(difflib.unified_diff(orig, new, lineterm='')) 494 return '\n'.join(difflib.unified_diff(orig, new, lineterm=''))
499 495
500 496
501 class PackageSpec(object): 497 class PackageSpec(object):
502 API_VERSION = 1 498 API_VERSION = 1
503 499
504 def __init__(self, project_id, recipes_path, deps, canonical_base_url): 500 def __init__(self, project_id, recipes_path, deps):
505 self._project_id = project_id 501 self._project_id = project_id
506 self._recipes_path = recipes_path 502 self._recipes_path = recipes_path
507 self._deps = deps 503 self._deps = deps
508 self._canonical_base_url = canonical_base_url
509 504
510 @classmethod 505 @classmethod
511 def load_proto(cls, proto_file): 506 def load_proto(cls, proto_file):
512 buf = proto_file.read() 507 buf = proto_file.read()
513 assert buf.api_version == cls.API_VERSION 508 assert buf.api_version == cls.API_VERSION
514 509
515 deps = { str(dep.project_id): cls.spec_for_dep(dep) 510 deps = { str(dep.project_id): cls.spec_for_dep(dep)
516 for dep in buf.deps } 511 for dep in buf.deps }
517 return cls(str(buf.project_id), str(buf.recipes_path), deps, 512 return cls(str(buf.project_id), str(buf.recipes_path), deps)
518 buf.canonical_base_url)
519 513
520 @classmethod 514 @classmethod
521 def spec_for_dep(cls, dep): 515 def spec_for_dep(cls, dep):
522 """Returns a RepoSpec for the given dependency protobuf.""" 516 """Returns a RepoSpec for the given dependency protobuf."""
523 url = str(dep.url) 517 url = str(dep.url)
524 if url.startswith("file://"): 518 if url.startswith("file://"):
525 return PathRepoSpec(str(dep.project_id), url[len("file://"):]) 519 return PathRepoSpec(str(dep.project_id), url[len("file://"):])
526 520
527 if dep.repo_type in (package_pb2.DepSpec.GIT, package_pb2.DepSpec.GITILES): 521 if dep.repo_type in (package_pb2.DepSpec.GIT, package_pb2.DepSpec.GITILES):
528 if dep.repo_type == package_pb2.DepSpec.GIT: 522 if dep.repo_type == package_pb2.DepSpec.GIT:
(...skipping 14 matching lines...) Expand all
543 return self._project_id 537 return self._project_id
544 538
545 @property 539 @property
546 def recipes_path(self): 540 def recipes_path(self):
547 return self._recipes_path 541 return self._recipes_path
548 542
549 @property 543 @property
550 def deps(self): 544 def deps(self):
551 return self._deps 545 return self._deps
552 546
553 @property
554 def canonical_base_url(self):
555 return self._canonical_base_url
556
557 def dump(self): 547 def dump(self):
558 return package_pb2.Package( 548 return package_pb2.Package(
559 api_version=self.API_VERSION, 549 api_version=self.API_VERSION,
560 project_id=self._project_id, 550 project_id=self._project_id,
561 recipes_path=self._recipes_path, 551 recipes_path=self._recipes_path,
562 deps=[ self._deps[dep].dump() for dep in sorted(self._deps.keys()) ], 552 deps=[ self._deps[dep].dump() for dep in sorted(self._deps.keys()) ])
563 canonical_base_url=self._canonical_base_url)
564 553
565 def roll_candidates(self, root_spec, context): 554 def roll_candidates(self, root_spec, context):
566 """Returns list of consistent roll candidates, and rejected roll candidates. 555 """Returns list of consistent roll candidates, and rejected roll candidates.
567 556
568 The first one is sorted by score, descending. The more commits are pulled by 557 The first one is sorted by score, descending. The more commits are pulled by
569 the roll, the higher score. 558 the roll, the higher score.
570 559
571 Second list is included to distinguish between a situation where there are 560 Second list is included to distinguish between a situation where there are
572 no roll candidates from one where there are updates but they're not 561 no roll candidates from one where there are updates but they're not
573 consistent. 562 consistent.
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after
628 are the override path. 617 are the override path.
629 """ 618 """
630 context = PackageContext.from_proto_file(repo_root, proto_file, allow_fetch, 619 context = PackageContext.from_proto_file(repo_root, proto_file, allow_fetch,
631 deps_path=deps_path) 620 deps_path=deps_path)
632 621
633 if overrides: 622 if overrides:
634 overrides = {project_id: PathRepoSpec(project_id, path) 623 overrides = {project_id: PathRepoSpec(project_id, path)
635 for project_id, path in overrides.iteritems()} 624 for project_id, path in overrides.iteritems()}
636 package_deps = cls(context, overrides=overrides) 625 package_deps = cls(context, overrides=overrides)
637 626
638 package_deps._root_package = package_deps._create_package( 627 package_deps._root_package = package_deps._create_package(RootRepoSpec(proto _file))
639 RootRepoSpec(proto_file))
640 628
641 return package_deps 629 return package_deps
642 630
643 def _create_package(self, repo_spec): 631 def _create_package(self, repo_spec):
644 repo_spec.checkout(self._context) 632 repo_spec.checkout(self._context)
645 package_spec = PackageSpec.load_proto(repo_spec.proto_file(self._context)) 633 package_spec = PackageSpec.load_proto(repo_spec.proto_file(self._context))
646 return self._create_from_spec(repo_spec, package_spec) 634 return self._create_from_spec(repo_spec, package_spec)
647 635
648 def _create_from_spec(self, repo_spec, package_spec): 636 def _create_from_spec(self, repo_spec, package_spec):
649 project_id = package_spec.project_id 637 project_id = package_spec.project_id
(...skipping 12 matching lines...) Expand all
662 self._packages[project_id] = None 650 self._packages[project_id] = None
663 651
664 deps = {} 652 deps = {}
665 for dep, dep_repo in sorted(package_spec.deps.items()): 653 for dep, dep_repo in sorted(package_spec.deps.items()):
666 dep_repo = self._overrides.get(dep, dep_repo) 654 dep_repo = self._overrides.get(dep, dep_repo)
667 deps[dep] = self._create_package(dep_repo) 655 deps[dep] = self._create_package(dep_repo)
668 656
669 package = Package( 657 package = Package(
670 project_id, repo_spec, deps, 658 project_id, repo_spec, deps,
671 repo_spec.repo_root(self._context), 659 repo_spec.repo_root(self._context),
672 package_spec.recipes_path, 660 package_spec.recipes_path)
673 package_spec.canonical_base_url)
674 661
675 self._packages[project_id] = package 662 self._packages[project_id] = package
676 return package 663 return package
677 664
678 # TODO(luqui): Remove this, so all accesses to packages are done 665 # TODO(luqui): Remove this, so all accesses to packages are done
679 # via other packages with properly scoped deps. 666 # via other packages with properly scoped deps.
680 def get_package(self, package_id): 667 def get_package(self, package_id):
681 return self._packages[package_id] 668 return self._packages[package_id]
682 669
683 @property 670 @property
(...skipping 12 matching lines...) Expand all
696 >>> d = { 'x': 1, 'y': 2 } 683 >>> d = { 'x': 1, 'y': 2 }
697 >>> sorted(_updated(d, { 'y': 3, 'z': 4 }).items()) 684 >>> sorted(_updated(d, { 'y': 3, 'z': 4 }).items())
698 [('x', 1), ('y', 3), ('z', 4)] 685 [('x', 1), ('y', 3), ('z', 4)]
699 >>> sorted(d.items()) 686 >>> sorted(d.items())
700 [('x', 1), ('y', 2)] 687 [('x', 1), ('y', 2)]
701 """ 688 """
702 689
703 d = copy.copy(d) 690 d = copy.copy(d)
704 d.update(updates) 691 d.update(updates)
705 return d 692 return d
OLDNEW
« no previous file with comments | « recipe_engine/package.proto ('k') | recipe_engine/package_pb2.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698