OLD | NEW |
---|---|
(Empty) | |
1 #!/usr/bin/env python | |
2 # Copyright (c) 2014 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 # 'redefining builtin __doc__' pylint: disable=W0622 | |
7 __doc__ = """ | |
agable
2014/04/18 00:17:08
wat why is this necessary. __doc__ is already corr
iannucci
2014/04/28 21:05:28
Hmmmmmmmmmmm...... Not sure why I did that... Done
| |
8 Merge/Revert changes to chromium release branches. | |
agable
2014/04/18 00:17:08
Chromium
iannucci
2014/04/28 21:05:28
Done.
| |
9 | |
10 This will use the git clone in the current directory if it matches the commit | |
11 you passed in. Alternately, run this script in an empty directory and it will | |
12 clone the appropriate repo for you (using `git cache` to do the smallest amount | |
13 of network IO possible). | |
14 | |
15 This tool is aware of the following repos: | |
16 """ | |
17 | |
18 import argparse | |
19 import collections | |
20 import multiprocessing | |
21 import os | |
22 import pprint | |
23 import re | |
24 import sys | |
25 import textwrap | |
26 import urllib2 | |
agable
2014/04/18 00:17:08
dammit depot_tools y u no have requests
iannucci
2014/04/28 21:05:28
IKR
| |
27 import urlparse | |
28 | |
29 from multiprocessing.pool import ThreadPool | |
30 | |
31 import git_cache | |
32 import git_common as git | |
33 | |
34 from third_party import fancy_urllib | |
35 | |
36 assert fancy_urllib.can_validate_certs() | |
37 | |
38 CA_CERTS_FILE = os.path.abspath(os.path.join( | |
39 os.path.dirname(__file__), 'third_party', 'boto', 'cacerts', 'cacerts.txt' | |
40 )) | |
41 | |
42 urllib2.install_opener(urllib2.build_opener( | |
43 fancy_urllib.FancyRedirectHandler(), | |
44 fancy_urllib.FancyHTTPSHandler())) | |
45 | |
46 | |
47 MISSING = object() | |
48 | |
49 OK_HOST_FMT = '%s.googlesource.com' | |
50 OK_REPOS = { | |
51 'chrome-internal': ('chrome/src-internal',), | |
52 'chromium': ('chromium/src', 'chromium/blink', | |
53 'native_client/src/native_client') | |
54 } | |
55 | |
56 def repo_url(host, repo): | |
57 assert host in OK_REPOS | |
58 assert repo in OK_REPOS[host] | |
59 return 'https://%s/%s.git' % (OK_HOST_FMT % host, repo) | |
60 | |
61 # lambda avoids polluting module with variable names, but still executes at | |
62 # import-time. | |
63 __doc__ += (lambda: '\n'.join([ | |
64 ' * %s' % repo_url(host, repo) | |
65 for host, repos in OK_REPOS.iteritems() | |
66 for repo in repos | |
67 ]))() | |
68 | |
69 | |
70 def die(msg, *args): | |
71 msg = textwrap.dedent(msg) | |
72 if args: | |
73 msg = msg % args | |
74 print >> sys.stderr, msg | |
75 sys.exit(1) | |
76 | |
77 | |
78 def retry(fn, args=(), kwargs=None, on=(), but_not=(), upto=3): | |
79 kwargs = kwargs or {} | |
80 for attempt in xrange(upto): | |
81 try: | |
82 return fn(*args, **kwargs) | |
83 except but_not: | |
84 raise | |
85 except on: | |
86 if attempt + 1 == upto: | |
87 raise | |
88 | |
89 | |
90 ################################################################################ | |
agable
2014/04/18 00:17:08
I see you like to live dangerously with your comme
iannucci
2014/04/28 21:05:28
https://www.youtube.com/watch?v=Dpjl4XJ91xY
| |
91 | |
92 | |
93 def announce(msg=None, msg_fn=lambda: None): | |
94 print | |
95 print | |
96 print '=' * 80 | |
97 if msg: | |
98 print msg | |
agable
2014/04/18 00:17:08
textwrap.dedent this one too
iannucci
2014/04/28 21:05:28
Done.
| |
99 msg_fn() | |
100 print '=' * 80 | |
101 print | |
102 | |
103 | |
104 def confirm(prompt='Is this correct?', abort='No changes have been made.'): | |
105 while True: | |
106 v = raw_input('%s (Y/n) ' % prompt) | |
107 if v == '' or v in 'Yy': | |
108 break | |
109 if v in 'Nn': | |
110 die('Aborting. %s' % abort) | |
111 | |
112 | |
113 def summarize_job(correct_url, commits, target_ref, action): | |
114 def _msg_fn(): | |
115 preposition = 'to' if action == 'merge' else 'from' | |
116 print "Planning to %s %d change%s %s branch %s of %s." % ( | |
117 action, len(commits), 's' if len(commits) > 1 else '', | |
118 preposition, target_ref.num, correct_url) | |
119 print | |
120 for commit in commits: | |
121 print git.run('show', '-s', '--format=%H\t%s', commit) | |
122 announce(msg_fn=_msg_fn) | |
123 | |
124 | |
125 def ensure_working_directory(commits, target_ref): | |
126 # TODO(iannucci): check all hashes locally after fetching first | |
127 | |
128 fetch_specs = [ | |
129 '%s:%s' % (target_ref.remote_full_ref, target_ref.remote_full_ref) | |
130 ] + commits | |
131 | |
132 if git.check('rev-parse', '--is-inside-work-tree'): | |
133 actual_url = git.get_remote_url('origin') | |
134 | |
135 if not actual_url or not is_ok_repo(actual_url): | |
136 die("""\ | |
137 Inside a git repo, but origin's remote URL doesn't match one of the | |
agable
2014/04/18 00:17:08
I suppose technically these should be indented +4?
iannucci
2014/04/28 21:05:28
meh... makes 80 col awkward
| |
138 supported git repos. | |
139 Current URL: %s""", actual_url) | |
140 | |
141 s = git.run('status', '--porcelain') | |
142 if s: | |
143 die("""\ | |
144 Your current directory is usable for the command you specified, but it | |
145 appears to be dirty (i.e. there are uncommitted changes). Please commit, | |
146 freeze, or stash these changes and run this command again. | |
147 | |
148 %s""", '\n'.join(' '+l for l in s.splitlines())) | |
149 | |
150 correct_url = get_correct_url(commits, actual_url) | |
151 if correct_url != actual_url: | |
152 die("""\ | |
153 Commits specified appear to be from a different repo than the repo | |
agable
2014/04/18 00:17:08
s/Commits specified/The specified commits/
iannucci
2014/04/28 21:05:28
Done.
| |
154 in the current directory. | |
155 Current Repo: %s | |
156 Expected Repo: %s | |
157 | |
158 Please re-run this script in an empty working directory and we'll fetch | |
159 the correct repo.""", actual_url, correct_url) | |
160 | |
161 m = git_cache.Mirror.from_repo('.') | |
162 if m: | |
163 m.populate(bootstrap=True, verbose=True) | |
164 m.populate(fetch_specs=fetch_specs) | |
165 | |
166 elif len(os.listdir('.')) == 0: | |
167 sample_path = "/path/to/cache" | |
agable
2014/04/18 00:17:08
single quotes
iannucci
2014/04/28 21:05:28
Done.
| |
168 if sys.platform.startswith('win'): | |
169 sample_path = r"X:\path\to\cache" | |
agable
2014/04/18 00:17:08
here too
iannucci
2014/04/28 21:05:28
Done.
| |
170 if not git.config('cache.cachepath'): | |
171 die("""\ | |
172 Automatic drover checkouts require that you configure your global | |
173 cachepath to make these automatic checkouts as fast as possible. Do this | |
174 by running: | |
175 git config --global cache.cachepath "%s" | |
176 | |
177 We recommend picking a non-network-mounted path with a decent amount of | |
178 space (at least 4GB).""" % sample_path) | |
179 | |
180 correct_url = get_correct_url(commits) | |
181 | |
182 m = git_cache.Mirror(correct_url) | |
183 m.populate(bootstrap=True, verbose=True) | |
184 m.populate(fetch_specs=fetch_specs) | |
185 git.run('clone', '-s', '--no-checkout', m.mirror_path, '.') | |
186 git.run('update-ref', '-d', 'refs/heads/master') | |
187 else: | |
188 die('You must either invoke this from a git repo, or from an empty dir.') | |
189 | |
190 for s in [target_ref.local_full_ref] + commits: | |
191 git.check('fetch', 'origin', s) | |
192 | |
193 return correct_url | |
194 | |
195 | |
196 def find_hash_urls(commits, presumed_url=None): | |
agable
2014/04/18 00:17:08
It's a 70-line function, with three embedded funct
iannucci
2014/04/28 21:05:28
rewrite it in go? :p
| |
197 pool = ThreadPool() | |
198 | |
199 def process_async_results(asr, results): | |
agable
2014/04/18 00:17:08
Yes, it processes async results... still the least
iannucci
2014/04/28 21:05:28
PTAL
| |
200 try: | |
201 lost_commits = [] | |
202 passes = 0 | |
203 while asr and passes <= 10: | |
204 new_asr = {} | |
205 for commit, attempts in asr.iteritems(): | |
206 new_attempts = [] | |
207 for attempt in attempts: | |
208 try: | |
209 attempt = attempt.get(.5) | |
210 if attempt is not MISSING: | |
211 results[attempt].add(commit) | |
212 break | |
213 except multiprocessing.TimeoutError: | |
214 new_attempts.append(attempt) | |
215 else: | |
216 if new_attempts: | |
217 new_asr[commit] = new_attempts | |
218 else: | |
219 lost_commits.append(commit) | |
220 asr = new_asr | |
221 passes += 1 | |
222 return lost_commits | |
223 except Exception: | |
224 import traceback | |
225 traceback.print_exc() | |
226 | |
227 # TODO(iannucci): Gather a summary from each commit | |
228 def exists(url, commit): | |
229 query_url = '%s/+/%s?format=JSON' % (url, commit) | |
230 return MISSING if GET(query_url) is MISSING else url | |
231 | |
232 def go_fish(commit, except_for=()): | |
233 async_results = {commit: set()} | |
234 for host, repos in OK_REPOS.iteritems(): | |
235 for repo in repos: | |
236 url = repo_url(host, repo) | |
237 if url in except_for: | |
238 continue | |
239 async_results[commit].add( | |
240 pool.apply_async(exists, args=(url, commit))) | |
241 | |
242 results = collections.defaultdict(set) | |
243 lost = process_async_results(async_results, results) | |
244 if not lost: | |
245 return results.popitem()[0] | |
246 | |
247 # map of url -> set(commits) | |
248 results = collections.defaultdict(set) | |
249 | |
250 # Try to find one hash which matches some repo | |
251 while commits and not presumed_url: | |
agable
2014/04/18 00:17:08
guessed_url? presumed means you're guessing withou
iannucci
2014/04/28 21:05:28
We ARE guessing without reason. In the event that
agable
2014/04/28 21:38:22
That's actually a really good reason, not no reaso
| |
252 presumed_url = go_fish(commits[0]) | |
253 results[presumed_url].add(commits[0]) | |
254 commits = commits[1:] | |
255 | |
256 # map of commit -> attempts | |
257 async_results = collections.defaultdict(list) | |
258 for commit in commits: | |
259 async_results[commit].append( | |
260 pool.apply_async(exists, args=(presumed_url, commit))) | |
261 | |
262 lost = process_async_results(async_results, results) | |
263 | |
264 if lost: | |
265 fishing_pool = ThreadPool() | |
266 async_results = collections.defaultdict(list) | |
267 for commit in lost: | |
268 async_results[commit].append( | |
269 fishing_pool.apply_async(go_fish, (commit,), | |
270 {'except_for': presumed_url}) | |
271 ) | |
272 lost = process_async_results(async_results, results) | |
273 if lost: | |
274 results[None].update(lost) | |
275 | |
276 return {(k or 'UNKNOWN'): list(v) for k, v in results.iteritems()} | |
277 | |
278 | |
279 def GET(url, **kwargs): | |
280 try: | |
281 kwargs.setdefault('timeout', 5) | |
282 request = fancy_urllib.FancyRequest(url) | |
283 request.set_ssl_info(ca_certs=CA_CERTS_FILE) | |
284 return retry(urllib2.urlopen, [request], kwargs, | |
285 on=urllib2.URLError, but_not=urllib2.HTTPError, upto=3) | |
286 except urllib2.HTTPError as e: | |
287 if e.getcode() / 100 == 4: | |
288 return MISSING | |
289 raise | |
290 | |
291 | |
292 def get_correct_url(commits, presumed_url=None): | |
293 unverified = commits | |
294 if presumed_url: | |
295 unverified = [c for c in unverified if not git.verify_commit(c)] | |
296 if not unverified: | |
297 return presumed_url | |
298 git.cached_fetch(unverified) | |
299 unverified = [c for c in unverified if not git.verify_commit(c)] | |
300 if not unverified: | |
301 return presumed_url | |
302 | |
303 url_hashes = find_hash_urls(unverified, presumed_url) | |
304 if None in url_hashes: | |
305 die("""\ | |
306 Could not determine what repo the following commits originate from: | |
307 %r""", url_hashes[None]) | |
308 | |
309 if len(url_hashes) > 1: | |
310 die("""\ | |
311 Ambiguous commits specified. You supplied multiple commits, but they | |
312 appear to be from more than one repo? | |
313 %s""", pprint.pformat(dict(url_hashes))) | |
314 | |
315 return url_hashes.popitem()[0] | |
316 | |
317 | |
318 def is_ok_repo(url): | |
319 parsed = urlparse.urlsplit(url) | |
320 host = None | |
321 if parsed.scheme == 'https': | |
agable
2014/04/18 00:17:08
See if you can reorganize this whole conditional.
iannucci
2014/04/28 21:05:28
Done.
| |
322 for host in OK_REPOS: | |
agable
2014/04/18 00:17:08
Turn this into
if not any(...):
return False
iannucci
2014/04/28 21:05:28
Not needed
| |
323 if (OK_HOST_FMT % host) == parsed.netloc: | |
324 break | |
agable
2014/04/18 00:17:08
relying on the fact that |host| will remain set to
iannucci
2014/04/28 21:05:28
Done.
| |
325 else: | |
326 return False | |
327 elif parsed.scheme == 'sso': | |
328 if parsed.netloc not in OK_REPOS: | |
329 return False | |
330 host = parsed.netloc | |
331 else: | |
332 return False | |
333 | |
334 path = parsed.path.strip('/') | |
335 if path.endswith('.git'): | |
336 path = path[:-4] | |
337 | |
338 return path in OK_REPOS[host] | |
339 | |
340 | |
341 class NumberedBranch(collections.namedtuple('NumberedBranch', 'num')): | |
342 # pylint: disable=W0232 | |
343 @property | |
344 def remote_full_ref(self): | |
345 return 'refs/branch-heads/%d' % self.num | |
346 | |
347 @property | |
348 def local_full_ref(self): | |
349 return 'refs/origin/branch-heads/%d' % self.num | |
350 | |
351 | |
352 ReleaseBranch = collections.namedtuple('ReleaseBranch', 'num') | |
353 | |
354 | |
355 Channel = collections.namedtuple('Channel', 'os channel') | |
356 | |
357 | |
358 def resolve_ref(ref): | |
359 def data(): | |
360 import json | |
361 silly = json.load(GET('http://omahaproxy.appspot.com/all.json')) | |
362 ret = {} | |
363 for os_blob in silly: | |
364 v = ret[os_blob['os']] = {} | |
365 for vers in os_blob['versions']: | |
366 full_version = map(int, vers['version'].split('.')) | |
367 b = full_version[2] | |
368 | |
369 # could be empty string or None | |
370 tb = vers.get('true_branch') or '' | |
371 tb = tb.split('_', 1)[0] | |
372 if tb and tb.isdigit(): | |
373 b = int(tb) | |
374 v[vers['channel']] = {'major': full_version[0], | |
375 'branch': NumberedBranch(b)} | |
376 return ret | |
377 | |
378 if isinstance(ref, NumberedBranch): | |
379 return ref | |
380 elif isinstance(ref, Channel): | |
agable
2014/04/18 00:17:08
This is all getting removed.
iannucci
2014/04/28 21:05:28
Done.
| |
381 d = data() | |
382 if ref.os not in d: | |
383 die('Unrecognized Channel: %r', ref) | |
384 | |
385 r = d[ref.os].get(ref.channel, {}).get('branch') | |
386 if r: | |
387 return r | |
388 | |
389 die("No channel %s for os %s found." % (ref.channel, ref.os)) | |
390 elif isinstance(ref, ReleaseBranch): | |
391 d = data() | |
392 b = set() | |
393 for channel_map in d.itervalues(): | |
394 for channel, vers in channel_map.iteritems(): | |
395 if channel == 'canary': | |
396 continue # not a trustworthy source of information. | |
397 if vers['major'] == ref.num: | |
398 new_branch = vers['branch'] | |
399 b.add(new_branch) | |
400 | |
401 if len(b) > 1: | |
402 die('Ambiguous release branch m%s: %r', ref.num, b) | |
403 if not b: | |
404 die("Couldn't find branch for m%s", ref.num) | |
405 | |
406 return b.pop() | |
407 else: | |
408 die('Unrecognized ref type: %r', ref) | |
409 | |
410 | |
411 def parse_opts(): | |
412 epilog = textwrap.dedent("""\ | |
413 REF in the above may take the form of: | |
414 DDDD - a numbered branch (i.e. refs/branch-heads/DDDD) | |
415 mDD - a release milestone (will consult omahaproxy, aborts if ambiguous) | |
agable
2014/04/18 00:17:08
remove?
iannucci
2014/04/28 21:05:28
Done.
| |
416 os,channel - consults omahaproxy for the current branch-head | |
agable
2014/04/18 00:17:08
remove
iannucci
2014/04/28 21:05:28
Done.
| |
417 os = android, ios, cros, cf, linux, mac, win, ... | |
418 channel = canary, dev, beta, stable, ... | |
419 """) | |
420 | |
421 commit_re = re.compile('^[0-9a-fA-F]{40}$') | |
422 def commit_type(s): | |
423 if not commit_re.match(s): | |
424 raise argparse.ArgumentTypeError("%r is not a valid commit hash" % s) | |
425 return s | |
426 | |
427 def ref_type(s): | |
428 if not s: | |
429 raise argparse.ArgumentTypeError("Empty ref: %r" % s) | |
430 if ',' in s: | |
431 bits = s.split(',') | |
432 if len(bits) != 2: | |
433 raise argparse.ArgumentTypeError("Invalid Channel ref: %r" % s) | |
434 return Channel(*bits) | |
435 elif s[0] in 'mM': | |
436 if not s[1:].isdigit(): | |
437 raise argparse.ArgumentTypeError("Invalid ReleaseBranch ref: %r" % s) | |
438 return ReleaseBranch(int(s[1:])) | |
439 elif s.isdigit(): | |
440 return NumberedBranch(int(s)) | |
441 raise argparse.ArgumentTypeError("Invalid ref: %r" % s) | |
442 | |
443 parser = argparse.ArgumentParser( | |
444 description=__doc__, epilog=epilog, | |
445 formatter_class=argparse.RawDescriptionHelpFormatter | |
446 ) | |
447 | |
448 parser.add_argument('commit', nargs=1, metavar='HASH', | |
449 type=commit_type, help='commit hash to revert/merge') | |
450 | |
451 parser.add_argument('--prep_only', action='store_true', default=False, | |
452 help=( | |
453 'Prep and upload the CL (without sending mail) but ' | |
454 'don\'t push.')) | |
455 | |
456 parser.add_argument('--bug', metavar='NUM', action='append', dest='bugs', | |
457 help='optional bug number(s)') | |
458 | |
459 grp = parser.add_mutually_exclusive_group(required=True) | |
460 grp.add_argument('--merge_to', metavar='REF', type=ref_type, | |
461 help='branch to merge to') | |
462 grp.add_argument('--revert_from', metavar='REF', type=ref_type, | |
463 help='branch ref to revert from') | |
464 opts = parser.parse_args() | |
465 | |
466 # TODO(iannucci): Support multiple commits | |
467 opts.commits = opts.commit | |
468 del opts.commit | |
469 | |
470 if opts.merge_to: | |
471 opts.action = 'merge' | |
472 opts.ref = resolve_ref(opts.merge_to) | |
473 elif opts.revert_from: | |
474 opts.action = 'revert' | |
475 opts.ref = resolve_ref(opts.revert_from) | |
476 else: | |
477 parser.error("?confusion? must specify either revert_from or merge_to") | |
agable
2014/04/18 00:17:08
s/\?confusion\?/You/
iannucci
2014/04/28 21:05:28
yeah, but argparse should have caught this already
| |
478 | |
479 del opts.merge_to | |
480 del opts.revert_from | |
481 | |
482 return opts | |
483 | |
484 | |
485 def main(): | |
486 opts = parse_opts() | |
487 | |
488 announce('Preparing working directory') | |
489 | |
490 correct_url = ensure_working_directory(opts.commits, opts.ref) | |
491 summarize_job(correct_url, opts.commits, opts.ref, opts.action) | |
492 confirm() | |
493 | |
494 announce('Checking out branches to %s changes' % opts.action) | |
495 | |
496 git.run('fetch', 'origin', | |
497 '%s:%s' % (opts.ref.remote_full_ref, opts.ref.local_full_ref)) | |
498 git.check('update-ref', '-d', 'refs/heads/__drover_base') | |
499 git.run('checkout', '-b', '__drover_base', opts.ref.local_full_ref, | |
500 stdout=None, stderr=None) | |
501 git.run('config', 'branch.__drover_base.remote', 'origin') | |
502 git.run('config', 'branch.__drover_base.merge', opts.ref.remote_full_ref) | |
503 git.check('branch', '-D', '__drover_change') | |
agable
2014/04/18 00:17:08
why branch -D here but update-ref -d for __drover_
iannucci
2014/04/28 21:05:28
Done.
| |
504 git.run('checkout', '-t', '__drover_base', '-b', '__drover_change', | |
505 stdout=None, stderr=None) | |
506 | |
507 announce('Performing %s' % opts.action) | |
508 | |
509 # TODO(iannucci): support --signoff ? | |
510 authors = [] | |
511 for commit in opts.commits: | |
512 success = False | |
513 if opts.action == 'merge': | |
514 success = git.check('cherry-pick', '-x', commit, verbose=True, | |
515 stdout=None, stderr=None) | |
516 else: # merge | |
agable
2014/04/18 00:17:08
revert
iannucci
2014/04/28 21:05:28
Done.
| |
517 success = git.check('revert', '--no-edit', commit, verbose=True, | |
518 stdout=None, stderr=None) | |
519 if not success: | |
520 die("""\ | |
agable
2014/04/18 00:17:08
This doesn't require a multiline string.
iannucci
2014/04/28 21:05:28
Says you.
| |
521 Aborting. Failed to %s. | |
522 """ % opts.action) | |
523 | |
524 email = git.run('show', '--format=%ae', '-s') | |
525 # git-svn email addresses take the form of: | |
526 # user@domain.com@<svn id> | |
527 authors.append('@'.join(email.split('@', 2)[:2])) | |
528 | |
529 announce('Success! Uploading to codereview.chromium.org') | |
agable
2014/04/18 00:17:08
Not true for src-internal
iannucci
2014/04/28 21:05:28
Done.
| |
530 | |
531 if opts.prep_only: | |
532 print "Prep only mode, uploading CL but not sending mail." | |
533 mail = [] | |
534 else: | |
535 mail = ['--send-mail', '--reviewers=' + ','.join(authors)] | |
536 | |
537 args = [ | |
538 '-c', 'gitcl.remotebranch=__drover_base', | |
539 '-c', 'branch.__drover_change.base-url=%s' % correct_url, | |
540 'cl', 'upload', '--bypass-hooks' | |
541 ] + mail | |
542 | |
543 # TODO(iannucci): option to not bypass hooks? | |
544 git.check(*args, stdout=None, stderr=None, stdin=None) | |
545 | |
546 if opts.prep_only: | |
547 announce('Issue created. To push to the branch, run `git cl push`') | |
548 else: | |
549 announce('About to push! This will make the commit live.') | |
550 confirm(abort=('Issue has been created, ' | |
551 'but change was not pushed to the repo.')) | |
552 | |
agable
2014/04/18 00:17:08
Missing the actual git push call.
iannucci
2014/04/28 21:05:28
^_^
| |
553 return 0 | |
554 | |
555 | |
556 if __name__ == '__main__': | |
557 sys.exit(main()) | |
OLD | NEW |