OLD | NEW |
1 # Copyright 2015 The Chromium Authors. All rights reserved. | 1 # Copyright 2015 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 | 5 |
6 """Make authenticated calls to gitiles.""" | 6 """Make authenticated calls to gitiles.""" |
7 | 7 |
8 | 8 |
9 import base64 | 9 import base64 |
10 import logging | 10 import logging |
| 11 import httplib |
11 import httplib2 | 12 import httplib2 |
12 import json | 13 import json |
13 import urlparse | 14 import urlparse |
14 import netrc | 15 import netrc |
15 import time | 16 import time |
16 | 17 |
17 | 18 |
18 LOGGER = logging.getLogger(__name__) | 19 LOGGER = logging.getLogger(__name__) |
19 | 20 |
20 | 21 |
| 22 # The default number of retries when a Gitiles error is encountered. |
| 23 DEFAULT_RETRIES = 10 |
| 24 |
| 25 |
21 class GitilesError(Exception): | 26 class GitilesError(Exception): |
22 pass | 27 pass |
23 | 28 |
24 | 29 |
25 def call_gitiles(url, response_format, netrc_path=None, max_attempts=10): | 30 def call_gitiles(url, response_format, netrc_path=None, max_attempts=None): |
26 """Invokes Gitiles API and parses the JSON result. | 31 """Invokes Gitiles API and parses the JSON result. |
27 | 32 |
28 Given a gitiles URL, makes a ?format=json or ?format=text call and interprets | 33 Given a gitiles URL, makes a ?format=json or ?format=text call and interprets |
29 the result. The 'json' format is parsed and returned as a python object, while | 34 the result. The 'json' format is parsed and returned as a python object, while |
30 'text' calls are automatically base64-decoded. | 35 'text' calls are automatically base64-decoded. |
31 | 36 |
32 url is the gitiles URL to call. It must not contain a query parameter. | 37 url is the gitiles URL to call. It must not contain a query parameter. |
33 | 38 |
34 response_format is either 'json' or 'text'. This controls whether JSON or | 39 response_format is either 'json' or 'text'. This controls whether JSON or |
35 textual output is desired. This usually depends on the query. | 40 textual output is desired. This usually depends on the query. |
36 | 41 |
37 netrc_path is the path to the netrc credentials used for authentication. If | 42 netrc_path is the path to the netrc credentials used for authentication. If |
38 not specified, no authentication is used. | 43 not specified, no authentication is used. |
39 | 44 |
40 max_attempts is the number of attempts to call gitiles before giving up on | 45 max_attempts is the number of attempts to call gitiles before giving up on |
41 error. | 46 error. If None/zero, DEFAULT_RETRIES will be used. |
42 """ | 47 """ |
43 assert response_format in ('json', 'text'), ( | 48 assert response_format in ('json', 'text'), ( |
44 'response must be either json or text') | 49 'response must be either json or text') |
45 assert '?' not in url, 'url must not have a query parameter (?)' | 50 assert '?' not in url, 'url must not have a query parameter (?)' |
46 | 51 |
| 52 max_attempts = max_attempts or DEFAULT_RETRIES |
47 http = httplib2.Http() | 53 http = httplib2.Http() |
48 headers = {} | 54 headers = {} |
49 if netrc_path: | 55 if netrc_path: |
50 token = get_oauth_token_from_netrc(url, netrc_path) | 56 token = get_oauth_token_from_netrc(url, netrc_path) |
51 headers['Authorization'] = 'OAuth %s' % token | 57 headers['Authorization'] = 'OAuth %s' % token |
52 attempt = 0 | 58 attempt = 0 |
53 while attempt < max_attempts: | 59 while attempt < max_attempts: |
54 time.sleep(attempt) | 60 time.sleep(attempt) |
55 attempt += 1 | 61 attempt += 1 |
56 LOGGER.debug('GET %s', url) | 62 LOGGER.debug('GET %s', url) |
57 response, content = http.request( | 63 response, content = http.request( |
58 '%s?format=%s' % (url, response_format), | 64 '%s?format=%s' % (url, response_format), |
59 'GET', | 65 'GET', |
60 headers=headers | 66 headers=headers |
61 ) | 67 ) |
62 if response['status'] != '200': | 68 if response.status != httplib.OK: |
63 LOGGER.warning('GET %s failed with HTTP code %s', url, response['status']) | 69 LOGGER.warning('GET %s failed with HTTP code %d', url, response.status) |
| 70 if response.status < httplib.INTERNAL_SERVER_ERROR: |
| 71 break |
64 if attempt != max_attempts: | 72 if attempt != max_attempts: |
65 LOGGER.warning('Retrying...') | 73 LOGGER.warning('Retrying...') |
66 continue # pragma: no cover (actually reached, see https://goo.gl/QA8B2U) | 74 continue # pragma: no cover (actually reached, see https://goo.gl/QA8B2U) |
67 if response_format == 'json': | 75 if response_format == 'json': |
68 if not content.startswith(')]}\'\n'): | 76 if not content.startswith(')]}\'\n'): |
69 raise GitilesError('Unexpected gitiles response: %s' % content) | 77 raise GitilesError('Unexpected gitiles response: %s' % content) |
70 prefix_removed = content.split('\n', 1)[1] | 78 prefix_removed = content.split('\n', 1)[1] |
71 return json.loads(prefix_removed) | 79 return json.loads(prefix_removed) |
72 elif response_format == 'text': | 80 elif response_format == 'text': |
73 return base64.b64decode(content) | 81 return base64.b64decode(content) |
74 else: | 82 else: |
75 raise AssertionError() | 83 raise AssertionError() |
76 raise GitilesError('Failed to fetch %s: %s' % (url, content)) | 84 raise GitilesError('Failed to fetch %s: %s' % (url, content)) |
77 | 85 |
78 | 86 |
79 def get_oauth_token_from_netrc(url, netrc_path): | 87 def get_oauth_token_from_netrc(url, netrc_path): |
80 """Looks up OAuth token for |url| in .netrc file at |netrc_path|.""" | 88 """Looks up OAuth token for |url| in .netrc file at |netrc_path|.""" |
81 parsed = urlparse.urlparse(url) | 89 parsed = urlparse.urlparse(url) |
82 auth = netrc.netrc(netrc_path).authenticators(parsed.hostname) | 90 auth = netrc.netrc(netrc_path).authenticators(parsed.hostname) |
83 if not auth: | 91 if not auth: |
84 raise GitilesError( | 92 raise GitilesError( |
85 'netrc file %s is missing an entry for %s' % ( | 93 'netrc file %s is missing an entry for %s' % ( |
86 netrc_path, parsed.hostname)) | 94 netrc_path, parsed.hostname)) |
87 return auth[2] | 95 return auth[2] |
| 96 |
| 97 |
| 98 class Repository(object): |
| 99 def __init__(self, base_url, netrc_path=None, max_attempts=None): |
| 100 self._base_url = self._trim_slashes(base_url) |
| 101 self._netrc_path = netrc_path |
| 102 self._max_attempts = max_attempts |
| 103 |
| 104 @staticmethod |
| 105 def _trim_slashes(v): |
| 106 return v.strip('/') |
| 107 |
| 108 def __call__(self, ref='master', subpath=None): |
| 109 url = [self._base_url, '+', ref] |
| 110 if subpath: |
| 111 url.append(self._trim_slashes(subpath)) |
| 112 url = '/'.join(url) |
| 113 return call_gitiles( |
| 114 url, |
| 115 'json', |
| 116 netrc_path=self._netrc_path, |
| 117 max_attempts=self._max_attempts) |
| 118 |
| 119 def ref_info(self, ref): |
| 120 return self(ref) |
OLD | NEW |