OLD | NEW |
(Empty) | |
| 1 #!/usr/bin/env python |
| 2 # Copyright 2017 The Chromium Authors. All rights reserved. |
| 3 # Use of this source code is governed by a BSD-style license that can be |
| 4 # found in the LICENSE file. |
| 5 |
| 6 import argparse |
| 7 import httplib2 |
| 8 import json |
| 9 import numpy |
| 10 from oauth2client import client |
| 11 from oauth2client import service_account # pylint: disable=no-name-in-module |
| 12 import time |
| 13 |
| 14 |
| 15 # TODO(rnephew): Integrate into catapult/experimental/benchmark_health_report. |
| 16 REQUEST_URL = 'https://chromeperf.appspot.com/api/' |
| 17 # pylint: disable=line-too-long |
| 18 HELP_SITE = 'https://developers.google.com/api-client-library/python/auth/servic
e-accounts#creatinganaccount' |
| 19 |
| 20 OAUTH_CLIENT_ID = ( |
| 21 '62121018386-h08uiaftreu4dr3c4alh3l7mogskvb7i.apps.googleusercontent.com') |
| 22 OAUTH_CLIENT_SECRET = 'vc1fZfV1cZC6mgDSHV-KSPOz' |
| 23 SCOPES = 'https://www.googleapis.com/auth/userinfo.email' |
| 24 |
| 25 |
| 26 def AuthorizeAccount(args): |
| 27 """A factory for authorized account credentials.""" |
| 28 if args.credentials: |
| 29 try: |
| 30 return AuthorizeAccountServiceAccount(args.credentials) |
| 31 except Exception: # pylint: disable=broad-except |
| 32 print ('Failure authenticating with service account. Falling back to user' |
| 33 ' authentication.') |
| 34 return AuthorizeAccountUserAccount() |
| 35 |
| 36 |
| 37 def AuthorizeAccountServiceAccount(json_key): |
| 38 """Used to create a service account connection with the performance dashboard. |
| 39 |
| 40 args: |
| 41 json_key: Path to json file that contains credentials. |
| 42 returns: |
| 43 An object that can be used to communicate with the dashboard. |
| 44 """ |
| 45 creds = service_account.ServiceAccountCredentials.from_json_keyfile_name( |
| 46 json_key, [SCOPES]) |
| 47 return creds.authorize(httplib2.Http()) |
| 48 |
| 49 |
| 50 def AuthorizeAccountUserAccount(): |
| 51 """Used to create an user account connection with the performance dashboard. |
| 52 |
| 53 returns: |
| 54 An object that can be used to communicate with the dashboard. |
| 55 """ |
| 56 flow = client.OAuth2WebServerFlow( |
| 57 OAUTH_CLIENT_ID, OAUTH_CLIENT_SECRET, [SCOPES], approval_prompt='force') |
| 58 flow.redirect_uri = client.OOB_CALLBACK_URN |
| 59 print('Go to the followinhg link in your browser:\n' |
| 60 ' %s\n' % flow.step1_get_authorize_url()) |
| 61 code = raw_input('Enter verification code: ').strip() |
| 62 try: |
| 63 creds = flow.step2_exchange(code) |
| 64 return creds.authorize(httplib2.Http()) |
| 65 except client.FlowExchangeError: |
| 66 print 'User authentication has failed.' |
| 67 raise |
| 68 |
| 69 |
| 70 |
| 71 def MakeApiRequest(credentials, request, retry=True): |
| 72 """Used to communicate with perf dashboard. |
| 73 |
| 74 args: |
| 75 credentials: Set of credentials generated by |
| 76 request: String that contains POST request to dashboard. |
| 77 returns: |
| 78 Contents of the response from the dashboard. |
| 79 """ |
| 80 print 'Making API request: %s' % request |
| 81 resp, content = credentials.request( |
| 82 REQUEST_URL + request, |
| 83 method="POST", |
| 84 headers={'Content-length': 0}) |
| 85 if resp['status'] != '200': |
| 86 print ('Error detected while making api request. Returned: %s' |
| 87 % (resp['status'])) |
| 88 if retry: |
| 89 print 'Retrying command after 3 seconds...' |
| 90 time.sleep(3) |
| 91 return MakeApiRequest(credentials, request, retry=False) |
| 92 return (resp, content) |
| 93 |
| 94 |
| 95 def _ProcessTimeseriesData(ts): |
| 96 """Does noise processing of timeseries data. |
| 97 args: |
| 98 ts: Timeseries from dashboard. |
| 99 returns: |
| 100 Dict of noise metrics. |
| 101 """ |
| 102 ts_values = [t[1] for t in ts] |
| 103 mean = numpy.mean(ts_values) |
| 104 std = numpy.std(ts_values) |
| 105 return { |
| 106 'count': len(ts_values), |
| 107 'sum': sum(ts_values), |
| 108 'mean': mean, |
| 109 'variance': numpy.var(ts_values), |
| 110 'stdev': std, |
| 111 'cv': std / mean * 100 if mean else None, |
| 112 } |
| 113 |
| 114 |
| 115 def GetBugData(creds, bug, cache): |
| 116 """Returns data for given bug.""" |
| 117 try: |
| 118 if not bug: |
| 119 return {'bug': {'state': None, 'status': None, 'summary': None}} |
| 120 if int(bug) == -1: |
| 121 return {'bug': {'state': None, 'status': None, 'summary': 'Invalid'}} |
| 122 if int(bug) == -2: |
| 123 return {'bug': {'state': None, 'status': None, 'summary': 'Ignored'}} |
| 124 r = 'bugs/%s' % bug |
| 125 _, output = MakeApiRequest(creds, r) |
| 126 |
| 127 if cache.get(bug): |
| 128 print 'Returning cached data for bug %s' % bug |
| 129 return cache[bug] |
| 130 data = json.loads(output) |
| 131 # Only care about date of comments, not connent. |
| 132 data['bug']['comments'] = [a['published'] for a in data['bug']['comments']] |
| 133 cache[bug] = data |
| 134 return data |
| 135 except Exception: # pylint: disable=broad-except |
| 136 print 'Problem when collecting bug data for bug %s: %s' % (bug, output) |
| 137 raise |
| 138 |
| 139 |
| 140 def GetAlertData(credentials, benchmark, days): |
| 141 """Returns alerts for given benchmark.""" |
| 142 r = 'alerts/history/%s/?benchmark=%s' %(str(days), benchmark) |
| 143 _, output = MakeApiRequest(credentials, r) |
| 144 try: |
| 145 data = json.loads(output)['anomalies'] |
| 146 return data |
| 147 except: |
| 148 print 'Problem getting alerts for benchmark %s: %s' % (benchmark, output) |
| 149 raise |
| 150 |
| 151 |
| 152 def GetNoiseData(credentials, metric, days): |
| 153 """Returns noise data for given metric.""" |
| 154 r = 'timeseries/%s?num_days=%s' % (metric, str(days)) |
| 155 if not metric: |
| 156 return None |
| 157 _, output = MakeApiRequest(credentials, r) |
| 158 try: |
| 159 data = json.loads(output) |
| 160 if not data: |
| 161 print 'No data found for metric %s in the last %s days.' % (metric, days) |
| 162 return None |
| 163 ts = data['timeseries'][1:] # First entry is book keeping. |
| 164 return _ProcessTimeseriesData(ts) |
| 165 except Exception: |
| 166 print 'Problem getting timeseries for %s: %s' % (metric, output) |
| 167 raise |
| 168 |
| 169 |
| 170 def Main(): |
| 171 parser = argparse.ArgumentParser() |
| 172 parser.add_argument('-b', '--benchmark', required=True, |
| 173 help='Benchmark to pull data for.') |
| 174 parser.add_argument('-d', '--days', required=False, default=30, |
| 175 help='Number of days to collect data for. Default 30') |
| 176 parser.add_argument('--credentials', |
| 177 help=('Path to json credentials file. See %s for ' |
| 178 'information about generating this.' % HELP_SITE)) |
| 179 parser.add_argument('--output-path', default='alert_analyzer.json', |
| 180 help='Path to save file to. Default: alert_analyzer.json') |
| 181 args = parser.parse_args() |
| 182 |
| 183 credentials = AuthorizeAccount(args) |
| 184 data = [] |
| 185 |
| 186 alerts = GetAlertData(credentials, args.benchmark, args.days) |
| 187 bug_cache = {} |
| 188 print '%s alerts found! Collecting data related to them...' % len(alerts) |
| 189 for alert in alerts: |
| 190 entry = {'alert': alert} |
| 191 bug_id = alert.get('bug_id') |
| 192 metric = '%s/%s/%s/%s' % (alert['master'], alert['bot'], alert['testsuite'], |
| 193 alert['test']) |
| 194 |
| 195 entry['noise'] = { |
| 196 'reg': GetNoiseData(credentials, metric, args.days), |
| 197 'ref': GetNoiseData(credentials, alert['ref_test'], args.days) |
| 198 } |
| 199 entry['bug'] = GetBugData(credentials, bug_id, bug_cache)['bug'] |
| 200 |
| 201 data.append(entry) |
| 202 |
| 203 # Save at end. |
| 204 with open(args.output_path, 'w') as fp: |
| 205 print 'Saving data to %s.' % args.output_path |
| 206 json.dump(data, fp, sort_keys=True, indent=2) |
| 207 |
| 208 |
| 209 if __name__ == '__main__': |
| 210 Main() |
OLD | NEW |