Chromium Code Reviews Chromium Code Reviews
Issue 9706088:
Some changes to performance test script. (Closed)
Base URL: http://dart.googlecode.com/svn/branches/bleeding_edge/dart/| OLD | NEW |
|---|---|
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 | 2 |
| 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 4 # for details. All rights reserved. Use of this source code is governed by a | 4 # for details. All rights reserved. Use of this source code is governed by a |
| 5 # BSD-style license that can be found in the LICENSE file. | 5 # BSD-style license that can be found in the LICENSE file. |
| 6 | 6 |
| 7 import datetime | 7 import datetime |
| 8 import getpass | |
| 8 import math | 9 import math |
| 9 from matplotlib.font_manager import FontProperties | 10 from matplotlib.font_manager import FontProperties |
| 10 import matplotlib.pyplot as plt | 11 import matplotlib.pyplot as plt |
| 11 import optparse | 12 import optparse |
| 12 import os | 13 import os |
| 13 from os.path import dirname, abspath | 14 from os.path import dirname, abspath |
| 14 import platform | 15 import platform |
| 15 import shutil | 16 import shutil |
| 16 import subprocess | 17 import subprocess |
| 17 import time | 18 import time |
| (...skipping 148 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 166 def get_os_directory(): | 167 def get_os_directory(): |
| 167 """Specifies the name of the directory for the testing build of dart, which | 168 """Specifies the name of the directory for the testing build of dart, which |
| 168 has yet a different naming convention from utils.getBuildRoot(...).""" | 169 has yet a different naming convention from utils.getBuildRoot(...).""" |
| 169 if platform.system() == 'Windows': | 170 if platform.system() == 'Windows': |
| 170 return 'windows' | 171 return 'windows' |
| 171 elif platform.system() == 'Darwin': | 172 elif platform.system() == 'Darwin': |
| 172 return 'macos' | 173 return 'macos' |
| 173 else: | 174 else: |
| 174 return 'linux' | 175 return 'linux' |
| 175 | 176 |
| 176 def upload_to_app_engine(): | 177 def upload_to_app_engine(username, password): |
| 177 """Upload our results to our appengine server.""" | 178 """Upload our results to our appengine server. |
| 179 Arguments: | |
| 180 username: App Engine username for uploading data to dartperf.googleplex.com | |
| 181 password: App Engine password | |
| 182 """ | |
| 178 # TODO(efortuna): This is the most basic way to get the data up | 183 # TODO(efortuna): This is the most basic way to get the data up |
| 179 # for others to view. Revisit this once we're serving nicer graphs (Google | 184 # for others to view. Revisit this once we're serving nicer graphs (Google |
| 180 # Chart Tools) and from multiple perfbots and once we're in a position to | 185 # Chart Tools) and from multiple perfbots and once we're in a position to |
| 181 # organize the data in a useful manner(!!). | 186 # organize the data in a useful manner(!!). |
| 182 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', | 187 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', |
| 183 'perf_testing')) | 188 'perf_testing')) |
| 189 for data in [BROWSER_PERF, TIME_SIZE, CL_PERF]: | |
| 190 path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS()) | |
| 191 shutil.rmtree(path, ignore_errors=True) | |
| 192 os.makedirs(path) | |
| 193 files = [] | |
| 194 # Copy the 1000 most recent trace files to be uploaded. | |
| 195 for f in os.listdir(data): | |
| 196 files += [(os.path.getmtime(os.path.join(data, f)), f)] | |
| 197 files.sort() | |
| 198 for i in xrange(1000): | |
| 199 if len(files) > 0: | |
| 200 f = files.pop() | |
|
vsm
2012/03/16 00:32:18
You can replace these 3 lines with:
for f in files
Emily Fortuna
2012/03/16 21:27:57
Done.
| |
| 201 shutil.copyfile(os.path.join(data, f[1]), | |
| 202 os.path.join(path, f[1]+'.txt')) | |
| 203 # Generate directory listing. | |
| 204 for data in [BROWSER_PERF, TIME_SIZE, CL_PERF]: | |
| 205 path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS()) | |
| 206 out = open(os.path.join('appengine', 'static', | |
| 207 '%s-%s.html' % (data, utils.GuessOS())), 'w') | |
| 208 out.write('<html>\n <body>\n <ul>\n') | |
| 209 for f in os.listdir(path): | |
| 210 if not f.startswith('.'): | |
| 211 out.write(' <li><a href=data' + \ | |
| 212 '''/%(data)s/%(os)s/%(file)s>%(file)s</a></li>\n''' % \ | |
| 213 {'data': data, 'os': utils.GuessOS(), 'file': f}) | |
| 214 out.write(' </ul>\n </body>\n</html>') | |
| 215 out.close() | |
| 216 | |
| 184 shutil.rmtree(os.path.join('appengine', 'static', 'graphs'), | 217 shutil.rmtree(os.path.join('appengine', 'static', 'graphs'), |
| 185 ignore_errors=True) | 218 ignore_errors=True) |
| 186 shutil.copytree('graphs', os.path.join('appengine', 'static', 'graphs')) | 219 shutil.copytree('graphs', os.path.join('appengine', 'static', 'graphs')) |
| 187 shutil.copyfile('index.html', os.path.join('appengine', 'static', | 220 shutil.copyfile('index.html', os.path.join('appengine', 'static', |
| 188 'index.html')) | 221 'index.html')) |
| 189 run_cmd(['../../../third_party/appengine-python/1.5.4/appcfg.py', 'update', | 222 shutil.copyfile('data.html', os.path.join('appengine', 'static', |
| 190 'appengine/']) | 223 'data.html')) |
| 224 p = subprocess.Popen([os.path.join('..', '..', '..', 'third_party', | |
| 225 'appengine-python', 'appcfg.py'), 'update', | |
| 226 'appengine/'], shell=HAS_SHELL, stdin=subprocess.PIPE) | |
| 227 p.stdin.write(username + '\n') | |
| 228 p.stdin.write(password + '\n') | |
| 229 p.communicate() | |
| 230 | |
| 191 | 231 |
| 192 class TestRunner(object): | 232 class TestRunner(object): |
| 193 """The base clas to provide shared code for different tests we will run and | 233 """The base class to provide shared code for different tests we will run and |
| 194 graph.""" | 234 graph.""" |
| 195 | 235 |
| 196 def __init__(self, result_folder_name, platform_list, v8_and_or_frog_list, | 236 def __init__(self, result_folder_name, platform_list, v8_and_or_frog_list, |
| 197 values_list): | 237 values_list): |
| 198 """Args: | 238 """Args: |
| 199 result_folder_name the name of the folder where a tracefile of | 239 result_folder_name the name of the folder where a tracefile of |
| 200 performance results will be stored. | 240 performance results will be stored. |
| 201 platform_list a list containing the platform(s) that our data has been | 241 platform_list a list containing the platform(s) that our data has been |
| 202 run on. (command line, firefox, chrome, etc) | 242 run on. (command line, firefox, chrome, etc) |
| 203 v8_and_or_frog_list a list specifying whether we hold data about Frog | 243 v8_and_or_frog_list a list specifying whether we hold data about Frog |
| (...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 283 for line in output.split('\n'): | 323 for line in output.split('\n'): |
| 284 if 'Revision' in line: | 324 if 'Revision' in line: |
| 285 run_cmd(['echo', line.strip()], outfile) | 325 run_cmd(['echo', line.strip()], outfile) |
| 286 return True | 326 return True |
| 287 return False | 327 return False |
| 288 | 328 |
| 289 if not search_for_revision(['svn', 'info']): | 329 if not search_for_revision(['svn', 'info']): |
| 290 if not search_for_revision(['git', 'svn', 'info']): | 330 if not search_for_revision(['git', 'svn', 'info']): |
| 291 run_cmd(['echo', 'Revision: unknown'], outfile) | 331 run_cmd(['echo', 'Revision: unknown'], outfile) |
| 292 | 332 |
| 293 def write_html(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, | |
| 294 cleanFile=False): | |
| 295 """Adds an html table to the webpage to display the data values. This method | |
| 296 will be removed when we have a nicer way to display data values.""" | |
| 297 #TODO(efortuna): fix this. | |
| 298 return | |
| 299 #TODO(efortuna): Take this method out when have finalized where the data is | |
| 300 # going to be displayed. | |
| 301 f = '' | |
| 302 out = '' | |
| 303 if cleanFile: | |
| 304 f = open('template.html') | |
| 305 else: | |
| 306 shutil.copy('index.html', 'temp.html') | |
| 307 f = open('temp.html') | |
| 308 out = open('index.html', 'w') | |
| 309 inTable = False | |
| 310 for line in f.readlines(): | |
| 311 if not inTable: | |
| 312 out.write(line) | |
| 313 if delimiter in line: | |
| 314 inTable = not inTable | |
| 315 if inTable: | |
| 316 out.write('<table border="1"> <tr> <td> svn revision </td>') | |
| 317 for revision in rev_nums: | |
| 318 out.write('<td>%d</td>' % revision) | |
| 319 out.write('</tr>\n<tr><td> %s</td>' % label_1) | |
| 320 for perf in dict_1: | |
| 321 out.write('<td>%f</td>' % perf) | |
| 322 out.write('</tr>\n<tr><td> %s</td>' % label_2) | |
| 323 for perf in dict_2: | |
| 324 out.write('<td>%f</td>' % perf) | |
| 325 out.write('</tr> </table>') | |
| 326 | |
| 327 def calculate_geometric_mean(self, platform, frog_or_v8, svn_revision): | 333 def calculate_geometric_mean(self, platform, frog_or_v8, svn_revision): |
| 328 """Calculate the aggregate geometric mean for V8 and frog benchmark sets, | 334 """Calculate the aggregate geometric mean for V8 and frog benchmark sets, |
| 329 given two benchmark dictionaries.""" | 335 given two benchmark dictionaries.""" |
| 330 geo_mean = 0 | 336 geo_mean = 0 |
| 331 for benchmark in get_benchmarks(): | 337 for benchmark in get_benchmarks(): |
| 332 geo_mean += math.log(self.values_dict[platform][frog_or_v8][benchmark][ | 338 geo_mean += math.log(self.values_dict[platform][frog_or_v8][benchmark][ |
| 333 len(self.values_dict[platform][frog_or_v8][benchmark]) - 1]) | 339 len(self.values_dict[platform][frog_or_v8][benchmark]) - 1]) |
| 334 | 340 |
| 335 mean = V8_MEAN | 341 mean = V8_MEAN |
| 336 if frog_or_v8 == FROG: | 342 if frog_or_v8 == FROG: |
| 337 mean = FROG_MEAN | 343 mean = FROG_MEAN |
| 338 self.values_dict[platform][frog_or_v8][mean] += \ | 344 self.values_dict[platform][frog_or_v8][mean] += \ |
| 339 [math.pow(math.e, geo_mean / len(get_benchmarks()))] | 345 [math.pow(math.e, geo_mean / len(get_benchmarks()))] |
| 340 self.revision_dict[platform][frog_or_v8][mean] += [svn_revision] | 346 self.revision_dict[platform][frog_or_v8][mean] += [svn_revision] |
| 341 | 347 |
| 342 def run(self): | 348 def run(self): |
| 343 """Run the benchmarks/tests from the command line and plot the | 349 """Run the benchmarks/tests from the command line and plot the |
| 344 results.""" | 350 results.""" |
| 345 plt.cla() # cla = clear current axes | 351 plt.cla() # cla = clear current axes |
| 346 os.chdir(DART_INSTALL_LOCATION) | 352 os.chdir(DART_INSTALL_LOCATION) |
| 347 ensure_output_directory(self.result_folder_name) | 353 ensure_output_directory(self.result_folder_name) |
| 348 ensure_output_directory(GRAPH_OUT_DIR) | 354 ensure_output_directory(GRAPH_OUT_DIR) |
| 349 self.run_tests() | 355 self.run_tests() |
| 356 | |
| 350 os.chdir(os.path.join('tools', 'testing', 'perf_testing')) | 357 os.chdir(os.path.join('tools', 'testing', 'perf_testing')) |
| 351 | 358 |
| 352 # TODO(efortuna): You will want to make this only use a subset of the files | 359 # TODO(efortuna): You will want to make this only use a subset of the files |
| 353 # eventually. | 360 # eventually. |
| 354 files = os.listdir(self.result_folder_name) | 361 files = os.listdir(self.result_folder_name) |
| 355 | 362 |
| 356 for afile in files: | 363 for afile in files: |
| 357 if not afile.startswith('.'): | 364 if not afile.startswith('.'): |
| 358 self.process_file(afile) | 365 self.process_file(afile) |
| 359 | 366 |
| 360 self.plot_results('%s.png' % self.result_folder_name) | 367 self.plot_results('%s.png' % self.result_folder_name) |
| 361 | 368 |
| 362 class PerformanceTest(TestRunner): | 369 class PerformanceTest(TestRunner): |
| 363 """Super class for all performance testing.""" | 370 """Super class for all performance testing.""" |
| 364 def __init__(self, result_folder_name, platform_list, platform_type): | 371 def __init__(self, result_folder_name, platform_list, platform_type): |
| 365 super(PerformanceTest, self).__init__(result_folder_name, | 372 super(PerformanceTest, self).__init__(result_folder_name, |
| 366 platform_list, get_versions(), get_benchmarks()) | 373 platform_list, get_versions(), get_benchmarks()) |
| 367 self.platform_list = platform_list | 374 self.platform_list = platform_list |
| 368 self.platform_type = platform_type | 375 self.platform_type = platform_type |
| 369 | 376 |
| 370 def plot_all_perf(self, png_filename): | 377 def plot_all_perf(self, png_filename): |
| 371 """Create a plot that shows the performance changes of individual benchmarks | 378 """Create a plot that shows the performance changes of individual benchmarks |
| 372 run by V8 and generated by frog, over svn history.""" | 379 run by V8 and generated by frog, over svn history.""" |
| 373 for benchmark in get_benchmarks(): | 380 for benchmark in get_benchmarks(): |
| 374 self.style_and_save_perf_plot( | 381 self.style_and_save_perf_plot( |
| 375 'Performance of %s over time on the %s' % (benchmark, | 382 'Performance of %s over time on the %s on %s' % (benchmark, |
| 376 self.platform_type), 'Speed (bigger = better)', 16, 14, 'lower left', | 383 self.platform_type, utils.GuessOS()), 'Speed (bigger = better)', 16, |
| 377 benchmark + png_filename, self.platform_list, get_versions(), | 384 14, 'lower left', benchmark + png_filename, self.platform_list, |
| 378 [benchmark]) | 385 get_versions(), [benchmark]) |
| 379 | 386 |
| 380 def plot_avg_perf(self, png_filename): | 387 def plot_avg_perf(self, png_filename): |
| 381 """Generate a plot that shows the performance changes of the geomentric mean | 388 """Generate a plot that shows the performance changes of the geomentric mean |
| 382 of V8 and frog benchmark performance over svn history.""" | 389 of V8 and frog benchmark performance over svn history.""" |
| 383 (title, y_axis, size_x, size_y, loc, filename) = \ | 390 (title, y_axis, size_x, size_y, loc, filename) = \ |
| 384 ('Geometric Mean of benchmark %s performance' % self.platform_type, | 391 ('Geometric Mean of benchmark %s performance' % self.platform_type, |
| 385 'Speed (bigger = better)', 16, 5, 'center', 'avg'+png_filename) | 392 'Speed (bigger = better)', 16, 5, 'lower left', 'avg'+png_filename) |
| 386 clear_axis = True | 393 clear_axis = True |
| 387 for platform in self.platform_list: | 394 for platform in self.platform_list: |
| 388 self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, | 395 self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, |
| 389 filename, [platform], [V8], [V8_MEAN], clear_axis) | 396 filename, [platform], [V8], [V8_MEAN], clear_axis) |
| 390 clear_axis = False | 397 clear_axis = False |
| 391 self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, | 398 self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, |
| 392 filename, [platform], [FROG], [FROG_MEAN], clear_axis) | 399 filename, [platform], [FROG], [FROG_MEAN], clear_axis) |
| 393 | 400 |
| 394 def plot_results(self, png_filename): | 401 def plot_results(self, png_filename): |
| 395 self.plot_all_perf(png_filename) | 402 self.plot_all_perf(png_filename) |
| 396 self.plot_avg_perf('2' + png_filename) | 403 self.plot_avg_perf('2' + png_filename) |
| 397 | 404 |
| 398 | 405 |
| 399 class CommandLinePerformanceTest(PerformanceTest): | 406 class CommandLinePerformanceTest(PerformanceTest): |
| 400 """Run performance tests from the command line.""" | 407 """Run performance tests from the command line.""" |
| 401 | 408 |
| 402 def __init__(self, result_folder_name): | 409 def __init__(self, result_folder_name): |
| 403 super(CommandLinePerformanceTest, self).__init__(result_folder_name, | 410 super(CommandLinePerformanceTest, self).__init__(result_folder_name, |
| 404 [COMMAND_LINE], 'command line') | 411 [COMMAND_LINE], 'command line') |
| 405 | 412 |
| 406 def process_file(self, afile): | 413 def process_file(self, afile): |
| 407 """Pull all the relevant information out of a given tracefile. | 414 """Pull all the relevant information out of a given tracefile. |
| 408 | 415 |
| 409 Args: | 416 Args: |
| 410 afile: The filename string we will be processing.""" | 417 afile: The filename string we will be processing.""" |
| 418 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', | |
| 419 'perf_testing')) | |
| 411 f = open(os.path.join(self.result_folder_name, afile)) | 420 f = open(os.path.join(self.result_folder_name, afile)) |
| 412 tabulate_data = False | 421 tabulate_data = False |
| 413 revision_num = 0 | 422 revision_num = 0 |
| 414 for line in f.readlines(): | 423 for line in f.readlines(): |
| 415 if 'Revision' in line: | 424 if 'Revision' in line: |
| 416 revision_num = int(line.split()[1]) | 425 revision_num = int(line.split()[1]) |
| 417 elif 'Benchmark' in line: | 426 elif 'Benchmark' in line: |
| 418 tabulate_data = True | 427 tabulate_data = True |
| 419 elif tabulate_data: | 428 elif tabulate_data: |
| 420 tokens = line.split() | 429 tokens = line.split() |
| (...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 468 'perf-%s-%s-%s' % (self.cur_time, browser, version)) | 477 'perf-%s-%s-%s' % (self.cur_time, browser, version)) |
| 469 self.add_svn_revision_to_trace(self.trace_file) | 478 self.add_svn_revision_to_trace(self.trace_file) |
| 470 file_path = os.path.join(os.getcwd(), 'internal', 'browserBenchmarks', | 479 file_path = os.path.join(os.getcwd(), 'internal', 'browserBenchmarks', |
| 471 'benchmark_page_%s.html' % version) | 480 'benchmark_page_%s.html' % version) |
| 472 run_cmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), | 481 run_cmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), |
| 473 '--out', file_path, '--browser', browser, | 482 '--out', file_path, '--browser', browser, |
| 474 '--timeout', '600', '--perf'], self.trace_file, append=True) | 483 '--timeout', '600', '--perf'], self.trace_file, append=True) |
| 475 | 484 |
| 476 def process_file(self, afile): | 485 def process_file(self, afile): |
| 477 """Comb through the html to find the performance results.""" | 486 """Comb through the html to find the performance results.""" |
| 487 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', | |
| 488 'perf_testing')) | |
| 478 parts = afile.split('-') | 489 parts = afile.split('-') |
| 479 browser = parts[2] | 490 browser = parts[2] |
| 480 version = parts[3] | 491 version = parts[3] |
| 481 f = open(os.path.join(self.result_folder_name, afile)) | 492 f = open(os.path.join(self.result_folder_name, afile)) |
| 482 lines = f.readlines() | 493 lines = f.readlines() |
| 483 line = '' | 494 line = '' |
| 484 i = 0 | 495 i = 0 |
| 485 revision_num = 0 | 496 revision_num = 0 |
| 486 while '<div id="results">' not in line and i < len(lines): | 497 while '<div id="results">' not in line and i < len(lines): |
| 487 if 'Revision' in line: | 498 if 'Revision' in line: |
| 488 revision_num = int(line.split()[1]) | 499 revision_num = int(line.split()[1].strip('"')) |
| 489 line = lines[i] | 500 line = lines[i] |
| 490 i += 1 | 501 i += 1 |
| 491 | 502 |
| 492 if i >= len(lines) or revision_num == 0: | 503 if i >= len(lines) or revision_num == 0: |
| 493 # Then this run did not complete. Ignore this tracefile. | 504 # Then this run did not complete. Ignore this tracefile. |
| 494 return | 505 return |
| 495 | 506 |
| 496 line = lines[i] | 507 line = lines[i] |
| 497 i += 1 | 508 i += 1 |
| 498 results = [] | 509 results = [] |
| (...skipping 10 matching lines...) Expand all Loading... | |
| 509 if version == V8: | 520 if version == V8: |
| 510 bench_dict = self.values_dict[browser][V8] | 521 bench_dict = self.values_dict[browser][V8] |
| 511 else: | 522 else: |
| 512 bench_dict = self.values_dict[browser][FROG] | 523 bench_dict = self.values_dict[browser][FROG] |
| 513 bench_dict[name] += [float(score)] | 524 bench_dict[name] += [float(score)] |
| 514 self.revision_dict[browser][version][name] += [revision_num] | 525 self.revision_dict[browser][version][name] += [revision_num] |
| 515 | 526 |
| 516 f.close() | 527 f.close() |
| 517 self.calculate_geometric_mean(browser, version, revision_num) | 528 self.calculate_geometric_mean(browser, version, revision_num) |
| 518 | 529 |
| 519 def write_html(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, | |
| 520 cleanFile=False): | |
| 521 #TODO(efortuna) | |
| 522 pass | |
| 523 | |
| 524 | 530 |
| 525 class BrowserCorrectnessTest(TestRunner): | 531 class BrowserCorrectnessTest(TestRunner): |
| 526 def __init__(self, test_type, result_folder_name): | 532 def __init__(self, test_type, result_folder_name): |
| 527 super(BrowserCorrectnessTest, self).__init__(result_folder_name, | 533 super(BrowserCorrectnessTest, self).__init__(result_folder_name, |
| 528 get_browsers(), [FROG], [CORRECTNESS]) | 534 get_browsers(), [FROG], [CORRECTNESS]) |
| 529 self.test_type = test_type | 535 self.test_type = test_type |
| 530 | 536 |
| 531 def run_tests(self): | 537 def run_tests(self): |
| 532 """Run a test of the latest svn revision.""" | 538 """Run a test of the latest svn revision.""" |
| 533 system = get_os_directory() | 539 system = get_os_directory() |
| 534 suffix = '' | 540 suffix = '' |
| 535 if platform.system() == 'Windows': | 541 if platform.system() == 'Windows': |
| 536 suffix = '.exe' | 542 suffix = '.exe' |
| 537 for browser in get_browsers(): | 543 for browser in get_browsers(): |
| 538 current_file = 'correctness%s-%s' % (self.cur_time, browser) | 544 current_file = 'correctness%s-%s' % (self.cur_time, browser) |
| 539 self.trace_file = os.path.join('tools', 'testing', | 545 self.trace_file = os.path.join('tools', 'testing', |
| 540 'perf_testing', self.result_folder_name, current_file) | 546 'perf_testing', self.result_folder_name, current_file) |
| 541 self.add_svn_revision_to_trace(self.trace_file) | 547 self.add_svn_revision_to_trace(self.trace_file) |
| 542 dart_sdk = os.path.join(os.getcwd(), utils.GetBuildRoot(utils.GuessOS(), | 548 dart_sdk = os.path.join(os.getcwd(), utils.GetBuildRoot(utils.GuessOS(), |
| 543 'release', 'ia32'), 'dart-sdk') | 549 'release', 'ia32'), 'dart-sdk') |
| 544 run_cmd([os.path.join('.', 'tools', 'testing', 'bin', system, | 550 run_cmd([os.path.join('.', 'tools', 'testing', 'bin', system, |
| 545 'dart' + suffix), os.path.join('tools', 'test.dart'), | 551 'dart' + suffix), os.path.join('tools', 'test.dart'), |
| 546 '--component=webdriver', | 552 '--component=webdriver', |
| 547 '--browser=%s' % browser, '--frog=%s' % os.path.join(dart_sdk, 'bin', | 553 '--browser=%s' % browser, '--frog=%s' % os.path.join(dart_sdk, 'bin', |
| 548 'frogc'), '--froglib=%s' % os.path.join(dart_sdk, 'lib'), '--report', | 554 'frogc'), '--froglib=%s' % os.path.join(dart_sdk, 'lib'), '--report', |
| 549 '--timeout=20', '--progress=color', '--mode=release', '-j1', | 555 '--timeout=20', '--progress=color', '--mode=release', |
| 550 self.test_type], self.trace_file, append=True) | 556 self.test_type], self.trace_file, append=True) |
| 551 | 557 |
| 552 def process_file(self, afile): | 558 def process_file(self, afile): |
| 553 """Given a trace file, extract all the relevant information out of it to | 559 """Given a trace file, extract all the relevant information out of it to |
| 554 determine the number of correctly passing tests. | 560 determine the number of correctly passing tests. |
| 555 | 561 |
| 556 Arguments: | 562 Arguments: |
| 557 afile: the filename string""" | 563 afile: the filename string""" |
| 564 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', | |
| 565 'perf_testing')) | |
| 558 browser = afile.rpartition('-')[2] | 566 browser = afile.rpartition('-')[2] |
| 559 f = open(os.path.join(self.result_folder_name, afile)) | 567 f = open(os.path.join(self.result_folder_name, afile)) |
| 560 revision_num = 0 | 568 revision_num = 0 |
| 561 lines = f.readlines() | 569 lines = f.readlines() |
| 562 total_tests = 0 | 570 total_tests = 0 |
| 563 num_failed = 0 | 571 num_failed = 0 |
| 564 expect_fail = 0 | 572 expect_fail = 0 |
| 565 for line in lines: | 573 for line in lines: |
| 566 if 'Total:' in line: | 574 if 'Total:' in line: |
| 567 total_tests = int(line.split('Total: ')[1].split()[0]) | 575 total_tests = int(line.split('Total: ')[1].split()[0]) |
| (...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 650 | 658 |
| 651 run_cmd(['echo', '%d Generated checked total size' % total_size], | 659 run_cmd(['echo', '%d Generated checked total size' % total_size], |
| 652 self.trace_file, append=True) | 660 self.trace_file, append=True) |
| 653 os.chdir('..') | 661 os.chdir('..') |
| 654 | 662 |
| 655 def process_file(self, afile): | 663 def process_file(self, afile): |
| 656 """Pull all the relevant information out of a given tracefile. | 664 """Pull all the relevant information out of a given tracefile. |
| 657 | 665 |
| 658 Args: | 666 Args: |
| 659 afile: is the filename string we will be processing.""" | 667 afile: is the filename string we will be processing.""" |
| 668 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', | |
| 669 'perf_testing')) | |
| 660 f = open(os.path.join(self.result_folder_name, afile)) | 670 f = open(os.path.join(self.result_folder_name, afile)) |
| 661 tabulate_data = False | 671 tabulate_data = False |
| 662 revision_num = 0 | 672 revision_num = 0 |
| 663 for line in f.readlines(): | 673 for line in f.readlines(): |
| 664 tokens = line.split() | 674 tokens = line.split() |
| 665 if 'Revision' in line: | 675 if 'Revision' in line: |
| 666 revision_num = int(line.split()[1]) | 676 revision_num = int(line.split()[1]) |
| 667 else: | 677 else: |
| 668 for metric in self.values_list: | 678 for metric in self.values_list: |
| 669 if metric in line: | 679 if metric in line: |
| (...skipping 12 matching lines...) Expand all Loading... | |
| 682 # Fill in 0 if compilation failed. | 692 # Fill in 0 if compilation failed. |
| 683 if self.values_dict[COMMAND_LINE][FROG][metric][-1] < \ | 693 if self.values_dict[COMMAND_LINE][FROG][metric][-1] < \ |
| 684 self.failure_threshold[metric]: | 694 self.failure_threshold[metric]: |
| 685 self.values_dict[COMMAND_LINE][FROG][metric] += [0] | 695 self.values_dict[COMMAND_LINE][FROG][metric] += [0] |
| 686 self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num] | 696 self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num] |
| 687 | 697 |
| 688 f.close() | 698 f.close() |
| 689 | 699 |
| 690 def plot_results(self, png_filename): | 700 def plot_results(self, png_filename): |
| 691 self.style_and_save_perf_plot('Compiled minfrog Sizes', | 701 self.style_and_save_perf_plot('Compiled minfrog Sizes', |
| 692 'Size (in bytes)', 10, 10, 'center', png_filename, [COMMAND_LINE], | 702 'Size (in bytes)', 10, 10, 'lower left', png_filename, [COMMAND_LINE], |
| 693 [FROG], ['swarm', 'total', 'minfrog']) | 703 [FROG], ['swarm', 'total', 'minfrog']) |
| 694 self.write_html('bar', self.revision_dict[COMMAND_LINE][FROG]['minfrog'], | |
| 695 'minfrog size', self.values_dict[COMMAND_LINE][FROG]['minfrog'], '', []) | |
| 696 | 704 |
| 697 self.style_and_save_perf_plot('Time to compile and bootstrap', | 705 self.style_and_save_perf_plot('Time to compile and bootstrap', |
| 698 'Seconds', 10, 10, 'center', '2' + png_filename, [COMMAND_LINE], [FROG], | 706 'Seconds', 10, 10, 'lower left', '2' + png_filename, [COMMAND_LINE], |
| 699 ['Bootstrapping', 'Compiling on Dart VM']) | 707 [FROG], ['Bootstrapping', 'Compiling on Dart VM']) |
| 700 self.write_html('baz', | |
| 701 self.revision_dict[COMMAND_LINE][FROG]['Bootstrapping'], | |
| 702 'Bootstrapping', self.values_dict[COMMAND_LINE][FROG]['Bootstrapping'], | |
| 703 'Compiling on Dart VM', | |
| 704 self.values_dict[COMMAND_LINE][FROG]['Compiling on Dart VM']) | |
| 705 | |
| 706 | 708 |
| 707 def parse_args(): | 709 def parse_args(): |
| 708 parser = optparse.OptionParser() | 710 parser = optparse.OptionParser() |
| 709 parser.add_option('--command-line', '-c', dest='cl', | 711 parser.add_option('--command-line', '-c', dest='cl', |
| 710 help = 'Run the command line tests', | 712 help='Run the command line tests', |
| 711 action = 'store_true', default = False) | 713 action='store_true', default=False) |
| 712 parser.add_option('--size-time', '-s', dest = 'size', | 714 parser.add_option('--size-time', '-s', dest='size', |
| 713 help = 'Run the code size and timing tests', | 715 help='Run the code size and timing tests', |
| 714 action = 'store_true', default = False) | 716 action='store_true', default=False) |
| 715 parser.add_option('--language', '-l', dest = 'language', | 717 parser.add_option('--language', '-l', dest='language', |
| 716 help = 'Run the language correctness tests', | 718 help='Run the language correctness tests', |
| 717 action = 'store_true', default = False) | 719 action='store_true', default=False) |
| 718 parser.add_option('--browser-perf', '-b', dest = 'perf', | 720 parser.add_option('--browser-perf', '-b', dest='perf', |
| 719 help = 'Run the browser performance tests', | 721 help='Run the browser performance tests', |
| 720 action = 'store_true', default = False) | 722 action='store_true', default=False) |
| 721 parser.add_option('--forever', '-f', dest = 'continuous', | 723 parser.add_option('--forever', '-f', dest='continuous', |
| 722 help = 'Run this script forever, always checking for the next svn ' | 724 help = 'Run this script forever, always checking for the next svn ' |
| 723 'checkin', action = 'store_true', default = False) | 725 'checkin', action='store_true', default=False) |
| 724 parser.add_option('--verbose', '-v', dest = 'verbose', | 726 parser.add_option('--verbose', '-v', dest='verbose', |
| 725 help = 'Print extra debug output', action = 'store_true', default = False) | 727 help = 'Print extra debug output', action='store_true', default=False) |
| 728 parser.add_option('--user', '-u', dest='username', | |
| 729 help='Username for submitting new data to App Engine', default='') | |
| 726 | 730 |
| 727 args, ignored = parser.parse_args() | 731 args, ignored = parser.parse_args() |
| 732 password = '' | |
| 733 if args.username != '': | |
| 734 password = getpass.getpass("App Engine Password: ") | |
| 735 else: | |
| 736 print 'Warning: performance data will not be uploaded to App Engine' + \ | |
| 737 ' if you do not provide a username.' | |
| 728 if not (args.cl or args.size or args.language or args.perf): | 738 if not (args.cl or args.size or args.language or args.perf): |
| 729 args.cl = args.size = args.language = args.perf = True | 739 args.cl = args.size = args.language = args.perf = True |
| 730 return (args.cl, args.size, args.language, args.perf, args.continuous, | 740 return (args.cl, args.size, args.language, args.perf, args.continuous, |
| 731 args.verbose) | 741 args.verbose, args.username, password) |
| 732 | 742 |
| 733 def run_test_sequence(cl, size, language, perf): | 743 def run_test_sequence(cl, size, language, perf, username, password): |
| 734 # The buildbot already builds and syncs to a specific revision. Don't fight | 744 # The buildbot already builds and syncs to a specific revision. Don't fight |
| 735 # with it or replicate work. | 745 # with it or replicate work. |
| 736 if sync_and_build() == 1: | 746 if sync_and_build() == 1: |
| 737 return # The build is broken. | 747 return # The build is broken. |
| 738 if size: | 748 if size: |
| 739 CompileTimeAndSizeTest(TIME_SIZE).run() | 749 CompileTimeAndSizeTest(TIME_SIZE).run() |
| 740 if cl: | 750 if cl: |
| 741 CommandLinePerformanceTest(CL_PERF).run() | 751 CommandLinePerformanceTest(CL_PERF).run() |
| 742 if language: | 752 if language: |
| 743 BrowserCorrectnessTest('language', BROWSER_CORRECTNESS).run() | 753 BrowserCorrectnessTest('language', BROWSER_CORRECTNESS).run() |
| 744 if perf: | 754 if perf: |
| 745 BrowserPerformanceTest(BROWSER_PERF).run() | 755 BrowserPerformanceTest(BROWSER_PERF).run() |
| 746 | 756 |
| 747 # TODO(efortuna): Temporarily disabled until you make a safe way to provide | 757 is username != '': |
| 748 # your username/password for the uploading process. | 758 upload_to_app_engine(username, password) |
| 749 #upload_to_app_engine() | |
| 750 | 759 |
| 751 def main(): | 760 def main(): |
| 752 global VERBOSE | 761 global VERBOSE |
| 753 (cl, size, language, perf, continuous, verbose) = parse_args() | 762 (cl, size, language, perf, continuous, verbose, username, password) = parse_ar gs() |
| 754 VERBOSE = verbose | 763 VERBOSE = verbose |
| 755 if continuous: | 764 if continuous: |
| 756 while True: | 765 while True: |
| 757 if has_new_code(): | 766 if has_new_code(): |
| 758 run_test_sequence(cl, size, language, perf) | 767 run_test_sequence(cl, size, language, perf, username, password) |
| 759 else: | 768 else: |
| 760 time.sleep(SLEEP_TIME) | 769 time.sleep(SLEEP_TIME) |
| 761 else: | 770 else: |
| 762 run_test_sequence(cl, size, language, perf) | 771 run_test_sequence(cl, size, language, perf, username, password) |
| 763 | 772 |
| 764 if __name__ == '__main__': | 773 if __name__ == '__main__': |
| 765 main() | 774 main() |
| OLD | NEW |
