| OLD | NEW |
|---|
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 | 2 |
| 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 4 # for details. All rights reserved. Use of this source code is governed by a | 4 # for details. All rights reserved. Use of this source code is governed by a |
| 5 # BSD-style license that can be found in the LICENSE file. | 5 # BSD-style license that can be found in the LICENSE file. |
| 6 | 6 |
| 7 import datetime | 7 import datetime |
| 8 import getpass |
| 8 import math | 9 import math |
| 9 from matplotlib.font_manager import FontProperties | 10 from matplotlib.font_manager import FontProperties |
| 10 import matplotlib.pyplot as plt | 11 import matplotlib.pyplot as plt |
| 11 import optparse | 12 import optparse |
| 12 import os | 13 import os |
| 13 from os.path import dirname, abspath | 14 from os.path import dirname, abspath |
| 14 import platform | 15 import platform |
| 15 import shutil | 16 import shutil |
| 16 import subprocess | 17 import subprocess |
| 17 import time | 18 import time |
| 18 import traceback | 19 import traceback |
| 19 import sys | 20 import sys |
| 20 | 21 |
| 21 TOOLS_PATH = os.path.join(dirname(dirname(dirname(abspath(__file__))))) | 22 TOOLS_PATH = os.path.join(dirname(dirname(dirname(abspath(__file__))))) |
| 22 sys.path.append(TOOLS_PATH) | 23 sys.path.append(TOOLS_PATH) |
| 23 import utils | 24 import utils |
| 24 | 25 |
| 25 """This script runs to track performance and correctness progress of | 26 """This script runs to track performance and correctness progress of |
| 26 different svn revisions. It tests to see if there a newer version of the code on | 27 different svn revisions. It tests to see if there a newer version of the code on |
| 27 the server, and will sync and run the performance tests if so.""" | 28 the server, and will sync and run the performance tests if so.""" |
| 28 | 29 |
| 29 DART_INSTALL_LOCATION = os.path.join(dirname(abspath(__file__)), | 30 DART_INSTALL_LOCATION = os.path.join(dirname(abspath(__file__)), |
| 30 '..', '..', '..') | 31 '..', '..', '..') |
| 31 V8_MEAN = 'V8 Mean' | 32 JS_MEAN = 'JS Mean' |
| 32 FROG_MEAN = 'frog Mean' | 33 FROG_MEAN = 'frog js Mean' |
| 33 COMMAND_LINE = 'commandline' | 34 COMMAND_LINE = 'commandline' |
| 34 V8 = 'v8' | 35 JS = 'js' |
| 35 FROG = 'frog' | 36 FROG = 'frog' |
| 36 V8_AND_FROG = [V8, FROG] | 37 JS_AND_FROG = [JS, FROG] |
| 37 CORRECTNESS = 'Percent passing' | 38 CORRECTNESS = 'Percent passing' |
| 38 COLORS = ['blue', 'green', 'red', 'cyan', 'magenta', 'black'] | 39 COLORS = ['blue', 'green', 'red', 'cyan', 'magenta', 'black'] |
| 39 GRAPH_OUT_DIR = 'graphs' | 40 GRAPH_OUT_DIR = 'graphs' |
| 40 | 41 |
| 41 BROWSER_PERF = 'browser-perf' | 42 BROWSER_PERF = 'browser-perf' |
| 42 TIME_SIZE = 'code-time-size' | 43 TIME_SIZE = 'code-time-size' |
| 43 CL_PERF = 'cl-results' | 44 CL_PERF = 'cl-results' |
| 44 BROWSER_CORRECTNESS = 'browser-correctness' | 45 BROWSER_CORRECTNESS = 'browser-correctness' |
| 45 | 46 |
| 46 SLEEP_TIME = 200 | 47 SLEEP_TIME = 200 |
| (...skipping 101 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 148 | 149 |
| 149 def get_browsers(): | 150 def get_browsers(): |
| 150 browsers = ['ff', 'chrome'] | 151 browsers = ['ff', 'chrome'] |
| 151 if platform.system() == 'Darwin': | 152 if platform.system() == 'Darwin': |
| 152 browsers += ['safari'] | 153 browsers += ['safari'] |
| 153 if platform.system() == 'Windows': | 154 if platform.system() == 'Windows': |
| 154 browsers += ['ie'] | 155 browsers += ['ie'] |
| 155 return browsers | 156 return browsers |
| 156 | 157 |
| 157 def get_versions(): | 158 def get_versions(): |
| 158 return V8_AND_FROG | 159 return JS_AND_FROG |
| 159 | 160 |
| 160 def get_benchmarks(): | 161 def get_benchmarks(): |
| 161 return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees', | 162 return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees', |
| 162 'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute', | 163 'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute', |
| 163 'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers', | 164 'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers', |
| 164 'TreeSort'] | 165 'TreeSort'] |
| 165 | 166 |
| 166 def get_os_directory(): | 167 def get_os_directory(): |
| 167 """Specifies the name of the directory for the testing build of dart, which | 168 """Specifies the name of the directory for the testing build of dart, which |
| 168 has yet a different naming convention from utils.getBuildRoot(...).""" | 169 has yet a different naming convention from utils.getBuildRoot(...).""" |
| 169 if platform.system() == 'Windows': | 170 if platform.system() == 'Windows': |
| 170 return 'windows' | 171 return 'windows' |
| 171 elif platform.system() == 'Darwin': | 172 elif platform.system() == 'Darwin': |
| 172 return 'macos' | 173 return 'macos' |
| 173 else: | 174 else: |
| 174 return 'linux' | 175 return 'linux' |
| 175 | 176 |
| 176 def upload_to_app_engine(): | 177 def upload_to_app_engine(username, password): |
| 177 """Upload our results to our appengine server.""" | 178 """Upload our results to our appengine server. |
| 179 Arguments: |
| 180 username: App Engine username for uploading data to dartperf.googleplex.com |
| 181 password: App Engine password |
| 182 """ |
| 178 # TODO(efortuna): This is the most basic way to get the data up | 183 # TODO(efortuna): This is the most basic way to get the data up |
| 179 # for others to view. Revisit this once we're serving nicer graphs (Google | 184 # for others to view. Revisit this once we're serving nicer graphs (Google |
| 180 # Chart Tools) and from multiple perfbots and once we're in a position to | 185 # Chart Tools) and from multiple perfbots and once we're in a position to |
| 181 # organize the data in a useful manner(!!). | 186 # organize the data in a useful manner(!!). |
| 182 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', | 187 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', |
| 183 'perf_testing')) | 188 'perf_testing')) |
| 189 for data in [BROWSER_PERF, TIME_SIZE, CL_PERF]: |
| 190 path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS()) |
| 191 shutil.rmtree(path, ignore_errors=True) |
| 192 os.makedirs(path) |
| 193 files = [] |
| 194 # Copy the 1000 most recent trace files to be uploaded. |
| 195 for f in os.listdir(data): |
| 196 files += [(os.path.getmtime(os.path.join(data, f)), f)] |
| 197 files.sort() |
| 198 for f in files[-1000]: |
| 199 shutil.copyfile(os.path.join(data, f[1]), |
| 200 os.path.join(path, f[1]+'.txt')) |
| 201 # Generate directory listing. |
| 202 for data in [BROWSER_PERF, TIME_SIZE, CL_PERF]: |
| 203 path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS()) |
| 204 out = open(os.path.join('appengine', 'static', |
| 205 '%s-%s.html' % (data, utils.GuessOS())), 'w') |
| 206 out.write('<html>\n <body>\n <ul>\n') |
| 207 for f in os.listdir(path): |
| 208 if not f.startswith('.'): |
| 209 out.write(' <li><a href=data' + \ |
| 210 '''/%(data)s/%(os)s/%(file)s>%(file)s</a></li>\n''' % \ |
| 211 {'data': data, 'os': utils.GuessOS(), 'file': f}) |
| 212 out.write(' </ul>\n </body>\n</html>') |
| 213 out.close() |
| 214 |
| 184 shutil.rmtree(os.path.join('appengine', 'static', 'graphs'), | 215 shutil.rmtree(os.path.join('appengine', 'static', 'graphs'), |
| 185 ignore_errors=True) | 216 ignore_errors=True) |
| 186 shutil.copytree('graphs', os.path.join('appengine', 'static', 'graphs')) | 217 shutil.copytree('graphs', os.path.join('appengine', 'static', 'graphs')) |
| 187 shutil.copyfile('index.html', os.path.join('appengine', 'static', | 218 shutil.copyfile('index.html', os.path.join('appengine', 'static', |
| 188 'index.html')) | 219 'index.html')) |
| 189 run_cmd(['../../../third_party/appengine-python/1.5.4/appcfg.py', 'update', | 220 shutil.copyfile('data.html', os.path.join('appengine', 'static', |
| 190 'appengine/']) | 221 'data.html')) |
| 222 p = subprocess.Popen([os.path.join('..', '..', '..', 'third_party', |
| 223 'appengine-python', 'appcfg.py'), 'update', |
| 224 'appengine/'], shell=HAS_SHELL, stdin=subprocess.PIPE) |
| 225 p.stdin.write(username + '\n') |
| 226 p.stdin.write(password + '\n') |
| 227 p.communicate() |
| 228 |
| 191 | 229 |
| 192 class TestRunner(object): | 230 class TestRunner(object): |
| 193 """The base clas to provide shared code for different tests we will run and | 231 """The base class to provide shared code for different tests we will run and |
| 194 graph.""" | 232 graph.""" |
| 195 | 233 |
| 196 def __init__(self, result_folder_name, platform_list, v8_and_or_frog_list, | 234 def __init__(self, result_folder_name, platform_list, v8_and_or_frog_list, |
| 197 values_list): | 235 values_list): |
| 198 """Args: | 236 """Args: |
| 199 result_folder_name the name of the folder where a tracefile of | 237 result_folder_name the name of the folder where a tracefile of |
| 200 performance results will be stored. | 238 performance results will be stored. |
| 201 platform_list a list containing the platform(s) that our data has been | 239 platform_list a list containing the platform(s) that our data has been |
| 202 run on. (command line, firefox, chrome, etc) | 240 run on. (command line, firefox, chrome, etc) |
| 203 v8_and_or_frog_list a list specifying whether we hold data about Frog | 241 v8_and_or_frog_list a list specifying whether we hold data about Frog |
| (...skipping 12 matching lines...) Expand all Loading... |
| 216 self.color_index = 0 | 254 self.color_index = 0 |
| 217 for platform in platform_list: | 255 for platform in platform_list: |
| 218 self.revision_dict[platform] = dict() | 256 self.revision_dict[platform] = dict() |
| 219 self.values_dict[platform] = dict() | 257 self.values_dict[platform] = dict() |
| 220 for f in v8_and_or_frog_list: | 258 for f in v8_and_or_frog_list: |
| 221 self.revision_dict[platform][f] = dict() | 259 self.revision_dict[platform][f] = dict() |
| 222 self.values_dict[platform][f] = dict() | 260 self.values_dict[platform][f] = dict() |
| 223 for val in values_list: | 261 for val in values_list: |
| 224 self.revision_dict[platform][f][val] = [] | 262 self.revision_dict[platform][f][val] = [] |
| 225 self.values_dict[platform][f][val] = [] | 263 self.values_dict[platform][f][val] = [] |
| 226 if V8 in v8_and_or_frog_list: | 264 if JS in v8_and_or_frog_list: |
| 227 self.revision_dict[platform][V8][V8_MEAN] = [] | 265 self.revision_dict[platform][JS][JS_MEAN] = [] |
| 228 self.values_dict[platform][V8][V8_MEAN] = [] | 266 self.values_dict[platform][JS][JS_MEAN] = [] |
| 229 if FROG in v8_and_or_frog_list: | 267 if FROG in v8_and_or_frog_list: |
| 230 self.revision_dict[platform][FROG][FROG_MEAN] = [] | 268 self.revision_dict[platform][FROG][FROG_MEAN] = [] |
| 231 self.values_dict[platform][FROG][FROG_MEAN] = [] | 269 self.values_dict[platform][FROG][FROG_MEAN] = [] |
| 232 | 270 |
| 233 def get_color(self): | 271 def get_color(self): |
| 234 color = COLORS[self.color_index] | 272 color = COLORS[self.color_index] |
| 235 self.color_index = (self.color_index + 1) % len(COLORS) | 273 self.color_index = (self.color_index + 1) % len(COLORS) |
| 236 return color | 274 return color |
| 237 | 275 |
| 238 def style_and_save_perf_plot(self, chart_title, y_axis_label, size_x, size_y, | 276 def style_and_save_perf_plot(self, chart_title, y_axis_label, size_x, size_y, |
| (...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 283 for line in output.split('\n'): | 321 for line in output.split('\n'): |
| 284 if 'Revision' in line: | 322 if 'Revision' in line: |
| 285 run_cmd(['echo', line.strip()], outfile) | 323 run_cmd(['echo', line.strip()], outfile) |
| 286 return True | 324 return True |
| 287 return False | 325 return False |
| 288 | 326 |
| 289 if not search_for_revision(['svn', 'info']): | 327 if not search_for_revision(['svn', 'info']): |
| 290 if not search_for_revision(['git', 'svn', 'info']): | 328 if not search_for_revision(['git', 'svn', 'info']): |
| 291 run_cmd(['echo', 'Revision: unknown'], outfile) | 329 run_cmd(['echo', 'Revision: unknown'], outfile) |
| 292 | 330 |
| 293 def write_html(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, | |
| 294 cleanFile=False): | |
| 295 """Adds an html table to the webpage to display the data values. This method | |
| 296 will be removed when we have a nicer way to display data values.""" | |
| 297 #TODO(efortuna): fix this. | |
| 298 return | |
| 299 #TODO(efortuna): Take this method out when have finalized where the data is | |
| 300 # going to be displayed. | |
| 301 f = '' | |
| 302 out = '' | |
| 303 if cleanFile: | |
| 304 f = open('template.html') | |
| 305 else: | |
| 306 shutil.copy('index.html', 'temp.html') | |
| 307 f = open('temp.html') | |
| 308 out = open('index.html', 'w') | |
| 309 inTable = False | |
| 310 for line in f.readlines(): | |
| 311 if not inTable: | |
| 312 out.write(line) | |
| 313 if delimiter in line: | |
| 314 inTable = not inTable | |
| 315 if inTable: | |
| 316 out.write('<table border="1"> <tr> <td> svn revision </td>') | |
| 317 for revision in rev_nums: | |
| 318 out.write('<td>%d</td>' % revision) | |
| 319 out.write('</tr>\n<tr><td> %s</td>' % label_1) | |
| 320 for perf in dict_1: | |
| 321 out.write('<td>%f</td>' % perf) | |
| 322 out.write('</tr>\n<tr><td> %s</td>' % label_2) | |
| 323 for perf in dict_2: | |
| 324 out.write('<td>%f</td>' % perf) | |
| 325 out.write('</tr> </table>') | |
| 326 | |
| 327 def calculate_geometric_mean(self, platform, frog_or_v8, svn_revision): | 331 def calculate_geometric_mean(self, platform, frog_or_v8, svn_revision): |
| 328 """Calculate the aggregate geometric mean for V8 and frog benchmark sets, | 332 """Calculate the aggregate geometric mean for JS and frog benchmark sets, |
| 329 given two benchmark dictionaries.""" | 333 given two benchmark dictionaries.""" |
| 330 geo_mean = 0 | 334 geo_mean = 0 |
| 331 for benchmark in get_benchmarks(): | 335 for benchmark in get_benchmarks(): |
| 332 geo_mean += math.log(self.values_dict[platform][frog_or_v8][benchmark][ | 336 geo_mean += math.log(self.values_dict[platform][frog_or_v8][benchmark][ |
| 333 len(self.values_dict[platform][frog_or_v8][benchmark]) - 1]) | 337 len(self.values_dict[platform][frog_or_v8][benchmark]) - 1]) |
| 334 | 338 |
| 335 mean = V8_MEAN | 339 mean = JS_MEAN |
| 336 if frog_or_v8 == FROG: | 340 if frog_or_v8 == FROG: |
| 337 mean = FROG_MEAN | 341 mean = FROG_MEAN |
| 338 self.values_dict[platform][frog_or_v8][mean] += \ | 342 self.values_dict[platform][frog_or_v8][mean] += \ |
| 339 [math.pow(math.e, geo_mean / len(get_benchmarks()))] | 343 [math.pow(math.e, geo_mean / len(get_benchmarks()))] |
| 340 self.revision_dict[platform][frog_or_v8][mean] += [svn_revision] | 344 self.revision_dict[platform][frog_or_v8][mean] += [svn_revision] |
| 341 | 345 |
| 342 def run(self): | 346 def run(self): |
| 343 """Run the benchmarks/tests from the command line and plot the | 347 """Run the benchmarks/tests from the command line and plot the |
| 344 results.""" | 348 results.""" |
| 345 plt.cla() # cla = clear current axes | 349 plt.cla() # cla = clear current axes |
| 346 os.chdir(DART_INSTALL_LOCATION) | 350 os.chdir(DART_INSTALL_LOCATION) |
| 347 ensure_output_directory(self.result_folder_name) | 351 ensure_output_directory(self.result_folder_name) |
| 348 ensure_output_directory(GRAPH_OUT_DIR) | 352 ensure_output_directory(GRAPH_OUT_DIR) |
| 349 self.run_tests() | 353 self.run_tests() |
| 354 |
| 350 os.chdir(os.path.join('tools', 'testing', 'perf_testing')) | 355 os.chdir(os.path.join('tools', 'testing', 'perf_testing')) |
| 351 | 356 |
| 352 # TODO(efortuna): You will want to make this only use a subset of the files | 357 # TODO(efortuna): You will want to make this only use a subset of the files |
| 353 # eventually. | 358 # eventually. |
| 354 files = os.listdir(self.result_folder_name) | 359 files = os.listdir(self.result_folder_name) |
| 355 | 360 |
| 356 for afile in files: | 361 for afile in files: |
| 357 if not afile.startswith('.'): | 362 if not afile.startswith('.'): |
| 358 self.process_file(afile) | 363 self.process_file(afile) |
| 359 | 364 |
| 360 self.plot_results('%s.png' % self.result_folder_name) | 365 self.plot_results('%s.png' % self.result_folder_name) |
| 361 | 366 |
| 362 class PerformanceTest(TestRunner): | 367 class PerformanceTest(TestRunner): |
| 363 """Super class for all performance testing.""" | 368 """Super class for all performance testing.""" |
| 364 def __init__(self, result_folder_name, platform_list, platform_type): | 369 def __init__(self, result_folder_name, platform_list, platform_type): |
| 365 super(PerformanceTest, self).__init__(result_folder_name, | 370 super(PerformanceTest, self).__init__(result_folder_name, |
| 366 platform_list, get_versions(), get_benchmarks()) | 371 platform_list, get_versions(), get_benchmarks()) |
| 367 self.platform_list = platform_list | 372 self.platform_list = platform_list |
| 368 self.platform_type = platform_type | 373 self.platform_type = platform_type |
| 369 | 374 |
| 370 def plot_all_perf(self, png_filename): | 375 def plot_all_perf(self, png_filename): |
| 371 """Create a plot that shows the performance changes of individual benchmarks | 376 """Create a plot that shows the performance changes of individual benchmarks |
| 372 run by V8 and generated by frog, over svn history.""" | 377 run by JS and generated by frog, over svn history.""" |
| 373 for benchmark in get_benchmarks(): | 378 for benchmark in get_benchmarks(): |
| 374 self.style_and_save_perf_plot( | 379 self.style_and_save_perf_plot( |
| 375 'Performance of %s over time on the %s' % (benchmark, | 380 'Performance of %s over time on the %s on %s' % (benchmark, |
| 376 self.platform_type), 'Speed (bigger = better)', 16, 14, 'lower left', | 381 self.platform_type, utils.GuessOS()), 'Speed (bigger = better)', 16, |
| 377 benchmark + png_filename, self.platform_list, get_versions(), | 382 14, 'lower left', benchmark + png_filename, self.platform_list, |
| 378 [benchmark]) | 383 get_versions(), [benchmark]) |
| 379 | 384 |
| 380 def plot_avg_perf(self, png_filename): | 385 def plot_avg_perf(self, png_filename): |
| 381 """Generate a plot that shows the performance changes of the geomentric mean | 386 """Generate a plot that shows the performance changes of the geomentric mean |
| 382 of V8 and frog benchmark performance over svn history.""" | 387 of JS and frog benchmark performance over svn history.""" |
| 383 (title, y_axis, size_x, size_y, loc, filename) = \ | 388 (title, y_axis, size_x, size_y, loc, filename) = \ |
| 384 ('Geometric Mean of benchmark %s performance' % self.platform_type, | 389 ('Geometric Mean of benchmark %s performance' % self.platform_type, |
| 385 'Speed (bigger = better)', 16, 5, 'center', 'avg'+png_filename) | 390 'Speed (bigger = better)', 16, 5, 'lower left', 'avg'+png_filename) |
| 386 clear_axis = True | 391 clear_axis = True |
| 387 for platform in self.platform_list: | 392 for platform in self.platform_list: |
| 388 self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, | 393 self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, |
| 389 filename, [platform], [V8], [V8_MEAN], clear_axis) | 394 filename, [platform], [JS], [JS_MEAN], clear_axis) |
| 390 clear_axis = False | 395 clear_axis = False |
| 391 self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, | 396 self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, |
| 392 filename, [platform], [FROG], [FROG_MEAN], clear_axis) | 397 filename, [platform], [FROG], [FROG_MEAN], clear_axis) |
| 393 | 398 |
| 394 def plot_results(self, png_filename): | 399 def plot_results(self, png_filename): |
| 395 self.plot_all_perf(png_filename) | 400 self.plot_all_perf(png_filename) |
| 396 self.plot_avg_perf('2' + png_filename) | 401 self.plot_avg_perf('2' + png_filename) |
| 397 | 402 |
| 398 | 403 |
| 399 class CommandLinePerformanceTest(PerformanceTest): | 404 class CommandLinePerformanceTest(PerformanceTest): |
| 400 """Run performance tests from the command line.""" | 405 """Run performance tests from the command line.""" |
| 401 | 406 |
| 402 def __init__(self, result_folder_name): | 407 def __init__(self, result_folder_name): |
| 403 super(CommandLinePerformanceTest, self).__init__(result_folder_name, | 408 super(CommandLinePerformanceTest, self).__init__(result_folder_name, |
| 404 [COMMAND_LINE], 'command line') | 409 [COMMAND_LINE], 'command line') |
| 405 | 410 |
| 406 def process_file(self, afile): | 411 def process_file(self, afile): |
| 407 """Pull all the relevant information out of a given tracefile. | 412 """Pull all the relevant information out of a given tracefile. |
| 408 | 413 |
| 409 Args: | 414 Args: |
| 410 afile: The filename string we will be processing.""" | 415 afile: The filename string we will be processing.""" |
| 416 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', |
| 417 'perf_testing')) |
| 411 f = open(os.path.join(self.result_folder_name, afile)) | 418 f = open(os.path.join(self.result_folder_name, afile)) |
| 412 tabulate_data = False | 419 tabulate_data = False |
| 413 revision_num = 0 | 420 revision_num = 0 |
| 414 for line in f.readlines(): | 421 for line in f.readlines(): |
| 415 if 'Revision' in line: | 422 if 'Revision' in line: |
| 416 revision_num = int(line.split()[1]) | 423 revision_num = int(line.split()[1]) |
| 417 elif 'Benchmark' in line: | 424 elif 'Benchmark' in line: |
| 418 tabulate_data = True | 425 tabulate_data = True |
| 419 elif tabulate_data: | 426 elif tabulate_data: |
| 420 tokens = line.split() | 427 tokens = line.split() |
| 421 if len(tokens) < 4 or tokens[0] not in get_benchmarks(): | 428 if len(tokens) < 4 or tokens[0] not in get_benchmarks(): |
| 422 #Done tabulating data. | 429 #Done tabulating data. |
| 423 break | 430 break |
| 424 v8_value = float(tokens[1]) | 431 v8_value = float(tokens[1]) |
| 425 frog_value = float(tokens[3]) | 432 frog_value = float(tokens[3]) |
| 426 if v8_value == 0 or frog_value == 0: | 433 if v8_value == 0 or frog_value == 0: |
| 427 #Then there was an error when this performance test was run. Do not | 434 #Then there was an error when this performance test was run. Do not |
| 428 #count it in our numbers. | 435 #count it in our numbers. |
| 429 return | 436 return |
| 430 benchmark = tokens[0] | 437 benchmark = tokens[0] |
| 431 self.revision_dict[COMMAND_LINE][V8][benchmark] += [revision_num] | 438 self.revision_dict[COMMAND_LINE][JS][benchmark] += [revision_num] |
| 432 self.values_dict[COMMAND_LINE][V8][benchmark] += [v8_value] | 439 self.values_dict[COMMAND_LINE][JS][benchmark] += [v8_value] |
| 433 self.revision_dict[COMMAND_LINE][FROG][benchmark] += [revision_num] | 440 self.revision_dict[COMMAND_LINE][FROG][benchmark] += [revision_num] |
| 434 self.values_dict[COMMAND_LINE][FROG][benchmark] += [frog_value] | 441 self.values_dict[COMMAND_LINE][FROG][benchmark] += [frog_value] |
| 435 f.close() | 442 f.close() |
| 436 | 443 |
| 437 self.calculate_geometric_mean(COMMAND_LINE, FROG, revision_num) | 444 self.calculate_geometric_mean(COMMAND_LINE, FROG, revision_num) |
| 438 self.calculate_geometric_mean(COMMAND_LINE, V8, revision_num) | 445 self.calculate_geometric_mean(COMMAND_LINE, JS, revision_num) |
| 439 | 446 |
| 440 def run_tests(self): | 447 def run_tests(self): |
| 441 """Run a performance test on our updated system.""" | 448 """Run a performance test on our updated system.""" |
| 442 os.chdir('frog') | 449 os.chdir('frog') |
| 443 self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing', | 450 self.trace_file = os.path.join('..', 'tools', 'testing', 'perf_testing', |
| 444 self.result_folder_name, 'result' + self.cur_time) | 451 self.result_folder_name, 'result' + self.cur_time) |
| 445 run_cmd(['python', os.path.join('benchmarks', 'perf_tests.py')], | 452 run_cmd(['python', os.path.join('benchmarks', 'perf_tests.py')], |
| 446 self.trace_file) | 453 self.trace_file) |
| 447 os.chdir('..') | 454 os.chdir('..') |
| 448 | 455 |
| (...skipping 19 matching lines...) Expand all Loading... |
| 468 'perf-%s-%s-%s' % (self.cur_time, browser, version)) | 475 'perf-%s-%s-%s' % (self.cur_time, browser, version)) |
| 469 self.add_svn_revision_to_trace(self.trace_file) | 476 self.add_svn_revision_to_trace(self.trace_file) |
| 470 file_path = os.path.join(os.getcwd(), 'internal', 'browserBenchmarks', | 477 file_path = os.path.join(os.getcwd(), 'internal', 'browserBenchmarks', |
| 471 'benchmark_page_%s.html' % version) | 478 'benchmark_page_%s.html' % version) |
| 472 run_cmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), | 479 run_cmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), |
| 473 '--out', file_path, '--browser', browser, | 480 '--out', file_path, '--browser', browser, |
| 474 '--timeout', '600', '--perf'], self.trace_file, append=True) | 481 '--timeout', '600', '--perf'], self.trace_file, append=True) |
| 475 | 482 |
| 476 def process_file(self, afile): | 483 def process_file(self, afile): |
| 477 """Comb through the html to find the performance results.""" | 484 """Comb through the html to find the performance results.""" |
| 485 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', |
| 486 'perf_testing')) |
| 478 parts = afile.split('-') | 487 parts = afile.split('-') |
| 479 browser = parts[2] | 488 browser = parts[2] |
| 480 version = parts[3] | 489 version = parts[3] |
| 481 f = open(os.path.join(self.result_folder_name, afile)) | 490 f = open(os.path.join(self.result_folder_name, afile)) |
| 482 lines = f.readlines() | 491 lines = f.readlines() |
| 483 line = '' | 492 line = '' |
| 484 i = 0 | 493 i = 0 |
| 485 revision_num = 0 | 494 revision_num = 0 |
| 486 while '<div id="results">' not in line and i < len(lines): | 495 while '<div id="results">' not in line and i < len(lines): |
| 487 if 'Revision' in line: | 496 if 'Revision' in line: |
| 488 revision_num = int(line.split()[1]) | 497 revision_num = int(line.split()[1].strip('"')) |
| 489 line = lines[i] | 498 line = lines[i] |
| 490 i += 1 | 499 i += 1 |
| 491 | 500 |
| 492 if i >= len(lines) or revision_num == 0: | 501 if i >= len(lines) or revision_num == 0: |
| 493 # Then this run did not complete. Ignore this tracefile. | 502 # Then this run did not complete. Ignore this tracefile. |
| 494 return | 503 return |
| 495 | 504 |
| 496 line = lines[i] | 505 line = lines[i] |
| 497 i += 1 | 506 i += 1 |
| 498 results = [] | 507 results = [] |
| 499 if line.find('<br>') > -1: | 508 if line.find('<br>') > -1: |
| 500 results = line.split('<br>') | 509 results = line.split('<br>') |
| 501 else: | 510 else: |
| 502 results = line.split('<br />') | 511 results = line.split('<br />') |
| 503 for result in results: | 512 for result in results: |
| 504 name_and_score = result.split(':') | 513 name_and_score = result.split(':') |
| 505 if len(name_and_score) < 2: | 514 if len(name_and_score) < 2: |
| 506 break | 515 break |
| 507 name = name_and_score[0].strip() | 516 name = name_and_score[0].strip() |
| 508 score = name_and_score[1].strip() | 517 score = name_and_score[1].strip() |
| 509 if version == V8: | 518 if version == JS or version == 'v8': |
| 510 bench_dict = self.values_dict[browser][V8] | 519 version = JS |
| 520 bench_dict = self.values_dict[browser][JS] |
| 511 else: | 521 else: |
| 512 bench_dict = self.values_dict[browser][FROG] | 522 bench_dict = self.values_dict[browser][FROG] |
| 513 bench_dict[name] += [float(score)] | 523 bench_dict[name] += [float(score)] |
| 514 self.revision_dict[browser][version][name] += [revision_num] | 524 self.revision_dict[browser][version][name] += [revision_num] |
| 515 | 525 |
| 516 f.close() | 526 f.close() |
| 517 self.calculate_geometric_mean(browser, version, revision_num) | 527 self.calculate_geometric_mean(browser, version, revision_num) |
| 518 | 528 |
| 519 def write_html(self, delimiter, rev_nums, label_1, dict_1, label_2, dict_2, | |
| 520 cleanFile=False): | |
| 521 #TODO(efortuna) | |
| 522 pass | |
| 523 | |
| 524 | 529 |
| 525 class BrowserCorrectnessTest(TestRunner): | 530 class BrowserCorrectnessTest(TestRunner): |
| 526 def __init__(self, test_type, result_folder_name): | 531 def __init__(self, test_type, result_folder_name): |
| 527 super(BrowserCorrectnessTest, self).__init__(result_folder_name, | 532 super(BrowserCorrectnessTest, self).__init__(result_folder_name, |
| 528 get_browsers(), [FROG], [CORRECTNESS]) | 533 get_browsers(), [FROG], [CORRECTNESS]) |
| 529 self.test_type = test_type | 534 self.test_type = test_type |
| 530 | 535 |
| 531 def run_tests(self): | 536 def run_tests(self): |
| 532 """Run a test of the latest svn revision.""" | 537 """Run a test of the latest svn revision.""" |
| 533 system = get_os_directory() | 538 system = get_os_directory() |
| 534 suffix = '' | 539 suffix = '' |
| 535 if platform.system() == 'Windows': | 540 if platform.system() == 'Windows': |
| 536 suffix = '.exe' | 541 suffix = '.exe' |
| 537 for browser in get_browsers(): | 542 for browser in get_browsers(): |
| 538 current_file = 'correctness%s-%s' % (self.cur_time, browser) | 543 current_file = 'correctness%s-%s' % (self.cur_time, browser) |
| 539 self.trace_file = os.path.join('tools', 'testing', | 544 self.trace_file = os.path.join('tools', 'testing', |
| 540 'perf_testing', self.result_folder_name, current_file) | 545 'perf_testing', self.result_folder_name, current_file) |
| 541 self.add_svn_revision_to_trace(self.trace_file) | 546 self.add_svn_revision_to_trace(self.trace_file) |
| 542 dart_sdk = os.path.join(os.getcwd(), utils.GetBuildRoot(utils.GuessOS(), | 547 dart_sdk = os.path.join(os.getcwd(), utils.GetBuildRoot(utils.GuessOS(), |
| 543 'release', 'ia32'), 'dart-sdk') | 548 'release', 'ia32'), 'dart-sdk') |
| 544 run_cmd([os.path.join('.', 'tools', 'testing', 'bin', system, | 549 run_cmd([os.path.join('.', 'tools', 'testing', 'bin', system, |
| 545 'dart' + suffix), os.path.join('tools', 'test.dart'), | 550 'dart' + suffix), os.path.join('tools', 'test.dart'), |
| 546 '--component=webdriver', | 551 '--component=webdriver', |
| 547 '--browser=%s' % browser, '--frog=%s' % os.path.join(dart_sdk, 'bin', | 552 '--browser=%s' % browser, '--frog=%s' % os.path.join(dart_sdk, 'bin', |
| 548 'frogc'), '--froglib=%s' % os.path.join(dart_sdk, 'lib'), '--report', | 553 'frogc'), '--froglib=%s' % os.path.join(dart_sdk, 'lib'), '--report', |
| 549 '--timeout=20', '--progress=color', '--mode=release', '-j1', | 554 '--timeout=20', '--progress=color', '--mode=release', |
| 550 self.test_type], self.trace_file, append=True) | 555 self.test_type], self.trace_file, append=True) |
| 551 | 556 |
| 552 def process_file(self, afile): | 557 def process_file(self, afile): |
| 553 """Given a trace file, extract all the relevant information out of it to | 558 """Given a trace file, extract all the relevant information out of it to |
| 554 determine the number of correctly passing tests. | 559 determine the number of correctly passing tests. |
| 555 | 560 |
| 556 Arguments: | 561 Arguments: |
| 557 afile: the filename string""" | 562 afile: the filename string""" |
| 563 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', |
| 564 'perf_testing')) |
| 558 browser = afile.rpartition('-')[2] | 565 browser = afile.rpartition('-')[2] |
| 559 f = open(os.path.join(self.result_folder_name, afile)) | 566 f = open(os.path.join(self.result_folder_name, afile)) |
| 560 revision_num = 0 | 567 revision_num = 0 |
| 561 lines = f.readlines() | 568 lines = f.readlines() |
| 562 total_tests = 0 | 569 total_tests = 0 |
| 563 num_failed = 0 | 570 num_failed = 0 |
| 564 expect_fail = 0 | 571 expect_fail = 0 |
| 565 for line in lines: | 572 for line in lines: |
| 566 if 'Total:' in line: | 573 if 'Total:' in line: |
| 567 total_tests = int(line.split('Total: ')[1].split()[0]) | 574 total_tests = int(line.split('Total: ')[1].split()[0]) |
| (...skipping 82 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 650 | 657 |
| 651 run_cmd(['echo', '%d Generated checked total size' % total_size], | 658 run_cmd(['echo', '%d Generated checked total size' % total_size], |
| 652 self.trace_file, append=True) | 659 self.trace_file, append=True) |
| 653 os.chdir('..') | 660 os.chdir('..') |
| 654 | 661 |
| 655 def process_file(self, afile): | 662 def process_file(self, afile): |
| 656 """Pull all the relevant information out of a given tracefile. | 663 """Pull all the relevant information out of a given tracefile. |
| 657 | 664 |
| 658 Args: | 665 Args: |
| 659 afile: is the filename string we will be processing.""" | 666 afile: is the filename string we will be processing.""" |
| 667 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', |
| 668 'perf_testing')) |
| 660 f = open(os.path.join(self.result_folder_name, afile)) | 669 f = open(os.path.join(self.result_folder_name, afile)) |
| 661 tabulate_data = False | 670 tabulate_data = False |
| 662 revision_num = 0 | 671 revision_num = 0 |
| 663 for line in f.readlines(): | 672 for line in f.readlines(): |
| 664 tokens = line.split() | 673 tokens = line.split() |
| 665 if 'Revision' in line: | 674 if 'Revision' in line: |
| 666 revision_num = int(line.split()[1]) | 675 revision_num = int(line.split()[1]) |
| 667 else: | 676 else: |
| 668 for metric in self.values_list: | 677 for metric in self.values_list: |
| 669 if metric in line: | 678 if metric in line: |
| (...skipping 12 matching lines...) Expand all Loading... |
| 682 # Fill in 0 if compilation failed. | 691 # Fill in 0 if compilation failed. |
| 683 if self.values_dict[COMMAND_LINE][FROG][metric][-1] < \ | 692 if self.values_dict[COMMAND_LINE][FROG][metric][-1] < \ |
| 684 self.failure_threshold[metric]: | 693 self.failure_threshold[metric]: |
| 685 self.values_dict[COMMAND_LINE][FROG][metric] += [0] | 694 self.values_dict[COMMAND_LINE][FROG][metric] += [0] |
| 686 self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num] | 695 self.revision_dict[COMMAND_LINE][FROG][metric] += [revision_num] |
| 687 | 696 |
| 688 f.close() | 697 f.close() |
| 689 | 698 |
| 690 def plot_results(self, png_filename): | 699 def plot_results(self, png_filename): |
| 691 self.style_and_save_perf_plot('Compiled minfrog Sizes', | 700 self.style_and_save_perf_plot('Compiled minfrog Sizes', |
| 692 'Size (in bytes)', 10, 10, 'center', png_filename, [COMMAND_LINE], | 701 'Size (in bytes)', 10, 10, 'lower left', png_filename, [COMMAND_LINE], |
| 693 [FROG], ['swarm', 'total', 'minfrog']) | 702 [FROG], ['swarm', 'total', 'minfrog']) |
| 694 self.write_html('bar', self.revision_dict[COMMAND_LINE][FROG]['minfrog'], | |
| 695 'minfrog size', self.values_dict[COMMAND_LINE][FROG]['minfrog'], '', []) | |
| 696 | 703 |
| 697 self.style_and_save_perf_plot('Time to compile and bootstrap', | 704 self.style_and_save_perf_plot('Time to compile and bootstrap', |
| 698 'Seconds', 10, 10, 'center', '2' + png_filename, [COMMAND_LINE], [FROG], | 705 'Seconds', 10, 10, 'lower left', '2' + png_filename, [COMMAND_LINE], |
| 699 ['Bootstrapping', 'Compiling on Dart VM']) | 706 [FROG], ['Bootstrapping', 'Compiling on Dart VM']) |
| 700 self.write_html('baz', | |
| 701 self.revision_dict[COMMAND_LINE][FROG]['Bootstrapping'], | |
| 702 'Bootstrapping', self.values_dict[COMMAND_LINE][FROG]['Bootstrapping'], | |
| 703 'Compiling on Dart VM', | |
| 704 self.values_dict[COMMAND_LINE][FROG]['Compiling on Dart VM']) | |
| 705 | |
| 706 | 707 |
| 707 def parse_args(): | 708 def parse_args(): |
| 708 parser = optparse.OptionParser() | 709 parser = optparse.OptionParser() |
| 709 parser.add_option('--command-line', '-c', dest='cl', | 710 parser.add_option('--command-line', '-c', dest='cl', |
| 710 help = 'Run the command line tests', | 711 help='Run the command line tests', |
| 711 action = 'store_true', default = False) | 712 action='store_true', default=False) |
| 712 parser.add_option('--size-time', '-s', dest = 'size', | 713 parser.add_option('--size-time', '-s', dest='size', |
| 713 help = 'Run the code size and timing tests', | 714 help='Run the code size and timing tests', |
| 714 action = 'store_true', default = False) | 715 action='store_true', default=False) |
| 715 parser.add_option('--language', '-l', dest = 'language', | 716 parser.add_option('--language', '-l', dest='language', |
| 716 help = 'Run the language correctness tests', | 717 help='Run the language correctness tests', |
| 717 action = 'store_true', default = False) | 718 action='store_true', default=False) |
| 718 parser.add_option('--browser-perf', '-b', dest = 'perf', | 719 parser.add_option('--browser-perf', '-b', dest='perf', |
| 719 help = 'Run the browser performance tests', | 720 help='Run the browser performance tests', |
| 720 action = 'store_true', default = False) | 721 action='store_true', default=False) |
| 721 parser.add_option('--forever', '-f', dest = 'continuous', | 722 parser.add_option('--forever', '-f', dest='continuous', |
| 722 help = 'Run this script forever, always checking for the next svn ' | 723 help = 'Run this script forever, always checking for the next svn ' |
| 723 'checkin', action = 'store_true', default = False) | 724 'checkin', action='store_true', default=False) |
| 724 parser.add_option('--verbose', '-v', dest = 'verbose', | 725 parser.add_option('--verbose', '-v', dest='verbose', |
| 725 help = 'Print extra debug output', action = 'store_true', default = False) | 726 help = 'Print extra debug output', action='store_true', default=False) |
| 727 parser.add_option('--user', '-u', dest='username', |
| 728 help='Username for submitting new data to App Engine', default='') |
| 726 | 729 |
| 727 args, ignored = parser.parse_args() | 730 args, ignored = parser.parse_args() |
| 731 password = '' |
| 732 if args.username != '': |
| 733 password = getpass.getpass("App Engine Password: ") |
| 734 else: |
| 735 print 'Warning: performance data will not be uploaded to App Engine' + \ |
| 736 ' if you do not provide a username.' |
| 728 if not (args.cl or args.size or args.language or args.perf): | 737 if not (args.cl or args.size or args.language or args.perf): |
| 729 args.cl = args.size = args.language = args.perf = True | 738 args.cl = args.size = args.language = args.perf = True |
| 730 return (args.cl, args.size, args.language, args.perf, args.continuous, | 739 return (args.cl, args.size, args.language, args.perf, args.continuous, |
| 731 args.verbose) | 740 args.verbose, args.username, password) |
| 732 | 741 |
| 733 def run_test_sequence(cl, size, language, perf): | 742 def run_test_sequence(cl, size, language, perf, username, password): |
| 734 # The buildbot already builds and syncs to a specific revision. Don't fight | 743 # The buildbot already builds and syncs to a specific revision. Don't fight |
| 735 # with it or replicate work. | 744 # with it or replicate work. |
| 736 if sync_and_build() == 1: | 745 if sync_and_build() == 1: |
| 737 return # The build is broken. | 746 return # The build is broken. |
| 738 if size: | 747 if size: |
| 739 CompileTimeAndSizeTest(TIME_SIZE).run() | 748 CompileTimeAndSizeTest(TIME_SIZE).run() |
| 740 if cl: | 749 if cl: |
| 741 CommandLinePerformanceTest(CL_PERF).run() | 750 CommandLinePerformanceTest(CL_PERF).run() |
| 742 if language: | 751 if language: |
| 743 BrowserCorrectnessTest('language', BROWSER_CORRECTNESS).run() | 752 BrowserCorrectnessTest('language', BROWSER_CORRECTNESS).run() |
| 744 if perf: | 753 if perf: |
| 745 BrowserPerformanceTest(BROWSER_PERF).run() | 754 BrowserPerformanceTest(BROWSER_PERF).run() |
| 746 | 755 |
| 747 # TODO(efortuna): Temporarily disabled until you make a safe way to provide | 756 if username != '': |
| 748 # your username/password for the uploading process. | 757 upload_to_app_engine(username, password) |
| 749 #upload_to_app_engine() | |
| 750 | 758 |
| 751 def main(): | 759 def main(): |
| 752 global VERBOSE | 760 global VERBOSE |
| 753 (cl, size, language, perf, continuous, verbose) = parse_args() | 761 (cl, size, language, perf, continuous, verbose, username, password) = parse_ar
gs() |
| 754 VERBOSE = verbose | 762 VERBOSE = verbose |
| 755 if continuous: | 763 if continuous: |
| 756 while True: | 764 while True: |
| 757 if has_new_code(): | 765 if has_new_code(): |
| 758 run_test_sequence(cl, size, language, perf) | 766 run_test_sequence(cl, size, language, perf, username, password) |
| 759 else: | 767 else: |
| 760 time.sleep(SLEEP_TIME) | 768 time.sleep(SLEEP_TIME) |
| 761 else: | 769 else: |
| 762 run_test_sequence(cl, size, language, perf) | 770 run_test_sequence(cl, size, language, perf, username, password) |
| 763 | 771 |
| 764 if __name__ == '__main__': | 772 if __name__ == '__main__': |
| 765 main() | 773 main() |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 9706088:
Some changes to performance test script. (Closed)
Base URL: http://dart.googlecode.com/svn/branches/bleeding_edge/dart/