Chromium Code Reviews Chromium Code Reviews
Issue 10340006:
Add uploading to appengine code back in. (Closed)
Base URL: http://dart.googlecode.com/svn/branches/bleeding_edge/dart/| OLD | NEW |
|---|---|
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 | 2 |
| 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 4 # for details. All rights reserved. Use of this source code is governed by a | 4 # for details. All rights reserved. Use of this source code is governed by a |
| 5 # BSD-style license that can be found in the LICENSE file. | 5 # BSD-style license that can be found in the LICENSE file. |
| 6 | 6 |
| 7 import datetime | 7 import datetime |
| 8 import math | |
| 9 try: | |
| 10 from matplotlib.font_manager import FontProperties | |
| 11 import matplotlib.pyplot as plt | |
| 12 except ImportError: | |
| 13 pass # Only needed if we want to make graphs. | |
| 8 import optparse | 14 import optparse |
| 9 import os | 15 import os |
| 10 from os.path import dirname, abspath | 16 from os.path import dirname, abspath |
| 11 import platform | 17 import platform |
| 12 import re | 18 import re |
| 13 import shutil | 19 import shutil |
| 14 import stat | 20 import stat |
| 15 import subprocess | 21 import subprocess |
| 16 import sys | 22 import sys |
| 17 import time | 23 import time |
| (...skipping 84 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 102 | 108 |
| 103 self.run_cmd(['gclient', 'sync']) | 109 self.run_cmd(['gclient', 'sync']) |
| 104 | 110 |
| 105 # On Windows, the output directory is marked as "Read Only," which causes an | 111 # On Windows, the output directory is marked as "Read Only," which causes an |
| 106 # error to be thrown when we use shutil.rmtree. This helper function changes | 112 # error to be thrown when we use shutil.rmtree. This helper function changes |
| 107 # the permissions so we can still delete the directory. | 113 # the permissions so we can still delete the directory. |
| 108 def on_rm_error(func, path, exc_info): | 114 def on_rm_error(func, path, exc_info): |
| 109 if os.path.exists(path): | 115 if os.path.exists(path): |
| 110 os.chmod(path, stat.S_IWRITE) | 116 os.chmod(path, stat.S_IWRITE) |
| 111 os.unlink(path) | 117 os.unlink(path) |
| 112 # TODO(efortuna): building the sdk locally is a band-aid until all build XXX | 118 # TODO(efortuna): building the sdk locally is a band-aid until all build |
| 113 # platform SDKs are hosted in Google storage. Pull from https://sandbox. | 119 # platform SDKs are hosted in Google storage. Pull from https://sandbox. |
| 114 # google.com/storage/?arg=dart-dump-render-tree/sdk/#dart-dump-render-tree%2 Fsdk | 120 # google.com/storage/?arg=dart-dump-render-tree/sdk/#dart-dump-render-tree%2 Fsdk |
| 115 # eventually. | 121 # eventually. |
| 116 # TODO(efortuna): Currently always building ia32 architecture because we | 122 # TODO(efortuna): Currently always building ia32 architecture because we |
| 117 # don't have test statistics for what's passing on x64. Eliminate arch | 123 # don't have test statistics for what's passing on x64. Eliminate arch |
| 118 # specification when we have tests running on x64, too. | 124 # specification when we have tests running on x64, too. |
| 119 shutil.rmtree(os.path.join(os.getcwd(), | 125 shutil.rmtree(os.path.join(os.getcwd(), |
| 120 utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32')), | 126 utils.GetBuildRoot(utils.GuessOS(), 'release', 'ia32')), |
| 121 onerror=on_rm_error) | 127 onerror=on_rm_error) |
| 122 | 128 |
| (...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 158 def get_os_directory(self): | 164 def get_os_directory(self): |
| 159 """Specifies the name of the directory for the testing build of dart, which | 165 """Specifies the name of the directory for the testing build of dart, which |
| 160 has yet a different naming convention from utils.getBuildRoot(...).""" | 166 has yet a different naming convention from utils.getBuildRoot(...).""" |
| 161 if platform.system() == 'Windows': | 167 if platform.system() == 'Windows': |
| 162 return 'windows' | 168 return 'windows' |
| 163 elif platform.system() == 'Darwin': | 169 elif platform.system() == 'Darwin': |
| 164 return 'macos' | 170 return 'macos' |
| 165 else: | 171 else: |
| 166 return 'linux' | 172 return 'linux' |
| 167 | 173 |
| 174 def upload_to_app_engine(self, suite_names): | |
| 175 """Upload our results to our appengine server. | |
| 176 Arguments: | |
| 177 suite_names: Directories to upload data from (should match directory | |
| 178 names).""" | |
| 179 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', | |
| 180 'perf_testing')) | |
| 181 for data in suite_names: | |
| 182 path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS()) | |
| 183 shutil.rmtree(path, ignore_errors=True) | |
| 184 os.makedirs(path) | |
| 185 files = [] | |
| 186 # Copy the 1000 most recent trace files to be uploaded. | |
| 187 for f in os.listdir(data): | |
| 188 files += [(os.path.getmtime(os.path.join(data, f)), f)] | |
| 189 files.sort() | |
| 190 for f in files[-1000:]: | |
| 191 shutil.copyfile(os.path.join(data, f[1]), | |
| 192 os.path.join(path, f[1]+'.txt')) | |
| 193 # Generate directory listing. | |
| 194 for data in suite_names: | |
| 195 path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS()) | |
| 196 out = open(os.path.join('appengine', 'static', | |
| 197 '%s-%s.html' % (data, utils.GuessOS())), 'w') | |
| 198 out.write('<html>\n <body>\n <ul>\n') | |
| 199 for f in os.listdir(path): | |
| 200 if not f.startswith('.'): | |
| 201 out.write(' <li><a href=data' + \ | |
| 202 '''/%(data)s/%(os)s/%(file)s>%(file)s</a></li>\n''' % \ | |
| 203 {'data': data, 'os': utils.GuessOS(), 'file': f}) | |
| 204 out.write(' </ul>\n </body>\n</html>') | |
| 205 out.close() | |
| 206 | |
| 207 shutil.rmtree(os.path.join('appengine', 'static', 'graphs'), | |
| 208 ignore_errors=True) | |
| 209 shutil.copytree('graphs', os.path.join('appengine', 'static', 'graphs')) | |
| 210 shutil.copyfile('index.html', os.path.join('appengine', 'static', | |
| 211 'index.html')) | |
| 212 shutil.copyfile('dromaeo.html', os.path.join('appengine', 'static', | |
| 213 'dromaeo.html')) | |
| 214 shutil.copyfile('data.html', os.path.join('appengine', 'static', | |
| 215 'data.html')) | |
| 216 self.run_cmd([os.path.join('..', '..', '..', 'third_party', | |
| 217 'appengine-python', 'appcfg.py'), '--oauth2', | |
| 218 'update', 'appengine/']) | |
| 219 | |
| 220 | |
| 168 def parse_args(self): | 221 def parse_args(self): |
| 169 parser = optparse.OptionParser() | 222 parser = optparse.OptionParser() |
| 170 parser.add_option('--suites', '-s', dest='suites', help='Run the specified ' | 223 parser.add_option('--suites', '-s', dest='suites', help='Run the specified ' |
| 171 'comma-separated test suites from set: %s' % \ | 224 'comma-separated test suites from set: %s' % \ |
| 172 ','.join(TestBuilder.available_suite_names()), | 225 ','.join(TestBuilder.available_suite_names()), |
| 173 action='store', default=None) | 226 action='store', default=None) |
| 174 parser.add_option('--forever', '-f', dest='continuous', help='Run this scri' | 227 parser.add_option('--forever', '-f', dest='continuous', help='Run this scri' |
| 175 'pt forever, always checking for the next svn checkin', | 228 'pt forever, always checking for the next svn checkin', |
| 176 action='store_true', default=False) | 229 action='store_true', default=False) |
| 177 parser.add_option('--nobuild', '-n', dest='no_build', action='store_true', | 230 parser.add_option('--nobuild', '-n', dest='no_build', action='store_true', |
| (...skipping 23 matching lines...) Expand all Loading... | |
| 201 self.suite_names = suites | 254 self.suite_names = suites |
| 202 self.no_build = args.no_build | 255 self.no_build = args.no_build |
| 203 self.no_upload = args.no_upload | 256 self.no_upload = args.no_upload |
| 204 self.no_test = args.no_test | 257 self.no_test = args.no_test |
| 205 self.verbose = args.verbose | 258 self.verbose = args.verbose |
| 206 return args.continuous | 259 return args.continuous |
| 207 | 260 |
| 208 def run_test_sequence(self): | 261 def run_test_sequence(self): |
| 209 """Run the set of commands to (possibly) build, run, and graph the results | 262 """Run the set of commands to (possibly) build, run, and graph the results |
| 210 of our tests. | 263 of our tests. |
| 211 | |
| 212 Args: | |
| 213 suite_names: The "display name" the user enters to specify which | |
| 214 benchmark(s) to run. | |
| 215 no_build: True if we should not check the repository and build the latest | |
| 216 version. | |
| 217 """ | 264 """ |
| 218 suites = [] | 265 suites = [] |
| 219 for name in self.suite_names: | 266 for name in self.suite_names: |
| 220 suites += [TestBuilder.make_test(name, self)] | 267 suites += [TestBuilder.make_test(name, self)] |
| 221 | 268 |
| 222 if not self.no_build and self.sync_and_build(suites) == 1: | 269 if not self.no_build and self.sync_and_build(suites) == 1: |
| 223 return # The build is broken. | 270 return # The build is broken. |
| 224 | 271 |
| 225 for test in suites: | 272 for test in suites: |
| 226 test.run() | 273 test.run() |
| 227 | 274 |
| 228 | 275 |
| 229 class Test(object): | 276 class Test(object): |
| 230 """The base class to provide shared code for different tests we will run and | 277 """The base class to provide shared code for different tests we will run and |
| 231 graph. At a high level, each test has three visitors (the tester, the | 278 graph. At a high level, each test has three visitors (the tester, the |
| 232 file_processor that perform operations on the test object.""" | 279 file_processor and the grapher) that perform operations on the test object.""" |
| 233 | 280 |
| 234 def __init__(self, result_folder_name, platform_list, variants, | 281 def __init__(self, result_folder_name, platform_list, variants, |
| 235 values_list, test_runner, tester, file_processor, | 282 values_list, test_runner, tester, file_processor, grapher, |
| 236 build_targets=['create_sdk']): | 283 extra_metrics=['Geo-Mean'], build_targets=['create_sdk']): |
| 237 """Args: | 284 """Args: |
| 238 result_folder_name: The name of the folder where a tracefile of | 285 result_folder_name: The name of the folder where a tracefile of |
| 239 performance results will be stored. | 286 performance results will be stored. |
| 240 platform_list: A list containing the platform(s) that our data has been | 287 platform_list: A list containing the platform(s) that our data has been |
| 241 run on. (command line, firefox, chrome, etc) | 288 run on. (command line, firefox, chrome, etc) |
| 242 variants: A list specifying whether we hold data about Frog | 289 variants: A list specifying whether we hold data about Frog |
| 243 generated code, plain JS code, or a combination of both, or | 290 generated code, plain JS code, or a combination of both, or |
| 244 Dart depending on the test. | 291 Dart depending on the test. |
| 245 values_list: A list containing the type of data we will be graphing | 292 values_list: A list containing the type of data we will be graphing |
| 246 (benchmarks, percentage passing, etc). | 293 (benchmarks, percentage passing, etc). |
| 247 test_runner: Reference to the parent test runner object that notifies a | 294 test_runner: Reference to the parent test runner object that notifies a |
| 248 test when to run. | 295 test when to run. |
| 249 tester: The visitor that actually performs the test running mechanics. | 296 tester: The visitor that actually performs the test running mechanics. |
| 250 file_processor: The visitor that processes files in the format | 297 file_processor: The visitor that processes files in the format |
| 251 appropriate for this test. | 298 appropriate for this test. |
| 299 grapher: The visitor that generates graphs given our test result data. | |
| 300 extra_metrics: A list of any additional measurements we wish to keep | |
| 301 track of (such as the geometric mean of a set, the sum, etc). | |
| 252 build_targets: The targets necessary to build to run these tests | 302 build_targets: The targets necessary to build to run these tests |
| 253 (default target is create_sdk).""" | 303 (default target is create_sdk).""" |
| 254 self.result_folder_name = result_folder_name | 304 self.result_folder_name = result_folder_name |
| 255 # cur_time is used as a timestamp of when this performance test was run. | 305 # cur_time is used as a timestamp of when this performance test was run. |
| 256 self.cur_time = str(time.mktime(datetime.datetime.now().timetuple())) | 306 self.cur_time = str(time.mktime(datetime.datetime.now().timetuple())) |
| 257 self.values_list = values_list | 307 self.values_list = values_list |
| 258 self.platform_list = platform_list | 308 self.platform_list = platform_list |
| 259 self.test_runner = test_runner | 309 self.test_runner = test_runner |
| 260 self.tester = tester | 310 self.tester = tester |
| 261 self.file_processor = file_processor | 311 self.file_processor = file_processor |
| 262 self.build_targets = build_targets | 312 self.build_targets = build_targets |
| 313 self.revision_dict = dict() | |
| 314 self.values_dict = dict() | |
| 315 self.grapher = grapher | |
| 316 self.extra_metrics = extra_metrics | |
| 317 # Initialize our values store. | |
| 318 for platform in platform_list: | |
| 319 self.revision_dict[platform] = dict() | |
| 320 self.values_dict[platform] = dict() | |
| 321 for f in variants: | |
| 322 self.revision_dict[platform][f] = dict() | |
| 323 self.values_dict[platform][f] = dict() | |
| 324 for val in values_list: | |
| 325 self.revision_dict[platform][f][val] = [] | |
| 326 self.values_dict[platform][f][val] = [] | |
| 327 for extra_metric in extra_metrics: | |
| 328 self.revision_dict[platform][f][extra_metric] = [] | |
| 329 self.values_dict[platform][f][extra_metric] = [] | |
| 263 | 330 |
| 264 def is_valid_combination(self, platform, variant): | 331 def is_valid_combination(self, platform, variant): |
| 265 """Check whether data should be captured for this platform/variant | 332 """Check whether data should be captured for this platform/variant |
| 266 combination. | 333 combination. |
| 267 """ | 334 """ |
| 268 return True | 335 return True |
| 269 | 336 |
| 270 def run(self): | 337 def run(self): |
| 271 """Run the benchmarks/tests from the command line and plot the | 338 """Run the benchmarks/tests from the command line and plot the |
| 272 results. | 339 results. |
| 273 """ | 340 """ |
| 274 for visitor in [self.tester, self.file_processor]: | 341 for visitor in [self.tester, self.file_processor, self.grapher]: |
| 275 visitor.prepare() | 342 visitor.prepare() |
| 276 | 343 |
| 277 os.chdir(DART_INSTALL_LOCATION) | 344 os.chdir(DART_INSTALL_LOCATION) |
| 278 self.test_runner.ensure_output_directory(self.result_folder_name) | 345 self.test_runner.ensure_output_directory(self.result_folder_name) |
| 279 if not self.test_runner.no_test: | 346 if not self.test_runner.no_test: |
| 280 self.tester.run_tests() | 347 self.tester.run_tests() |
| 281 | 348 |
| 282 os.chdir(os.path.join('tools', 'testing', 'perf_testing')) | 349 os.chdir(os.path.join('tools', 'testing', 'perf_testing')) |
| 283 | 350 |
| 284 files = os.listdir(self.result_folder_name) | 351 files = os.listdir(self.result_folder_name) |
| 285 for afile in files: | 352 for afile in files: |
| 286 if not afile.startswith('.'): | 353 if not afile.startswith('.'): |
| 287 if self.file_processor.process_file(afile): | 354 self.file_processor.process_file(afile) |
| 288 os.remove(os.path.join(self.result_folder_name, afile)) | 355 |
| 356 if 'plt' in globals(): | |
| 357 # Only run Matplotlib if it is installed. | |
| 358 self.grapher.plot_results('%s.png' % self.result_folder_name) | |
| 289 | 359 |
| 290 | 360 |
| 291 class Tester(object): | 361 class Tester(object): |
| 292 """The base level visitor class that runs tests. It contains convenience | 362 """The base level visitor class that runs tests. It contains convenience |
| 293 methods that many Tester objects use. Any class that would like to be a | 363 methods that many Tester objects use. Any class that would like to be a |
| 294 TesterVisitor must implement the run_tests() method.""" | 364 TesterVisitor must implement the run_tests() method.""" |
| 295 | 365 |
| 296 def __init__(self, test): | 366 def __init__(self, test): |
| 297 self.test = test | 367 self.test = test |
| 298 | 368 |
| (...skipping 39 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 338 COMPILE_TIME = 'CompileTime' | 408 COMPILE_TIME = 'CompileTime' |
| 339 CODE_SIZE = 'CodeSize' | 409 CODE_SIZE = 'CodeSize' |
| 340 | 410 |
| 341 def __init__(self, test): | 411 def __init__(self, test): |
| 342 self.test = test | 412 self.test = test |
| 343 | 413 |
| 344 def prepare(self): | 414 def prepare(self): |
| 345 """Perform any initial setup required before the test is run.""" | 415 """Perform any initial setup required before the test is run.""" |
| 346 pass | 416 pass |
| 347 | 417 |
| 418 def should_report_results(self, afile): | |
| 419 """We store all trace files locally, but we don't want to post all of the | |
| 420 results every time, so we only attempt to post results for recent runs.""" | |
| 421 cur_time = time.time() | |
| 422 file_mod_time = os.path.getmtime(os.path.join( | |
| 423 self.test.result_folder_name, afile)) | |
| 424 return cur_time - file_mod_time < 1000 # Files modified in the last ~15 min. | |
| 425 | |
| 348 def report_results(self, benchmark_name, score, platform, variant, | 426 def report_results(self, benchmark_name, score, platform, variant, |
| 349 revision_number, metric): | 427 revision_number, metric): |
| 350 """Store the results of the benchmark run. | 428 """Store the results of the benchmark run. |
| 351 Args: | 429 Args: |
| 352 benchmark_name: The name of the individual benchmark. | 430 benchmark_name: The name of the individual benchmark. |
| 353 score: The numerical value of this benchmark. | 431 score: The numerical value of this benchmark. |
| 354 platform: The platform the test was run on (firefox, command line, etc). | 432 platform: The platform the test was run on (firefox, command line, etc). |
| 355 variant: Specifies whether the data was about generated Frog, js, a | 433 variant: Specifies whether the data was about generated Frog, js, a |
| 356 combination of both, or Dart depending on the test. | 434 combination of both, or Dart depending on the test. |
| 357 revision_number: The revision of the code (and sometimes the revision of | 435 revision_number: The revision of the code (and sometimes the revision of |
| 358 dartium). | 436 dartium). |
| 359 | 437 |
| 360 Returns: True if the post was successful.""" | 438 Returns: True if the post was successful file.""" |
| 361 # TODO(efortuna): delete results file if returns True. | |
| 362 return post_results.report_results(benchmark_name, score, platform, variant, | 439 return post_results.report_results(benchmark_name, score, platform, variant, |
| 363 revision_number, metric) | 440 revision_number, metric) |
| 364 | 441 |
| 442 def calculate_geometric_mean(self, platform, variant, svn_revision): | |
| 443 """Calculate the aggregate geometric mean for JS and frog benchmark sets, | |
| 444 given two benchmark dictionaries.""" | |
| 445 geo_mean = 0 | |
| 446 # TODO(vsm): Suppress graphing this combination altogether. For | |
| 447 # now, we feed a geomean of 0. | |
| 448 if self.test.is_valid_combination(platform, variant): | |
| 449 for benchmark in self.test.values_list: | |
| 450 geo_mean += math.log( | |
| 451 self.test.values_dict[platform][variant][benchmark][ | |
| 452 len(self.test.values_dict[platform][variant][benchmark]) - 1]) | |
| 453 | |
| 454 self.test.values_dict[platform][variant]['Geo-Mean'] += \ | |
| 455 [math.pow(math.e, geo_mean / len(self.test.values_list))] | |
| 456 self.test.revision_dict[platform][variant]['Geo-Mean'] += [svn_revision] | |
| 457 | |
| 458 | |
| 459 class Grapher(object): | |
| 460 """The base level visitor class that generates graphs for data. It contains | |
| 461 convenience methods that many Grapher objects use. Any class that would like | |
| 462 to be a GrapherVisitor must implement the plot_results() method.""" | |
| 463 | |
| 464 graph_out_dir = 'graphs' | |
| 465 | |
| 466 def __init__(self, test): | |
| 467 self.color_index = 0 | |
| 468 self.test = test | |
| 469 | |
| 470 def prepare(self): | |
| 471 """Perform any initial setup required before the test is run.""" | |
| 472 if 'plt' in globals(): | |
| 473 plt.cla() # cla = clear current axes | |
| 474 else: | |
| 475 print 'Unable to import Matplotlib and therefore unable to generate ' + \ | |
| 476 'graphs. Please install it for this version of Python.' | |
| 477 self.test.test_runner.ensure_output_directory(Grapher.graph_out_dir) | |
| 478 | |
| 479 def style_and_save_perf_plot(self, chart_title, y_axis_label, size_x, size_y, | |
| 480 legend_loc, filename, platform_list, variants, | |
| 481 values_list, should_clear_axes=True): | |
| 482 """Sets style preferences for chart boilerplate that is consistent across | |
| 483 all charts, and saves the chart as a png. | |
| 484 Args: | |
| 485 size_x: the size of the printed chart, in inches, in the horizontal | |
| 486 direction | |
| 487 size_y: the size of the printed chart, in inches in the vertical direction | |
| 488 legend_loc: the location of the legend in on the chart. See suitable | |
| 489 arguments for the loc argument in matplotlib | |
| 490 filename: the filename that we want to save the resulting chart as | |
| 491 platform_list: a list containing the platform(s) that our data has been | |
| 492 run on. (command line, firefox, chrome, etc) | |
| 493 values_list: a list containing the type of data we will be graphing | |
| 494 (performance, percentage passing, etc) | |
| 495 should_clear_axes: True if we want to create a fresh graph, instead of | |
| 496 plotting additional lines on the current graph.""" | |
| 497 if should_clear_axes: | |
| 498 plt.cla() # cla = clear current axes | |
| 499 for platform in platform_list: | |
| 500 for f in variants: | |
| 501 for val in values_list: | |
| 502 plt.plot(self.test.revision_dict[platform][f][val], | |
| 503 self.test.values_dict[platform][f][val], | |
| 504 color=self.get_color(), label='%s-%s-%s' % (platform, f, val)) | |
| 505 | |
| 506 plt.xlabel('Revision Number') | |
| 507 plt.ylabel(y_axis_label) | |
| 508 plt.title(chart_title) | |
| 509 fontP = FontProperties() | |
| 510 fontP.set_size('small') | |
| 511 plt.legend(loc=legend_loc, prop = fontP) | |
| 512 | |
| 513 fig = plt.gcf() | |
| 514 fig.set_size_inches(size_x, size_y) | |
| 515 fig.savefig(os.path.join(Grapher.graph_out_dir, filename)) | |
| 516 | |
| 517 def get_color(self): | |
| 518 # Just a bunch of distinct colors for a potentially large number of values | |
| 519 # we wish to graph. | |
| 520 colors = [ | |
| 521 'blue', 'green', 'red', 'cyan', 'magenta', 'black', '#3366CC', | |
| 522 '#DC3912', '#FF9900', '#109618', '#990099', '#0099C6', '#DD4477', | |
| 523 '#66AA00', '#B82E2E', '#316395', '#994499', '#22AA99', '#AAAA11', | |
| 524 '#6633CC', '#E67300', '#8B0707', '#651067', '#329262', '#5574A6', | |
| 525 '#3B3EAC', '#B77322', '#16D620', '#B91383', '#F4359E', '#9C5935', | |
| 526 '#A9C413', '#2A778D', '#668D1C', '#BEA413', '#0C5922', '#743411', | |
| 527 '#45AFE2', '#FF3300', '#FFCC00', '#14C21D', '#DF51FD', '#15CBFF', | |
| 528 '#FF97D2', '#97FB00', '#DB6651', '#518BC6', '#BD6CBD', '#35D7C2', | |
| 529 '#E9E91F', '#9877DD', '#FF8F20', '#D20B0B', '#B61DBA', '#40BD7E', | |
| 530 '#6AA7C4', '#6D70CD', '#DA9136', '#2DEA36', '#E81EA6', '#F558AE', | |
| 531 '#C07145', '#D7EE53', '#3EA7C6', '#97D129', '#E9CA1D', '#149638', | |
| 532 '#C5571D'] | |
| 533 color = colors[self.color_index] | |
| 534 self.color_index = (self.color_index + 1) % len(colors) | |
| 535 return color | |
| 365 | 536 |
| 366 class RuntimePerformanceTest(Test): | 537 class RuntimePerformanceTest(Test): |
| 367 """Super class for all runtime performance testing.""" | 538 """Super class for all runtime performance testing.""" |
| 368 | 539 |
| 369 def __init__(self, result_folder_name, platform_list, platform_type, | 540 def __init__(self, result_folder_name, platform_list, platform_type, |
| 370 versions, benchmarks, test_runner, tester, file_processor, | 541 versions, benchmarks, test_runner, tester, file_processor, |
| 371 build_targets=['create_sdk']): | 542 build_targets=['create_sdk']): |
| 372 """Args: | 543 """Args: |
| 373 result_folder_name: The name of the folder where a tracefile of | 544 result_folder_name: The name of the folder where a tracefile of |
| 374 performance results will be stored. | 545 performance results will be stored. |
| 375 platform_list: A list containing the platform(s) that our data has been | 546 platform_list: A list containing the platform(s) that our data has been |
| 376 run on. (command line, firefox, chrome, etc) | 547 run on. (command line, firefox, chrome, etc) |
| 377 variants: A list specifying whether we hold data about Frog | 548 variants: A list specifying whether we hold data about Frog |
| 378 generated code, plain JS code, or a combination of both, or | 549 generated code, plain JS code, or a combination of both, or |
| 379 Dart depending on the test. | 550 Dart depending on the test. |
| 380 values_list: A list containing the type of data we will be graphing | 551 values_list: A list containing the type of data we will be graphing |
| 381 (benchmarks, percentage passing, etc). | 552 (benchmarks, percentage passing, etc). |
| 382 test_runner: Reference to the parent test runner object that notifies a | 553 test_runner: Reference to the parent test runner object that notifies a |
| 383 test when to run. | 554 test when to run. |
| 384 tester: The visitor that actually performs the test running mechanics. | 555 tester: The visitor that actually performs the test running mechanics. |
| 385 file_processor: The visitor that processes files in the format | 556 file_processor: The visitor that processes files in the format |
| 386 appropriate for this test. | 557 appropriate for this test. |
| 558 grapher: The visitor that generates graphs given our test result data. | |
| 559 extra_metrics: A list of any additional measurements we wish to keep | |
| 560 track of (such as the geometric mean of a set, the sum, etc). | |
| 387 build_targets: The targets necessary to build to run these tests | 561 build_targets: The targets necessary to build to run these tests |
| 388 (default target is create_sdk).""" | 562 (default target is create_sdk).""" |
| 389 super(RuntimePerformanceTest, self).__init__(result_folder_name, | 563 super(RuntimePerformanceTest, self).__init__(result_folder_name, |
| 390 platform_list, versions, benchmarks, test_runner, tester, | 564 platform_list, versions, benchmarks, test_runner, tester, |
| 391 file_processor, build_targets=build_targets) | 565 file_processor, self.RuntimePerfGrapher(self), |
| 566 build_targets=build_targets) | |
| 392 self.platform_list = platform_list | 567 self.platform_list = platform_list |
| 393 self.platform_type = platform_type | 568 self.platform_type = platform_type |
| 394 self.versions = versions | 569 self.versions = versions |
| 395 self.benchmarks = benchmarks | 570 self.benchmarks = benchmarks |
| 396 | 571 |
| 572 class RuntimePerfGrapher(Grapher): | |
| 573 def plot_all_perf(self, png_filename): | |
| 574 """Create a plot that shows the performance changes of individual | |
| 575 benchmarks run by JS and generated by frog, over svn history.""" | |
| 576 for benchmark in self.test.benchmarks: | |
| 577 self.style_and_save_perf_plot( | |
| 578 'Performance of %s over time on the %s on %s' % (benchmark, | |
| 579 self.test.platform_type, utils.GuessOS()), | |
| 580 'Speed (bigger = better)', 16, 14, 'lower left', | |
| 581 benchmark + png_filename, self.test.platform_list, | |
| 582 self.test.versions, [benchmark]) | |
| 583 | |
| 584 def plot_avg_perf(self, png_filename): | |
| 585 """Generate a plot that shows the performance changes of the geomentric | |
| 586 mean of JS and frog benchmark performance over svn history.""" | |
| 587 (title, y_axis, size_x, size_y, loc, filename) = \ | |
| 588 ('Geometric Mean of benchmark %s performance on %s ' % | |
| 589 (self.test.platform_type, utils.GuessOS()), 'Speed (bigger = better)', | |
| 590 16, 5, 'lower left', 'avg'+png_filename) | |
| 591 clear_axis = True | |
| 592 for platform in self.test.platform_list: | |
| 593 for version in self.test.versions: | |
| 594 if self.test.is_valid_combination(platform, version): | |
| 595 for metric in self.test.extra_metrics: | |
| 596 self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, | |
| 597 filename, [platform], [version], | |
| 598 [metric], clear_axis) | |
| 599 clear_axis = False | |
| 600 | |
| 601 def plot_results(self, png_filename): | |
| 602 self.plot_all_perf(png_filename) | |
| 603 self.plot_avg_perf('2' + png_filename) | |
| 604 | |
| 397 | 605 |
| 398 class BrowserTester(Tester): | 606 class BrowserTester(Tester): |
| 399 @staticmethod | 607 @staticmethod |
| 400 def get_browsers(add_dartium=True): | 608 def get_browsers(add_dartium=True): |
| 401 browsers = ['ff', 'chrome'] | 609 browsers = ['ff', 'chrome'] |
| 402 if add_dartium: | 610 if add_dartium: |
| 403 browsers += ['dartium'] | 611 browsers += ['dartium'] |
| 404 has_shell = False | 612 has_shell = False |
| 405 if platform.system() == 'Darwin': | 613 if platform.system() == 'Darwin': |
| 406 browsers += ['safari'] | 614 browsers += ['safari'] |
| (...skipping 100 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 507 return True | 715 return True |
| 508 upload_success = True | 716 upload_success = True |
| 509 for result in results: | 717 for result in results: |
| 510 name_and_score = result.split(':') | 718 name_and_score = result.split(':') |
| 511 if len(name_and_score) < 2: | 719 if len(name_and_score) < 2: |
| 512 break | 720 break |
| 513 name = name_and_score[0].strip() | 721 name = name_and_score[0].strip() |
| 514 score = name_and_score[1].strip() | 722 score = name_and_score[1].strip() |
| 515 if version == 'js' or version == 'v8': | 723 if version == 'js' or version == 'v8': |
| 516 version = 'js' | 724 version = 'js' |
| 517 upload_success = upload_success and self.report_results( | 725 bench_dict = self.test.values_dict[browser][version] |
| 518 name, score, browser, version, revision_num, self.SCORE) | 726 bench_dict[name] += [float(score)] |
| 727 self.test.revision_dict[browser][version][name] += [revision_num] | |
| 728 if self.should_report_results(afile): | |
| 729 upload_success = upload_success and self.report_results( | |
| 730 name, score, browser, version, revision_num, self.SCORE) | |
| 731 else: | |
| 732 upload_success = False | |
| 519 | 733 |
| 520 f.close() | 734 f.close() |
| 735 self.calculate_geometric_mean(browser, version, revision_num) | |
| 521 return upload_success | 736 return upload_success |
| 522 | 737 |
| 523 | 738 |
| 524 class DromaeoTester(Tester): | 739 class DromaeoTester(Tester): |
| 525 DROMAEO_BENCHMARKS = { | 740 DROMAEO_BENCHMARKS = { |
| 526 'attr': ('attributes', [ | 741 'attr': ('attributes', [ |
| 527 'getAttribute', | 742 'getAttribute', |
| 528 'element.property', | 743 'element.property', |
| 529 'setAttribute', | 744 'setAttribute', |
| 530 'element.property = value']), | 745 'element.property = value']), |
| (...skipping 124 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 655 class DromaeoFileProcessor(Processor): | 870 class DromaeoFileProcessor(Processor): |
| 656 def process_file(self, afile): | 871 def process_file(self, afile): |
| 657 """Comb through the html to find the performance results. | 872 """Comb through the html to find the performance results. |
| 658 Returns: True if we successfully posted our data to storage.""" | 873 Returns: True if we successfully posted our data to storage.""" |
| 659 if self.test.test_runner.no_upload: | 874 if self.test.test_runner.no_upload: |
| 660 return | 875 return |
| 661 parts = afile.split('-') | 876 parts = afile.split('-') |
| 662 browser = parts[2] | 877 browser = parts[2] |
| 663 version = parts[3] | 878 version = parts[3] |
| 664 | 879 |
| 880 bench_dict = self.test.values_dict[browser][version] | |
| 881 | |
| 665 f = open(os.path.join(self.test.result_folder_name, afile)) | 882 f = open(os.path.join(self.test.result_folder_name, afile)) |
| 666 lines = f.readlines() | 883 lines = f.readlines() |
| 667 i = 0 | 884 i = 0 |
| 668 revision_num = 0 | 885 revision_num = 0 |
| 669 revision_pattern = r'Revision: (\d+)' | 886 revision_pattern = r'Revision: (\d+)' |
| 670 suite_pattern = r'<div class="result-item done">(.+?)</ol></div>' | 887 suite_pattern = r'<div class="result-item done">(.+?)</ol></div>' |
| 671 result_pattern = r'<b>(.+?)</b>(.+?)<small> runs/s(.+)' | 888 result_pattern = r'<b>(.+?)</b>(.+?)<small> runs/s(.+)' |
| 672 | 889 |
| 673 upload_success = True | 890 upload_success = True |
| 674 for line in lines: | 891 for line in lines: |
| 675 rev = re.match(revision_pattern, line.strip()) | 892 rev = re.match(revision_pattern, line.strip()) |
| 676 if rev: | 893 if rev: |
| 677 revision_num = int(rev.group(1)) | 894 revision_num = int(rev.group(1)) |
| 678 continue | 895 continue |
| 679 | 896 |
| 680 suite_results = re.findall(suite_pattern, line) | 897 suite_results = re.findall(suite_pattern, line) |
| 681 if suite_results: | 898 if suite_results: |
| 682 for suite_result in suite_results: | 899 for suite_result in suite_results: |
| 683 results = re.findall(r'<li>(.*?)</li>', suite_result) | 900 results = re.findall(r'<li>(.*?)</li>', suite_result) |
| 684 if results: | 901 if results: |
| 685 for result in results: | 902 for result in results: |
| 686 r = re.match(result_pattern, result) | 903 r = re.match(result_pattern, result) |
| 687 name = DromaeoTester.legalize_filename(r.group(1).strip(':')) | 904 name = DromaeoTester.legalize_filename(r.group(1).strip(':')) |
| 688 score = float(r.group(2)) | 905 score = float(r.group(2)) |
| 689 upload_success = upload_success and self.report_results( | 906 bench_dict[name] += [float(score)] |
| 690 name, score, browser, version, revision_num, self.SCORE) | 907 self.test.revision_dict[browser][version][name] += \ |
| 908 [revision_num] | |
| 909 if self.should_report_results(afile): | |
| 910 upload_success = upload_success and self.report_results( | |
| 911 name, score, browser, version, revision_num, self.SCORE) | |
| 912 else: | |
| 913 upload_success = False | |
| 691 | 914 |
| 692 f.close() | 915 f.close() |
| 916 self.calculate_geometric_mean(browser, version, revision_num) | |
| 693 return upload_success | 917 return upload_success |
| 694 | 918 |
| 695 | 919 |
| 696 class DromaeoSizeTest(Test): | 920 class DromaeoSizeTest(Test): |
| 697 """Run tests to determine the compiled file output size of Dromaeo.""" | 921 """Run tests to determine the compiled file output size of Dromaeo.""" |
| 698 def __init__(self, test_runner): | 922 def __init__(self, test_runner): |
| 699 super(DromaeoSizeTest, self).__init__( | 923 super(DromaeoSizeTest, self).__init__( |
| 700 self.name(), | 924 self.name(), |
| 701 ['commandline'], ['dart', 'frog_dom', 'frog_html', | 925 ['commandline'], ['dart', 'frog_dom', 'frog_html', |
| 702 'frog_htmlidiomatic'], | 926 'frog_htmlidiomatic'], |
| 703 DromaeoTester.DROMAEO_BENCHMARKS.keys(), test_runner, | 927 DromaeoTester.DROMAEO_BENCHMARKS.keys(), test_runner, |
| 704 self.DromaeoSizeTester(self), | 928 self.DromaeoSizeTester(self), |
| 705 self.DromaeoSizeProcessor(self)) | 929 self.DromaeoSizeProcessor(self), |
| 930 self.DromaeoSizeGrapher(self), extra_metrics=['sum']) | |
| 706 | 931 |
| 707 @staticmethod | 932 @staticmethod |
| 708 def name(): | 933 def name(): |
| 709 return 'dromaeo-size' | 934 return 'dromaeo-size' |
| 710 | 935 |
| 711 | 936 |
| 712 class DromaeoSizeTester(DromaeoTester): | 937 class DromaeoSizeTester(DromaeoTester): |
| 713 def run_tests(self): | 938 def run_tests(self): |
| 714 # Build tests. | 939 # Build tests. |
| 715 dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo') | 940 dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo') |
| (...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 756 js_size = os.path.getsize(os.path.join(frog_path, name)) | 981 js_size = os.path.getsize(os.path.join(frog_path, name)) |
| 757 except OSError: | 982 except OSError: |
| 758 pass #If compilation failed, continue on running other tests. | 983 pass #If compilation failed, continue on running other tests. |
| 759 | 984 |
| 760 total_size[variant] += js_size | 985 total_size[variant] += js_size |
| 761 self.test.test_runner.run_cmd( | 986 self.test.test_runner.run_cmd( |
| 762 ['echo', 'Size (%s, %s): %s' % (variant, suite, str(js_size))], | 987 ['echo', 'Size (%s, %s): %s' % (variant, suite, str(js_size))], |
| 763 self.test.trace_file, append=True) | 988 self.test.trace_file, append=True) |
| 764 | 989 |
| 765 self.test.test_runner.run_cmd( | 990 self.test.test_runner.run_cmd( |
| 766 ['echo', 'Size (dart, %s): %s' % (total_dart_size, 'sum')], | 991 ['echo', 'Size (dart, %s): %s' % (total_dart_size, |
| 992 self.test.extra_metrics[0])], | |
| 767 self.test.trace_file, append=True) | 993 self.test.trace_file, append=True) |
| 768 for (variant, _) in variants: | 994 for (variant, _) in variants: |
| 769 self.test.test_runner.run_cmd( | 995 self.test.test_runner.run_cmd( |
| 770 ['echo', 'Size (%s, %s): %s' % (variant, 'sum', | 996 ['echo', 'Size (%s, %s): %s' % (variant, self.test.extra_metrics[0], |
| 771 total_size[variant])], | 997 total_size[variant])], |
| 772 self.test.trace_file, append=True) | 998 self.test.trace_file, append=True) |
| 773 | 999 |
| 774 | 1000 |
| 775 class DromaeoSizeProcessor(Processor): | 1001 class DromaeoSizeProcessor(Processor): |
| 776 def process_file(self, afile): | 1002 def process_file(self, afile): |
| 777 """Pull all the relevant information out of a given tracefile. | 1003 """Pull all the relevant information out of a given tracefile. |
| 778 | 1004 |
| 779 Args: | 1005 Args: |
| 780 afile: is the filename string we will be processing. | 1006 afile: is the filename string we will be processing. |
| (...skipping 17 matching lines...) Expand all Loading... | |
| 798 | 1024 |
| 799 result = re.match(result_pattern, line.strip()) | 1025 result = re.match(result_pattern, line.strip()) |
| 800 if result: | 1026 if result: |
| 801 variant = result.group(1) | 1027 variant = result.group(1) |
| 802 metric = result.group(2) | 1028 metric = result.group(2) |
| 803 num = result.group(3) | 1029 num = result.group(3) |
| 804 if num.find('.') == -1: | 1030 if num.find('.') == -1: |
| 805 num = int(num) | 1031 num = int(num) |
| 806 else: | 1032 else: |
| 807 num = float(num) | 1033 num = float(num) |
| 808 upload_success = upload_success and self.report_results( | 1034 self.test.values_dict['commandline'][variant][metric] += [num] |
| 809 metric, num, 'commandline', variant, revision_num, self.CODE_SIZE) | 1035 self.test.revision_dict['commandline'][variant][metric] += \ |
| 1036 [revision_num] | |
| 1037 if self.should_report_results(afile): | |
| 1038 upload_success = upload_success and self.report_results( | |
| 1039 metric, num, 'commandline', variant, revision_num, | |
| 1040 self.CODE_SIZE) | |
| 1041 else: | |
| 1042 upload_success = False | |
| 810 | 1043 |
| 811 f.close() | 1044 f.close() |
| 812 return upload_success | 1045 return upload_success |
| 813 | 1046 |
| 1047 class DromaeoSizeGrapher(Grapher): | |
| 1048 def plot_results(self, png_filename): | |
| 1049 self.style_and_save_perf_plot( | |
| 1050 'Compiled Dromaeo Sizes', | |
| 1051 'Size (in bytes)', 10, 10, 'lower left', png_filename, | |
| 1052 ['commandline'], | |
| 1053 ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'], | |
| 1054 DromaeoTester.DROMAEO_BENCHMARKS.keys()) | |
| 1055 | |
| 1056 self.style_and_save_perf_plot( | |
| 1057 'Compiled Dromaeo Sizes', | |
| 1058 'Size (in bytes)', 10, 10, 'lower left', '2' + png_filename, | |
| 1059 ['commandline'], | |
| 1060 ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'], | |
| 1061 [self.test.extra_metrics[0]]) | |
| 814 | 1062 |
| 815 class CompileTimeAndSizeTest(Test): | 1063 class CompileTimeAndSizeTest(Test): |
| 816 """Run tests to determine how long frogc takes to compile, and the compiled | 1064 """Run tests to determine how long frogc takes to compile, and the compiled |
| 817 file output size of some benchmarking files.""" | 1065 file output size of some benchmarking files.""" |
| 818 def __init__(self, test_runner): | 1066 def __init__(self, test_runner): |
| 819 """Reference to the test_runner object that notifies us when to begin | 1067 """Reference to the test_runner object that notifies us when to begin |
| 820 testing.""" | 1068 testing.""" |
| 821 super(CompileTimeAndSizeTest, self).__init__( | 1069 super(CompileTimeAndSizeTest, self).__init__( |
| 822 self.name(), ['commandline'], ['frog'], ['swarm', 'total'], | 1070 self.name(), ['commandline'], ['frog'], ['swarm', 'total'], |
| 823 test_runner, self.CompileTester(self), | 1071 test_runner, self.CompileTester(self), |
| 824 self.CompileProcessor(self)) | 1072 self.CompileProcessor(self), self.CompileGrapher(self)) |
| 825 self.dart_compiler = os.path.join( | 1073 self.dart_compiler = os.path.join( |
| 826 DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(), | 1074 DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(), |
| 827 'release', 'ia32'), 'dart-sdk', 'bin', 'frogc') | 1075 'release', 'ia32'), 'dart-sdk', 'bin', 'frogc') |
| 828 _suffix = '' | 1076 _suffix = '' |
| 829 if platform.system() == 'Windows': | 1077 if platform.system() == 'Windows': |
| 830 _suffix = '.exe' | 1078 _suffix = '.exe' |
| 831 self.dart_vm = os.path.join( | 1079 self.dart_vm = os.path.join( |
| 832 DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(), | 1080 DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(), |
| 833 'release', 'ia32'), 'dart-sdk', 'bin','dart' + _suffix) | 1081 'release', 'ia32'), 'dart-sdk', 'bin','dart' + _suffix) |
| 834 self.failure_threshold = {'swarm' : 100, 'total' : 100} | 1082 self.failure_threshold = {'swarm' : 100, 'total' : 100} |
| (...skipping 65 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 900 if 'Revision' in line: | 1148 if 'Revision' in line: |
| 901 revision_num = int(line.split()[1]) | 1149 revision_num = int(line.split()[1]) |
| 902 else: | 1150 else: |
| 903 for metric in self.test.values_list: | 1151 for metric in self.test.values_list: |
| 904 if metric in line: | 1152 if metric in line: |
| 905 num = tokens[0] | 1153 num = tokens[0] |
| 906 if num.find('.') == -1: | 1154 if num.find('.') == -1: |
| 907 num = int(num) | 1155 num = int(num) |
| 908 else: | 1156 else: |
| 909 num = float(num) | 1157 num = float(num) |
| 1158 self.test.values_dict['commandline']['frog'][metric] += [num] | |
| 1159 self.test.revision_dict['commandline']['frog'][metric] += \ | |
| 1160 [revision_num] | |
| 910 score_type = self.CODE_SIZE | 1161 score_type = self.CODE_SIZE |
| 911 if 'Compiling' in metric or 'Bootstrapping' in metric: | 1162 if 'Compiling' in metric or 'Bootstrapping' in metric: |
| 912 score_type = self.COMPILE_TIME | 1163 score_type = self.COMPILE_TIME |
| 913 upload_success = upload_success and self.report_results( | 1164 if self.should_report_results(afile): |
| 914 metric, num, 'commandline', 'frog', revision_num, score_type) | 1165 upload_success = upload_success and self.report_results( |
| 1166 metric, num, 'commandline', 'frog', revision_num, | |
| 1167 score_type) | |
| 1168 else: | |
| 1169 upload_success = False | |
| 1170 if revision_num != 0:» » | |
|
vsm
2012/05/02 22:29:18
It looks like there is extra trailing whitespace o
| |
| 1171 for metric in self.test.values_list:» » | |
| 1172 self.test.revision_dict['commandline']['frog'][metric].pop()» » | |
| 1173 self.test.revision_dict['commandline']['frog'][metric] += \ | |
| 1174 [revision_num]» » | |
| 1175 # Fill in 0 if compilation failed.» » | |
| 1176 if self.test.values_dict['commandline']['frog'][metric][-1] < \ | |
| 1177 self.test.failure_threshold[metric]:» » | |
| 1178 self.test.values_dict['commandline']['frog'][metric] += [0] | |
| 1179 self.test.revision_dict['commandline']['frog'][metric] += \ | |
| 1180 [revision_num] | |
| 915 | 1181 |
| 916 f.close() | 1182 f.close() |
| 917 return upload_success | 1183 return upload_success |
| 918 | 1184 |
| 1185 class CompileGrapher(Grapher): | |
| 1186 | |
| 1187 def plot_results(self, png_filename): | |
| 1188 self.style_and_save_perf_plot( | |
| 1189 'Compiled frog sizes', 'Size (in bytes)', 10, 10, 'lower left', | |
| 1190 png_filename, ['commandline'], ['frog'], ['swarm', 'total']) | |
| 1191 | |
| 919 | 1192 |
| 920 class TestBuilder(object): | 1193 class TestBuilder(object): |
| 921 """Construct the desired test object.""" | 1194 """Construct the desired test object.""" |
| 922 available_suites = dict((suite.name(), suite) for suite in [ | 1195 available_suites = dict((suite.name(), suite) for suite in [ |
| 923 CompileTimeAndSizeTest, CommonBrowserTest, DromaeoTest, DromaeoSizeTest]) | 1196 CompileTimeAndSizeTest, CommonBrowserTest, DromaeoTest, DromaeoSizeTest]) |
| 924 | 1197 |
| 925 @staticmethod | 1198 @staticmethod |
| 926 def make_test(test_name, test_runner): | 1199 def make_test(test_name, test_runner): |
| 927 return TestBuilder.available_suites[test_name](test_runner) | 1200 return TestBuilder.available_suites[test_name](test_runner) |
| 928 | 1201 |
| 929 @staticmethod | 1202 @staticmethod |
| 930 def available_suite_names(): | 1203 def available_suite_names(): |
| 931 return TestBuilder.available_suites.keys() | 1204 return TestBuilder.available_suites.keys() |
| 932 | 1205 |
| 933 | 1206 |
| 934 def main(): | 1207 def main(): |
| 935 runner = TestRunner() | 1208 runner = TestRunner() |
| 936 continuous = runner.parse_args() | 1209 continuous = runner.parse_args() |
| 937 if continuous: | 1210 if continuous: |
| 938 while True: | 1211 while True: |
| 939 if runner.has_new_code(): | 1212 if runner.has_new_code(): |
| 940 runner.run_test_sequence() | 1213 runner.run_test_sequence() |
| 941 else: | 1214 else: |
| 942 time.sleep(200) | 1215 time.sleep(200) |
| 943 else: | 1216 else: |
| 944 runner.run_test_sequence() | 1217 runner.run_test_sequence() |
| 945 | 1218 |
| 946 if __name__ == '__main__': | 1219 if __name__ == '__main__': |
| 947 main() | 1220 main() |
| OLD | NEW |
