Chromium Code Reviews Chromium Code Reviews
Issue 10164009:
Don't upload data to appengine. (Closed)
Base URL: http://dart.googlecode.com/svn/branches/bleeding_edge/dart/| OLD | NEW |
|---|---|
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 | 2 |
| 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 4 # for details. All rights reserved. Use of this source code is governed by a | 4 # for details. All rights reserved. Use of this source code is governed by a |
| 5 # BSD-style license that can be found in the LICENSE file. | 5 # BSD-style license that can be found in the LICENSE file. |
| 6 | 6 |
| 7 import datetime | 7 import datetime |
| 8 import getpass | |
| 9 import math | |
| 10 try: | |
| 11 from matplotlib.font_manager import FontProperties | |
| 12 import matplotlib.pyplot as plt | |
| 13 except ImportError: | |
| 14 pass # Only needed if we want to make graphs. | |
| 15 import optparse | 8 import optparse |
| 16 import os | 9 import os |
| 17 from os.path import dirname, abspath | 10 from os.path import dirname, abspath |
| 18 import platform | 11 import platform |
| 19 import re | 12 import re |
| 20 import shutil | 13 import shutil |
| 21 import stat | 14 import stat |
| 22 import subprocess | 15 import subprocess |
| 23 import sys | 16 import sys |
| 24 import time | 17 import time |
| 25 import traceback | |
| 26 | 18 |
| 27 TOOLS_PATH = os.path.join(dirname(dirname(dirname(abspath(__file__))))) | 19 TOOLS_PATH = os.path.join(dirname(dirname(dirname(abspath(__file__))))) |
| 28 DART_INSTALL_LOCATION = abspath(os.path.join(dirname(abspath(__file__)), | 20 DART_INSTALL_LOCATION = abspath(os.path.join(dirname(abspath(__file__)), |
| 29 '..', '..', '..')) | 21 '..', '..', '..')) |
| 30 sys.path.append(TOOLS_PATH) | 22 sys.path.append(TOOLS_PATH) |
| 23 sys.path.append(os.path.join(DART_INSTALL_LOCATION, 'internal', 'tests')) | |
| 24 import post_results | |
| 31 import utils | 25 import utils |
| 32 | 26 |
| 33 """This script runs to track performance and size progress of | 27 """This script runs to track performance and size progress of |
| 34 different svn revisions. It tests to see if there a newer version of the code on | 28 different svn revisions. It tests to see if there a newer version of the code on |
| 35 the server, and will sync and run the performance tests if so.""" | 29 the server, and will sync and run the performance tests if so.""" |
| 36 class TestRunner(object): | 30 class TestRunner(object): |
| 37 | 31 |
| 38 def __init__(self): | 32 def __init__(self): |
| 39 self.verbose = False | 33 self.verbose = False |
| 40 self.has_shell = False | 34 self.has_shell = False |
| (...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 164 def get_os_directory(self): | 158 def get_os_directory(self): |
| 165 """Specifies the name of the directory for the testing build of dart, which | 159 """Specifies the name of the directory for the testing build of dart, which |
| 166 has yet a different naming convention from utils.getBuildRoot(...).""" | 160 has yet a different naming convention from utils.getBuildRoot(...).""" |
| 167 if platform.system() == 'Windows': | 161 if platform.system() == 'Windows': |
| 168 return 'windows' | 162 return 'windows' |
| 169 elif platform.system() == 'Darwin': | 163 elif platform.system() == 'Darwin': |
| 170 return 'macos' | 164 return 'macos' |
| 171 else: | 165 else: |
| 172 return 'linux' | 166 return 'linux' |
| 173 | 167 |
| 174 def upload_to_app_engine(self, suite_names): | |
| 175 """Upload our results to our appengine server. | |
| 176 Arguments: | |
| 177 suite_names: Directories to upload data from (should match directory | |
| 178 names).""" | |
| 179 # TODO(efortuna): This is the most basic way to get the data up | |
| 180 # for others to view. Revisit this once we're serving nicer graphs (Google | |
| 181 # Chart Tools) and from multiple perfbots and once we're in a position to | |
| 182 # organize the data in a useful manner(!!). | |
| 183 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', | |
| 184 'perf_testing')) | |
| 185 for data in suite_names: | |
| 186 path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS()) | |
| 187 shutil.rmtree(path, ignore_errors=True) | |
| 188 os.makedirs(path) | |
| 189 files = [] | |
| 190 # Copy the 1000 most recent trace files to be uploaded. | |
| 191 for f in os.listdir(data): | |
| 192 files += [(os.path.getmtime(os.path.join(data, f)), f)] | |
| 193 files.sort() | |
| 194 for f in files[-1000:]: | |
| 195 shutil.copyfile(os.path.join(data, f[1]), | |
| 196 os.path.join(path, f[1]+'.txt')) | |
| 197 # Generate directory listing. | |
| 198 for data in suite_names: | |
| 199 path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS()) | |
| 200 out = open(os.path.join('appengine', 'static', | |
| 201 '%s-%s.html' % (data, utils.GuessOS())), 'w') | |
| 202 out.write('<html>\n <body>\n <ul>\n') | |
| 203 for f in os.listdir(path): | |
| 204 if not f.startswith('.'): | |
| 205 out.write(' <li><a href=data' + \ | |
| 206 '''/%(data)s/%(os)s/%(file)s>%(file)s</a></li>\n''' % \ | |
| 207 {'data': data, 'os': utils.GuessOS(), 'file': f}) | |
| 208 out.write(' </ul>\n </body>\n</html>') | |
| 209 out.close() | |
| 210 | |
| 211 shutil.rmtree(os.path.join('appengine', 'static', 'graphs'), | |
| 212 ignore_errors=True) | |
| 213 shutil.copytree('graphs', os.path.join('appengine', 'static', 'graphs')) | |
| 214 shutil.copyfile('index.html', os.path.join('appengine', 'static', | |
| 215 'index.html')) | |
| 216 shutil.copyfile('dromaeo.html', os.path.join('appengine', 'static', | |
| 217 'dromaeo.html')) | |
| 218 shutil.copyfile('data.html', os.path.join('appengine', 'static', | |
| 219 'data.html')) | |
| 220 self.run_cmd([os.path.join('..', '..', '..', 'third_party', | |
| 221 'appengine-python', 'appcfg.py'), '--oauth2', | |
| 222 'update', 'appengine/']) | |
| 223 | |
| 224 def parse_args(self): | 168 def parse_args(self): |
| 225 parser = optparse.OptionParser() | 169 parser = optparse.OptionParser() |
| 226 parser.add_option('--suites', '-s', dest='suites', help='Run the specified ' | 170 parser.add_option('--suites', '-s', dest='suites', help='Run the specified ' |
| 227 'comma-separated test suites from set: %s' % \ | 171 'comma-separated test suites from set: %s' % \ |
| 228 ','.join(TestBuilder.available_suite_names()), | 172 ','.join(TestBuilder.available_suite_names()), |
| 229 action='store', default=None) | 173 action='store', default=None) |
| 230 parser.add_option('--forever', '-f', dest='continuous', help='Run this scri' | 174 parser.add_option('--forever', '-f', dest='continuous', help='Run this scri' |
| 231 'pt forever, always checking for the next svn checkin', | 175 'pt forever, always checking for the next svn checkin', |
| 232 action='store_true', default=False) | 176 action='store_true', default=False) |
| 233 parser.add_option('--graph-only', '-g', dest='graph_only', default=False, | |
| 234 help='Do not run tests, only regenerate graphs', | |
| 235 action='store_true') | |
| 236 parser.add_option('--nobuild', '-n', dest='no_build', action='store_true', | 177 parser.add_option('--nobuild', '-n', dest='no_build', action='store_true', |
| 237 help='Do not sync with the repository and do not ' | 178 help='Do not sync with the repository and do not ' |
| 238 'rebuild.', default=False) | 179 'rebuild.', default=False) |
| 239 parser.add_option('--upload', '-u', dest='upload', help='Upload data to ' | |
| 240 'app engine (will require authentication).', | |
| 241 action='store_true', default=False) | |
| 242 parser.add_option('--verbose', '-v', dest='verbose', help='Print extra ' | 180 parser.add_option('--verbose', '-v', dest='verbose', help='Print extra ' |
| 243 'debug output', action='store_true', default=False) | 181 'debug output', action='store_true', default=False) |
| 244 | 182 |
| 245 args, ignored = parser.parse_args() | 183 args, ignored = parser.parse_args() |
| 246 | 184 |
| 247 if not args.suites: | 185 if not args.suites: |
| 248 suites = TestBuilder.available_suite_names() | 186 suites = TestBuilder.available_suite_names() |
| 249 else: | 187 else: |
| 250 suites = [] | 188 suites = [] |
| 251 suitelist = args.suites.split(',') | 189 suitelist = args.suites.split(',') |
| 252 for name in suitelist: | 190 for name in suitelist: |
| 253 if name in TestBuilder.available_suite_names(): | 191 if name in TestBuilder.available_suite_names(): |
| 254 suites.append(name) | 192 suites.append(name) |
| 255 else: | 193 else: |
| 256 print ('Error: Invalid suite %s not in ' % name) + \ | 194 print ('Error: Invalid suite %s not in ' % name) + \ |
| 257 '%s' % ','.join(TestBuilder.available_suite_names()) | 195 '%s' % ','.join(TestBuilder.available_suite_names()) |
| 258 sys.exit(1) | 196 sys.exit(1) |
| 259 self.suite_names = suites | 197 self.suite_names = suites |
| 260 self.no_build = args.no_build | 198 self.no_build = args.no_build |
| 261 self.graph_only = args.graph_only | |
| 262 self.upload = args.upload | |
| 263 self.verbose = args.verbose | 199 self.verbose = args.verbose |
| 264 return args.continuous | 200 return args.continuous |
| 265 | 201 |
| 266 def run_test_sequence(self): | 202 def run_test_sequence(self): |
| 267 """Run the set of commands to (possibly) build, run, and graph the results | 203 """Run the set of commands to (possibly) build, run, and graph the results |
| 268 of our tests. | 204 of our tests. |
| 269 | 205 |
| 270 Args: | 206 Args: |
| 271 suite_names: The "display name" the user enters to specify which | 207 suite_names: The "display name" the user enters to specify which |
| 272 benchmark(s) to run. | 208 benchmark(s) to run. |
| 273 no_build: True if we should not check the repository and build the latest | 209 no_build: True if we should not check the repository and build the latest |
| 274 version. | 210 version. |
| 275 graph_only: True if we should not run the tests, just (re)generate graphs. | 211 """ |
| 276 upload: True if we should upload our results to appengine.""" | |
| 277 suites = [] | 212 suites = [] |
| 278 for name in self.suite_names: | 213 for name in self.suite_names: |
| 279 suites += [TestBuilder.make_test(name, self)] | 214 suites += [TestBuilder.make_test(name, self)] |
| 280 | 215 |
| 281 if not self.no_build and self.sync_and_build(suites) == 1: | 216 if not self.no_build and self.sync_and_build(suites) == 1: |
| 282 return # The build is broken. | 217 return # The build is broken. |
| 283 | 218 |
| 284 for test in suites: | 219 for test in suites: |
| 285 test.run(self.graph_only) | 220 test.run() |
| 286 | |
| 287 if self.upload: | |
| 288 self.upload_to_app_engine(TestBuilder.available_suite_names()) | |
| 289 | 221 |
| 290 | 222 |
| 291 class Test(object): | 223 class Test(object): |
| 292 """The base class to provide shared code for different tests we will run and | 224 """The base class to provide shared code for different tests we will run and |
| 293 graph. At a high level, each test has three visitors (the tester, the | 225 graph. At a high level, each test has three visitors (the tester, the |
| 294 file_processor, and the grapher) that perform operations on the test | 226 file_processor that perform operations on the test object.""" |
| 295 object.""" | |
| 296 | 227 |
| 297 def __init__(self, result_folder_name, platform_list, variants, | 228 def __init__(self, result_folder_name, platform_list, variants, |
| 298 values_list, test_runner, tester, file_processor, grapher, | 229 values_list, test_runner, tester, file_processor, |
| 299 extra_metrics=['Geo-Mean'], build_targets=['create_sdk']): | 230 build_targets=['create_sdk']): |
| 300 """Args: | 231 """Args: |
| 301 result_folder_name: The name of the folder where a tracefile of | 232 result_folder_name: The name of the folder where a tracefile of |
| 302 performance results will be stored. | 233 performance results will be stored. |
| 303 platform_list: A list containing the platform(s) that our data has been | 234 platform_list: A list containing the platform(s) that our data has been |
| 304 run on. (command line, firefox, chrome, etc) | 235 run on. (command line, firefox, chrome, etc) |
| 305 variants: A list specifying whether we hold data about Frog | 236 variants: A list specifying whether we hold data about Frog |
| 306 generated code, plain JS code, or a combination of both, or | 237 generated code, plain JS code, or a combination of both, or |
| 307 Dart depending on the test. | 238 Dart depending on the test. |
| 308 values_list: A list containing the type of data we will be graphing | 239 values_list: A list containing the type of data we will be graphing |
| 309 (benchmarks, percentage passing, etc). | 240 (benchmarks, percentage passing, etc). |
| 310 test_runner: Reference to the parent test runner object that notifies a | 241 test_runner: Reference to the parent test runner object that notifies a |
| 311 test when to run. | 242 test when to run. |
| 312 tester: The visitor that actually performs the test running mechanics. | 243 tester: The visitor that actually performs the test running mechanics. |
| 313 file_processor: The visitor that processes files in the format | 244 file_processor: The visitor that processes files in the format |
| 314 appropriate for this test. | 245 appropriate for this test. |
| 315 grapher: The visitor that generates graphs given our test result data. | |
| 316 extra_metrics: A list of any additional measurements we wish to keep | |
| 317 track of (such as the geometric mean of a set, the sum, etc). | |
| 318 build_targets: The targets necessary to build to run these tests | 246 build_targets: The targets necessary to build to run these tests |
| 319 (default target is create_sdk).""" | 247 (default target is create_sdk).""" |
| 320 self.result_folder_name = result_folder_name | 248 self.result_folder_name = result_folder_name |
| 321 # cur_time is used as a timestamp of when this performance test was run. | 249 # cur_time is used as a timestamp of when this performance test was run. |
| 322 self.cur_time = str(time.mktime(datetime.datetime.now().timetuple())) | 250 self.cur_time = str(time.mktime(datetime.datetime.now().timetuple())) |
| 323 self.values_list = values_list | 251 self.values_list = values_list |
| 324 self.platform_list = platform_list | 252 self.platform_list = platform_list |
| 325 self.revision_dict = dict() | |
| 326 self.values_dict = dict() | |
| 327 self.test_runner = test_runner | 253 self.test_runner = test_runner |
| 328 self.tester = tester | 254 self.tester = tester |
| 329 self.file_processor = file_processor | 255 self.file_processor = file_processor |
| 330 self.grapher = grapher | |
| 331 self.extra_metrics = extra_metrics | |
| 332 self.build_targets = build_targets | 256 self.build_targets = build_targets |
| 333 # Initialize our values store. | |
| 334 for platform in platform_list: | |
| 335 self.revision_dict[platform] = dict() | |
| 336 self.values_dict[platform] = dict() | |
| 337 for f in variants: | |
| 338 self.revision_dict[platform][f] = dict() | |
| 339 self.values_dict[platform][f] = dict() | |
| 340 for val in values_list: | |
| 341 self.revision_dict[platform][f][val] = [] | |
| 342 self.values_dict[platform][f][val] = [] | |
| 343 for extra_metric in extra_metrics: | |
| 344 self.revision_dict[platform][f][extra_metric] = [] | |
| 345 self.values_dict[platform][f][extra_metric] = [] | |
| 346 | 257 |
| 347 def is_valid_combination(self, platform, variant): | 258 def is_valid_combination(self, platform, variant): |
| 348 """Check whether data should be captured for this platform/variant | 259 """Check whether data should be captured for this platform/variant |
| 349 combination. | 260 combination. |
| 350 """ | 261 """ |
| 351 return True | 262 return True |
| 352 | 263 |
| 353 def run(self, graph_only): | 264 def run(self): |
| 354 """Run the benchmarks/tests from the command line and plot the | 265 """Run the benchmarks/tests from the command line and plot the |
| 355 results. | 266 results. |
| 356 | 267 """ |
| 357 Args: | 268 for visitor in [self.tester, self.file_processor]: |
| 358 graph_only: True if we should just graph the results instead of also | |
| 359 running tests.""" | |
| 360 for visitor in [self.tester, self.file_processor, self.grapher]: | |
| 361 visitor.prepare() | 269 visitor.prepare() |
| 362 | 270 |
| 363 os.chdir(DART_INSTALL_LOCATION) | 271 os.chdir(DART_INSTALL_LOCATION) |
| 364 self.test_runner.ensure_output_directory(self.result_folder_name) | 272 self.test_runner.ensure_output_directory(self.result_folder_name) |
| 365 if not graph_only: | 273 self.tester.run_tests() |
| 366 self.tester.run_tests() | |
| 367 | 274 |
| 368 os.chdir(os.path.join('tools', 'testing', 'perf_testing')) | 275 os.chdir(os.path.join('tools', 'testing', 'perf_testing')) |
| 369 | 276 |
| 370 # TODO(efortuna): You will want to make this only use a subset of the files | 277 # TODO(efortuna): Remove trace files once uploaded. This will happen in a |
| 371 # eventually. | 278 # future CL. |
| 372 files = os.listdir(self.result_folder_name) | 279 files = os.listdir(self.result_folder_name) |
| 373 | |
| 374 for afile in files: | 280 for afile in files: |
| 375 if not afile.startswith('.'): | 281 if not afile.startswith('.'): |
| 376 self.file_processor.process_file(afile) | 282 self.file_processor.process_file(afile) |
| 377 | 283 |
| 378 if 'plt' in globals(): | |
| 379 # Only run Matplotlib if it is installed. | |
| 380 self.grapher.plot_results('%s.png' % self.result_folder_name) | |
| 381 | |
| 382 | 284 |
| 383 class Tester(object): | 285 class Tester(object): |
| 384 """The base level visitor class that runs tests. It contains convenience | 286 """The base level visitor class that runs tests. It contains convenience |
| 385 methods that many Tester objects use. Any class that would like to be a | 287 methods that many Tester objects use. Any class that would like to be a |
| 386 TesterVisitor must implement the run_tests() method.""" | 288 TesterVisitor must implement the run_tests() method.""" |
| 387 | 289 |
| 388 def __init__(self, test): | 290 def __init__(self, test): |
| 389 self.test = test | 291 self.test = test |
| 390 | 292 |
| 391 def prepare(self): | 293 def prepare(self): |
| (...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 426 methods that many File Processor objects use. Any class that would like to be | 328 methods that many File Processor objects use. Any class that would like to be |
| 427 a ProcessorVisitor must implement the process_file() method.""" | 329 a ProcessorVisitor must implement the process_file() method.""" |
| 428 | 330 |
| 429 def __init__(self, test): | 331 def __init__(self, test): |
| 430 self.test = test | 332 self.test = test |
| 431 | 333 |
| 432 def prepare(self): | 334 def prepare(self): |
| 433 """Perform any initial setup required before the test is run.""" | 335 """Perform any initial setup required before the test is run.""" |
| 434 pass | 336 pass |
| 435 | 337 |
| 436 def calculate_geometric_mean(self, platform, variant, svn_revision): | 338 def report_results(self, benchmark_name, score, platform, variant, |
| 437 """Calculate the aggregate geometric mean for JS and frog benchmark sets, | 339 revision_number): |
| 438 given two benchmark dictionaries.""" | 340 """Store the results of the benchmark run. |
| 439 geo_mean = 0 | 341 Args: |
| 440 # TODO(vsm): Suppress graphing this combination altogether. For | 342 benchmark_name: The name of the individual benchmark. |
| 441 # now, we feed a geomean of 0. | 343 score: The numerical value of this benchmark. |
| 442 if self.test.is_valid_combination(platform, variant): | 344 platform: The platform the test was run on (firefox, command line, etc). |
| 443 for benchmark in self.test.values_list: | 345 variant: Specifies whether the data was about generated Frog, js, a |
| 444 geo_mean += math.log( | 346 combination of both, or Dart depending on the test. |
| 445 self.test.values_dict[platform][variant][benchmark][ | 347 revision_number: The revision of the code (and sometimes the revision of |
| 446 len(self.test.values_dict[platform][variant][benchmark]) - 1]) | 348 dartium). |
| 447 | |
| 448 self.test.values_dict[platform][variant]['Geo-Mean'] += \ | |
| 449 [math.pow(math.e, geo_mean / len(self.test.values_list))] | |
| 450 self.test.revision_dict[platform][variant]['Geo-Mean'] += [svn_revision] | |
| 451 | |
| 452 | |
| 453 class Grapher(object): | |
| 454 """The base level visitor class that generates graphs for data. It contains | |
| 455 convenience methods that many Grapher objects use. Any class that would like | |
| 456 to be a GrapherVisitor must implement the plot_results() method.""" | |
| 457 | 349 |
| 458 graph_out_dir = 'graphs' | 350 Returns: True if the post was successful.""" |
| 459 | 351 return post_results.report_results(benchmark_name, score, platform, variant, |
| 460 def __init__(self, test): | 352 revision_number) |
| 461 self.color_index = 0 | |
| 462 self.test = test | |
| 463 | |
| 464 def prepare(self): | |
| 465 """Perform any initial setup required before the test is run.""" | |
| 466 if 'plt' in globals(): | |
| 467 plt.cla() # cla = clear current axes | |
| 468 else: | |
| 469 print 'Unable to import Matplotlib and therefore unable to generate ' + \ | |
| 470 'graphs. Please install it for this version of Python.' | |
| 471 self.test.test_runner.ensure_output_directory(Grapher.graph_out_dir) | |
| 472 | |
| 473 def style_and_save_perf_plot(self, chart_title, y_axis_label, size_x, size_y, | |
| 474 legend_loc, filename, platform_list, variants, | |
| 475 values_list, should_clear_axes=True): | |
| 476 """Sets style preferences for chart boilerplate that is consistent across | |
| 477 all charts, and saves the chart as a png. | |
| 478 | |
| 479 Args: | |
| 480 size_x: the size of the printed chart, in inches, in the horizontal | |
| 481 direction | |
| 482 size_y: the size of the printed chart, in inches in the vertical direction | |
| 483 legend_loc: the location of the legend in on the chart. See suitable | |
| 484 arguments for the loc argument in matplotlib | |
| 485 filename: the filename that we want to save the resulting chart as | |
| 486 platform_list: a list containing the platform(s) that our data has been | |
| 487 run on. (command line, firefox, chrome, etc) | |
| 488 values_list: a list containing the type of data we will be graphing | |
| 489 (performance, percentage passing, etc) | |
| 490 should_clear_axes: True if we want to create a fresh graph, instead of | |
| 491 plotting additional lines on the current graph.""" | |
| 492 if should_clear_axes: | |
| 493 plt.cla() # cla = clear current axes | |
| 494 for platform in platform_list: | |
| 495 for f in variants: | |
| 496 for val in values_list: | |
| 497 plt.plot(self.test.revision_dict[platform][f][val], | |
| 498 self.test.values_dict[platform][f][val], | |
| 499 color=self.get_color(), label='%s-%s-%s' % (platform, f, val)) | |
| 500 | |
| 501 plt.xlabel('Revision Number') | |
| 502 plt.ylabel(y_axis_label) | |
| 503 plt.title(chart_title) | |
| 504 fontP = FontProperties() | |
| 505 fontP.set_size('small') | |
| 506 plt.legend(loc=legend_loc, prop = fontP) | |
| 507 | |
| 508 fig = plt.gcf() | |
| 509 fig.set_size_inches(size_x, size_y) | |
| 510 fig.savefig(os.path.join(Grapher.graph_out_dir, filename)) | |
| 511 | |
| 512 def get_color(self): | |
| 513 # Just a bunch of distinct colors for a potentially large number of values | |
| 514 # we wish to graph. | |
| 515 colors = [ | |
| 516 'blue', 'green', 'red', 'cyan', 'magenta', 'black', '#3366CC', | |
| 517 '#DC3912', '#FF9900', '#109618', '#990099', '#0099C6', '#DD4477', | |
| 518 '#66AA00', '#B82E2E', '#316395', '#994499', '#22AA99', '#AAAA11', | |
| 519 '#6633CC', '#E67300', '#8B0707', '#651067', '#329262', '#5574A6', | |
| 520 '#3B3EAC', '#B77322', '#16D620', '#B91383', '#F4359E', '#9C5935', | |
| 521 '#A9C413', '#2A778D', '#668D1C', '#BEA413', '#0C5922', '#743411', | |
| 522 '#45AFE2', '#FF3300', '#FFCC00', '#14C21D', '#DF51FD', '#15CBFF', | |
| 523 '#FF97D2', '#97FB00', '#DB6651', '#518BC6', '#BD6CBD', '#35D7C2', | |
| 524 '#E9E91F', '#9877DD', '#FF8F20', '#D20B0B', '#B61DBA', '#40BD7E', | |
| 525 '#6AA7C4', '#6D70CD', '#DA9136', '#2DEA36', '#E81EA6', '#F558AE', | |
| 526 '#C07145', '#D7EE53', '#3EA7C6', '#97D129', '#E9CA1D', '#149638', | |
| 527 '#C5571D'] | |
| 528 color = colors[self.color_index] | |
| 529 self.color_index = (self.color_index + 1) % len(colors) | |
| 530 return color | |
| 531 | 353 |
| 532 | 354 |
| 533 class RuntimePerformanceTest(Test): | 355 class RuntimePerformanceTest(Test): |
| 534 """Super class for all runtime performance testing.""" | 356 """Super class for all runtime performance testing.""" |
| 535 | 357 |
| 536 def __init__(self, result_folder_name, platform_list, platform_type, | 358 def __init__(self, result_folder_name, platform_list, platform_type, |
| 537 versions, benchmarks, test_runner, tester, file_processor, | 359 versions, benchmarks, test_runner, tester, file_processor, |
| 538 build_targets=['create_sdk']): | 360 build_targets=['create_sdk']): |
| 539 """Args: | 361 """Args: |
| 540 result_folder_name: The name of the folder where a tracefile of | 362 result_folder_name: The name of the folder where a tracefile of |
| 541 performance results will be stored. | 363 performance results will be stored. |
| 542 platform_list: A list containing the platform(s) that our data has been | 364 platform_list: A list containing the platform(s) that our data has been |
| 543 run on. (command line, firefox, chrome, etc) | 365 run on. (command line, firefox, chrome, etc) |
| 544 variants: A list specifying whether we hold data about Frog | 366 variants: A list specifying whether we hold data about Frog |
| 545 generated code, plain JS code, or a combination of both, or | 367 generated code, plain JS code, or a combination of both, or |
| 546 Dart depending on the test. | 368 Dart depending on the test. |
| 547 values_list: A list containing the type of data we will be graphing | 369 values_list: A list containing the type of data we will be graphing |
| 548 (benchmarks, percentage passing, etc). | 370 (benchmarks, percentage passing, etc). |
| 549 test_runner: Reference to the parent test runner object that notifies a | 371 test_runner: Reference to the parent test runner object that notifies a |
| 550 test when to run. | 372 test when to run. |
| 551 tester: The visitor that actually performs the test running mechanics. | 373 tester: The visitor that actually performs the test running mechanics. |
| 552 file_processor: The visitor that processes files in the format | 374 file_processor: The visitor that processes files in the format |
| 553 appropriate for this test. | 375 appropriate for this test. |
| 554 grapher: The visitor that generates graphs given our test result data. | |
| 555 extra_metrics: A list of any additional measurements we wish to keep | |
| 556 track of (such as the geometric mean of a set, the sum, etc). | |
| 557 build_targets: The targets necessary to build to run these tests | 376 build_targets: The targets necessary to build to run these tests |
| 558 (default target is create_sdk).""" | 377 (default target is create_sdk).""" |
| 559 super(RuntimePerformanceTest, self).__init__(result_folder_name, | 378 super(RuntimePerformanceTest, self).__init__(result_folder_name, |
| 560 platform_list, versions, benchmarks, test_runner, tester, | 379 platform_list, versions, benchmarks, test_runner, tester, |
| 561 file_processor, self.RuntimePerfGrapher(self), | 380 file_processor, build_targets=build_targets) |
| 562 build_targets=build_targets) | |
| 563 self.platform_list = platform_list | 381 self.platform_list = platform_list |
| 564 self.platform_type = platform_type | 382 self.platform_type = platform_type |
| 565 self.versions = versions | 383 self.versions = versions |
| 566 self.benchmarks = benchmarks | 384 self.benchmarks = benchmarks |
| 567 | 385 |
| 568 class RuntimePerfGrapher(Grapher): | |
| 569 def plot_all_perf(self, png_filename): | |
| 570 """Create a plot that shows the performance changes of individual | |
| 571 benchmarks run by JS and generated by frog, over svn history.""" | |
| 572 for benchmark in self.test.benchmarks: | |
| 573 self.style_and_save_perf_plot( | |
| 574 'Performance of %s over time on the %s on %s' % (benchmark, | |
| 575 self.test.platform_type, utils.GuessOS()), | |
| 576 'Speed (bigger = better)', 16, 14, 'lower left', | |
| 577 benchmark + png_filename, self.test.platform_list, | |
| 578 self.test.versions, [benchmark]) | |
| 579 | |
| 580 def plot_avg_perf(self, png_filename): | |
| 581 """Generate a plot that shows the performance changes of the geomentric | |
| 582 mean of JS and frog benchmark performance over svn history.""" | |
| 583 (title, y_axis, size_x, size_y, loc, filename) = \ | |
| 584 ('Geometric Mean of benchmark %s performance on %s ' % | |
| 585 (self.test.platform_type, utils.GuessOS()), 'Speed (bigger = better)', | |
| 586 16, 5, 'lower left', 'avg'+png_filename) | |
| 587 clear_axis = True | |
| 588 for platform in self.test.platform_list: | |
| 589 for version in self.test.versions: | |
| 590 if self.test.is_valid_combination(platform, version): | |
| 591 for metric in self.test.extra_metrics: | |
| 592 self.style_and_save_perf_plot(title, y_axis, size_x, size_y, loc, | |
| 593 filename, [platform], [version], | |
| 594 [metric], clear_axis) | |
| 595 clear_axis = False | |
| 596 | |
| 597 def plot_results(self, png_filename): | |
| 598 self.plot_all_perf(png_filename) | |
| 599 self.plot_avg_perf('2' + png_filename) | |
| 600 | |
| 601 | |
| 602 class CommonCommandLineTest(RuntimePerformanceTest): | |
| 603 """Run the basic performance tests (Benchpress, some V8 benchmarks) from the | |
| 604 command line.""" | |
| 605 | |
| 606 def __init__(self, test_runner): | |
| 607 """Args: | |
| 608 test_runner: Reference to the object that notfies this test when to | |
| 609 run.""" | |
| 610 super(CommonCommandLineTest, self).__init__( | |
| 611 self.name(), ['commandline'], | |
| 612 'command line', ['js', 'frog'], self.get_standalone_benchmarks(), | |
| 613 test_runner, self.CommonCommandLineTester(self), | |
| 614 self.CommonCommandLineFileProcessor(self), | |
| 615 build_targets=['create_sdk', 'dart2js']) | |
| 616 | |
| 617 @staticmethod | |
| 618 def name(): | |
| 619 return 'cl-perf' | |
| 620 | |
| 621 @staticmethod | |
| 622 def get_standalone_benchmarks(): | |
| 623 return ['Mandelbrot', 'DeltaBlue', 'Richards', 'NBody', 'BinaryTrees', | |
| 624 'Fannkuch', 'Meteor', 'BubbleSort', 'Fibonacci', 'Loop', 'Permute', | |
| 625 'Queens', 'QuickSort', 'Recurse', 'Sieve', 'Sum', 'Tak', 'Takl', 'Towers', | |
| 626 'TreeSort'] | |
| 627 | |
| 628 class CommonCommandLineTester(Tester): | |
| 629 def run_tests(self): | |
| 630 """Run a performance test on our updated system.""" | |
| 631 os.chdir('frog') | |
| 632 self.test.trace_file = os.path.join( | |
| 633 '..', 'tools', 'testing', 'perf_testing', | |
| 634 self.test.result_folder_name, 'result' + self.test.cur_time) | |
| 635 self.test.test_runner.run_cmd(['python', os.path.join('benchmarks', | |
| 636 'perf_tests.py')], self.test.trace_file) | |
| 637 os.chdir('..') | |
| 638 | |
| 639 class CommonCommandLineFileProcessor(Processor): | |
| 640 def process_file(self, afile): | |
| 641 """Pull all the relevant information out of a given tracefile. | |
| 642 | |
| 643 Args: | |
| 644 afile: The filename string we will be processing.""" | |
| 645 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', | |
| 646 'testing', 'perf_testing')) | |
| 647 f = open(os.path.join(self.test.result_folder_name, afile)) | |
| 648 tabulate_data = False | |
| 649 revision_num = 0 | |
| 650 for line in f.readlines(): | |
| 651 if 'Revision' in line: | |
| 652 revision_num = int(line.split()[1]) | |
| 653 elif 'Benchmark' in line: | |
| 654 tabulate_data = True | |
| 655 elif tabulate_data: | |
| 656 tokens = line.split() | |
| 657 if len(tokens) < 4 or tokens[0] not in self.test.benchmarks: | |
| 658 #Done tabulating data. | |
| 659 break | |
| 660 js_value = float(tokens[1]) | |
| 661 frog_value = float(tokens[3]) | |
| 662 if js_value == 0 or frog_value == 0: | |
| 663 #Then there was an error when this performance test was run. Do not | |
| 664 #count it in our numbers. | |
| 665 return | |
| 666 benchmark = tokens[0] | |
| 667 self.test.revision_dict['commandline']['js'][benchmark] += \ | |
| 668 [revision_num] | |
| 669 self.test.values_dict['commandline']['js'][benchmark] += [js_value] | |
| 670 self.test.revision_dict['commandline']['frog'][benchmark] += \ | |
| 671 [revision_num] | |
| 672 self.test.values_dict['commandline']['frog'][benchmark] += \ | |
| 673 [frog_value] | |
| 674 f.close() | |
| 675 | |
| 676 self.calculate_geometric_mean('commandline', 'frog', revision_num) | |
| 677 self.calculate_geometric_mean('commandline', 'js', revision_num) | |
| 678 | |
| 679 | 386 |
| 680 class BrowserTester(Tester): | 387 class BrowserTester(Tester): |
| 681 @staticmethod | 388 @staticmethod |
| 682 def get_browsers(): | 389 def get_browsers(): |
| 683 browsers = ['dartium', 'ff', 'chrome'] | 390 browsers = ['dartium', 'ff', 'chrome'] |
| 684 has_shell = False | 391 has_shell = False |
| 685 if platform.system() == 'Darwin': | 392 if platform.system() == 'Darwin': |
| 686 browsers += ['safari'] | 393 browsers += ['safari'] |
| 687 if platform.system() == 'Windows': | 394 if platform.system() == 'Windows': |
| 688 browsers += ['ie'] | 395 browsers += ['ie'] |
| (...skipping 90 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 779 else: | 486 else: |
| 780 results = line.split('<br />') | 487 results = line.split('<br />') |
| 781 for result in results: | 488 for result in results: |
| 782 name_and_score = result.split(':') | 489 name_and_score = result.split(':') |
| 783 if len(name_and_score) < 2: | 490 if len(name_and_score) < 2: |
| 784 break | 491 break |
| 785 name = name_and_score[0].strip() | 492 name = name_and_score[0].strip() |
| 786 score = name_and_score[1].strip() | 493 score = name_and_score[1].strip() |
| 787 if version == 'js' or version == 'v8': | 494 if version == 'js' or version == 'v8': |
| 788 version = 'js' | 495 version = 'js' |
| 789 bench_dict = self.test.values_dict[browser]['js'] | 496 self.report_results(name, score, browser, version, revision_num) |
| 790 else: | |
| 791 bench_dict = self.test.values_dict[browser]['frog'] | |
| 792 bench_dict[name] += [float(score)] | |
| 793 self.test.revision_dict[browser][version][name] += [revision_num] | |
| 794 | 497 |
| 795 f.close() | 498 f.close() |
| 796 self.calculate_geometric_mean(browser, version, revision_num) | 499 |
| 797 | 500 |
| 798 class DromaeoTester(Tester): | 501 class DromaeoTester(Tester): |
| 799 DROMAEO_BENCHMARKS = { | 502 DROMAEO_BENCHMARKS = { |
| 800 'attr': ('attributes', [ | 503 'attr': ('attributes', [ |
| 801 'getAttribute', | 504 'getAttribute', |
| 802 'element.property', | 505 'element.property', |
| 803 'setAttribute', | 506 'setAttribute', |
| 804 'element.property = value']), | 507 'element.property = value']), |
| 805 'modify': ('modify', [ | 508 'modify': ('modify', [ |
| 806 'createElement', | 509 'createElement', |
| (...skipping 13 matching lines...) Expand all Loading... | |
| 820 'getElementsByName', | 523 'getElementsByName', |
| 821 'getElementsByName (not in document)']), | 524 'getElementsByName (not in document)']), |
| 822 'traverse': ('traverse', [ | 525 'traverse': ('traverse', [ |
| 823 'firstChild', | 526 'firstChild', |
| 824 'lastChild', | 527 'lastChild', |
| 825 'nextSibling', | 528 'nextSibling', |
| 826 'previousSibling', | 529 'previousSibling', |
| 827 'childNodes']) | 530 'childNodes']) |
| 828 } | 531 } |
| 829 | 532 |
| 830 # Use legal appengine filenames for benchmark names. | |
| 831 @staticmethod | |
| 832 def legalize_filename(str): | |
| 833 remap = { | |
| 834 ' ': '_', | |
| 835 '(': '_', | |
| 836 ')': '_', | |
| 837 '*': 'ALL', | |
| 838 '=': 'ASSIGN', | |
| 839 } | |
| 840 for (old, new) in remap.iteritems(): | |
| 841 str = str.replace(old, new) | |
| 842 return str | |
| 843 | |
| 844 # TODO(vsm): This is a hack to skip breaking tests. Triage this | 533 # TODO(vsm): This is a hack to skip breaking tests. Triage this |
| 845 # failure properly. The modify suite fails on 32-bit chrome on | 534 # failure properly. The modify suite fails on 32-bit chrome on |
| 846 # the mac. | 535 # the mac. |
| 847 @staticmethod | 536 @staticmethod |
| 848 def get_valid_dromaeo_tags(): | 537 def get_valid_dromaeo_tags(): |
| 849 tags = [tag for (tag, _) in DromaeoTester.DROMAEO_BENCHMARKS.values()] | 538 tags = [tag for (tag, _) in DromaeoTester.DROMAEO_BENCHMARKS.values()] |
| 850 if platform.system() == 'Darwin': | 539 if platform.system() == 'Darwin': |
| 851 tags.remove('modify') | 540 tags.remove('modify') |
| 852 return tags | 541 return tags |
| 853 | 542 |
| 854 @staticmethod | 543 @staticmethod |
| 855 def get_dromaeo_benchmarks(): | 544 def get_dromaeo_benchmarks(): |
| 856 valid = DromaeoTester.get_valid_dromaeo_tags() | 545 valid = DromaeoTester.get_valid_dromaeo_tags() |
| 857 benchmarks = reduce(lambda l1,l2: l1+l2, | 546 benchmarks = reduce(lambda l1,l2: l1+l2, |
| 858 [tests for (tag, tests) in | 547 [tests for (tag, tests) in |
| 859 DromaeoTester.DROMAEO_BENCHMARKS.values() | 548 DromaeoTester.DROMAEO_BENCHMARKS.values() |
| 860 if tag in valid]) | 549 if tag in valid]) |
| 861 return map(DromaeoTester.legalize_filename, benchmarks) | 550 return benchmarks |
|
vsm
2012/04/20 19:13:33
This might mess up some of the existing data files
Emily Fortuna
2012/04/20 19:47:17
Ah, in that case, I'll leave it as it was, just so
| |
| 862 | 551 |
| 863 @staticmethod | 552 @staticmethod |
| 864 def get_dromaeo_versions(): | 553 def get_dromaeo_versions(): |
| 865 return ['js', 'frog_dom', 'frog_html'] | 554 return ['js', 'frog_dom', 'frog_html'] |
| 866 | 555 |
| 867 | 556 |
| 868 class DromaeoTest(RuntimePerformanceTest): | 557 class DromaeoTest(RuntimePerformanceTest): |
| 869 """Runs Dromaeo tests, in the browser.""" | 558 """Runs Dromaeo tests, in the browser.""" |
| 870 def __init__(self, test_runner): | 559 def __init__(self, test_runner): |
| 871 super(DromaeoTest, self).__init__( | 560 super(DromaeoTest, self).__init__( |
| (...skipping 22 matching lines...) Expand all Loading... | |
| 894 dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo') | 583 dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo') |
| 895 current_path = os.getcwd() | 584 current_path = os.getcwd() |
| 896 os.chdir(dromaeo_path) | 585 os.chdir(dromaeo_path) |
| 897 self.test.test_runner.run_cmd(['python', 'generate_frog_tests.py']) | 586 self.test.test_runner.run_cmd(['python', 'generate_frog_tests.py']) |
| 898 os.chdir(current_path) | 587 os.chdir(current_path) |
| 899 | 588 |
| 900 versions = DromaeoTester.get_dromaeo_versions() | 589 versions = DromaeoTester.get_dromaeo_versions() |
| 901 | 590 |
| 902 for browser in BrowserTester.get_browsers(): | 591 for browser in BrowserTester.get_browsers(): |
| 903 for version_name in versions: | 592 for version_name in versions: |
| 904 if not self.test.is_valid_combination(browser, version): | 593 if not self.test.is_valid_combination(browser, version_name): |
|
vsm
2012/04/20 19:13:33
Thanks! :-)
| |
| 905 continue | 594 continue |
| 906 version = DromaeoTest.DromaeoPerfTester.get_dromaeo_url_query( | 595 version = DromaeoTest.DromaeoPerfTester.get_dromaeo_url_query( |
| 907 browser, version_name) | 596 browser, version_name) |
| 908 self.test.trace_file = os.path.join( | 597 self.test.trace_file = os.path.join( |
| 909 'tools', 'testing', 'perf_testing', self.test.result_folder_name, | 598 'tools', 'testing', 'perf_testing', self.test.result_folder_name, |
| 910 'dromaeo-%s-%s-%s' % (self.test.cur_time, browser, version_name)) | 599 'dromaeo-%s-%s-%s' % (self.test.cur_time, browser, version_name)) |
| 911 self.add_svn_revision_to_trace(self.test.trace_file, browser) | 600 self.add_svn_revision_to_trace(self.test.trace_file, browser) |
| 912 file_path = '"%s"' % os.path.join(os.getcwd(), dromaeo_path, | 601 file_path = '"%s"' % os.path.join(os.getcwd(), dromaeo_path, |
| 913 'index-js.html?%s' % version) | 602 'index-js.html?%s' % version) |
| 914 self.test.test_runner.run_cmd( | 603 self.test.test_runner.run_cmd( |
| (...skipping 11 matching lines...) Expand all Loading... | |
| 926 return '|'.join([ '%s&%s' % (version, tag) for tag in tags]) | 615 return '|'.join([ '%s&%s' % (version, tag) for tag in tags]) |
| 927 | 616 |
| 928 | 617 |
| 929 class DromaeoFileProcessor(Processor): | 618 class DromaeoFileProcessor(Processor): |
| 930 def process_file(self, afile): | 619 def process_file(self, afile): |
| 931 """Comb through the html to find the performance results.""" | 620 """Comb through the html to find the performance results.""" |
| 932 parts = afile.split('-') | 621 parts = afile.split('-') |
| 933 browser = parts[2] | 622 browser = parts[2] |
| 934 version = parts[3] | 623 version = parts[3] |
| 935 | 624 |
| 936 bench_dict = self.test.values_dict[browser][version] | |
| 937 | |
| 938 f = open(os.path.join(self.test.result_folder_name, afile)) | 625 f = open(os.path.join(self.test.result_folder_name, afile)) |
| 939 lines = f.readlines() | 626 lines = f.readlines() |
| 940 i = 0 | 627 i = 0 |
| 941 revision_num = 0 | 628 revision_num = 0 |
| 942 revision_pattern = r'Revision: (\d+)' | 629 revision_pattern = r'Revision: (\d+)' |
| 943 suite_pattern = r'<div class="result-item done">(.+?)</ol></div>' | 630 suite_pattern = r'<div class="result-item done">(.+?)</ol></div>' |
| 944 result_pattern = r'<b>(.+?)</b>(.+?)<small> runs/s(.+)' | 631 result_pattern = r'<b>(.+?)</b>(.+?)<small> runs/s(.+)' |
| 945 | 632 |
| 946 for line in lines: | 633 for line in lines: |
| 947 rev = re.match(revision_pattern, line.strip()) | 634 rev = re.match(revision_pattern, line.strip()) |
| 948 if rev: | 635 if rev: |
| 949 revision_num = int(rev.group(1)) | 636 revision_num = int(rev.group(1)) |
| 950 continue | 637 continue |
| 951 | 638 |
| 952 suite_results = re.findall(suite_pattern, line) | 639 suite_results = re.findall(suite_pattern, line) |
| 953 if suite_results: | 640 if suite_results: |
| 954 for suite_result in suite_results: | 641 for suite_result in suite_results: |
| 955 results = re.findall(r'<li>(.*?)</li>', suite_result) | 642 results = re.findall(r'<li>(.*?)</li>', suite_result) |
| 956 if results: | 643 if results: |
| 957 for result in results: | 644 for result in results: |
| 958 r = re.match(result_pattern, result) | 645 r = re.match(result_pattern, result) |
| 959 name = DromaeoTester.legalize_filename( | 646 name = r.group(1).strip(':') |
| 960 r.group(1).strip(':')) | |
| 961 score = float(r.group(2)) | 647 score = float(r.group(2)) |
| 962 bench_dict[name] += [float(score)] | 648 self.report_results(name, score, browser, version, revision_num) |
| 963 self.test.revision_dict[browser][version][name] += \ | |
| 964 [revision_num] | |
| 965 | 649 |
| 966 f.close() | 650 f.close() |
| 967 self.calculate_geometric_mean(browser, version, revision_num) | |
| 968 | 651 |
| 969 | 652 |
| 970 class DromaeoSizeTest(Test): | 653 class DromaeoSizeTest(Test): |
| 971 """Run tests to determine the compiled file output size of Dromaeo.""" | 654 """Run tests to determine the compiled file output size of Dromaeo.""" |
| 972 def __init__(self, test_runner): | 655 def __init__(self, test_runner): |
| 973 super(DromaeoSizeTest, self).__init__( | 656 super(DromaeoSizeTest, self).__init__( |
| 974 self.name(), | 657 self.name(), |
| 975 ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'], | 658 ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'], |
| 976 DromaeoTester.DROMAEO_BENCHMARKS.keys(), test_runner, | 659 DromaeoTester.DROMAEO_BENCHMARKS.keys(), test_runner, |
| 977 self.DromaeoSizeTester(self), | 660 self.DromaeoSizeTester(self), |
| 978 self.DromaeoSizeProcessor(self), | 661 self.DromaeoSizeProcessor(self)) |
| 979 self.DromaeoSizeGrapher(self), extra_metrics=['sum']) | |
| 980 | 662 |
| 981 @staticmethod | 663 @staticmethod |
| 982 def name(): | 664 def name(): |
| 983 return 'dromaeo-size' | 665 return 'dromaeo-size' |
| 984 | 666 |
| 985 | 667 |
| 986 class DromaeoSizeTester(DromaeoTester): | 668 class DromaeoSizeTester(DromaeoTester): |
| 987 def run_tests(self): | 669 def run_tests(self): |
| 988 # Build tests. | 670 # Build tests. |
| 989 dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo') | 671 dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo') |
| (...skipping 40 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1030 js_size = os.path.getsize(os.path.join(frog_path, name)) | 712 js_size = os.path.getsize(os.path.join(frog_path, name)) |
| 1031 except OSError: | 713 except OSError: |
| 1032 pass #If compilation failed, continue on running other tests. | 714 pass #If compilation failed, continue on running other tests. |
| 1033 | 715 |
| 1034 total_size[variant] += js_size | 716 total_size[variant] += js_size |
| 1035 self.test.test_runner.run_cmd( | 717 self.test.test_runner.run_cmd( |
| 1036 ['echo', 'Size (%s, %s): %s' % (variant, suite, str(js_size))], | 718 ['echo', 'Size (%s, %s): %s' % (variant, suite, str(js_size))], |
| 1037 self.test.trace_file, append=True) | 719 self.test.trace_file, append=True) |
| 1038 | 720 |
| 1039 self.test.test_runner.run_cmd( | 721 self.test.test_runner.run_cmd( |
| 1040 ['echo', 'Size (dart, %s): %s' % (total_dart_size, | 722 ['echo', 'Size (dart, %s): %s' % (total_dart_size, 'sum')], |
| 1041 self.test.extra_metrics[0])], | |
| 1042 self.test.trace_file, append=True) | 723 self.test.trace_file, append=True) |
| 1043 for (variant, _) in variants: | 724 for (variant, _) in variants: |
| 1044 self.test.test_runner.run_cmd( | 725 self.test.test_runner.run_cmd( |
| 1045 ['echo', 'Size (%s, %s): %s' % (variant, self.test.extra_metrics[0], | 726 ['echo', 'Size (%s, %s): %s' % (variant, 'sum', |
| 1046 total_size[variant])], | 727 total_size[variant])], |
| 1047 self.test.trace_file, append=True) | 728 self.test.trace_file, append=True) |
| 1048 | 729 |
| 730 | |
| 1049 class DromaeoSizeProcessor(Processor): | 731 class DromaeoSizeProcessor(Processor): |
| 1050 def process_file(self, afile): | 732 def process_file(self, afile): |
| 1051 """Pull all the relevant information out of a given tracefile. | 733 """Pull all the relevant information out of a given tracefile. |
| 1052 | 734 |
| 1053 Args: | 735 Args: |
| 1054 afile: is the filename string we will be processing.""" | 736 afile: is the filename string we will be processing.""" |
| 1055 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', | 737 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', |
| 1056 'testing', 'perf_testing')) | 738 'testing', 'perf_testing')) |
| 1057 f = open(os.path.join(self.test.result_folder_name, afile)) | 739 f = open(os.path.join(self.test.result_folder_name, afile)) |
| 1058 tabulate_data = False | 740 tabulate_data = False |
| 1059 revision_num = 0 | 741 revision_num = 0 |
| 1060 revision_pattern = r'Revision: (\d+)' | 742 revision_pattern = r'Revision: (\d+)' |
| 1061 result_pattern = r'Size \((\w+), ([a-zA-Z0-9-]+)\): (\d+)' | 743 result_pattern = r'Size \((\w+), ([a-zA-Z0-9-]+)\): (\d+)' |
| 1062 | 744 |
| 1063 for line in f.readlines(): | 745 for line in f.readlines(): |
| 1064 rev = re.match(revision_pattern, line.strip()) | 746 rev = re.match(revision_pattern, line.strip()) |
| 1065 if rev: | 747 if rev: |
| 1066 revision_num = int(rev.group(1)) | 748 revision_num = int(rev.group(1)) |
| 1067 continue | 749 continue |
| 1068 | 750 |
| 1069 result = re.match(result_pattern, line.strip()) | 751 result = re.match(result_pattern, line.strip()) |
| 1070 if result: | 752 if result: |
| 1071 variant = result.group(1) | 753 variant = result.group(1) |
| 1072 metric = result.group(2) | 754 metric = result.group(2) |
| 1073 num = result.group(3) | 755 num = result.group(3) |
| 1074 if num.find('.') == -1: | 756 if num.find('.') == -1: |
| 1075 num = int(num) | 757 num = int(num) |
| 1076 else: | 758 else: |
| 1077 num = float(num) | 759 num = float(num) |
| 1078 self.test.values_dict['browser'][variant][metric] += [num] | 760 self.report_results(metric, num, 'browser', variant, revision_num) |
| 1079 self.test.revision_dict['browser'][variant][metric] += [revision_num] | |
| 1080 | 761 |
| 1081 f.close() | 762 f.close() |
| 1082 class DromaeoSizeGrapher(Grapher): | |
| 1083 def plot_results(self, png_filename): | |
| 1084 self.style_and_save_perf_plot( | |
| 1085 'Compiled Dromaeo Sizes', | |
| 1086 'Size (in bytes)', 10, 10, 'lower left', png_filename, | |
| 1087 ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'], | |
| 1088 DromaeoTester.DROMAEO_BENCHMARKS.keys()) | |
| 1089 | |
| 1090 self.style_and_save_perf_plot( | |
| 1091 'Compiled Dromaeo Sizes', | |
| 1092 'Size (in bytes)', 10, 10, 'lower left', '2' + png_filename, | |
| 1093 ['browser'], ['dart', 'frog_dom', 'frog_html', 'frog_htmlidiomatic'], | |
| 1094 [self.test.extra_metrics[0]]) | |
| 1095 | 763 |
| 1096 | 764 |
| 1097 class CompileTimeAndSizeTest(Test): | 765 class CompileTimeAndSizeTest(Test): |
| 1098 """Run tests to determine how long minfrog takes to compile, and the compiled | 766 """Run tests to determine how long minfrog takes to compile, and the compiled |
| 1099 file output size of some benchmarking files.""" | 767 file output size of some benchmarking files.""" |
| 1100 def __init__(self, test_runner): | 768 def __init__(self, test_runner): |
| 1101 """Reference to the test_runner object that notifies us when to begin | 769 """Reference to the test_runner object that notifies us when to begin |
| 1102 testing.""" | 770 testing.""" |
| 1103 super(CompileTimeAndSizeTest, self).__init__( | 771 super(CompileTimeAndSizeTest, self).__init__( |
| 1104 self.name(), ['commandline'], ['frog'], | 772 self.name(), ['commandline'], ['frog'], |
| 1105 ['Compiling on Dart VM', 'Bootstrapping', 'minfrog', 'swarm', 'total'], | 773 ['Compiling on Dart VM', 'Bootstrapping', 'minfrog', 'swarm', 'total'], |
| 1106 test_runner, self.CompileTester(self), | 774 test_runner, self.CompileTester(self), |
| 1107 self.CompileProcessor(self), self.CompileGrapher(self)) | 775 self.CompileProcessor(self)) |
| 1108 self.dart_compiler = os.path.join( | 776 self.dart_compiler = os.path.join( |
| 1109 DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(), | 777 DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(), |
| 1110 'release', 'ia32'), 'dart-sdk', 'bin', 'frogc') | 778 'release', 'ia32'), 'dart-sdk', 'bin', 'frogc') |
| 1111 _suffix = '' | 779 _suffix = '' |
| 1112 if platform.system() == 'Windows': | 780 if platform.system() == 'Windows': |
| 1113 _suffix = '.exe' | 781 _suffix = '.exe' |
| 1114 self.dart_vm = os.path.join( | 782 self.dart_vm = os.path.join( |
| 1115 DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(), | 783 DART_INSTALL_LOCATION, utils.GetBuildRoot(utils.GuessOS(), |
| 1116 'release', 'ia32'), 'dart-sdk', 'bin','dart' + _suffix) | 784 'release', 'ia32'), 'dart-sdk', 'bin','dart' + _suffix) |
| 1117 self.failure_threshold = { | 785 self.failure_threshold = { |
| (...skipping 59 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
| 1177 self.test.test_runner.run_cmd( | 845 self.test.test_runner.run_cmd( |
| 1178 ['echo', '%d Generated checked swarm size' % swarm_size], | 846 ['echo', '%d Generated checked swarm size' % swarm_size], |
| 1179 self.test.trace_file, append=True) | 847 self.test.trace_file, append=True) |
| 1180 | 848 |
| 1181 self.test.test_runner.run_cmd( | 849 self.test.test_runner.run_cmd( |
| 1182 ['echo', '%d Generated checked total size' % total_size], | 850 ['echo', '%d Generated checked total size' % total_size], |
| 1183 self.test.trace_file, append=True) | 851 self.test.trace_file, append=True) |
| 1184 | 852 |
| 1185 #Revert our newly built minfrog to prevent conflicts when we update | 853 #Revert our newly built minfrog to prevent conflicts when we update |
| 1186 self.test.test_runner.run_cmd( | 854 self.test.test_runner.run_cmd( |
| 1187 ['svn', 'revert', os.path.join(os.getcwd(), 'frog', 'minfrog')]) | 855 ['svn', 'revert', os.path.join(os.getcwd(), 'minfrog')]) |
| 1188 | |
| 1189 os.chdir('..') | 856 os.chdir('..') |
| 1190 | 857 |
| 858 | |
| 1191 class CompileProcessor(Processor): | 859 class CompileProcessor(Processor): |
| 860 | |
| 1192 def process_file(self, afile): | 861 def process_file(self, afile): |
| 1193 """Pull all the relevant information out of a given tracefile. | 862 """Pull all the relevant information out of a given tracefile. |
| 1194 | 863 |
| 1195 Args: | 864 Args: |
| 1196 afile: is the filename string we will be processing.""" | 865 afile: is the filename string we will be processing.""" |
| 1197 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', | 866 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', |
| 1198 'testing', 'perf_testing')) | 867 'testing', 'perf_testing')) |
| 1199 f = open(os.path.join(self.test.result_folder_name, afile)) | 868 f = open(os.path.join(self.test.result_folder_name, afile)) |
| 1200 tabulate_data = False | 869 tabulate_data = False |
| 1201 revision_num = 0 | 870 revision_num = 0 |
| 1202 for line in f.readlines(): | 871 for line in f.readlines(): |
| 1203 tokens = line.split() | 872 tokens = line.split() |
| 1204 if 'Revision' in line: | 873 if 'Revision' in line: |
| 1205 revision_num = int(line.split()[1]) | 874 revision_num = int(line.split()[1]) |
| 1206 else: | 875 else: |
| 1207 for metric in self.test.values_list: | 876 for metric in self.test.values_list: |
| 1208 if metric in line: | 877 if metric in line: |
| 1209 num = tokens[0] | 878 num = tokens[0] |
| 1210 if num.find('.') == -1: | 879 if num.find('.') == -1: |
| 1211 num = int(num) | 880 num = int(num) |
| 1212 else: | 881 else: |
| 1213 num = float(num) | 882 num = float(num) |
| 1214 self.test.values_dict['commandline']['frog'][metric] += [num] | 883 self.report_results(metric, num, 'commandline', 'frog', |
| 1215 self.test.revision_dict['commandline']['frog'][metric] += \ | 884 revision_num) |
| 1216 [revision_num] | |
| 1217 | |
| 1218 if revision_num != 0: | |
| 1219 for metric in self.test.values_list: | |
| 1220 self.test.revision_dict['commandline']['frog'][metric].pop() | |
| 1221 self.test.revision_dict['commandline']['frog'][metric] += \ | |
| 1222 [revision_num] | |
| 1223 # Fill in 0 if compilation failed. | |
| 1224 if self.test.values_dict['commandline']['frog'][metric][-1] < \ | |
| 1225 self.test.failure_threshold[metric]: | |
| 1226 self.test.values_dict['commandline']['frog'][metric] += [0] | |
| 1227 self.test.revision_dict['commandline']['frog'][metric] += \ | |
| 1228 [revision_num] | |
| 1229 | 885 |
| 1230 f.close() | 886 f.close() |
| 1231 | 887 |
| 1232 class CompileGrapher(Grapher): | |
| 1233 | |
| 1234 def plot_results(self, png_filename): | |
| 1235 self.style_and_save_perf_plot( | |
| 1236 'Compiled minfrog Sizes', 'Size (in bytes)', 10, 10, 'lower left', | |
| 1237 png_filename, ['commandline'], ['frog'], | |
| 1238 ['swarm', 'total', 'minfrog']) | |
| 1239 | |
| 1240 self.style_and_save_perf_plot( | |
| 1241 'Time to compile and bootstrap', | |
| 1242 'Seconds', 10, 10, 'lower left', '2' + png_filename, ['commandline'], | |
| 1243 ['frog'], ['Bootstrapping', 'Compiling on Dart VM']) | |
| 1244 | |
| 1245 | 888 |
| 1246 class TestBuilder(object): | 889 class TestBuilder(object): |
| 1247 """Construct the desired test object.""" | 890 """Construct the desired test object.""" |
| 1248 available_suites = dict((suite.name(), suite) for suite in [ | 891 available_suites = dict((suite.name(), suite) for suite in [ |
| 1249 CommonCommandLineTest, CompileTimeAndSizeTest, | 892 CompileTimeAndSizeTest, CommonBrowserTest, DromaeoTest, DromaeoSizeTest]) |
| 1250 CommonBrowserTest, DromaeoTest, DromaeoSizeTest]) | |
| 1251 | 893 |
| 1252 @staticmethod | 894 @staticmethod |
| 1253 def make_test(test_name, test_runner): | 895 def make_test(test_name, test_runner): |
| 1254 return TestBuilder.available_suites[test_name](test_runner) | 896 return TestBuilder.available_suites[test_name](test_runner) |
| 1255 | 897 |
| 1256 @staticmethod | 898 @staticmethod |
| 1257 def available_suite_names(): | 899 def available_suite_names(): |
| 1258 return TestBuilder.available_suites.keys() | 900 return TestBuilder.available_suites.keys() |
| 1259 | 901 |
| 1260 | 902 |
| 1261 def main(): | 903 def main(): |
| 1262 runner = TestRunner() | 904 runner = TestRunner() |
| 1263 continuous = runner.parse_args() | 905 continuous = runner.parse_args() |
| 1264 if continuous: | 906 if continuous: |
| 1265 while True: | 907 while True: |
| 1266 if runner.has_new_code(): | 908 if runner.has_new_code(): |
| 1267 runner.run_test_sequence() | 909 runner.run_test_sequence() |
| 1268 else: | 910 else: |
| 1269 time.sleep(200) | 911 time.sleep(200) |
| 1270 else: | 912 else: |
| 1271 runner.run_test_sequence() | 913 runner.run_test_sequence() |
| 1272 | 914 |
| 1273 if __name__ == '__main__': | 915 if __name__ == '__main__': |
| 1274 main() | 916 main() |
| OLD | NEW |
