| OLD | NEW |
|---|
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 | 2 |
| 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file |
| 4 # for details. All rights reserved. Use of this source code is governed by a | 4 # for details. All rights reserved. Use of this source code is governed by a |
| 5 # BSD-style license that can be found in the LICENSE file. | 5 # BSD-style license that can be found in the LICENSE file. |
| 6 | 6 |
| 7 import datetime | 7 import datetime |
| 8 import getpass | 8 import getpass |
| 9 import math | 9 import math |
| 10 from matplotlib.font_manager import FontProperties | 10 from matplotlib.font_manager import FontProperties |
| (...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 48 | 48 |
| 49 GEO_MEAN = 'Geo-Mean' | 49 GEO_MEAN = 'Geo-Mean' |
| 50 COMMAND_LINE = 'commandline' | 50 COMMAND_LINE = 'commandline' |
| 51 JS = 'js' | 51 JS = 'js' |
| 52 FROG = 'frog' | 52 FROG = 'frog' |
| 53 JS_AND_FROG = [JS, FROG] | 53 JS_AND_FROG = [JS, FROG] |
| 54 COLORS = ['blue', 'green', 'red', 'cyan', 'magenta', 'black'] | 54 COLORS = ['blue', 'green', 'red', 'cyan', 'magenta', 'black'] |
| 55 GRAPH_OUT_DIR = 'graphs' | 55 GRAPH_OUT_DIR = 'graphs' |
| 56 | 56 |
| 57 BROWSER_PERF = 'browser-perf' | 57 BROWSER_PERF = 'browser-perf' |
| 58 TIME_SIZE = 'code-time-size' | 58 TIME_SIZE = 'time-size' |
| 59 CL_PERF = 'cl-results' | 59 CL_PERF = 'cl-perf' |
| 60 # TODO(vsm): Merge these? | 60 # TODO(vsm): Merge these? |
| 61 DROMAEO = 'dromaeo' | 61 DROMAEO = 'dromaeo' |
| 62 DROMAEO_SIZE = 'dromaeo-size' | 62 DROMAEO_SIZE = 'dromaeo-size' |
| 63 | 63 |
| 64 SLEEP_TIME = 200 | 64 SLEEP_TIME = 200 |
| 65 VERBOSE = False | 65 VERBOSE = False |
| 66 HAS_SHELL = False | 66 HAS_SHELL = False |
| 67 if platform.system() == 'Windows': | 67 if platform.system() == 'Windows': |
| 68 # On Windows, shell must be true to get the correct environment variables. | 68 # On Windows, shell must be true to get the correct environment variables. |
| 69 HAS_SHELL = True | 69 HAS_SHELL = True |
| (...skipping 114 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 184 def get_os_directory(): | 184 def get_os_directory(): |
| 185 """Specifies the name of the directory for the testing build of dart, which | 185 """Specifies the name of the directory for the testing build of dart, which |
| 186 has yet a different naming convention from utils.getBuildRoot(...).""" | 186 has yet a different naming convention from utils.getBuildRoot(...).""" |
| 187 if platform.system() == 'Windows': | 187 if platform.system() == 'Windows': |
| 188 return 'windows' | 188 return 'windows' |
| 189 elif platform.system() == 'Darwin': | 189 elif platform.system() == 'Darwin': |
| 190 return 'macos' | 190 return 'macos' |
| 191 else: | 191 else: |
| 192 return 'linux' | 192 return 'linux' |
| 193 | 193 |
| 194 def upload_to_app_engine(username, password): | 194 def upload_to_app_engine(username, password, suite_names): |
| 195 """Upload our results to our appengine server. | 195 """Upload our results to our appengine server. |
| 196 Arguments: | 196 Arguments: |
| 197 username: App Engine username for uploading data to dartperf.googleplex.com | 197 username: App Engine username for uploading data to dartperf.googleplex.com |
| 198 password: App Engine password | 198 password: App Engine password |
| 199 suite_names: Directories to upload data from (should match suite names) |
| 199 """ | 200 """ |
| 200 # TODO(efortuna): This is the most basic way to get the data up | 201 # TODO(efortuna): This is the most basic way to get the data up |
| 201 # for others to view. Revisit this once we're serving nicer graphs (Google | 202 # for others to view. Revisit this once we're serving nicer graphs (Google |
| 202 # Chart Tools) and from multiple perfbots and once we're in a position to | 203 # Chart Tools) and from multiple perfbots and once we're in a position to |
| 203 # organize the data in a useful manner(!!). | 204 # organize the data in a useful manner(!!). |
| 204 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', | 205 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', |
| 205 'perf_testing')) | 206 'perf_testing')) |
| 206 # TODO(vsm): Factor out this list. | 207 for data in suite_names: |
| 207 for data in [BROWSER_PERF, TIME_SIZE, CL_PERF, DROMAEO, DROMAEO_SIZE]: | |
| 208 path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS()) | 208 path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS()) |
| 209 shutil.rmtree(path, ignore_errors=True) | 209 shutil.rmtree(path, ignore_errors=True) |
| 210 os.makedirs(path) | 210 os.makedirs(path) |
| 211 files = [] | 211 files = [] |
| 212 # Copy the 1000 most recent trace files to be uploaded. | 212 # Copy the 1000 most recent trace files to be uploaded. |
| 213 for f in os.listdir(data): | 213 for f in os.listdir(data): |
| 214 files += [(os.path.getmtime(os.path.join(data, f)), f)] | 214 files += [(os.path.getmtime(os.path.join(data, f)), f)] |
| 215 files.sort() | 215 files.sort() |
| 216 for f in files[-1000:]: | 216 for f in files[-1000:]: |
| 217 shutil.copyfile(os.path.join(data, f[1]), | 217 shutil.copyfile(os.path.join(data, f[1]), |
| 218 os.path.join(path, f[1]+'.txt')) | 218 os.path.join(path, f[1]+'.txt')) |
| 219 # Generate directory listing. | 219 # Generate directory listing. |
| 220 # TODO(vsm): Factor out this list. | 220 for data in suite_names: |
| 221 for data in [BROWSER_PERF, TIME_SIZE, CL_PERF, DROMAEO, DROMAEO_SIZE]: | |
| 222 path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS()) | 221 path = os.path.join('appengine', 'static', 'data', data, utils.GuessOS()) |
| 223 out = open(os.path.join('appengine', 'static', | 222 out = open(os.path.join('appengine', 'static', |
| 224 '%s-%s.html' % (data, utils.GuessOS())), 'w') | 223 '%s-%s.html' % (data, utils.GuessOS())), 'w') |
| 225 out.write('<html>\n <body>\n <ul>\n') | 224 out.write('<html>\n <body>\n <ul>\n') |
| 226 for f in os.listdir(path): | 225 for f in os.listdir(path): |
| 227 if not f.startswith('.'): | 226 if not f.startswith('.'): |
| 228 out.write(' <li><a href=data' + \ | 227 out.write(' <li><a href=data' + \ |
| 229 '''/%(data)s/%(os)s/%(file)s>%(file)s</a></li>\n''' % \ | 228 '''/%(data)s/%(os)s/%(file)s>%(file)s</a></li>\n''' % \ |
| 230 {'data': data, 'os': utils.GuessOS(), 'file': f}) | 229 {'data': data, 'os': utils.GuessOS(), 'file': f}) |
| 231 out.write(' </ul>\n </body>\n</html>') | 230 out.write(' </ul>\n </body>\n</html>') |
| (...skipping 259 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 491 for browser in get_browsers(): | 490 for browser in get_browsers(): |
| 492 for version in self.versions: | 491 for version in self.versions: |
| 493 self.trace_file = os.path.join('tools', 'testing', 'perf_testing', | 492 self.trace_file = os.path.join('tools', 'testing', 'perf_testing', |
| 494 self.result_folder_name, | 493 self.result_folder_name, |
| 495 'perf-%s-%s-%s' % (self.cur_time, browser, version)) | 494 'perf-%s-%s-%s' % (self.cur_time, browser, version)) |
| 496 self.add_svn_revision_to_trace(self.trace_file) | 495 self.add_svn_revision_to_trace(self.trace_file) |
| 497 file_path = os.path.join(os.getcwd(), 'internal', 'browserBenchmarks', | 496 file_path = os.path.join(os.getcwd(), 'internal', 'browserBenchmarks', |
| 498 'benchmark_page_%s.html' % version) | 497 'benchmark_page_%s.html' % version) |
| 499 run_cmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), | 498 run_cmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), |
| 500 '--out', file_path, '--browser', browser, | 499 '--out', file_path, '--browser', browser, |
| 501 '--timeout', '600', '--perf'], self.trace_file, append=True) | 500 '--timeout', '600', '--mode', 'perf'], self.trace_file, append=True) |
| 502 | 501 |
| 503 def process_file(self, afile): | 502 def process_file(self, afile): |
| 504 """Comb through the html to find the performance results.""" | 503 """Comb through the html to find the performance results.""" |
| 505 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', | 504 os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools', 'testing', |
| 506 'perf_testing')) | 505 'perf_testing')) |
| 507 parts = afile.split('-') | 506 parts = afile.split('-') |
| 508 browser = parts[2] | 507 browser = parts[2] |
| 509 version = parts[3] | 508 version = parts[3] |
| 510 f = open(os.path.join(self.result_folder_name, afile)) | 509 f = open(os.path.join(self.result_folder_name, afile)) |
| 511 lines = f.readlines() | 510 lines = f.readlines() |
| (...skipping 74 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 586 for browser in get_browsers(): | 585 for browser in get_browsers(): |
| 587 for version_name in versions: | 586 for version_name in versions: |
| 588 version = version_name.replace('_','&') | 587 version = version_name.replace('_','&') |
| 589 self.trace_file = os.path.join('tools', 'testing', 'perf_testing', | 588 self.trace_file = os.path.join('tools', 'testing', 'perf_testing', |
| 590 self.result_folder_name, | 589 self.result_folder_name, |
| 591 'dromaeo-%s-%s-%s' % (self.cur_time, browser, version_name)) | 590 'dromaeo-%s-%s-%s' % (self.cur_time, browser, version_name)) |
| 592 self.add_svn_revision_to_trace(self.trace_file) | 591 self.add_svn_revision_to_trace(self.trace_file) |
| 593 file_path = os.path.join(os.getcwd(), dromaeo_path, | 592 file_path = os.path.join(os.getcwd(), dromaeo_path, |
| 594 'index-js.html?%s' % version) | 593 'index-js.html?%s' % version) |
| 595 run_cmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), | 594 run_cmd(['python', os.path.join('tools', 'testing', 'run_selenium.py'), |
| 596 '--out', file_path, '--browser', browser, | 595 '--out', file_path, '--browser', browser, |
| 597 '--timeout', '200', '--dromaeo'], self.trace_file, append=True) | 596 '--timeout', '200', '--mode', 'dromaeo'], self.trace_file, |
| 597 append=True) |
| 598 | 598 |
| 599 def process_file(self, afile): | 599 def process_file(self, afile): |
| 600 """Comb through the html to find the performance results.""" | 600 """Comb through the html to find the performance results.""" |
| 601 parts = afile.split('-') | 601 parts = afile.split('-') |
| 602 browser = parts[2] | 602 browser = parts[2] |
| 603 version = parts[3] | 603 version = parts[3] |
| 604 | 604 |
| 605 bench_dict = self.values_dict[browser][version] | 605 bench_dict = self.values_dict[browser][version] |
| 606 | 606 |
| 607 f = open(os.path.join(self.result_folder_name, afile)) | 607 f = open(os.path.join(self.result_folder_name, afile)) |
| (...skipping 238 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 846 | 846 |
| 847 def plot_results(self, png_filename): | 847 def plot_results(self, png_filename): |
| 848 self.style_and_save_perf_plot('Compiled minfrog Sizes', | 848 self.style_and_save_perf_plot('Compiled minfrog Sizes', |
| 849 'Size (in bytes)', 10, 10, 'lower left', png_filename, [COMMAND_LINE], | 849 'Size (in bytes)', 10, 10, 'lower left', png_filename, [COMMAND_LINE], |
| 850 [FROG], ['swarm', 'total', 'minfrog']) | 850 [FROG], ['swarm', 'total', 'minfrog']) |
| 851 | 851 |
| 852 self.style_and_save_perf_plot('Time to compile and bootstrap', | 852 self.style_and_save_perf_plot('Time to compile and bootstrap', |
| 853 'Seconds', 10, 10, 'lower left', '2' + png_filename, [COMMAND_LINE], | 853 'Seconds', 10, 10, 'lower left', '2' + png_filename, [COMMAND_LINE], |
| 854 [FROG], ['Bootstrapping', 'Compiling on Dart VM']) | 854 [FROG], ['Bootstrapping', 'Compiling on Dart VM']) |
| 855 | 855 |
| 856 # TODO(vsm): Make these names consistent with BROWSER_PERF, CL_PERF, |
| 857 # etc. above. |
| 858 SUITES = { |
| 859 CL_PERF: CommandLinePerformanceTest, |
| 860 TIME_SIZE: CompileTimeAndSizeTest, |
| 861 BROWSER_PERF: BrowserStandalonePerformanceTest, |
| 862 DROMAEO: DromaeoTest, |
| 863 DROMAEO_SIZE: DromaeoSizeTest, |
| 864 } |
| 865 |
| 856 def parse_args(): | 866 def parse_args(): |
| 857 parser = optparse.OptionParser() | 867 parser = optparse.OptionParser() |
| 858 # TODO(vsm): Change to a list to scale. | 868 # TODO(vsm): Change to a list to scale. |
| 859 parser.add_option('--command-line', '-c', dest='cl', | 869 parser.add_option('--suites', '-s', dest='suites', |
| 860 help='Run the command line tests', | 870 help='Run the specified comma-separated test suites from set: %s' % \ |
| 861 action='store_true', default=False) | 871 ','.join(SUITES.keys()), |
| 862 parser.add_option('--size-time', '-s', dest='size', | 872 action='store', default=None) |
| 863 help='Run the code size and timing tests', | |
| 864 action='store_true', default=False) | |
| 865 parser.add_option('--browser-perf', '-b', dest='perf', | |
| 866 help='Run the browser performance tests', | |
| 867 action='store_true', default=False) | |
| 868 parser.add_option('--dromaeo', '-d', dest='dromaeo', | |
| 869 help='Run the Dromaeo performance tests', | |
| 870 action='store_true', default=False) | |
| 871 parser.add_option('--dromaeo-size', '-D', dest='dsize', | |
| 872 help='Run the Dromaeo size tests', | |
| 873 action='store_true', default=False) | |
| 874 parser.add_option('--forever', '-f', dest='continuous', | 873 parser.add_option('--forever', '-f', dest='continuous', |
| 875 help='Run this script forever, always checking for the next svn ' | 874 help='Run this script forever, always checking for the next svn ' |
| 876 'checkin', action='store_true', default=False) | 875 'checkin', action='store_true', default=False) |
| 877 parser.add_option('--verbose', '-v', dest='verbose', | 876 parser.add_option('--verbose', '-v', dest='verbose', |
| 878 help='Print extra debug output', action='store_true', default=False) | 877 help='Print extra debug output', action='store_true', default=False) |
| 879 parser.add_option('--nobuild', '-n', dest='no_build', action='store_true', | 878 parser.add_option('--nobuild', '-n', dest='no_build', action='store_true', |
| 880 help='Do not sync with the repository and do not rebuild.', default=False) | 879 help='Do not sync with the repository and do not rebuild.', default=False) |
| 881 parser.add_option('--graph-only', '-g', dest='graph_only', default=False, | 880 parser.add_option('--graph-only', '-g', dest='graph_only', default=False, |
| 882 help='Do not run tests, only regenerate graphs', action='store_true') | 881 help='Do not run tests, only regenerate graphs', action='store_true') |
| 883 parser.add_option('--user', '-u', dest='username', | 882 parser.add_option('--user', '-u', dest='username', |
| 884 help='Username for submitting new data to App Engine', default='') | 883 help='Username for submitting new data to App Engine', default='') |
| 885 | 884 |
| 886 args, ignored = parser.parse_args() | 885 args, ignored = parser.parse_args() |
| 887 password = '' | 886 password = '' |
| 888 if args.username != '': | 887 if args.username != '': |
| 889 password = getpass.getpass("App Engine Password: ") | 888 password = getpass.getpass("App Engine Password: ") |
| 890 else: | 889 else: |
| 891 print 'Warning: performance data will not be uploaded to App Engine' + \ | 890 print 'Warning: performance data will not be uploaded to App Engine' + \ |
| 892 ' if you do not provide a username.' | 891 ' if you do not provide a username.' |
| 893 if not (args.cl or args.size or args.perf or args.dromaeo or args.dsize): | |
| 894 args.cl = args.size = args.perf = args.dromaeo = args.dsize = True | |
| 895 return (args.cl, args.size, args.perf, args.dromaeo, args.dsize, | |
| 896 args.continuous, args.verbose, args.no_build, args.graph_only, | |
| 897 args.username, password) | |
| 898 | 892 |
| 899 def run_test_sequence(cl, size, perf, dromaeo, dsize, no_build, graph_only, | 893 if not args.suites: |
| 894 suites = SUITES.values() |
| 895 else: |
| 896 suites = [] |
| 897 suitelist = args.suites.split(',') |
| 898 for name in suitelist: |
| 899 if name in SUITES: |
| 900 suites.append(SUITES[name]) |
| 901 else: |
| 902 print 'Error: Invalid suite %s not in %s' % (name, |
| 903 ','.join(SUITES.keys())) |
| 904 sys.exit(1) |
| 905 return (suites, args.continuous, args.verbose, args.no_build, |
| 906 args.graph_only, args.username, password) |
| 907 |
| 908 def run_test_sequence(suites, no_build, graph_only, |
| 900 username, password): | 909 username, password): |
| 901 # The buildbot already builds and syncs to a specific revision. Don't fight | 910 # The buildbot already builds and syncs to a specific revision. Don't fight |
| 902 # with it or replicate work. | 911 # with it or replicate work. |
| 903 if (not no_build or not graph_only) and sync_and_build() == 1: | 912 if (not no_build or not graph_only) and sync_and_build() == 1: |
| 904 return # The build is broken. | 913 return # The build is broken. |
| 905 if size: | 914 |
| 906 CompileTimeAndSizeTest().run(graph_only) | 915 for test in suites: |
| 907 if cl: | 916 test().run(graph_only) |
| 908 CommandLinePerformanceTest().run(graph_only) | |
| 909 if perf: | |
| 910 BrowserStandalonePerformanceTest().run(graph_only) | |
| 911 if dromaeo: | |
| 912 DromaeoTest().run(graph_only) | |
| 913 if dsize: | |
| 914 DromaeoSizeTest().run(graph_only) | |
| 915 | 917 |
| 916 if username != '': | 918 if username != '': |
| 917 upload_to_app_engine(username, password) | 919 upload_to_app_engine(username, password, SUITES.keys()) |
| 918 | 920 |
| 919 def main(): | 921 def main(): |
| 920 global VERBOSE | 922 global VERBOSE |
| 921 (cl, size, perf, dromaeo, dsize, continuous, verbose, no_build, graph_only, | 923 (suites, continuous, verbose, no_build, graph_only, |
| 922 username, password) = parse_args() | 924 username, password) = parse_args() |
| 923 VERBOSE = verbose | 925 VERBOSE = verbose |
| 924 if continuous: | 926 if continuous: |
| 925 while True: | 927 while True: |
| 926 if has_new_code(): | 928 if has_new_code(): |
| 927 run_test_sequence(cl, size, perf, dromaeo, dsize, no_build, graph_only, | 929 run_test_sequence(suites, no_build, graph_only, |
| 928 username, password) | 930 username, password) |
| 929 else: | 931 else: |
| 930 time.sleep(SLEEP_TIME) | 932 time.sleep(SLEEP_TIME) |
| 931 else: | 933 else: |
| 932 run_test_sequence(cl, size, perf, dromaeo, dsize, no_build, graph_only, | 934 run_test_sequence(suites, no_build, graph_only, |
| 933 username, password) | 935 username, password) |
| 934 | 936 |
| 935 if __name__ == '__main__': | 937 if __name__ == '__main__': |
| 936 main() | 938 main() |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 9960018:
Various perf/selenium script cleanup (Closed)
Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart