OLD | NEW |
---|---|
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 | 2 |
3 # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 3 # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
4 # for details. All rights reserved. Use of this source code is governed by a | 4 # for details. All rights reserved. Use of this source code is governed by a |
5 # BSD-style license that can be found in the LICENSE file. | 5 # BSD-style license that can be found in the LICENSE file. |
6 | 6 |
7 import datetime | 7 import datetime |
8 import math | 8 import math |
9 import optparse | 9 import optparse |
10 import os | 10 import os |
11 from os.path import dirname, abspath | 11 from os.path import dirname, abspath |
12 import pickle | 12 import pickle |
13 import platform | 13 import platform |
14 import random | 14 import random |
15 import re | 15 import re |
16 import shutil | 16 import shutil |
17 import stat | 17 import stat |
18 import subprocess | 18 import subprocess |
19 import sys | 19 import sys |
20 import time | 20 import time |
21 | 21 |
22 TOOLS_PATH = os.path.join(dirname(dirname(dirname(abspath(__file__))))) | 22 TOOLS_PATH = os.path.join(dirname(dirname(dirname(abspath(__file__))))) |
23 TOP_LEVEL_DIR = abspath(os.path.join(dirname(abspath(__file__)), '..', '..', | 23 TOP_LEVEL_DIR = abspath(os.path.join(dirname(abspath(__file__)), '..', '..', |
24 '..')) | 24 '..')) |
25 DART_REPO_LOC = abspath(os.path.join(dirname(abspath(__file__)), '..', '..', | 25 DART_REPO_LOC = abspath(os.path.join(dirname(abspath(__file__)), '..', '..', |
26 '..', '..', '..', | 26 '..', '..', '..', |
27 'dart_checkout_for_perf_testing', | 27 'dart_checkout_for_perf_testing', |
28 'dart')) | 28 'dart')) |
29 # The earliest stored version of Dartium. Don't try to test earlier than this. | |
30 EARLIEST_REVISION = 4285 | |
29 sys.path.append(TOOLS_PATH) | 31 sys.path.append(TOOLS_PATH) |
30 sys.path.append(os.path.join(TOP_LEVEL_DIR, 'internal', 'tests')) | 32 sys.path.append(os.path.join(TOP_LEVEL_DIR, 'internal', 'tests')) |
31 import post_results | 33 import post_results |
32 import utils | 34 import utils |
33 | 35 |
34 """This script runs to track performance and size progress of | 36 """This script runs to track performance and size progress of |
35 different svn revisions. It tests to see if there a newer version of the code on | 37 different svn revisions. It tests to see if there a newer version of the code on |
36 the server, and will sync and run the performance tests if so.""" | 38 the server, and will sync and run the performance tests if so.""" |
37 class TestRunner(object): | 39 class TestRunner(object): |
38 | 40 |
(...skipping 298 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
337 self.revision_dict[platform][f][val] = [] | 339 self.revision_dict[platform][f][val] = [] |
338 self.values_dict[platform][f][val] = [] | 340 self.values_dict[platform][f][val] = [] |
339 for extra_metric in extra_metrics: | 341 for extra_metric in extra_metrics: |
340 self.revision_dict[platform][f][extra_metric] = [] | 342 self.revision_dict[platform][f][extra_metric] = [] |
341 self.values_dict[platform][f][extra_metric] = [] | 343 self.values_dict[platform][f][extra_metric] = [] |
342 | 344 |
343 def is_valid_combination(self, platform, variant): | 345 def is_valid_combination(self, platform, variant): |
344 """Check whether data should be captured for this platform/variant | 346 """Check whether data should be captured for this platform/variant |
345 combination. | 347 combination. |
346 """ | 348 """ |
349 # TODO(vsm): This avoids a bug in 32-bit Chrome (dartium) | |
350 # running JS dromaeo. | |
351 if platform == 'dartium' and variant == 'js': | |
352 return False | |
353 if platform == 'safari' and variant == 'dart2js' and \ | |
sra1
2012/08/18 02:59:02
Preferred Python style is to use parentheses to ma
| |
354 int(self.test_runner.current_revision_num) < 10193: | |
355 # In revision 10193 we fixed a bug that allows Safari 6 to run dart2js | |
356 # code. Since we can't change the Safari version on the machine, we're | |
357 # just not running | |
358 # for this case. | |
359 return False | |
347 return True | 360 return True |
348 | 361 |
349 def run(self): | 362 def run(self): |
350 """Run the benchmarks/tests from the command line and plot the | 363 """Run the benchmarks/tests from the command line and plot the |
351 results. | 364 results. |
352 """ | 365 """ |
353 for visitor in [self.tester, self.file_processor]: | 366 for visitor in [self.tester, self.file_processor]: |
354 visitor.prepare() | 367 visitor.prepare() |
355 | 368 |
356 os.chdir(TOP_LEVEL_DIR) | 369 os.chdir(TOP_LEVEL_DIR) |
(...skipping 348 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
705 'browser', | 718 'browser', |
706 DromaeoTester.get_dromaeo_versions(), | 719 DromaeoTester.get_dromaeo_versions(), |
707 DromaeoTester.get_dromaeo_benchmarks(), test_runner, | 720 DromaeoTester.get_dromaeo_benchmarks(), test_runner, |
708 self.DromaeoPerfTester(self), | 721 self.DromaeoPerfTester(self), |
709 self.DromaeoFileProcessor(self)) | 722 self.DromaeoFileProcessor(self)) |
710 | 723 |
711 @staticmethod | 724 @staticmethod |
712 def name(): | 725 def name(): |
713 return 'dromaeo' | 726 return 'dromaeo' |
714 | 727 |
715 def is_valid_combination(self, browser, version): | |
716 # TODO(vsm): This avoids a bug in 32-bit Chrome (dartium) | |
717 # running JS dromaeo. | |
718 if browser == 'dartium' and version == 'js': | |
719 return False | |
720 # dart:dom has been removed from Dartium. | |
721 if browser == 'dartium' and 'dom' in version: | |
722 return False | |
723 return True | |
724 | |
725 | |
726 class DromaeoPerfTester(DromaeoTester): | 728 class DromaeoPerfTester(DromaeoTester): |
727 def move_chrome_driver_if_needed(self, browser): | 729 def move_chrome_driver_if_needed(self, browser): |
728 """Move the appropriate version of ChromeDriver onto the path. | 730 """Move the appropriate version of ChromeDriver onto the path. |
729 TODO(efortuna): This is a total hack because the latest version of Chrome | 731 TODO(efortuna): This is a total hack because the latest version of Chrome |
730 (Dartium builds) requires a different version of ChromeDriver, that is | 732 (Dartium builds) requires a different version of ChromeDriver, that is |
731 incompatible with the release or beta Chrome and vice versa. Remove these | 733 incompatible with the release or beta Chrome and vice versa. Remove these |
732 shenanigans once we're back to both versions of Chrome using the same | 734 shenanigans once we're back to both versions of Chrome using the same |
733 version of ChromeDriver. IMPORTANT NOTE: This assumes your chromedriver is | 735 version of ChromeDriver. IMPORTANT NOTE: This assumes your chromedriver is |
734 in the default location (inside depot_tools). | 736 in the default location (inside depot_tools). |
735 """ | 737 """ |
(...skipping 26 matching lines...) Expand all Loading... | |
762 if not os.path.exists(os.path.dirname(to_dir)): | 764 if not os.path.exists(os.path.dirname(to_dir)): |
763 os.makedirs(os.path.dirname(to_dir)) | 765 os.makedirs(os.path.dirname(to_dir)) |
764 shutil.copyfile(from_dir, to_dir) | 766 shutil.copyfile(from_dir, to_dir) |
765 | 767 |
766 for loc in path: | 768 for loc in path: |
767 if 'depot_tools' in loc: | 769 if 'depot_tools' in loc: |
768 if browser == 'chrome': | 770 if browser == 'chrome': |
769 if os.path.exists(orig_chromedriver_path): | 771 if os.path.exists(orig_chromedriver_path): |
770 move_chromedriver(loc) | 772 move_chromedriver(loc) |
771 elif browser == 'dartium': | 773 elif browser == 'dartium': |
772 if not os.path.exists(dartium_chromedriver_path): | 774 if self.test.test_runner.current_revision_num < 7823: |
775 # 7823 is the first recorded version we have of a different | |
776 # chromedriver. If before that, just use the regular chromedriver. | |
777 self.test.test_runner.run_cmd(os.path.join( | |
778 TOP_LEVEL_DIR, 'tools', 'testing', 'webdriver_test_setup.py'), | |
779 '-f', '-s', '-p') | |
780 elif not os.path.exists(dartium_chromedriver_path): | |
773 self.test.test_runner.get_archive('chromedriver') | 781 self.test.test_runner.get_archive('chromedriver') |
774 # Move original chromedriver for storage. | 782 # Move original chromedriver for storage. |
775 if not os.path.exists(orig_chromedriver_path): | 783 if not os.path.exists(orig_chromedriver_path): |
776 move_chromedriver(loc, copy_to_depot_tools_dir=False) | 784 move_chromedriver(loc, copy_to_depot_tools_dir=False) |
777 # Copy Dartium chromedriver into depot_tools | 785 if self.test.test_runner.current_revision_num >= 7823: |
sra1
2012/08/18 02:59:02
Since you mention 7823 twice, make it a constant,
| |
778 move_chromedriver(loc, from_path=os.path.join( | 786 # Copy Dartium chromedriver into depot_tools |
779 dartium_chromedriver_path, 'chromedriver')) | 787 move_chromedriver(loc, from_path=os.path.join( |
788 dartium_chromedriver_path, 'chromedriver')) | |
780 os.chdir(current_dir) | 789 os.chdir(current_dir) |
781 | 790 |
782 def run_tests(self): | 791 def run_tests(self): |
783 """Run dromaeo in the browser.""" | 792 """Run dromaeo in the browser.""" |
784 | 793 |
785 self.test.test_runner.get_archive('dartium') | 794 self.test.test_runner.get_archive('dartium') |
786 | 795 |
787 # Build tests. | 796 # Build tests. |
788 dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo') | 797 dromaeo_path = os.path.join('samples', 'third_party', 'dromaeo') |
789 current_path = os.getcwd() | 798 current_path = os.getcwd() |
(...skipping 181 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
971 | 980 |
972 if random.choice([True, False]): | 981 if random.choice([True, False]): |
973 # Select a random CL number, with greater likelihood of selecting a CL in | 982 # Select a random CL number, with greater likelihood of selecting a CL in |
974 # the more recent history than the distant past (using a simplified weighted | 983 # the more recent history than the distant past (using a simplified weighted |
975 # bucket algorithm). If that CL has less than 10 runs, run additional. If it | 984 # bucket algorithm). If that CL has less than 10 runs, run additional. If it |
976 # already has 10 runs, look for another CL number that is not yet have all | 985 # already has 10 runs, look for another CL number that is not yet have all |
977 # of its additional runs (do this up to 15 times). | 986 # of its additional runs (do this up to 15 times). |
978 tries = 0 | 987 tries = 0 |
979 # Select which "thousands bucket" we're going to run additional tests for. | 988 # Select which "thousands bucket" we're going to run additional tests for. |
980 bucket_size = 1000 | 989 bucket_size = 1000 |
981 thousands_list = range(1, int(revision_num)/bucket_size + 1) | 990 thousands_list = range(EARLIEST_REVISION/bucket_size, |
991 int(revision_num)/bucket_size + 1) | |
982 weighted_total = sum(thousands_list) | 992 weighted_total = sum(thousands_list) |
983 generated_random_number = random.randint(0, weighted_total - 1) | 993 generated_random_number = random.randint(0, weighted_total - 1) |
984 for i in list(reversed(thousands_list)): | 994 for i in list(reversed(thousands_list)): |
985 thousands = thousands_list[i - 1] | 995 thousands = thousands_list[i - 1] |
986 weighted_total -= thousands_list[i - 1] | 996 weighted_total -= thousands_list[i - 1] |
987 if weighted_total <= generated_random_number: | 997 if weighted_total <= generated_random_number: |
988 break | 998 break |
989 while tries < 15 and not has_run_extra: | 999 while tries < 15 and not has_run_extra: |
990 # Now select a particular revision in that bucket. | 1000 # Now select a particular revision in that bucket. |
991 if thousands == int(revision_num)/bucket_size: | 1001 if thousands == int(revision_num)/bucket_size: |
992 max_range = 1 + revision_num % bucket_size | 1002 max_range = 1 + revision_num % bucket_size |
993 else: | 1003 else: |
994 max_range = bucket_size | 1004 max_range = bucket_size |
995 rev = thousands * bucket_size + random.randrange(0, max_range) | 1005 rev = thousands * bucket_size + random.randrange(0, max_range) |
996 if rev not in results_set: | 1006 if rev not in results_set: |
997 has_run_extra = try_to_run_additional(rev) | 1007 has_run_extra = try_to_run_additional(rev) |
998 tries += 1 | 1008 tries += 1 |
999 | 1009 |
1000 if not has_run_extra: | 1010 if not has_run_extra: |
1001 # Try to get up to 10 runs of each CL, starting with the most recent | 1011 # Try to get up to 10 runs of each CL, starting with the most recent |
1002 # CL that does not yet have 10 runs. But only perform a set of extra | 1012 # CL that does not yet have 10 runs. But only perform a set of extra |
1003 # runs at most 2 at a time before checking to see if new code has been | 1013 # runs at most 2 at a time before checking to see if new code has been |
1004 # checked in. | 1014 # checked in. |
1005 while revision_num > 0 and not has_run_extra: | 1015 while revision_num > EARLIEST_REVISION and not has_run_extra: |
1006 if revision_num not in results_set: | 1016 if revision_num not in results_set: |
1007 has_run_extra = try_to_run_additional(revision_num) | 1017 has_run_extra = try_to_run_additional(revision_num) |
1008 revision_num -= 1 | 1018 revision_num -= 1 |
1009 if not has_run_extra: | 1019 if not has_run_extra: |
1010 # No more extra back-runs to do (for now). Wait for new code. | 1020 # No more extra back-runs to do (for now). Wait for new code. |
1011 time.sleep(200) | 1021 time.sleep(200) |
1012 return results_set | 1022 return results_set |
1013 | 1023 |
1014 def main(): | 1024 def main(): |
1015 runner = TestRunner() | 1025 runner = TestRunner() |
(...skipping 12 matching lines...) Expand all Loading... | |
1028 results_set = update_set_of_done_cls() | 1038 results_set = update_set_of_done_cls() |
1029 if runner.has_interesting_code(): | 1039 if runner.has_interesting_code(): |
1030 runner.run_test_sequence() | 1040 runner.run_test_sequence() |
1031 else: | 1041 else: |
1032 results_set = fill_in_back_history(results_set, runner) | 1042 results_set = fill_in_back_history(results_set, runner) |
1033 else: | 1043 else: |
1034 runner.run_test_sequence() | 1044 runner.run_test_sequence() |
1035 | 1045 |
1036 if __name__ == '__main__': | 1046 if __name__ == '__main__': |
1037 main() | 1047 main() |
OLD | NEW |