OLD | NEW |
1 # Copyright (C) 2010 Google Inc. All rights reserved. | 1 # Copyright (C) 2010 Google Inc. All rights reserved. |
2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze
ged | 2 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Sze
ged |
3 # Copyright (C) 2011 Apple Inc. All rights reserved. | 3 # Copyright (C) 2011 Apple Inc. All rights reserved. |
4 # | 4 # |
5 # Redistribution and use in source and binary forms, with or without | 5 # Redistribution and use in source and binary forms, with or without |
6 # modification, are permitted provided that the following conditions are | 6 # modification, are permitted provided that the following conditions are |
7 # met: | 7 # met: |
8 # | 8 # |
9 # * Redistributions of source code must retain the above copyright | 9 # * Redistributions of source code must retain the above copyright |
10 # notice, this list of conditions and the following disclaimer. | 10 # notice, this list of conditions and the following disclaimer. |
(...skipping 129 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
140 "This option implies --pixel-tests. If specified, only those te
sts " | 140 "This option implies --pixel-tests. If specified, only those te
sts " |
141 "will be executed as pixel tests that are located in one of the
" | 141 "will be executed as pixel tests that are located in one of the
" |
142 "directories enumerated with the option. Some ports may ignore
this " | 142 "directories enumerated with the option. Some ports may ignore
this " |
143 "option while others can have a default value that can be overr
idden here."), | 143 "option while others can have a default value that can be overr
idden here."), |
144 | 144 |
145 optparse.make_option("--skip-failing-tests", action="store_true", | 145 optparse.make_option("--skip-failing-tests", action="store_true", |
146 default=False, help="Skip tests that are expected to fail. " | 146 default=False, help="Skip tests that are expected to fail. " |
147 "Note: When using this option, you might miss new crashes " | 147 "Note: When using this option, you might miss new crashes " |
148 "in these tests."), | 148 "in these tests."), |
149 optparse.make_option("--additional-drt-flag", action="append", | 149 optparse.make_option("--additional-drt-flag", action="append", |
150 default=[], help="Additional command line flag to pass to DumpRender
Tree " | 150 default=[], help="Additional command line flag to pass to the driver
" |
151 "Specify multiple times to add multiple flags."), | 151 "Specify multiple times to add multiple flags."), |
152 optparse.make_option("--driver-name", type="string", | 152 optparse.make_option("--driver-name", type="string", |
153 help="Alternative DumpRenderTree binary to use"), | 153 help="Alternative driver binary to use"), |
154 optparse.make_option("--content-shell", action="store_true", | |
155 help="Use Content Shell instead of DumpRenderTree"), | |
156 optparse.make_option("--dump-render-tree", action="store_true", | |
157 help="Use DumpRenderTree instead of Content Shell"), | |
158 optparse.make_option("--additional-platform-directory", action="append", | 154 optparse.make_option("--additional-platform-directory", action="append", |
159 default=[], help="Additional directory where to look for test " | 155 default=[], help="Additional directory where to look for test " |
160 "baselines (will take precendence over platform baselines). " | 156 "baselines (will take precendence over platform baselines). " |
161 "Specify multiple times to add multiple search path entries."), | 157 "Specify multiple times to add multiple search path entries."), |
162 optparse.make_option("--additional-expectations", action="append", defau
lt=[], | 158 optparse.make_option("--additional-expectations", action="append", defau
lt=[], |
163 help="Path to a test_expectations file that will override previous e
xpectations. " | 159 help="Path to a test_expectations file that will override previous e
xpectations. " |
164 "Specify multiple times for multiple sets of overrides."), | 160 "Specify multiple times for multiple sets of overrides."), |
165 optparse.make_option("--compare-port", action="store", default=None, | 161 optparse.make_option("--compare-port", action="store", default=None, |
166 help="Use the specified port's baselines first"), | 162 help="Use the specified port's baselines first"), |
167 optparse.make_option("--no-show-results", action="store_false", | 163 optparse.make_option("--no-show-results", action="store_false", |
168 default=True, dest="show_results", | 164 default=True, dest="show_results", |
169 help="Don't launch a browser with results after the tests " | 165 help="Don't launch a browser with results after the tests " |
170 "are done"), | 166 "are done"), |
171 optparse.make_option("--full-results-html", action="store_true", | 167 optparse.make_option("--full-results-html", action="store_true", |
172 default=False, | 168 default=False, |
173 help="Show all failures in results.html, rather than only regression
s"), | 169 help="Show all failures in results.html, rather than only regression
s"), |
174 optparse.make_option("--clobber-old-results", action="store_true", | 170 optparse.make_option("--clobber-old-results", action="store_true", |
175 default=False, help="Clobbers test results from previous runs."), | 171 default=False, help="Clobbers test results from previous runs."), |
176 ])) | 172 ])) |
177 | 173 |
178 option_group_definitions.append(("Testing Options", [ | 174 option_group_definitions.append(("Testing Options", [ |
179 optparse.make_option("--build", dest="build", | 175 optparse.make_option("--build", dest="build", |
180 action="store_true", default=True, | 176 action="store_true", default=True, |
181 help="Check to ensure the DumpRenderTree build is up-to-date " | 177 help="Check to ensure the build is up-to-date (default)."), |
182 "(default)."), | |
183 optparse.make_option("--no-build", dest="build", | 178 optparse.make_option("--no-build", dest="build", |
184 action="store_false", help="Don't check to see if the " | 179 action="store_false", help="Don't check to see if the build is up-to
-date."), |
185 "DumpRenderTree build is up-to-date."), | |
186 optparse.make_option("-n", "--dry-run", action="store_true", | 180 optparse.make_option("-n", "--dry-run", action="store_true", |
187 default=False, | 181 default=False, |
188 help="Do everything but actually run the tests or upload results."), | 182 help="Do everything but actually run the tests or upload results."), |
189 optparse.make_option("--wrapper", | 183 optparse.make_option("--wrapper", |
190 help="wrapper command to insert before invocations of " | 184 help="wrapper command to insert before invocations of " |
191 "DumpRenderTree; option is split on whitespace before " | 185 "the driver; option is split on whitespace before " |
192 "running. (Example: --wrapper='valgrind --smc-check=all')"), | 186 "running. (Example: --wrapper='valgrind --smc-check=all')"), |
193 optparse.make_option("-i", "--ignore-tests", action="append", default=[]
, | 187 optparse.make_option("-i", "--ignore-tests", action="append", default=[]
, |
194 help="directories or test to ignore (may specify multiple times)"), | 188 help="directories or test to ignore (may specify multiple times)"), |
195 optparse.make_option("--ignore-flaky-tests", action="store", | 189 optparse.make_option("--ignore-flaky-tests", action="store", |
196 help=("Control whether tests that are flaky on the bots get ignored.
" | 190 help=("Control whether tests that are flaky on the bots get ignored.
" |
197 "'very-flaky' == Ignore any tests that flaked more than once on
the bot." | 191 "'very-flaky' == Ignore any tests that flaked more than once on
the bot." |
198 "'maybe-flaky' == Ignore any tests that flaked once on the bot." | 192 "'maybe-flaky' == Ignore any tests that flaked once on the bot." |
199 "'unexpected' == Ignore any tests that had unexpected results on
the bot.")), | 193 "'unexpected' == Ignore any tests that had unexpected results on
the bot.")), |
200 optparse.make_option("--ignore-builder-category", action="store", | 194 optparse.make_option("--ignore-builder-category", action="store", |
201 help=("The category of builders to use with the --ignore-flaky-tests
" | 195 help=("The category of builders to use with the --ignore-flaky-tests
" |
(...skipping 13 matching lines...) Expand all Loading... |
215 "'none' == use the order in which the tests were listed either
in arguments or test list, " | 209 "'none' == use the order in which the tests were listed either
in arguments or test list, " |
216 "'natural' == use the natural order (default), " | 210 "'natural' == use the natural order (default), " |
217 "'random' == randomize the test order.")), | 211 "'random' == randomize the test order.")), |
218 optparse.make_option("--run-chunk", | 212 optparse.make_option("--run-chunk", |
219 help=("Run a specified chunk (n:l), the nth of len l, " | 213 help=("Run a specified chunk (n:l), the nth of len l, " |
220 "of the layout tests")), | 214 "of the layout tests")), |
221 optparse.make_option("--run-part", help=("Run a specified part (n:m), " | 215 optparse.make_option("--run-part", help=("Run a specified part (n:m), " |
222 "the nth of m parts, of the layout tests")), | 216 "the nth of m parts, of the layout tests")), |
223 optparse.make_option("--batch-size", | 217 optparse.make_option("--batch-size", |
224 help=("Run a the tests in batches (n), after every n tests, " | 218 help=("Run a the tests in batches (n), after every n tests, " |
225 "DumpRenderTree is relaunched."), type="int", default=None), | 219 "the driver is relaunched."), type="int", default=None), |
226 optparse.make_option("--run-singly", action="store_true", | 220 optparse.make_option("--run-singly", action="store_true", |
227 default=False, help="run a separate DumpRenderTree for each test (im
plies --verbose)"), | 221 default=False, help="run a separate driver for each test (implies --
verbose)"), |
228 optparse.make_option("--child-processes", | 222 optparse.make_option("--child-processes", |
229 help="Number of DumpRenderTrees to run in parallel."), | 223 help="Number of drivers to run in parallel."), |
230 # FIXME: Display default number of child processes that will run. | 224 # FIXME: Display default number of child processes that will run. |
231 optparse.make_option("-f", "--fully-parallel", action="store_true", | 225 optparse.make_option("-f", "--fully-parallel", action="store_true", |
232 help="run all tests in parallel"), | 226 help="run all tests in parallel"), |
233 optparse.make_option("--exit-after-n-failures", type="int", default=None
, | 227 optparse.make_option("--exit-after-n-failures", type="int", default=None
, |
234 help="Exit after the first N failures instead of running all " | 228 help="Exit after the first N failures instead of running all " |
235 "tests"), | 229 "tests"), |
236 optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int", | 230 optparse.make_option("--exit-after-n-crashes-or-timeouts", type="int", |
237 default=None, help="Exit after the first N crashes instead of " | 231 default=None, help="Exit after the first N crashes instead of " |
238 "running all tests"), | 232 "running all tests"), |
239 optparse.make_option("--iterations", type="int", default=1, help="Number
of times to run the set of tests (e.g. ABCABCABC)"), | 233 optparse.make_option("--iterations", type="int", default=1, help="Number
of times to run the set of tests (e.g. ABCABCABC)"), |
(...skipping 110 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
350 printer.print_config(port.results_directory()) | 344 printer.print_config(port.results_directory()) |
351 | 345 |
352 run_details = manager.run(args) | 346 run_details = manager.run(args) |
353 _log.debug("Testing completed, Exit status: %d" % run_details.exit_code) | 347 _log.debug("Testing completed, Exit status: %d" % run_details.exit_code) |
354 return run_details | 348 return run_details |
355 finally: | 349 finally: |
356 printer.cleanup() | 350 printer.cleanup() |
357 | 351 |
358 if __name__ == '__main__': | 352 if __name__ == '__main__': |
359 sys.exit(main(sys.argv[1:], sys.stdout, sys.stderr)) | 353 sys.exit(main(sys.argv[1:], sys.stdout, sys.stderr)) |
OLD | NEW |