OLD | NEW |
| (Empty) |
1 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | |
2 # for details. All rights reserved. Use of this source code is governed by a | |
3 # BSD-style license that can be found in the LICENSE file. | |
4 # | |
5 | |
6 import os | |
7 import platform | |
8 import re | |
9 import shutil | |
10 import subprocess | |
11 import sys | |
12 import tempfile | |
13 | |
14 import utils | |
15 | |
16 OS_GUESS = utils.GuessOS() | |
17 | |
18 HTML_CONTENTS = """ | |
19 <html> | |
20 <head> | |
21 <title> Test %(title)s </title> | |
22 <style> | |
23 .unittest-table { font-family:monospace; border:1px; } | |
24 .unittest-pass { background: #6b3;} | |
25 .unittest-fail { background: #d55;} | |
26 .unittest-error { background: #a11;} | |
27 </style> | |
28 </head> | |
29 <body> | |
30 <h1> Running %(title)s </h1> | |
31 <script type="text/javascript" src="%(controller_script)s"></script> | |
32 <script type="text/javascript"> | |
33 // If nobody intercepts the error, finish the test. | |
34 onerror = function() { window.layoutTestController.notifyDone() }; | |
35 | |
36 document.onreadystatechange = function() { | |
37 if (document.readyState != "loaded") return; | |
38 // If 'startedDartTest' is not set, that means that the test did not have | |
39 // a chance to load. This will happen when a load error occurs in the VM. | |
40 // Give the machine time to start up. | |
41 setTimeout(function() { | |
42 // A window.postMessage might have been enqueued after this timeout. | |
43 // Just sleep another time to give the browser the time to process the | |
44 // posted message. | |
45 setTimeout(function() { | |
46 if (layoutTestController && !layoutTestController.startedDartTest) { | |
47 layoutTestController.notifyDone(); | |
48 } | |
49 }, 0); | |
50 }, 50); | |
51 }; | |
52 </script> | |
53 <script type="%(script_type)s" src="%(source_script)s"></script> | |
54 </body> | |
55 </html> | |
56 """ | |
57 | |
58 DART_TEST_AS_LIBRARY = """ | |
59 #library('test'); | |
60 #source('%(test)s'); | |
61 """ | |
62 | |
63 DART_CONTENTS = """ | |
64 #library('test'); | |
65 | |
66 #import('%(dom_library)s'); | |
67 #import('%(test_framework)s'); | |
68 | |
69 #import('%(library)s', prefix: "Test"); | |
70 | |
71 waitForDone() { | |
72 window.postMessage('unittest-suite-wait-for-done', '*'); | |
73 } | |
74 | |
75 pass() { | |
76 document.body.innerHTML = 'PASS'; | |
77 window.postMessage('unittest-suite-done', '*'); | |
78 } | |
79 | |
80 fail(e, trace) { | |
81 document.body.innerHTML = 'FAIL: $e, $trace'; | |
82 window.postMessage('unittest-suite-done', '*'); | |
83 } | |
84 | |
85 main() { | |
86 bool needsToWait = false; | |
87 bool mainIsFinished = false; | |
88 TestRunner.waitForDoneCallback = () { needsToWait = true; }; | |
89 TestRunner.doneCallback = () { | |
90 if (mainIsFinished) { | |
91 pass(); | |
92 } else { | |
93 needsToWait = false; | |
94 } | |
95 }; | |
96 try { | |
97 Test.main(); | |
98 if (needsToWait) { | |
99 waitForDone(); | |
100 } else { | |
101 pass(); | |
102 } | |
103 mainIsFinished = true; | |
104 } catch(var e, var trace) { | |
105 fail(e, trace); | |
106 } | |
107 } | |
108 """ | |
109 | |
110 | |
111 # Patterns for matching test options in .dart files. | |
112 DART_OPTIONS_PATTERN = re.compile(r'// DartOptions=(.*)') | |
113 | |
114 # Pattern for checking if the test is a web test. | |
115 DOM_IMPORT_PATTERN = re.compile(r'#import.*(dart:(dom|html)|html\.dart).*\);', | |
116 re.MULTILINE) | |
117 | |
118 # Pattern for matching the output of a browser test. | |
119 BROWSER_OUTPUT_PASS_PATTERN = re.compile(r'^Content-Type: text/plain\nPASS$', | |
120 re.MULTILINE) | |
121 | |
122 # Pattern for matching flaky errors of browser tests. xvfb-run by default uses | |
123 # DISPLAY=:99, we keep that in the error pattern to avoid matching real | |
124 # errors when DISPLAY is set incorrectly. | |
125 BROWSER_FLAKY_DISPLAY_ERR_PATTERN = re.compile( | |
126 r'Gtk-WARNING \*\*: cannot open display: :99', re.MULTILINE) | |
127 | |
128 # Pattern for checking if the test is a library in itself. | |
129 LIBRARY_DEFINITION_PATTERN = re.compile(r'^#library\(.*\);', | |
130 re.MULTILINE) | |
131 SOURCE_OR_IMPORT_PATTERN = re.compile(r'^#(source|import)\(.*\);', | |
132 re.MULTILINE) | |
133 | |
134 | |
135 class Error(Exception): | |
136 """Base class for exceptions in this module.""" | |
137 pass | |
138 | |
139 | |
140 def _IsWebTest(source): | |
141 """Returns True if the source includes a dart dom library #import.""" | |
142 return DOM_IMPORT_PATTERN.search(source) | |
143 | |
144 | |
145 def IsLibraryDefinition(test, source): | |
146 """Returns True if the source has a #library statement.""" | |
147 if LIBRARY_DEFINITION_PATTERN.search(source): | |
148 return True | |
149 if SOURCE_OR_IMPORT_PATTERN.search(source): | |
150 print ('WARNING for %s: Browser tests need a #library ' | |
151 'for a file that #import or #source' % test) | |
152 return False | |
153 | |
154 | |
155 class Architecture(object): | |
156 """Definitions for different ways to test based on the component flag.""" | |
157 | |
158 def __init__(self, root_path, arch, mode, component, test): | |
159 self.root_path = root_path | |
160 self.arch = arch | |
161 self.mode = mode | |
162 self.component = component | |
163 self.test = test | |
164 self.build_root = utils.GetBuildRoot(OS_GUESS, self.mode, self.arch) | |
165 source = file(test).read() | |
166 self.vm_options = [] | |
167 self.dart_options = utils.ParseTestOptions(DART_OPTIONS_PATTERN, | |
168 source, | |
169 root_path) | |
170 self.is_web_test = _IsWebTest(source) | |
171 self.temp_dir = None | |
172 | |
173 def GetVMOption(self, option): | |
174 for flag in self.vm_options: | |
175 if flag.startswith('--%s=' % option): | |
176 return flag.split('=')[1] | |
177 return None | |
178 | |
179 def HasFatalTypeErrors(self): | |
180 """Returns True if this type of component supports --fatal-type-errors.""" | |
181 return False | |
182 | |
183 def GetTestFrameworkPath(self): | |
184 """Path to dart source (TestFramework.dart) for testing framework.""" | |
185 return os.path.join(self.root_path, 'tests', 'isolate', 'src', | |
186 'TestFramework.dart') | |
187 | |
188 | |
189 class BrowserArchitecture(Architecture): | |
190 """Architecture that runs compiled dart->JS through a browser.""" | |
191 | |
192 def __init__(self, root_path, arch, mode, component, test): | |
193 super(BrowserArchitecture, self).__init__(root_path, arch, mode, component, | |
194 test) | |
195 self.temp_dir = tempfile.mkdtemp() | |
196 if not self.is_web_test: self.GenerateWebTestScript() | |
197 | |
198 def GetTestScriptFile(self): | |
199 """Returns the name of the .dart file to compile.""" | |
200 if self.is_web_test: return os.path.abspath(self.test) | |
201 return os.path.join(self.temp_dir, 'test.dart') | |
202 | |
203 def GetHtmlContents(self): | |
204 """Fills in the HTML_CONTENTS template with info for this architecture.""" | |
205 script_type = self.GetScriptType() | |
206 controller_path = os.path.join(self.root_path, 'client', 'testing', | |
207 'unittest', 'test_controller.js') | |
208 return HTML_CONTENTS % { | |
209 'title': self.test, | |
210 'controller_script': controller_path, | |
211 'script_type': script_type, | |
212 'source_script': self.GetScriptPath() | |
213 } | |
214 | |
215 def GetHtmlPath(self): | |
216 """Creates a path for the generated .html file. | |
217 | |
218 Resources for web tests are relative to the 'html' file. We | |
219 output the 'html' file in the 'out' directory instead of the temporary | |
220 directory because we can easily go the the resources in 'client' through | |
221 'out'. | |
222 | |
223 Returns: | |
224 Created path for the generated .html file. | |
225 """ | |
226 if self.is_web_test: | |
227 html_path = os.path.join(self.root_path, 'client', self.build_root) | |
228 if not os.path.exists(html_path): | |
229 os.makedirs(html_path) | |
230 return html_path | |
231 | |
232 return self.temp_dir | |
233 | |
234 def GetTestContents(self, library_file): | |
235 """Pastes a preamble on the front of the .dart file before testing.""" | |
236 unittest_path = os.path.join(self.root_path, 'client', 'testing', | |
237 'unittest', 'unittest.dart') | |
238 | |
239 if self.component == 'chromium': | |
240 dom_path = os.path.join(self.root_path, 'client', 'testing', | |
241 'unittest', 'dom_for_unittest.dart') | |
242 else: | |
243 dom_path = os.path.join('dart:dom') | |
244 | |
245 test_framework_path = self.GetTestFrameworkPath() | |
246 test_path = os.path.abspath(self.test) | |
247 | |
248 inputs = { | |
249 'unittest': unittest_path, | |
250 'test': test_path, | |
251 'dom_library': dom_path, | |
252 'test_framework': test_framework_path, | |
253 'library': library_file | |
254 } | |
255 return DART_CONTENTS % inputs | |
256 | |
257 def GenerateWebTestScript(self): | |
258 """Creates a .dart file to run in the test.""" | |
259 if IsLibraryDefinition(self.test, file(self.test).read()): | |
260 library_file = os.path.abspath(self.test) | |
261 else: | |
262 library_file = 'test_as_library.dart' | |
263 test_as_library = DART_TEST_AS_LIBRARY % { | |
264 'test': os.path.abspath(self.test) | |
265 } | |
266 test_as_library_file = os.path.join(self.temp_dir, library_file) | |
267 f = open(test_as_library_file, 'w') | |
268 f.write(test_as_library) | |
269 f.close() | |
270 | |
271 app_output_file = self.GetTestScriptFile() | |
272 f = open(app_output_file, 'w') | |
273 f.write(self.GetTestContents(library_file)) | |
274 f.close() | |
275 | |
276 def GetRunCommand(self, fatal_static_type_errors=False): | |
277 """Returns a command line to execute for the test.""" | |
278 fatal_static_type_errors = fatal_static_type_errors # shutup lint! | |
279 # Find DRT | |
280 # For some reason, DRT needs to be called via an absolute path | |
281 drt_location = self.GetVMOption('browser') | |
282 if drt_location is not None: | |
283 drt_location = os.path.abspath(drt_location) | |
284 else: | |
285 drt_location = os.path.join(self.root_path, 'client', 'tests', 'drt', | |
286 'DumpRenderTree') | |
287 | |
288 # On Mac DumpRenderTree is a .app folder | |
289 if platform.system() == 'Darwin': | |
290 drt_location += '.app/Contents/MacOS/DumpRenderTree' | |
291 | |
292 drt_flags = ['--no-timeout'] | |
293 if len(self.vm_options) > 0: | |
294 dart_flags = '--dart-flags=' | |
295 dart_flags += ' '.join(self.vm_options) | |
296 drt_flags.append(dart_flags) | |
297 | |
298 html_output_file = os.path.join(self.GetHtmlPath(), self.GetHtmlName()) | |
299 f = open(html_output_file, 'w') | |
300 f.write(self.GetHtmlContents()) | |
301 f.close() | |
302 | |
303 drt_flags.append(html_output_file) | |
304 | |
305 return [drt_location] + drt_flags | |
306 | |
307 def HasFailed(self, output): | |
308 """Return True if the 'PASS' result string isn't in the output.""" | |
309 return not BROWSER_OUTPUT_PASS_PATTERN.search(output) | |
310 | |
311 def WasFlakyDrt(self, error): | |
312 """Return whether the error indicates a flaky error from running. | |
313 DumpRenderTree within xvfb-run. | |
314 """ | |
315 return BROWSER_FLAKY_DISPLAY_ERR_PATTERN.search(error) | |
316 | |
317 def RunTest(self, verbose): | |
318 """Calls GetRunCommand() and executes the returned commandline. | |
319 | |
320 Args: | |
321 verbose: if True, print additional diagnostics to stdout. | |
322 | |
323 Returns: | |
324 Return code from executable. 0 == PASS, 253 = CRASH, anything | |
325 else is treated as FAIL | |
326 """ | |
327 retcode = self.Compile() | |
328 if retcode != 0: return 1 | |
329 | |
330 command = self.GetRunCommand() | |
331 | |
332 unused_status, output, err = ExecutePipedCommand(command, verbose) | |
333 if not self.HasFailed(output): | |
334 return 0 | |
335 | |
336 # TODO(sigmund): print better error message, including how to run test | |
337 # locally, and translate error traces using source map info. | |
338 print '(FAIL) test page:\033[31m %s \033[0m' % command[2] | |
339 if verbose: | |
340 print 'Additional info: ' | |
341 print output | |
342 print err | |
343 return 1 | |
344 | |
345 def Cleanup(self): | |
346 """Removes temporary files created for the test.""" | |
347 if self.temp_dir: | |
348 shutil.rmtree(self.temp_dir) | |
349 self.temp_dir = None | |
350 | |
351 | |
352 class ChromiumArchitecture(BrowserArchitecture): | |
353 """Architecture that runs compiled dart->JS through a chromium DRT.""" | |
354 | |
355 def __init__(self, root_path, arch, mode, component, test): | |
356 super(ChromiumArchitecture, self).__init__(root_path, arch, mode, component,
test) | |
357 | |
358 def GetScriptType(self): | |
359 return 'text/javascript' | |
360 | |
361 def GetScriptPath(self): | |
362 """Returns the name of the output .js file to create.""" | |
363 path = self.GetTestScriptFile() | |
364 return os.path.abspath(os.path.join(self.temp_dir, | |
365 os.path.basename(path) + '.js')) | |
366 | |
367 def GetHtmlName(self): | |
368 """Returns the name of the output .html file to create.""" | |
369 relpath = os.path.relpath(self.test, self.root_path) | |
370 return relpath.replace(os.sep, '_') + '.html' | |
371 | |
372 def Compile(self): | |
373 return ExecuteCommand(self.GetCompileCommand()) | |
374 | |
375 class DartcChromiumArchitecture(ChromiumArchitecture): | |
376 """ChromiumArchitecture that compiles code using dartc.""" | |
377 | |
378 def __init__(self, root_path, arch, mode, component, test): | |
379 super(DartcChromiumArchitecture, self).__init__( | |
380 root_path, arch, mode, component, test) | |
381 | |
382 def GetCompileCommand(self, fatal_static_type_errors=False): | |
383 """Returns cmdline as an array to invoke the compiler on this test.""" | |
384 | |
385 # We need an absolute path because the compilation will run | |
386 # in a temporary directory. | |
387 build_root = utils.GetBuildRoot(OS_GUESS, self.mode, 'ia32') | |
388 dartc = os.path.abspath(os.path.join(build_root, 'compiler', 'bin', | |
389 'dartc')) | |
390 if utils.IsWindows(): dartc += '.exe' | |
391 cmd = [dartc, '--work', self.temp_dir] | |
392 if self.mode == 'release': | |
393 cmd += ['--optimize'] | |
394 cmd += self.vm_options | |
395 cmd += ['--out', self.GetScriptPath()] | |
396 if fatal_static_type_errors: | |
397 # TODO(zundel): update to --fatal_type_errors for both VM and Compiler | |
398 cmd.append('-fatal-type-errors') | |
399 cmd.append(self.GetTestScriptFile()) | |
400 return cmd | |
401 | |
402 | |
403 class FrogChromiumArchitecture(ChromiumArchitecture): | |
404 """ChromiumArchitecture that compiles code using frog.""" | |
405 | |
406 def __init__(self, root_path, arch, mode, component, test): | |
407 super(FrogChromiumArchitecture, self).__init__( | |
408 root_path, arch, mode, component, test) | |
409 | |
410 def GetCompileCommand(self, fatal_static_type_errors=False): | |
411 """Returns cmdline as an array to invoke the compiler on this test.""" | |
412 | |
413 # Get a frog executable from the command line. Default to frogsh. | |
414 # We need an absolute path because the compilation will run | |
415 # in a temporary directory. | |
416 frog = self.GetVMOption('frog') | |
417 if frog is not None: | |
418 frog = os.path.abspath(frog) | |
419 else: | |
420 frog = os.path.abspath(utils.GetDartRunner(self.mode, self.arch, 'frogsh')
) | |
421 frog_libdir = self.GetVMOption('froglib') | |
422 if frog_libdir is not None: | |
423 frog_libdir = os.path.abspath(frog_libdir) | |
424 else: | |
425 frog_libdir = os.path.abspath(os.path.join(self.root_path, 'frog', 'lib')) | |
426 cmd = [frog, | |
427 '--libdir=%s' % frog_libdir, | |
428 '--compile-only', | |
429 '--out=%s' % self.GetScriptPath()] | |
430 cmd.extend(self.vm_options) | |
431 cmd.append(self.GetTestScriptFile()) | |
432 return cmd | |
433 | |
434 | |
435 class DartiumArchitecture(BrowserArchitecture): | |
436 """Architecture that runs dart in an VM embedded in DumpRenderTree.""" | |
437 | |
438 def __init__(self, root_path, arch, mode, component, test): | |
439 super(DartiumArchitecture, self).__init__(root_path, arch, mode, component,
test) | |
440 | |
441 def GetScriptType(self): | |
442 return 'application/dart' | |
443 | |
444 def GetScriptPath(self): | |
445 return 'file:///' + self.GetTestScriptFile() | |
446 | |
447 def GetHtmlName(self): | |
448 path = os.path.relpath(self.test, self.root_path).replace(os.sep, '_') | |
449 return path + '.dartium.html' | |
450 | |
451 def GetCompileCommand(self, fatal_static_type_errors=False): | |
452 fatal_static_type_errors = fatal_static_type_errors # shutup lint! | |
453 return None | |
454 | |
455 def Compile(self): | |
456 return 0 | |
457 | |
458 | |
459 class WebDriverArchitecture(FrogChromiumArchitecture): | |
460 """Architecture that runs compiled dart->JS (via frog) through a variety of | |
461 real browsers using WebDriver.""" | |
462 | |
463 def __init__(self, root_path, arch, mode, component, test): | |
464 super(WebDriverArchitecture, self).__init__(root_path, arch, mode, | |
465 component, test) | |
466 | |
467 def GetRunCommand(self, fatal_static_type_errors=False): | |
468 """Returns a command line to execute for the test.""" | |
469 flags = self.vm_options | |
470 browser_flag = 'chrome' | |
471 if 'ff' in flags or 'firefox' in flags: | |
472 browser_flag = 'ff' | |
473 elif 'ie' in flags or 'explorer' in flags or 'internet-explorer' in flags: | |
474 browser_flag = 'ie' | |
475 elif 'safari' in flags: | |
476 browser_flag = 'safari' | |
477 | |
478 selenium_location = os.path.join(self.root_path, 'tools', 'testing', | |
479 'run_selenium.py') | |
480 | |
481 html_output_file = os.path.join(self.GetHtmlPath(), self.GetHtmlName()) | |
482 f = open(html_output_file, 'w') | |
483 f.write(self.GetHtmlContents()) | |
484 f.close() | |
485 return [selenium_location, '--out', html_output_file, '--browser', | |
486 browser_flag] | |
487 | |
488 | |
489 class StandaloneArchitecture(Architecture): | |
490 """Base class for architectures that run tests without a browser.""" | |
491 | |
492 def __init__(self, root_path, arch, mode, component, test): | |
493 super(StandaloneArchitecture, self).__init__(root_path, arch, mode, componen
t, | |
494 test) | |
495 | |
496 def GetExecutable(self): | |
497 """Returns the path to the Dart test runner (executes the .dart file).""" | |
498 return utils.GetDartRunner(self.mode, self.arch, self.component) | |
499 | |
500 | |
501 def GetCompileCommand(self, fatal_static_type_errors=False): | |
502 fatal_static_type_errors = fatal_static_type_errors # shutup lint! | |
503 return None | |
504 | |
505 def GetOptions(self): | |
506 return [] | |
507 | |
508 def GetRunCommand(self, fatal_static_type_errors=False): | |
509 """Returns a command line to execute for the test.""" | |
510 dart = self.GetExecutable() | |
511 command = [dart] + self.GetOptions() + self.vm_options | |
512 if fatal_static_type_errors: | |
513 command += self.GetFatalTypeErrorsFlags() | |
514 | |
515 if self.dart_options: | |
516 command += self.dart_options | |
517 else: | |
518 command += [self.test] | |
519 | |
520 return command | |
521 | |
522 def GetFatalTypeErrorsFlags(self): | |
523 return [] | |
524 | |
525 def RunTest(self, verbose): | |
526 command = self.GetRunCommand() | |
527 return ExecuteCommand(command, verbose) | |
528 | |
529 def Cleanup(self): | |
530 return | |
531 | |
532 | |
533 class LegArchitecture(StandaloneArchitecture): | |
534 | |
535 def __init__(self, root_path, arch, mode, component, test): | |
536 super(LegArchitecture, self).__init__(root_path, arch, mode, component, | |
537 test) | |
538 def GetOptions(self): | |
539 return ['--leg_only'] | |
540 | |
541 def GetExecutable(self): | |
542 """Returns the path to the Dart test runner (executes the .dart file).""" | |
543 return utils.GetDartRunner(self.mode, self.arch, 'frog') | |
544 | |
545 | |
546 # Long term, we should do the running machinery that is currently in | |
547 # DartRunner.java | |
548 class DartcArchitecture(StandaloneArchitecture): | |
549 """Runs the Dart ->JS compiler then runs the result in a standalone JS VM.""" | |
550 | |
551 def __init__(self, root_path, arch, mode, component, test): | |
552 super(DartcArchitecture, self).__init__(root_path, arch, mode, component, te
st) | |
553 | |
554 def GetOptions(self): | |
555 if self.mode == 'release': return ['--optimize'] | |
556 return [] | |
557 | |
558 def GetFatalTypeErrorsFlags(self): | |
559 return ['--fatal-type-errors'] | |
560 | |
561 def HasFatalTypeErrors(self): | |
562 return True | |
563 | |
564 | |
565 def ExecutePipedCommand(cmd, verbose): | |
566 """Execute a command in a subprocess.""" | |
567 if verbose: | |
568 print 'Executing: ' + ' '.join(cmd) | |
569 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
570 (output, err) = pipe.communicate() | |
571 if pipe.returncode != 0 and verbose: | |
572 print 'Execution failed: ' + output + '\n' + err | |
573 print output | |
574 print err | |
575 return pipe.returncode, output, err | |
576 | |
577 | |
578 def ExecuteCommand(cmd, verbose=False): | |
579 """Execute a command in a subprocess.""" | |
580 if verbose: print 'Executing: ' + ' '.join(cmd) | |
581 return subprocess.call(cmd) | |
582 | |
583 | |
584 def GetArchitecture(arch, mode, component, test): | |
585 root_path = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) | |
586 if component == 'chromium': | |
587 return DartcChromiumArchitecture(root_path, arch, mode, component, test) | |
588 | |
589 elif component == 'dartium': | |
590 return DartiumArchitecture(root_path, arch, mode, component, test) | |
591 | |
592 elif component == 'frogium': | |
593 return FrogChromiumArchitecture(root_path, arch, mode, component, test) | |
594 | |
595 elif component == 'webdriver': | |
596 return WebDriverArchitecture(root_path, arch, mode, component, test) | |
597 | |
598 elif component in ['vm', 'frog', 'frogsh']: | |
599 return StandaloneArchitecture(root_path, arch, mode, component, test) | |
600 | |
601 elif component == 'leg': | |
602 return LegArchitecture(root_path, arch, mode, component, test) | |
603 | |
604 elif component == 'dartc': | |
605 return DartcArchitecture(root_path, arch, mode, component, test) | |
OLD | NEW |