tools/test.py: Add support for JUnit compatible XML output

tools/test.py

 #!/usr/bin/env python
 #
 # Copyright 2012 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 25 matching lines @@
 import re
 import signal
 import subprocess
 import sys
 import tempfile
 import time
 import threading
 import utils
 from Queue import Queue, Empty
 
+from unittest_output import UnitTestOutput
 
 VERBOSE = False
+XMLOUT = None

[Jakob Kummerow, 2012/09/28 17:38:52]

No global variables. Please.

+XMLTESTSUITE = None
 
 
 # ---------------------------------------------
 # --- P r o g r e s s   I n d i c a t o r s ---
 # ---------------------------------------------
 
 
 class ProgressIndicator(object):
 
   def __init__(self, cases):
     self.cases = cases
     self.queue = Queue(len(cases))
     for case in cases:
       self.queue.put_nowait(case)
     self.succeeded = 0
     self.remaining = len(cases)
     self.total = len(cases)
     self.failed = [ ]
     self.crashed = 0
     self.terminate = False
     self.lock = threading.Lock()
+    if XMLOUT:

[Jakob Kummerow, 2012/09/28 17:38:52]

This looks weird. The UnitTestProgressIndicator is-a ProgressIndicator according
to its class definition (which is good!). The additional has-a relation here is
counter-intuitive. Why doesn't the UnitTestProgressIndicator simply replace any
other progress indicator in use? If you want to print progress information to
stdout at the same time as dumping the XML file, you can always derive from one
of the printing progress indicators (and call the superclass methods when you
override them).

This comment also applies to the next few edits below.

+      self.xmlOutputter = UnitTestProgressIndicator()
 
   def PrintFailureHeader(self, test):
     if test.IsNegative():
       negative_marker = '[negative] '
     else:
       negative_marker = ''
     print "=== %(label)s %(negative)s===" % {
       'label': test.GetLabel(),
       'negative': negative_marker
     }
     print "Path: %s" % "/".join(test.path)
 
   def Run(self, tasks):
     self.Starting()
+    if XMLOUT:
+      self.xmlOutputter.Starting()
     threads = []
     # Spawn N-1 threads and then use this thread as the last one.
     # That way -j1 avoids threading altogether which is a nice fallback
     # in case of threading problems.
     for i in xrange(tasks - 1):
       thread = threading.Thread(target=self.RunSingle, args=[])
       threads.append(thread)
       thread.start()
     try:
       self.RunSingle()
       # Wait for the remaining threads
       for thread in threads:
         # Use a timeout so that signals (ctrl-c) will be processed.
         thread.join(timeout=10000000)
     except Exception, e:
       # If there's an exception we schedule an interruption for any
       # remaining threads.
       self.terminate = True
       # ...and then reraise the exception to bail out
       raise
     self.Done()
+    if XMLOUT:
+      self.xmlOutputter.Done()
     return not self.failed
 
   def RunSingle(self):
     while not self.terminate:
       try:
         test = self.queue.get_nowait()
       except Empty:
         return
       case = test.case
       self.lock.acquire()
       self.AboutToRun(case)
+      if XMLOUT:
+        self.xmlOutputter.AboutToRun(case)
       self.lock.release()
       try:
         start = time.time()
         output = case.Run()
         case.duration = (time.time() - start)
       except BreakNowException:
         self.terminate = True
       except IOError, e:
         assert self.terminate
         return
       if self.terminate:
         return
       self.lock.acquire()
       if output.UnexpectedOutput():
         self.failed.append(output)
         if output.HasCrashed():
           self.crashed += 1
       else:
         self.succeeded += 1
       self.remaining -= 1
       self.HasRun(output)
+      if XMLOUT:
+        self.xmlOutputter.HasRun(output)
       self.lock.release()
 
 
 def EscapeCommand(command):
   parts = []
   for part in command:
     if ' ' in part:
       # Escape spaces and double quotes. We may need to escape more characters
       # for this to work properly.
       parts.append('"%s"' % part.replace('"', '\\"'))
@@ skipping 151 matching lines @@
   def __init__(self, cases):
     templates = {
       'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s",
       'stdout': '%s',
       'stderr': '%s',
     }
     super(MonochromeProgressIndicator, self).__init__(cases, templates)
 
   def ClearLine(self, last_line_length):
     print ("\r" + (" " * last_line_length) + "\r"),
 

[Jakob Kummerow, 2012/09/28 17:38:52]

nit: two newlines between top-level definitions

+class UnitTestProgressIndicator(ProgressIndicator):
+
+  def __init__(self):
+    self.outputter = UnitTestOutput(XMLTESTSUITE)
+    if XMLOUT:
+      self.outfile = open(XMLOUT, "w")
+    else:
+      self.outfile = sys.stdout
+
+  def Starting(self):
+    pass
+
+  def AboutToRun(self, case):
+    self.outputter.startNewTest(case.GetName())
+
+  def Done(self):
+    self.outputter.finishAndWrite(self.outfile)
+    if self.outfile != sys.stdout:
+      self.outfile.close()
+
+  def HasRun(self, output):
+    if output.UnexpectedOutput():
+      failtext = ""
+      stdout = output.output.stdout.strip()
+      if len(stdout):
+        failtext += "stdout:\n"
+        failtext += stdout
+        failtext += "\n"
+      stderr = output.output.stderr.strip()
+      if len(stderr):
+        failtext += "stderr:\n"
+        failtext += stderr
+        failtext += "\n"
+      if output.HasCrashed():
+        failtext += "--- CRASHED ---"
+      if output.HasTimedOut():
+        failtext += "--- TIMEOUT ---"
+      self.outputter.finishCurrentTest(output.command, True, failtext)
+    else:
+      self.outputter.finishCurrentTest(output.command, False)
+
 
 PROGRESS_INDICATORS = {
   'verbose': VerboseProgressIndicator,
   'dots': DotsProgressIndicator,
   'color': ColorProgressIndicator,
   'mono': MonochromeProgressIndicator
 }
 
 
 # -------------------------
@@ skipping 913 matching lines @@
         dest="suppress_dialogs", action="store_false")
   result.add_option("--mips-arch-variant", help="mips architecture variant: mips32r1/mips32r2", default="mips32r2");
   result.add_option("--shell", help="Path to V8 shell", default="d8")
   result.add_option("--isolates", help="Whether to test isolates", default=False, action="store_true")
   result.add_option("--store-unexpected-output",
       help="Store the temporary JS files from tests that fails",
       dest="store_unexpected_output", default=True, action="store_true")
   result.add_option("--no-store-unexpected-output",
       help="Deletes the temporary JS files from tests that fails",
       dest="store_unexpected_output", action="store_false")
+  result.add_option("--xmlout", help="File name of the UnitTest output")
+  result.add_option("--xmltestsuite", help="Name of the testsuite in the UnitTest XML output", default="v8tests")
   result.add_option("--stress-only",
                     help="Only run tests with --always-opt --stress-opt",
                     default=False, action="store_true")
   result.add_option("--nostress",
                     help="Don't run crankshaft --always-opt --stress-op test",
                     default=False, action="store_true")
   result.add_option("--shard-count",
                     help="Split testsuites into this number of shards",
                     default=1, type="int")
   result.add_option("--shard-run",
@@ skipping 33 matching lines @@
     if options.arch in ['android', 'arm', 'mipsel']:
       options.timeout = 2 * TIMEOUT_DEFAULT;
     else:
       options.timeout = TIMEOUT_DEFAULT;
   if options.snapshot:
     options.scons_flags.append("snapshot=on")
   global VARIANT_FLAGS
   if options.mips_arch_variant:
     options.scons_flags.append("mips_arch_variant=" + options.mips_arch_variant)
 
+  global XMLOUT
+  XMLOUT = options.xmlout
+  global XMLTESTSUITE
+  XMLTESTSUITE = options.xmltestsuite
+
   if options.stress_only:
     VARIANT_FLAGS = [['--stress-opt', '--always-opt']]
   if options.nostress:
     VARIANT_FLAGS = [[],['--nocrankshaft']]
   if options.shell.endswith("d8"):
     if options.special_command:
       options.special_command += " --test"
     else:
       options.special_command = "@ --test"
   if options.noprof:
@@ skipping 241 matching lines @@
     index = 1
     for entry in timed_tests[:20]:
       t = FormatTime(entry.duration)
       sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel()))
       index += 1
 
   return result
 
 
 if __name__ == '__main__':
-  sys.exit(Main())
+  if XMLOUT:
+    Main()
+    sys.exit(0)
+  else:
+    sys.exit(Main())