Fix dealing with incomplete files as the first set of files encountered.

tools/testing/perf_testing/run_perf_tests.py

 #!/usr/bin/python
 
-# Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
+# Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
 import math             
 try:            
   from matplotlib.font_manager import FontProperties            
   import matplotlib.pyplot as plt               
 except ImportError:             
   pass # Only needed if we want to make graphs.
@@ skipping 406 matching lines @@
   COMPILE_TIME = 'CompileTime'
   CODE_SIZE = 'CodeSize'
 
   def __init__(self, test):
     self.test = test
 
   def prepare(self):
     """Perform any initial setup required before the test is run."""
     pass
 
+  def open_trace_file(self, afile, not_yet_uploaded):
+    """Find the correct location for the trace file, and open it.
+    Args: 
+      afile: The tracefile name.
+      not_yet_uploaded: True if this file is to be found in a directory that
+         contains un-uploaded data.
+    Returns: A file object corresponding to the given file name."""
+    file_path = os.path.join(self.test.result_folder_name, afile)
+    if not not_yet_uploaded:
+      file_path = os.path.join('old', file_path)
+    return open(file_path)
+
   def report_results(self, benchmark_name, score, platform, variant, 
                      revision_number, metric):
     """Store the results of the benchmark run.
     Args:
       benchmark_name: The name of the individual benchmark.
       score: The numerical value of this benchmark.
       platform: The platform the test was run on (firefox, command line, etc).
       variant: Specifies whether the data was about generated Frog, js, a
           combination of both, or Dart depending on the test.
       revision_number: The revision of the code (and sometimes the revision of
@@ skipping 244 matching lines @@
   class CommonBrowserFileProcessor(Processor):
     def process_file(self, afile, should_post_file):
       """Comb through the html to find the performance results.
       Returns: True if we successfully posted our data to storage and/or we can
           delete the trace file."""
       os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools',
                             'testing', 'perf_testing'))
       parts = afile.split('-')
       browser = parts[2]
       version = parts[3]
-      if should_post_file:
-        f = open(os.path.join(self.test.result_folder_name, afile))
-      else:
-        f = open(os.path.join('old', self.test.result_folder_name, afile))
+      f = self.open_trace_file(afile, should_post_file)
       lines = f.readlines()
       line = ''
       i = 0
       revision_num = 0
       while '<div id="results">' not in line and i < len(lines):
         if 'Revision' in line:
           revision_num = int(line.split()[1].strip('"'))
         line = lines[i]
         i += 1
 
@@ skipping 167 matching lines @@
   class DromaeoFileProcessor(Processor):
     def process_file(self, afile, should_post_file):
       """Comb through the html to find the performance results.
       Returns: True if we successfully posted our data to storage."""
       parts = afile.split('-')
       browser = parts[2]
       version = parts[3]
 
       bench_dict = self.test.values_dict[browser][version]
 
-      f = open(os.path.join(self.test.result_folder_name, afile))
+      f = self.open_trace_file(afile, should_post_file)
       lines = f.readlines()
       i = 0
       revision_num = 0
       revision_pattern = r'Revision: (\d+)'
       suite_pattern = r'<div class="result-item done">(.+?)</ol></div>'
       result_pattern = r'<b>(.+?)</b>(.+?)<small> runs/s(.+)'
 
       upload_success = True
       for line in lines:
         rev = re.match(revision_pattern, line.strip())
@@ skipping 107 matching lines @@
 
   class DromaeoSizeProcessor(Processor):
     def process_file(self, afile, should_post_file):
       """Pull all the relevant information out of a given tracefile.
 
       Args:
         afile: is the filename string we will be processing.
       Returns: True if we successfully posted our data to storage."""
       os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools',
           'testing', 'perf_testing'))
-      f = open(os.path.join(self.test.result_folder_name, afile))
+      f = self.open_trace_file(afile, should_post_file)
       tabulate_data = False
       revision_num = 0
       revision_pattern = r'Revision: (\d+)'
       result_pattern = r'Size \((\w+), ([a-zA-Z0-9-]+)\): (\d+)'
 
       upload_success = True
       for line in f.readlines():
         rev = re.match(revision_pattern, line.strip())
         if rev:
           revision_num = int(rev.group(1))
@@ skipping 108 matching lines @@
   class CompileProcessor(Processor):
 
     def process_file(self, afile, should_post_file):
       """Pull all the relevant information out of a given tracefile.
 
       Args:
         afile: is the filename string we will be processing.
       Returns: True if we successfully posted our data to storage."""
       os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools',
           'testing', 'perf_testing'))
-      f = open(os.path.join(self.test.result_folder_name, afile))
+      f = self.open_trace_file(afile, should_post_file)
       tabulate_data = False
       revision_num = 0
       upload_success = True
       for line in f.readlines():
         tokens = line.split()
         if 'Revision' in line:
           revision_num = int(line.split()[1])
         else:
           for metric in self.test.values_list:
             if metric in line:
@@ skipping 11 matching lines @@
               if not self.test.test_runner.no_upload and should_post_file:
                 if num < self.test.failure_threshold[metric]:
                   num = 0
                 upload_success = upload_success and self.report_results(
                     metric, num, 'commandline', 'frog', revision_num,
                     score_type)
               else:
                 upload_success = False
       if revision_num != 0:
         for metric in self.test.values_list:
-          self.test.revision_dict['commandline']['frog'][metric].pop()
-          self.test.revision_dict['commandline']['frog'][metric] += \
-              [revision_num]
-          # Fill in 0 if compilation failed.
-          if self.test.values_dict['commandline']['frog'][metric][-1] < \
-              self.test.failure_threshold[metric]:
-            self.test.values_dict['commandline']['frog'][metric] += [0]
+          try:
+            self.test.revision_dict['commandline']['frog'][metric].pop()
             self.test.revision_dict['commandline']['frog'][metric] += \
                 [revision_num]
+            # Fill in 0 if compilation failed.
+            if self.test.values_dict['commandline']['frog'][metric][-1] < \
+                self.test.failure_threshold[metric]:
+              self.test.values_dict['commandline']['frog'][metric] += [0]
+              self.test.revision_dict['commandline']['frog'][metric] += \
+                  [revision_num]
+          except IndexError:
+            # We tried to pop from an empty list. This happens if the first
+            # trace file we encounter is incomplete.
+            pass
 
       f.close()
       return upload_success
 
   class CompileGrapher(Grapher):
 
     def plot_results(self, png_filename):               
       self.style_and_save_perf_plot(            
           'Compiled frog sizes', 'Size (in bytes)', 10, 10, 'lower left',
           png_filename, ['commandline'], ['frog'], ['swarm', 'total'])
@@ skipping 20 matching lines @@
     while True:
       if runner.has_new_code():
         runner.run_test_sequence()
       else:
         time.sleep(200)
   else:
     runner.run_test_sequence()
 
 if __name__ == '__main__':
   main()