Fix dealing with incomplete files as the first set of files encountered.

tools/testing/perf_testing/run_perf_tests.py

 #!/usr/bin/python
 
 # Copyright (c) 2011, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 import datetime
 import math             
 try:            
   from matplotlib.font_manager import FontProperties            
@@ skipping 673 matching lines @@
   class CommonBrowserFileProcessor(Processor):
     def process_file(self, afile, should_post_file):
       """Comb through the html to find the performance results.
       Returns: True if we successfully posted our data to storage and/or we can
           delete the trace file."""
       os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools',
                             'testing', 'perf_testing'))
       parts = afile.split('-')
       browser = parts[2]
       version = parts[3]
-      if should_post_file:
-        f = open(os.path.join(self.test.result_folder_name, afile))
-      else:
-        f = open(os.path.join('old', self.test.result_folder_name, afile))
+      file_path = os.path.join(self.test.result_folder_name, afile)

[vsm, 2012/05/09 20:53:11]

Consider refactoring into Processor base class.

[Emily Fortuna, 2012/05/09 20:57:07]

Will do!

+      if not should_post_file:
+        file_path = os.path.join('old', file_path)
+      f = open(file_path)
       lines = f.readlines()
       line = ''
       i = 0
       revision_num = 0
       while '<div id="results">' not in line and i < len(lines):
         if 'Revision' in line:
           revision_num = int(line.split()[1].strip('"'))
         line = lines[i]
         i += 1
 
       if i >= len(lines) or revision_num == 0:
         # Then this run did not complete. Ignore this tracefile.
         return True
 
       line = lines[i]
       i += 1
       results = []
       if line.find('<br>') > -1:
         results = line.split('<br>')
       else:
         results = line.split('<br />')
       if results == []:
         return True
       upload_success = True
       for result in results:
         name_and_score = result.split(':')
         if len(name_and_score) < 2:
-          break
+          return True
         name = name_and_score[0].strip()
         score = name_and_score[1].strip()
         if version == 'js' or version == 'v8':
           version = 'js'
         bench_dict = self.test.values_dict[browser][version]
         bench_dict[name] += [float(score)]
         self.test.revision_dict[browser][version][name] += [revision_num]
         if not self.test.test_runner.no_upload and should_post_file:
           upload_success = upload_success and self.report_results(
               name, score, browser, version, revision_num, self.SCORE)
@@ skipping 139 matching lines @@
   class DromaeoFileProcessor(Processor):
     def process_file(self, afile, should_post_file):
       """Comb through the html to find the performance results.
       Returns: True if we successfully posted our data to storage."""
       parts = afile.split('-')
       browser = parts[2]
       version = parts[3]
 
       bench_dict = self.test.values_dict[browser][version]
 
-      f = open(os.path.join(self.test.result_folder_name, afile))
+      file_path = os.path.join(self.test.result_folder_name, afile)
+      if not should_post_file:
+        file_path = os.path.join('old', file_path)
+      f = open(file_path)
       lines = f.readlines()
       i = 0
       revision_num = 0
       revision_pattern = r'Revision: (\d+)'
       suite_pattern = r'<div class="result-item done">(.+?)</ol></div>'
       result_pattern = r'<b>(.+?)</b>(.+?)<small> runs/s(.+)'
 
       upload_success = True
       for line in lines:
         rev = re.match(revision_pattern, line.strip())
@@ skipping 107 matching lines @@
 
   class DromaeoSizeProcessor(Processor):
     def process_file(self, afile, should_post_file):
       """Pull all the relevant information out of a given tracefile.
 
       Args:
         afile: is the filename string we will be processing.
       Returns: True if we successfully posted our data to storage."""
       os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools',
           'testing', 'perf_testing'))
-      f = open(os.path.join(self.test.result_folder_name, afile))
+      file_path = os.path.join(self.test.result_folder_name, afile)
+      if not should_post_file:
+        file_path = os.path.join('old', file_path)
+      f = open(file_path)
       tabulate_data = False
       revision_num = 0
       revision_pattern = r'Revision: (\d+)'
       result_pattern = r'Size \((\w+), ([a-zA-Z0-9-]+)\): (\d+)'
 
       upload_success = True
       for line in f.readlines():
         rev = re.match(revision_pattern, line.strip())
         if rev:
           revision_num = int(rev.group(1))
@@ skipping 108 matching lines @@
   class CompileProcessor(Processor):
 
     def process_file(self, afile, should_post_file):
       """Pull all the relevant information out of a given tracefile.
 
       Args:
         afile: is the filename string we will be processing.
       Returns: True if we successfully posted our data to storage."""
       os.chdir(os.path.join(DART_INSTALL_LOCATION, 'tools',
           'testing', 'perf_testing'))
-      f = open(os.path.join(self.test.result_folder_name, afile))
+      file_path = os.path.join(self.test.result_folder_name, afile)
+      if not should_post_file:
+        file_path = os.path.join('old', file_path)
+      f = open(file_path)
       tabulate_data = False
       revision_num = 0
       upload_success = True
       for line in f.readlines():
         tokens = line.split()
         if 'Revision' in line:
           revision_num = int(line.split()[1])
         else:
           for metric in self.test.values_list:
             if metric in line:
@@ skipping 11 matching lines @@
               if not self.test.test_runner.no_upload and should_post_file:
                 if num < self.test.failure_threshold[metric]:
                   num = 0
                 upload_success = upload_success and self.report_results(
                     metric, num, 'commandline', 'frog', revision_num,
                     score_type)
               else:
                 upload_success = False
       if revision_num != 0:
         for metric in self.test.values_list:
-          self.test.revision_dict['commandline']['frog'][metric].pop()
-          self.test.revision_dict['commandline']['frog'][metric] += \
-              [revision_num]
-          # Fill in 0 if compilation failed.
-          if self.test.values_dict['commandline']['frog'][metric][-1] < \
-              self.test.failure_threshold[metric]:
-            self.test.values_dict['commandline']['frog'][metric] += [0]
+          try:
+            self.test.revision_dict['commandline']['frog'][metric].pop()
             self.test.revision_dict['commandline']['frog'][metric] += \
                 [revision_num]
+            # Fill in 0 if compilation failed.
+            if self.test.values_dict['commandline']['frog'][metric][-1] < \
+                self.test.failure_threshold[metric]:
+              self.test.values_dict['commandline']['frog'][metric] += [0]
+              self.test.revision_dict['commandline']['frog'][metric] += \
+                  [revision_num]
+          except IndexError:
+            # We tried to pop from an empty list. This happens if the first
+            # trace file we encounter is incomplete.
+            pass
 
       f.close()
       return upload_success
 
   class CompileGrapher(Grapher):
 
     def plot_results(self, png_filename):               
       self.style_and_save_perf_plot(            
           'Compiled frog sizes', 'Size (in bytes)', 10, 10, 'lower left',
           png_filename, ['commandline'], ['frog'], ['swarm', 'total'])
@@ skipping 20 matching lines @@
     while True:
       if runner.has_new_code():
         runner.run_test_sequence()
       else:
         time.sleep(200)
   else:
     runner.run_test_sequence()
 
 if __name__ == '__main__':
   main()