Stop generating and uploading incremental_results.json.

Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

 # Copyright (C) 2010 Google Inc. All rights reserved.
 # Copyright (C) 2010 Gabor Rapcsanyi (rgabor@inf.u-szeged.hu), University of Szeged
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 # notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
@@ skipping 22 matching lines @@
 objects to the Manager. The Manager then aggregates the TestFailures to
 create a final report.
 """
 
 import json
 import logging
 import random
 import sys
 import time
 
+from webkitpy.common.net.file_uploader import FileUploader
 from webkitpy.layout_tests.controllers.layout_test_finder import LayoutTestFinder
 from webkitpy.layout_tests.controllers.layout_test_runner import LayoutTestRunner
 from webkitpy.layout_tests.controllers.test_result_writer import TestResultWriter
-from webkitpy.layout_tests.layout_package import json_layout_results_generator
 from webkitpy.layout_tests.layout_package import json_results_generator
 from webkitpy.layout_tests.models import test_expectations
 from webkitpy.layout_tests.models import test_failures
 from webkitpy.layout_tests.models import test_run_results
 from webkitpy.layout_tests.models.test_input import TestInput
 
 _log = logging.getLogger(__name__)
 
 # Builder base URL where we have the archived test results.
 BUILDER_BASE_URL = "http://build.chromium.org/buildbot/layout_test_results/"
@@ skipping 171 matching lines @@
         self._look_for_new_crash_logs(initial_results, start_time)
         if retry_results:
             self._look_for_new_crash_logs(retry_results, start_time)
 
         _log.debug("summarizing results")
         summarized_results = test_run_results.summarize_results(self._port, self._expectations, initial_results, retry_results, enabled_pixel_tests_in_retry)
         self._printer.print_results(end_time - start_time, initial_results, summarized_results)
 
         if not self._options.dry_run:
             self._port.print_leaks_summary()
-            self._upload_json_files(summarized_results, initial_results)
+            self._write_json_files(summarized_results, initial_results)
+            self._upload_json_files()
 
             results_path = self._filesystem.join(self._results_directory, "results.html")
             self._copy_results_html_file(results_path)
             if self._options.show_results and (initial_results.unexpected_results_by_name or
                                                (self._options.full_results_html and initial_results.total_failures)):
                 self._port.show_results_html_file(results_path)
 
         return test_run_results.RunDetails(self._port.exit_code_from_summarized_results(summarized_results),
                                            summarized_results, initial_results, retry_results, enabled_pixel_tests_in_retry)
 
@@ skipping 69 matching lines @@
         for dirname in possible_dirs:
             if self._filesystem.isdir(self._filesystem.join(layout_tests_dir, dirname)):
                 self._filesystem.rmtree(self._filesystem.join(self._results_directory, dirname))
 
     def _tests_to_retry(self, run_results, include_crashes):
         return [result.test_name for result in run_results.unexpected_results_by_name.values() if
                    ((result.type != test_expectations.PASS) and
                     (result.type != test_expectations.MISSING) and
                     (result.type != test_expectations.CRASH or include_crashes))]
 
-    def _upload_json_files(self, summarized_results, initial_results):
-        """Writes the results of the test run as JSON files into the results

[Julie Parent, 2013/05/17 20:38:29]

Why did you remove the parameter descriptions?

[ojan, 2013/05/17 20:41:37]

They didn't seem terribly accurate or useful. You can't know from the
description what these things really are, so you have to go find the calling
location anyways.

what comments == useless comments.

-        dir and upload the files to the appengine server.
-
-        Args:
-          summarized_results: dict of results
-          initial_results: full summary object
-        """
+    def _write_json_files(self, summarized_results, initial_results):
         _log.debug("Writing JSON files in %s." % self._results_directory)
 
         # FIXME: Upload stats.json to the server and delete times_ms.
         times_trie = json_results_generator.test_timings_trie(self._port, initial_results.results_by_name.values())
         times_json_path = self._filesystem.join(self._results_directory, "times_ms.json")
         json_results_generator.write_json(self._filesystem, times_trie, times_json_path)
 
         stats_trie = self._stats_trie(initial_results)
         stats_path = self._filesystem.join(self._results_directory, "stats.json")
         self._filesystem.write_text_file(stats_path, json.dumps(stats_trie))
 
         full_results_path = self._filesystem.join(self._results_directory, "full_results.json")
         # We write full_results.json out as jsonp because we need to load it from a file url and Chromium doesn't allow that.
         json_results_generator.write_json(self._filesystem, summarized_results, full_results_path, callback="ADD_RESULTS")
 
-        generator = json_layout_results_generator.JSONLayoutResultsGenerator(
-            self._port, self._options.builder_name, self._options.build_name,
-            self._options.build_number, self._results_directory,
-            BUILDER_BASE_URL,
-            self._expectations, initial_results,
-            self._options.test_results_server,
-            "layout-tests",
-            self._options.master_name)
-
         _log.debug("Finished writing JSON files.")
 
+    def _upload_json_files(self):
+        if not self._options.test_results_server:
+            return
 
-        json_files = ["incremental_results.json", "full_results.json", "times_ms.json"]
+        if not self._options.master_name:
+            _log.error("--test-results-server was set, but --master-name was not.  Not uploading JSON files.")
+            return
 
-        generator.upload_json_files(json_files)
+        _log.debug("Uploading JSON files for builder: %s", self._options.builder_name)
+        attrs = [("builder", self._options.builder_name),
+                 ("testtype", "layout-tests"),
+                 ("master", self._options.master_name)]
 
-        incremental_results_path = self._filesystem.join(self._results_directory, "incremental_results.json")
+        files = [(file, self._filesystem.join(self._results_directory, file)) for file in ["full_results.json", "times_ms.json"]]
 
-        # Remove these files from the results directory so they don't take up too much space on the buildbot.
-        # The tools use the version we uploaded to the results server anyway.
-        self._filesystem.remove(times_json_path)
-        self._filesystem.remove(incremental_results_path)
+        url = "http://%s/testfile/upload" % self._options.test_results_server
+        # Set uploading timeout in case appengine server is having problems.
+        # 120 seconds are more than enough to upload test results.
+        uploader = FileUploader(url, 120)
+        try:
+            response = uploader.upload_as_multipart_form_data(self._filesystem, files, attrs)
+            if response:
+                if response.code == 200:
+                    _log.debug("JSON uploaded.")
+                else:
+                    _log.debug("JSON upload failed, %d: '%s'" % (response.code, response.read()))
+            else:
+                _log.error("JSON upload failed; no response returned")
+        except Exception, err:
+            _log.error("Upload failed: %s" % err)
 
     def _copy_results_html_file(self, destination_path):
         base_dir = self._port.path_from_webkit_base('LayoutTests', 'fast', 'harness')
         results_file = self._filesystem.join(base_dir, 'results.html')
         # Note that the results.html template file won't exist when we're using a MockFileSystem during unit tests,
         # so make sure it exists before we try to copy it.
         if self._filesystem.exists(results_file):
             self._filesystem.copyfile(results_file, destination_path)
 
     def _stats_trie(self, initial_results):
         def _worker_number(worker_name):
             return int(worker_name.split('/')[1]) if worker_name else -1
 
         stats = {}
         for result in initial_results.results_by_name.values():
             if result.type != test_expectations.SKIP:
                 stats[result.test_name] = {'results': (_worker_number(result.worker_name), result.test_number, result.pid, int(result.test_run_time * 1000), int(result.total_run_time * 1000))}
         stats_trie = {}
         for name, value in stats.iteritems():
             json_results_generator.add_path_to_trie(name, value, stats_trie)
         return stats_trie