Rename process_log_utils and related things.

scripts/slave/runtest.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """A tool used to run a Chrome test executable and process the output.
 
 This script is used by the buildbot slaves. It must be run from the outer
 build directory, e.g. chrome-release/build/.
 
@@ skipping 33 matching lines @@
 from common import chromium_utils
 from common import gtest_utils
 
 # TODO(crbug.com/403564). We almost certainly shouldn't be importing this.
 import config
 
 from slave import annotation_utils
 from slave import build_directory
 from slave import crash_utils
 from slave import gtest_slave_utils
-from slave import process_log_utils
+from slave import performance_log_processor
 from slave import results_dashboard
 from slave import slave_utils
 from slave import xvfb
 
 USAGE = '%s [options] test.exe [test args]' % os.path.basename(sys.argv[0])
 
 CHROME_SANDBOX_PATH = '/opt/chromium/chrome_sandbox'
 
 # Directory to write JSON for test results into.
 DEST_DIR = 'gtest_results'
 
 # Names of httpd configuration file under different platforms.
 HTTPD_CONF = {
     'linux': 'httpd2_linux.conf',
     'mac': 'httpd2_mac.conf',
     'win': 'httpd.conf'
 }
 # Regex matching git comment lines containing svn revision info.
 GIT_SVN_ID_RE = re.compile('^git-svn-id: .*@([0-9]+) .*$')
 # Regex for the master branch commit position.
 GIT_CR_POS_RE = re.compile('^Cr-Commit-Position: refs/heads/master@{#(\d+)}$')
 
 # The directory that this script is in.
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 
+LOG_PROCESSOR_CLASSES = {
+    'gtest': gtest_utils.GTestLogParser,
+    'graphing': performance_log_processor.GraphingLogProcessor,
+    'pagecycler': performance_log_processor.GraphingPageCyclerLogProcessor,
+}
+
 
 def _ShouldEnableSandbox(sandbox_path):
   """Checks whether the current slave should use the sandbox.
 
   This is based on should_enable_sandbox in src/testing/test_env.py.
 
   Args:
     sandbox_path: Path to sandbox file.
 
   Returns:
@@ skipping 203 matching lines @@
     The git SHA1 hash string.
   """
   git_exe = 'git.bat' if sys.platform.startswith('win') else 'git'
   p = subprocess.Popen(
       [git_exe, 'rev-parse', 'HEAD'],
       cwd=in_directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
   (stdout, _) = p.communicate()
   return stdout.strip()
 
 
-def _GenerateJSONForTestResults(options, results_tracker):
+def _GenerateJSONForTestResults(options, log_processor):
   """Generates or updates a JSON file from the gtest results XML and upload the
   file to the archive server.
 
   The archived JSON file will be placed at:
     www-dir/DEST_DIR/buildname/testname/results.json
   on the archive server. NOTE: This will be deprecated.
 
   Args:
     options: command-line options that are supposed to have build_dir,
         results_directory, builder_name, build_name and test_output_xml values.
+    log_processor: An instance of PerformanceLogProcessor or similar class.
 
   Returns:
     True upon success, False upon failure.
   """
   results_map = None
   try:
     if (os.path.exists(options.test_output_xml) and
         not _UsingGtestJson(options)):
       results_map = gtest_slave_utils.GetResultsMapFromXML(
           options.test_output_xml)
     else:
       if _UsingGtestJson(options):
         sys.stderr.write('using JSON summary output instead of gtest XML\n')
       else:
         sys.stderr.write(
             ('"%s" \ "%s" doesn\'t exist: Unable to generate JSON from XML, '
              'using log output.\n') % (os.getcwd(), options.test_output_xml))
       # The file did not get generated. See if we can generate a results map
       # from the log output.
-      results_map = gtest_slave_utils.GetResultsMap(results_tracker)
+      results_map = gtest_slave_utils.GetResultsMap(log_processor)
   except Exception as e:
     # This error will be caught by the following 'not results_map' statement.
     print 'Error: ', e
 
   if not results_map:
     print 'No data was available to update the JSON results'
     # Consider this non-fatal.
     return True
 
   build_dir = os.path.abspath(options.build_dir)
   slave_name = slave_utils.SlaveBuildName(build_dir)
 
   generate_json_options = copy.copy(options)
   generate_json_options.build_name = slave_name
   generate_json_options.input_results_xml = options.test_output_xml
   generate_json_options.builder_base_url = '%s/%s/%s/%s' % (
       config.Master.archive_url, DEST_DIR, slave_name, options.test_type)
   generate_json_options.master_name = options.master_class_name or _GetMaster()
   generate_json_options.test_results_server = config.Master.test_results_server
 
-  # Print out master name for log_parser
   print _GetMasterString(generate_json_options.master_name)
 
   generator = None
 
   try:
     if options.revision:
       generate_json_options.chrome_revision = options.revision
     else:
       chrome_dir = chromium_utils.FindUpwardParent(build_dir, 'third_party')
       generate_json_options.chrome_revision = _GetRevision(chrome_dir)
@@ skipping 151 matching lines @@
   return http_server
 
 
 def _UsingGtestJson(options):
   """Returns True if we're using GTest JSON summary."""
   return (options.annotate == 'gtest' and
           not options.run_python_script and
           not options.run_shell_script)
 
 
-def _GetParsers():
-  """Returns a dictionary mapping strings to log parser classes."""
-  parsers = {
-      'gtest': gtest_utils.GTestLogParser,
-      'graphing': process_log_utils.GraphingLogProcessor,
-      'pagecycler': process_log_utils.GraphingPageCyclerLogProcessor,
-  }
-  return parsers
-
-
-def _ListParsers(selection):
-  """Prints a list of available log parser classes iff the input is 'list'.
+def _ListLogProcessors(selection):
+  """Prints a list of available log processor classes iff the input is 'list'.
 
   Args:
-    selection: A log parser name, or the string "list".
+    selection: A log processor name, or the string "list".
 
   Returns:
     True if a list was printed, False otherwise.
   """
-  parsers = _GetParsers()
   shouldlist = selection and selection == 'list'
   if shouldlist:
     print
-    print 'Available log parsers:'
-    for p in parsers:
-      print ' ', p, parsers[p].__name__
+    print 'Available log processors:'
+    for p in LOG_PROCESSOR_CLASSES:
+      print ' ', p, LOG_PROCESSOR_CLASSES[p].__name__
 
   return shouldlist
 
 
-def _SelectResultsTracker(options):
-  """Returns a log parser class (aka results tracker class).
+def _SelectLogProcessor(options):
+  """Returns a log processor class based on the command line options.
 
   Args:
     options: Command-line options (from OptionParser).
 
   Returns:
-    A log parser class (aka results tracker class), or None.
+    A log processor class, or None.
   """
   if _UsingGtestJson(options):
     return gtest_utils.GTestJSONParser
 
-  parsers = _GetParsers()
   if options.annotate:
-    if options.annotate in parsers:
+    if options.annotate in LOG_PROCESSOR_CLASSES:
       if options.generate_json_file and options.annotate != 'gtest':
         raise NotImplementedError('"%s" doesn\'t make sense with '
                                   'options.generate_json_file.')
       else:
-        return parsers[options.annotate]
+        return LOG_PROCESSOR_CLASSES[options.annotate]
     else:
       raise KeyError('"%s" is not a valid GTest parser!' % options.annotate)
   elif options.generate_json_file:
-    return parsers['gtest']
+    return LOG_PROCESSOR_CLASSES['gtest']
 
   return None
 
 
 def _GetCommitPos(build_properties):
-  """Extract the commit position from the build properties, if its there."""
+  """Extracts the commit position from the build properties, if its there."""
   if 'got_revision_cp' not in build_properties:
     return None
   commit_pos = build_properties['got_revision_cp']
   return int(re.search(r'{#(\d+)}', commit_pos).group(1))
 
 
-def _CreateResultsTracker(tracker_class, options):
-  """Instantiate a log parser (aka results tracker).
+def _CreateLogProcessor(log_processor_class, options):
+  """Creates a log processor instance.
 
   Args:
-    tracker_class: A log parser class.
+    log_processor_class: A subclass of PerformanceLogProcessor or similar class.
     options: Command-line options (from OptionParser).
 
   Returns:
-    An instance of a log parser class, or None.
+    An instance of a log processor class, or None.
   """
-  if not tracker_class:
+  if not log_processor_class:
     return None
 
-  if tracker_class.__name__ in ('GTestLogParser',):
-    tracker_obj = tracker_class()
-  elif tracker_class.__name__ in ('GTestJSONParser',):
-    tracker_obj = tracker_class(options.build_properties.get('mastername'))
+  if log_processor_class.__name__ in ('GTestLogParser',):
+    tracker_obj = log_processor_class()
+  elif log_processor_class.__name__ in ('GTestJSONParser',):
+    tracker_obj = log_processor_class(
+        options.build_properties.get('mastername'))
   else:
     build_dir = os.path.abspath(options.build_dir)
 
     if options.webkit_revision:
       webkit_revision = options.webkit_revision
     else:
       try:
         webkit_dir = chromium_utils.FindUpward(
             build_dir, 'third_party', 'WebKit', 'Source')
         webkit_revision = _GetRevision(webkit_dir)
       except Exception:
         webkit_revision = 'undefined'
 
     commit_pos_num = _GetCommitPos(options.build_properties)
     if commit_pos_num is not None:
       revision = commit_pos_num
     elif options.revision:
       revision = options.revision
     else:
       revision = _GetRevision(os.path.dirname(build_dir))
 
-    tracker_obj = tracker_class(
+    tracker_obj = log_processor_class(
         revision=revision,
         build_properties=options.build_properties,
         factory_properties=options.factory_properties,
         webkit_revision=webkit_revision)
 
   if options.annotate and options.generate_json_file:
     tracker_obj.ProcessLine(_GetMasterString(_GetMaster()))
 
   return tracker_obj
 
@@ skipping 12 matching lines @@
   supplemental_columns = {}
   supplemental_columns_file = os.path.join(build_dir,
                                            results_dashboard.CACHE_DIR,
                                            supplemental_colummns_file_name)
   if os.path.exists(supplemental_columns_file):
     with file(supplemental_columns_file, 'r') as f:
       supplemental_columns = json.loads(f.read())
   return supplemental_columns
 
 
-def _SendResultsToDashboard(results_tracker, system, test, url, build_dir,
+def _SendResultsToDashboard(log_processor, system, test, url, build_dir,
                             mastername, buildername, buildnumber,
                             supplemental_columns_file, extra_columns=None):
-  """Sends results from a results tracker (aka log parser) to the dashboard.
+  """Sends results from a log processor instance to the dashboard.
 
   Args:
-    results_tracker: An instance of a log parser class, which has been used to
+    log_processor: An instance of a log processor class, which has been used to
         process the test output, so it contains the test results.
     system: A string such as 'linux-release', which comes from perf_id.
     test: Test "suite" name string.
     url: Dashboard URL.
     build_dir: Build dir name (used for cache file by results_dashboard).
     mastername: Buildbot master name, e.g. 'chromium.perf'.
         WARNING! This is incorrectly called "masterid" in some parts of the
         dashboard code.
     buildername: Builder name, e.g. 'Linux QA Perf (1)'
     buildnumber: Build number (as a string).
     supplemental_columns_file: Filename for JSON supplemental columns file.
     extra_columns: A dict of extra values to add to the supplemental columns
         dict.
   """
   if system is None:
     # perf_id not specified in factory properties.
     print 'Error: No system name (perf_id) specified when sending to dashboard.'
     return
   supplemental_columns = _GetSupplementalColumns(
       build_dir, supplemental_columns_file)
   if extra_columns:
     supplemental_columns.update(extra_columns)
 
-  charts = _GetDataFromLogProcessor(results_tracker)
+  charts = _GetDataFromLogProcessor(log_processor)
   points = results_dashboard.MakeListOfPoints(
       charts, system, test, mastername, buildername, buildnumber,
       supplemental_columns)
   results_dashboard.SendResults(points, url, build_dir)
 
 
 def _GetDataFromLogProcessor(log_processor):
   """Returns a mapping of chart names to chart data.
 
   Args:
     log_processor: A log processor (aka results tracker) object.
 
   Returns:
     A dictionary mapping chart name to lists of chart data.
-    put together in process_log_utils. Each chart data dictionary contains:
+    put together in log_processor. Each chart data dictionary contains:
       "traces": A dictionary mapping trace names to value, stddev pairs.
       "units": Units for the chart.
       "rev": A revision number or git hash.
       Plus other revision keys, e.g. webkit_rev, ver, v8_rev.
   """
   charts = {}
   for log_file_name, line_list in log_processor.PerformanceLogs().iteritems():
     if not log_file_name.endswith('-summary.dat'):
       # The log processor data also contains "graphs list" file contents,
       # which we can ignore.
       continue
     chart_name = log_file_name.replace('-summary.dat', '')
 
     # It's assumed that the log lines list has length one, because for each
-    # graph name only one line is added in process_log_utils in the method
+    # graph name only one line is added in log_processor in the method
     # GraphingLogProcessor._CreateSummaryOutput.
     if len(line_list) != 1:
       print 'Error: Unexpected log processor line list: %s' % str(line_list)
       continue
     line = line_list[0].rstrip()
     try:
       charts[chart_name] = json.loads(line)
     except ValueError:
       print 'Error: Could not parse JSON: %s' % line
   return charts
@@ skipping 211 matching lines @@
     perf_id = options.factory_properties.get('perf_id')
     if options.factory_properties.get('add_perf_id_suffix'):
       perf_id += options.build_properties.get('perf_id_suffix')
   return perf_id
 
 
 def _MainParse(options, _args):
   """Run input through annotated test parser.
 
   This doesn't execute a test, but reads test input from a file and runs it
-  through the specified annotation parser.
+  through the specified annotation parser (aka log processor).
   """
   if not options.annotate:
     raise chromium_utils.MissingArgument('--parse-input doesn\'t make sense '
                                          'without --annotate.')
 
-  # If --annotate=list was passed, list the log parser classes and exit.
-  if _ListParsers(options.annotate):
+  # If --annotate=list was passed, list the log processor classes and exit.
+  if _ListLogProcessors(options.annotate):
     return 0
 
-  tracker_class = _SelectResultsTracker(options)
-  results_tracker = _CreateResultsTracker(tracker_class, options)
+  log_processor_class = _SelectLogProcessor(options)
+  results_tracker = _CreateLogProcessor(log_processor_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   if options.parse_input == '-':
     f = sys.stdin
   else:
     try:
@@ skipping 42 matching lines @@
   elif options.run_shell_script:
     command = ['bash', test_exe_path]
   elif options.run_python_script:
     command = [sys.executable, test_exe]
   else:
     command = [test_exe_path]
     if options.annotate == 'gtest':
       command.extend(['--brave-new-test-launcher', '--test-launcher-bot-mode'])
   command.extend(args[1:])
 
-  # If --annotate=list was passed, list the log parser classes and exit.
-  if _ListParsers(options.annotate):
+  # If --annotate=list was passed, list the log processor classes and exit.
+  if _ListLogProcessors(options.annotate):
     return 0
-  tracker_class = _SelectResultsTracker(options)
-  results_tracker = _CreateResultsTracker(tracker_class, options)
+  log_processor_class = _SelectLogProcessor(options)
+  results_tracker = _CreateLogProcessor(log_processor_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     http_server = None
     if options.document_root:
       http_server = _StartHttpServer('mac', build_dir=build_dir,
@@ skipping 95 matching lines @@
   tmpdir = tempfile.mkdtemp()
   command = [test_exe_path,
       '-d', device,
       '-s', ios_version,
       '-t', '120',
       '-u', tmpdir,
       app_exe_path, '--'
   ]
   command.extend(args[1:])
 
-  # If --annotate=list was passed, list the log parser classes and exit.
-  if _ListParsers(options.annotate):
+  # If --annotate=list was passed, list the log processor classes and exit.
+  if _ListLogProcessors(options.annotate):
     return 0
-  results_tracker = _CreateResultsTracker(_GetParsers()['gtest'], options)
+  results_tracker = _CreateLogProcessor(LOG_PROCESSOR_CLASSES['gtest'], options)
 
   # Make sure the simulator isn't running.
   kill_simulator()
 
   # Nuke anything that appears to be stale chrome items in the temporary
   # directory from previous test runs (i.e.- from crashes or unittest leaks).
   slave_utils.RemoveChromeTemporaryFiles()
 
   dirs_to_cleanup = [tmpdir]
   crash_files_before = set([])
@@ skipping 109 matching lines @@
   elif options.run_shell_script:
     command = ['bash', test_exe_path]
   elif options.run_python_script:
     command = [sys.executable, test_exe]
   else:
     command = [test_exe_path]
     if options.annotate == 'gtest':
       command.extend(['--brave-new-test-launcher', '--test-launcher-bot-mode'])
   command.extend(args[1:])
 
-  # If --annotate=list was passed, list the log parser classes and exit.
-  if _ListParsers(options.annotate):
+  # If --annotate=list was passed, list the log processor classes and exit.
+  if _ListLogProcessors(options.annotate):
     return 0
-  tracker_class = _SelectResultsTracker(options)
-  results_tracker = _CreateResultsTracker(tracker_class, options)
+  log_processor_class = _SelectLogProcessor(options)
+  results_tracker = _CreateLogProcessor(log_processor_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     start_xvfb = False
     http_server = None
     json_file_name = None
@@ skipping 124 matching lines @@
                                    'agent_logger.exe'),
                'start',
                '--output-file=%s' % logfile,
                '--'] + command
   command.extend(args[1:])
 
   # Nuke anything that appears to be stale chrome items in the temporary
   # directory from previous test runs (i.e.- from crashes or unittest leaks).
   slave_utils.RemoveChromeTemporaryFiles()
 
-  # If --annotate=list was passed, list the log parser classes and exit.
-  if _ListParsers(options.annotate):
+  # If --annotate=list was passed, list the log processor classes and exit.
+  if _ListLogProcessors(options.annotate):
     return 0
-  tracker_class = _SelectResultsTracker(options)
-  results_tracker = _CreateResultsTracker(tracker_class, options)
+  log_processor_class = _SelectLogProcessor(options)
+  results_tracker = _CreateLogProcessor(log_processor_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   try:
     http_server = None
     if options.document_root:
       http_server = _StartHttpServer('win', build_dir=build_dir,
@@ skipping 57 matching lines @@
 
   Returns:
     Exit status code.
   """
   if options.run_python_script:
     return _MainLinux(options, args, extra_env)
 
   if len(args) < 1:
     raise chromium_utils.MissingArgument('Usage: %s' % USAGE)
 
-  if _ListParsers(options.annotate):
+  if _ListLogProcessors(options.annotate):
     return 0
-  tracker_class = _SelectResultsTracker(options)
-  results_tracker = _CreateResultsTracker(tracker_class, options)
+  log_processor_class = _SelectLogProcessor(options)
+  results_tracker = _CreateLogProcessor(log_processor_class, options)
 
   if options.generate_json_file:
     if os.path.exists(options.test_output_xml):
       # remove the old XML output file.
       os.remove(options.test_output_xml)
 
   # Assume it's a gtest apk, so use the android harness.
   test_suite = args[0]
   run_test_target_option = '--release'
   if options.target == 'Debug':
@@ skipping 226 matching lines @@
     options.perf_dashboard_id = options.factory_properties.get('test_name')
 
   options.test_type = options.test_type or options.factory_properties.get(
       'step_name', '')
 
   if options.run_shell_script and options.run_python_script:
     sys.stderr.write('Use either --run-shell-script OR --run-python-script, '
                      'not both.')
     return 1
 
-  # Print out builder name for log_parser
   print '[Running on builder: "%s"]' % options.builder_name
 
   did_launch_dbus = False
   if not options.no_spawn_dbus:
     did_launch_dbus = _LaunchDBus()
 
   try:
     options.build_dir = build_directory.GetBuildOutputDirectory()
 
     if options.pass_target and options.target:
@@ skipping 175 matching lines @@
   finally:
     if did_launch_dbus:
       # It looks like the command line argument --exit-with-session
       # isn't working to clean up the spawned dbus-daemon. Kill it
       # manually.
       _ShutdownDBus()
 
 
 if '__main__' == __name__:
   sys.exit(main())