Adding pyauto-based memory usage tests for ChromeOS.

chrome/test/functional/perf.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Basic pyauto performance tests.
 
 For tests that need to be run for multiple iterations (e.g., so that average
 and standard deviation values can be reported), the default number of iterations
 run for each of these tests is specified by |_DEFAULT_NUM_ITERATIONS|.
@@ skipping 72 matching lines @@
     self._seen_graph_lines = {}
 
     pyauto.PyUITest.setUp(self)
 
     # Flush all buffers to disk and wait until system calms down.  Must be done
     # *after* calling pyauto.PyUITest.setUp, since that is where Chrome is
     # killed and re-initialized for a new test.
     # TODO(dennisjeffrey): Implement wait for idle CPU on Windows/Mac.
     if self.IsLinux():  # IsLinux() also implies IsChromeOS().
       os.system('sync')
-      self._WaitForIdleCPU(60.0, 0.03)
+      self._WaitForIdleCPU(60.0, 0.05)
 
   def _WaitForIdleCPU(self, timeout, utilization):
     """Waits for the CPU to become idle (< utilization).
 
     Args:
       timeout: The longest time in seconds to wait before throwing an error.
       utilization: The CPU usage below which the system should be considered
           idle (between 0 and 1.0 independent of cores/hyperthreads).
     """
     time_passed = 0.0
@@ skipping 1654 matching lines @@
   def testIntl2File(self):
     self._RunPageCyclerTest('intl2', self._num_iterations, 'Intl2File')
 
   def testMozFile(self):
     self._RunPageCyclerTest('moz', self._num_iterations, 'MozFile')
 
   def testMoz2File(self):
     self._RunPageCyclerTest('moz2', self._num_iterations, 'Moz2File')
 
 
+class MemoryTest(BasePerfTest):
+  """Tests to measure memory consumption under different usage scenarios."""
+
+  def setUp(self):
+    pyauto.PyUITest.setUp(self)
+
+    # Log in to get a fresh Chrome instance with a clean memory state (if
+    # already logged in, log out first).
+    if self.GetLoginInfo()['is_logged_in']:
+      self.Logout()
+      self.assertFalse(self.GetLoginInfo()['is_logged_in'],
+                      msg='Failed to log out.')
+
+    credentials = self.GetPrivateInfo()['test_google_account']
+    self.Login(credentials['username'], credentials['password'])
+    self.assertTrue(self.GetLoginInfo()['is_logged_in'],
+                    msg='Failed to log in.')
+
+  def _GetMemoryStats(self, duration):
+    """Identifies and returns different kinds of current memory usage stats.
+
+    This function samples values each second for |duration| seconds, then
+    outputs the min, max, and ending values for each measurement type.
+
+    Args:
+      duration: The number of seconds to sample data before outputting the
+          minimum, maximum, and ending values for each measurement type.
+
+    Returns:
+      A dictionary containing memory usage information.  Each measurement type
+      is associated with the min, max, and ending values from among all
+      sampled values.  Values are specified in KB.
+      {
+        'gtt': {  # GPU memory usage (graphics translation table)
+          'min': ...,
+          'max': ...,
+          'end': ...,
+        },
+        'mem_available': { ... },  # CPU available memory
+        'mem_shared': { ... },  # CPU shared memory
+      }
+    """
+    logging.debug('Sampling memory information for %d seconds...' % duration)
+    stats = {
+      'gtt': [],
+      'mem_available': [],
+      'mem_shared': [],
+    }
+
+    for _ in xrange(duration):
+      p = subprocess.Popen('grep bytes /sys/kernel/debug/dri/0/i915_gem_gtt',

[Sonny, 2012/04/26 04:55:18]

GTT isn't guaranteed to always be there.  It'll only be present on systems using
Intel graphics, so you should test to see if it's there before opening that
file.  We also may just want to ignore GTT because it ends up being included in
Shmem -- see below comment.

[ihf, 2012/04/26 05:19:17]

Good catch on the checking for file existence.

[dennis_jeffrey, 2012/04/26 18:47:39]

I modified the code to exclude GTT values if the file doesn't exist.

+                           stdout=subprocess.PIPE,
+                           shell=True)
+      stdout = p.communicate()[0]
+
+      # GPU memory.
+      gtt_used = re.search(
+          'Total [\d]+ objects, ([\d]+) bytes', stdout).group(1)
+      stats['gtt'].append(int(gtt_used) / 1024.0)
+
+      # CPU memory.
+      stdout = ''
+      with open('/proc/meminfo') as f:
+        stdout = f.read()
+      mem_free = re.search('MemFree:\s*([\d]+) kB', stdout).group(1)
+      mem_dirty = re.search('Dirty:\s*([\d]+) kB', stdout).group(1)

[Sonny, 2012/04/26 04:55:18]

Dirty  Memory isn't really very interesting because it'll change quite a bit
based on when we write the pages out to disk.  More interesting would probaby be
AnonPages and Slab

[Sonny, 2012/04/26 05:38:14]

Now that I understand what mem_available is supposed to be, i see why you're
reading these particular entries from meminfo.  But I still think we really want
a bunch more than just those numbers which are included in that calculation.

[dennis_jeffrey, 2012/04/26 18:47:39]

See my comment at line 1846 below.

+      mem_active_file = re.search(
+          'Active\(file\):\s*([\d]+) kB', stdout).group(1)
+      mem_inactive_file = re.search(
+          'Inactive\(file\):\s*([\d]+) kB', stdout).group(1)
+      stats['mem_available'].append(
+          (int(mem_active_file) + int(mem_inactive_file)) - int(mem_dirty) +
+          int(mem_free))

[Sonny, 2012/04/26 04:55:18]

I'm not sure what mem_available is supposed to mean here?

[ihf, 2012/04/26 05:19:17]

This was suggested by James as it is used for discarding tabs.

[James Cook, 2012/04/26 05:25:59]

It's apparently what Luigi uses in the kernel to provide the low memory signal
to chrome.

[Sonny, 2012/04/26 05:38:14]

Ah.. I see.  That's not actually exactly true either, because Luigi is also
using another number which is our floor on how much file cache we should allow
to stay.
If you really want that number, you need to read another proc file and factor it
in here.

[dennis_jeffrey, 2012/04/26 18:47:39]

Let me know how the computation for available memory needs to change here.

[Sonny, 2012/04/26 22:25:40]

so it should be
MemFree + Inactive file + active file - dirty - min_file_mem

and min_file_mem comes from /proc/sys/vm/min_filelist_kbytes

[dennis_jeffrey, 2012/04/28 00:21:35]

Done.

+
+      mem_shared = re.search('Shmem:\s*([\d]+) kB', stdout).group(1)
+      stats['mem_shared'].append(int(mem_shared))
+
+      time.sleep(1)
+

[Sonny, 2012/04/26 04:55:18]

Also weren't we going to get the memory stats from Chrome using GetProcessInfo?

[ihf, 2012/04/26 05:19:17]

First time I read about it.

[Sonny, 2012/04/26 05:38:14]

we want the private and shared and possibly the javascript memory in use for
each renderer.  Dennis knows what I'm talking about, as he has some code to go
do this.  Actually that reminds me, we also want a chrome flag which forces
every tab to get it's own renderer process.

[dennis_jeffrey, 2012/04/26 18:47:39]

Yes, there is lots more information that people want to collect.  There's the
information Sonny mentioned (which as he said, we've already implemented before
when conducting other experiments).  Additionally, Puneet mentioned that he'd
like to see some other info too (need to check in with him again to get the
details).

I recommend getting a first version of this test checked in and running, then we
can add or modify the set of measurements taken in another CL, once we've hashed
out what we need.

[Sonny, 2012/04/26 22:25:40]

Ok... I just feel like we'll be adding more stuff for collection soon after this
lands, because what we are collecting now doesn't help us figure out what is
consuming the memory beyond GTT and Shmem.

[dennis_jeffrey, 2012/04/28 00:21:35]

We're now recording more different types of memory.  We can decide later which
subset of this to actually plot on graphs.

+    # Compute min, max, and ending values to return.
+    result = {}
+    for measurement_type in stats:
+      values = stats[measurement_type]
+      result[measurement_type] = {
+        'min': min(values),
+        'max': max(values),
+        'end': values[-1],
+      }
+
+    return result
+
+  def _RecordMemoryStats(self, description, when, duration):
+    """Outputs memory statistics to be graphed.
+
+    Args:
+      description: A string description for the test.  Should not contain
+          spaces.  For example, 'MemCtrl'.
+      when: A string description of when the memory stats are being recorded
+          during test execution (since memory stats may be recorded multiple
+          times during a test execution at certain "interesting" times).  Should
+          not contain spaces.
+      duration: The number of seconds to sample data before outputting the
+          memory statistics.
+    """
+    mem = self._GetMemoryStats(duration)
+    measurement_types = [
+      ('gtt', 'GTT'),
+      ('mem_available', 'MemAvail'),
+      ('mem_shared', 'MemShare'),
+    ]
+    for type_key, type_string in measurement_types:
+      self._OutputPerfGraphValue(
+          '%s-Min%s-%s' % (description, type_string, when),
+          mem[type_key]['min'], 'KB', '%s-%s' % (description, type_string))
+      self._OutputPerfGraphValue(
+          '%s-Max%s-%s' % (description, type_string, when),
+          mem[type_key]['max'], 'KB', '%s-%s' % (description, type_string))
+      self._OutputPerfGraphValue(
+          '%s-End%s-%s' % (description, type_string, when),
+          mem[type_key]['end'], 'KB', '%s-%s' % (description, type_string))
+
+  def _RunTest(self, tabs, description, duration):
+    """Runs a general memory test.
+
+    Args:
+      tabs: A list of strings representing the URLs of the websites to open
+          during this test.
+      description: A string description for the test.  Should not contain
+          spaces.  For example, 'MemCtrl'.
+      duration: The number of seconds to sample data before outputting memory
+          statistics.
+    """
+    self._RecordMemoryStats(description, '0Tabs0', duration)
+
+    for iteration_num in xrange(2):
+      for site in tabs:
+        self.AppendTab(pyauto.GURL(site))
+
+      self._RecordMemoryStats(description,
+                              '%dTabs%d' % (len(tabs), iteration_num + 1),
+                              duration)

[Sonny, 2012/04/26 04:55:18]

I'm not sure we want to record after opening every tab, probably just when the
whole set is opened?  I'm just worried it'll be noisy and too much information
to grok easily.

[ihf, 2012/04/26 05:19:17]

This works exactly as you want it.

[Sonny, 2012/04/26 05:38:14]

Ahh oops, didn't see the indent, thanks.

[dennis_jeffrey, 2012/04/26 18:47:39]

Done - working as intended.

+
+      for _ in xrange(len(tabs)):
+        self.GetBrowserWindow(0).GetTab(1).Close(True)
+
+      self._RecordMemoryStats(description, '0Tabs%d' % (iteration_num + 1),
+                              duration)
+
+  def testOpenCloseTabsControl(self):
+    """Measures memory usage when opening/closing tabs to about:blank."""
+    tabs = ['about:blank'] * 10
+    self._RunTest(tabs, 'MemCtrl', 15)
+
+
 class PerfTestServerRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
   """Request handler for the local performance test server."""
 
   def _IgnoreHandler(self, unused_args):
     """A GET request handler that simply replies with status code 200.
 
     Args:
       unused_args: A dictionary of arguments for the current GET request.
           The arguments are ignored.
     """
@@ skipping 212 matching lines @@
     """Identifies the port number to which the server is currently bound.
 
     Returns:
       The numeric port number to which the server is currently bound.
     """
     return self._server.server_address[1]
 
 
 if __name__ == '__main__':
   pyauto_functional.Main()