Adding pyauto-based memory usage tests for ChromeOS.

chrome/test/functional/perf.py

Index: chrome/test/functional/perf.py
diff --git a/chrome/test/functional/perf.py b/chrome/test/functional/perf.py
index 174f73131618b659a1e2bb39407460cda8360210..6d909ab27e5426b011b2139fcdc0a05f7729eeed 100755
--- a/chrome/test/functional/perf.py
+++ b/chrome/test/functional/perf.py
@@ -90,7 +90,7 @@ class BasePerfTest(pyauto.PyUITest):
     # TODO(dennisjeffrey): Implement wait for idle CPU on Windows/Mac.
     if self.IsLinux():  # IsLinux() also implies IsChromeOS().
       os.system('sync')
-      self._WaitForIdleCPU(60.0, 0.03)
+      self._WaitForIdleCPU(60.0, 0.05)
 
   def _WaitForIdleCPU(self, timeout, utilization):
     """Waits for the CPU to become idle (< utilization).
@@ -1765,6 +1765,160 @@ class PageCyclerTest(BasePerfTest):
     self._RunPageCyclerTest('moz2', self._num_iterations, 'Moz2File')
 
 
+class MemoryTest(BasePerfTest):
+  """Tests to measure memory consumption under different usage scenarios."""
+
+  def setUp(self):
+    pyauto.PyUITest.setUp(self)
+
+    # Log in to get a fresh Chrome instance with a clean memory state (if
+    # already logged in, log out first).
+    if self.GetLoginInfo()['is_logged_in']:
+      self.Logout()
+      self.assertFalse(self.GetLoginInfo()['is_logged_in'],
+                      msg='Failed to log out.')
+
+    credentials = self.GetPrivateInfo()['test_google_account']
+    self.Login(credentials['username'], credentials['password'])
+    self.assertTrue(self.GetLoginInfo()['is_logged_in'],
+                    msg='Failed to log in.')
+
+  def _GetMemoryStats(self, duration):
+    """Identifies and returns different kinds of current memory usage stats.
+
+    This function samples values each second for |duration| seconds, then
+    outputs the min, max, and ending values for each measurement type.
+
+    Args:
+      duration: The number of seconds to sample data before outputting the
+          minimum, maximum, and ending values for each measurement type.
+
+    Returns:
+      A dictionary containing memory usage information.  Each measurement type
+      is associated with the min, max, and ending values from among all
+      sampled values.  Values are specified in KB.
+      {
+        'gtt': {  # GPU memory usage (graphics translation table)
+          'min': ...,
+          'max': ...,
+          'end': ...,
+        },
+        'mem_available': { ... },  # CPU available memory
+        'mem_shared': { ... },  # CPU shared memory
+      }
+    """
+    logging.debug('Sampling memory information for %d seconds...' % duration)
+    stats = {
+      'gtt': [],
+      'mem_available': [],
+      'mem_shared': [],
+    }
+
+    for _ in xrange(duration):
+      p = subprocess.Popen('grep bytes /sys/kernel/debug/dri/0/i915_gem_gtt',

[Sonny, 2012/04/26 04:55:18]

GTT isn't guaranteed to always be there.  It'll only be present on systems using
Intel graphics, so you should test to see if it's there before opening that
file.  We also may just want to ignore GTT because it ends up being included in
Shmem -- see below comment.

[ihf, 2012/04/26 05:19:17]

Good catch on the checking for file existence.

[dennis_jeffrey, 2012/04/26 18:47:39]

I modified the code to exclude GTT values if the file doesn't exist.

+                           stdout=subprocess.PIPE,
+                           shell=True)
+      stdout = p.communicate()[0]
+
+      # GPU memory.
+      gtt_used = re.search(
+          'Total [\d]+ objects, ([\d]+) bytes', stdout).group(1)
+      stats['gtt'].append(int(gtt_used) / 1024.0)
+
+      # CPU memory.
+      stdout = ''
+      with open('/proc/meminfo') as f:
+        stdout = f.read()
+      mem_free = re.search('MemFree:\s*([\d]+) kB', stdout).group(1)
+      mem_dirty = re.search('Dirty:\s*([\d]+) kB', stdout).group(1)

[Sonny, 2012/04/26 04:55:18]

Dirty  Memory isn't really very interesting because it'll change quite a bit
based on when we write the pages out to disk.  More interesting would probaby be
AnonPages and Slab

[Sonny, 2012/04/26 05:38:14]

Now that I understand what mem_available is supposed to be, i see why you're
reading these particular entries from meminfo.  But I still think we really want
a bunch more than just those numbers which are included in that calculation.

[dennis_jeffrey, 2012/04/26 18:47:39]

See my comment at line 1846 below.

+      mem_active_file = re.search(
+          'Active\(file\):\s*([\d]+) kB', stdout).group(1)
+      mem_inactive_file = re.search(
+          'Inactive\(file\):\s*([\d]+) kB', stdout).group(1)
+      stats['mem_available'].append(
+          (int(mem_active_file) + int(mem_inactive_file)) - int(mem_dirty) +
+          int(mem_free))

[Sonny, 2012/04/26 04:55:18]

I'm not sure what mem_available is supposed to mean here?

[ihf, 2012/04/26 05:19:17]

This was suggested by James as it is used for discarding tabs.

[James Cook, 2012/04/26 05:25:59]

It's apparently what Luigi uses in the kernel to provide the low memory signal
to chrome.

[Sonny, 2012/04/26 05:38:14]

Ah.. I see.  That's not actually exactly true either, because Luigi is also
using another number which is our floor on how much file cache we should allow
to stay.
If you really want that number, you need to read another proc file and factor it
in here.

[dennis_jeffrey, 2012/04/26 18:47:39]

Let me know how the computation for available memory needs to change here.

[Sonny, 2012/04/26 22:25:40]

so it should be
MemFree + Inactive file + active file - dirty - min_file_mem

and min_file_mem comes from /proc/sys/vm/min_filelist_kbytes

[dennis_jeffrey, 2012/04/28 00:21:35]

Done.

+
+      mem_shared = re.search('Shmem:\s*([\d]+) kB', stdout).group(1)
+      stats['mem_shared'].append(int(mem_shared))
+
+      time.sleep(1)
+

[Sonny, 2012/04/26 04:55:18]

Also weren't we going to get the memory stats from Chrome using GetProcessInfo?

[ihf, 2012/04/26 05:19:17]

First time I read about it.

[Sonny, 2012/04/26 05:38:14]

we want the private and shared and possibly the javascript memory in use for
each renderer.  Dennis knows what I'm talking about, as he has some code to go
do this.  Actually that reminds me, we also want a chrome flag which forces
every tab to get it's own renderer process.

[dennis_jeffrey, 2012/04/26 18:47:39]

Yes, there is lots more information that people want to collect.  There's the
information Sonny mentioned (which as he said, we've already implemented before
when conducting other experiments).  Additionally, Puneet mentioned that he'd
like to see some other info too (need to check in with him again to get the
details).

I recommend getting a first version of this test checked in and running, then we
can add or modify the set of measurements taken in another CL, once we've hashed
out what we need.

[Sonny, 2012/04/26 22:25:40]

Ok... I just feel like we'll be adding more stuff for collection soon after this
lands, because what we are collecting now doesn't help us figure out what is
consuming the memory beyond GTT and Shmem.

[dennis_jeffrey, 2012/04/28 00:21:35]

We're now recording more different types of memory.  We can decide later which
subset of this to actually plot on graphs.

+    # Compute min, max, and ending values to return.
+    result = {}
+    for measurement_type in stats:
+      values = stats[measurement_type]
+      result[measurement_type] = {
+        'min': min(values),
+        'max': max(values),
+        'end': values[-1],
+      }
+
+    return result
+
+  def _RecordMemoryStats(self, description, when, duration):
+    """Outputs memory statistics to be graphed.
+
+    Args:
+      description: A string description for the test.  Should not contain
+          spaces.  For example, 'MemCtrl'.
+      when: A string description of when the memory stats are being recorded
+          during test execution (since memory stats may be recorded multiple
+          times during a test execution at certain "interesting" times).  Should
+          not contain spaces.
+      duration: The number of seconds to sample data before outputting the
+          memory statistics.
+    """
+    mem = self._GetMemoryStats(duration)
+    measurement_types = [
+      ('gtt', 'GTT'),
+      ('mem_available', 'MemAvail'),
+      ('mem_shared', 'MemShare'),
+    ]
+    for type_key, type_string in measurement_types:
+      self._OutputPerfGraphValue(
+          '%s-Min%s-%s' % (description, type_string, when),
+          mem[type_key]['min'], 'KB', '%s-%s' % (description, type_string))
+      self._OutputPerfGraphValue(
+          '%s-Max%s-%s' % (description, type_string, when),
+          mem[type_key]['max'], 'KB', '%s-%s' % (description, type_string))
+      self._OutputPerfGraphValue(
+          '%s-End%s-%s' % (description, type_string, when),
+          mem[type_key]['end'], 'KB', '%s-%s' % (description, type_string))
+
+  def _RunTest(self, tabs, description, duration):
+    """Runs a general memory test.
+
+    Args:
+      tabs: A list of strings representing the URLs of the websites to open
+          during this test.
+      description: A string description for the test.  Should not contain
+          spaces.  For example, 'MemCtrl'.
+      duration: The number of seconds to sample data before outputting memory
+          statistics.
+    """
+    self._RecordMemoryStats(description, '0Tabs0', duration)
+
+    for iteration_num in xrange(2):
+      for site in tabs:
+        self.AppendTab(pyauto.GURL(site))
+
+      self._RecordMemoryStats(description,
+                              '%dTabs%d' % (len(tabs), iteration_num + 1),
+                              duration)

[Sonny, 2012/04/26 04:55:18]

I'm not sure we want to record after opening every tab, probably just when the
whole set is opened?  I'm just worried it'll be noisy and too much information
to grok easily.

[ihf, 2012/04/26 05:19:17]

This works exactly as you want it.

[Sonny, 2012/04/26 05:38:14]

Ahh oops, didn't see the indent, thanks.

[dennis_jeffrey, 2012/04/26 18:47:39]

Done - working as intended.

+
+      for _ in xrange(len(tabs)):
+        self.GetBrowserWindow(0).GetTab(1).Close(True)
+
+      self._RecordMemoryStats(description, '0Tabs%d' % (iteration_num + 1),
+                              duration)
+
+  def testOpenCloseTabsControl(self):
+    """Measures memory usage when opening/closing tabs to about:blank."""
+    tabs = ['about:blank'] * 10
+    self._RunTest(tabs, 'MemCtrl', 15)
+
+
 class PerfTestServerRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
   """Request handler for the local performance test server."""