Upstream the android python scripts changes that enable the new Forwarder2 to be used in our test f…

build/android/pylib/forwarder.py

Index: build/android/pylib/forwarder.py
diff --git a/build/android/pylib/forwarder.py b/build/android/pylib/forwarder.py
index 03b16f17a84b487d2c41a025bcecdc9aaa607d4d..9b23d8c351f7ac471e8cc34acb8193f75280a166 100644
--- a/build/android/pylib/forwarder.py
+++ b/build/android/pylib/forwarder.py
@@ -7,14 +7,20 @@ import os
 import pexpect
 import re
 import sys
+import time
 
 import android_commands
+import cmd_helper
 import constants
+import ports
 
 class Forwarder(object):
   """Class to manage port forwards from the device to the host."""
 
-  _FORWARDER_PATH = constants.TEST_EXECUTABLE_DIR + '/forwarder'
+  _DEVICE_FORWARDER_PATH = constants.TEST_EXECUTABLE_DIR + '/device_forwarder'
+
+  # Unix Abstract socket path:
+  _DEVICE_ADB_CONTROL_PORT = 'chrome_device_forwarder'
   _TIMEOUT_SECS = 30
 
   def __init__(self, adb, port_pairs, tool, host_name, build_type):
@@ -40,75 +46,147 @@ class Forwarder(object):
     """
     self._adb = adb
     self._host_to_device_port_map = dict()
-    self._process = None
+    self._host_process = None
+    self._device_process = None
+    self._adb_forward_process = None
+
+    self._host_adb_control_port = ports.AllocateTestServerPort()
+    if not self._host_adb_control_port:
+      raise Exception('Failed to allocate a TCP port in the host machine.')
     adb.PushIfNeeded(
-        os.path.join(constants.CHROME_DIR, 'out', build_type, 'forwarder'),
-        Forwarder._FORWARDER_PATH)
+        os.path.join(constants.CHROME_DIR, 'out', build_type,
+                     'device_forwarder'),
+        Forwarder._DEVICE_FORWARDER_PATH)
+    self._host_forwarder_path = os.path.join(constants.CHROME_DIR,
+                                             'out',
+                                             build_type,
+                                             'host_forwarder')
     forward_string = ['%d:%d:%s' %
                       (device, host, host_name) for device, host in port_pairs]
+    logging.info('Forwarding ports: %s', forward_string)
+    timeout_sec = 5
+    host_pattern = 'host_forwarder.*' + ' '.join(forward_string)
+    # TODO(felipeg): Rather than using a blocking kill() here, the device
+    # forwarder could try to bind the Unix Domain Socket until it succeeds or
+    # while it fails because the socket is already bound (with appropriate
+    # timeout handling obviously).
+    self._KillHostForwarderBlocking(host_pattern, timeout_sec)
+    self._KillDeviceForwarderBlocking(timeout_sec)
+    self._adb_forward_process = pexpect.spawn(
+        'adb', ['-s',
+                adb._adb.GetSerialNumber(),
+                'forward',
+                'tcp:%s' % self._host_adb_control_port,
+                'localabstract:%s' % Forwarder._DEVICE_ADB_CONTROL_PORT])
+    self._device_process = pexpect.spawn(
+        'adb', ['-s',
+                adb._adb.GetSerialNumber(),
+                'shell',
+                '%s %s -D --adb_sock=%s' % (
+                    tool.GetUtilWrapper(),
+                    Forwarder._DEVICE_FORWARDER_PATH,
+                    Forwarder._DEVICE_ADB_CONTROL_PORT)])
 
-    # Kill off any existing forwarders on conflicting non-dynamically allocated
-    # ports.
-    for device_port, _ in port_pairs:
-      if device_port != 0:
-        self._KillForwardersUsingDevicePort(device_port)
+    device_success_re = re.compile('Starting Device Forwarder.')
+    device_failure_re = re.compile('.*:ERROR:(.*)')
+    index = self._device_process.expect([device_success_re,
+                                         device_failure_re,
+                                         pexpect.EOF,
+                                         pexpect.TIMEOUT],
+                                        Forwarder._TIMEOUT_SECS)
+    if index == 1:
+      # Failure
+      error_msg = str(self._device_process.match.group(1))
+      logging.error(self._device_process.before)
+      self._CloseProcess()
+      raise Exception('Failed to start Device Forwarder with Error: %s' %
+                      error_msg)
+    elif index == 2:
+      logging.error(self._device_process.before)
+      self._CloseProcess()
+      raise Exception('Unexpected EOF while trying to start Device Forwarder.')
+    elif index == 3:
+      logging.error(self._device_process.before)
+      self._CloseProcess()
+      raise Exception('Timeout while trying start Device Forwarder')
 
-    logging.info('Forwarding ports: %s' % (forward_string))
-    process = pexpect.spawn(
-      'adb', ['-s', adb._adb.GetSerialNumber(),
-              'shell', '%s %s -D %s' % (
-          tool.GetUtilWrapper(), Forwarder._FORWARDER_PATH,
-          ' '.join(forward_string))])
+    self._host_process = pexpect.spawn(self._host_forwarder_path,
+                                       ['--adb_port=%s' % (
+                                           self._host_adb_control_port)] +
+                                       forward_string)
 
     # Read the output of the command to determine which device ports where
     # forwarded to which host ports (necessary if
-    success_re = re.compile('Forwarding device port (\d+) to host (\d+):')
-    failure_re = re.compile('Couldn\'t start forwarder server for port spec: '
-                            '(\d+):(\d+)')
+    host_success_re = re.compile('Forwarding device port (\d+) to host (\d+):')
+    host_failure_re = re.compile('Couldn\'t start forwarder server for port '
+                                 'spec: (\d+):(\d+)')
     for pair in port_pairs:
-      index = process.expect([success_re, failure_re, pexpect.EOF,
-                              pexpect.TIMEOUT],
-                             Forwarder._TIMEOUT_SECS)
+      index = self._host_process.expect([host_success_re,
+                                         host_failure_re,
+                                         pexpect.EOF,
+                                         pexpect.TIMEOUT],
+                                        Forwarder._TIMEOUT_SECS)
       if index == 0:
         # Success
-        device_port = int(process.match.group(1))
-        host_port = int(process.match.group(2))
+        device_port = int(self._host_process.match.group(1))
+        host_port = int(self._host_process.match.group(2))
         self._host_to_device_port_map[host_port] = device_port
         logging.info("Forwarding device port: %d to host port: %d." %
                      (device_port, host_port))
       elif index == 1:
         # Failure
-        device_port = int(process.match.group(1))
-        host_port = int(process.match.group(2))
-        process.close()
+        device_port = int(self._host_process.match.group(1))
+        host_port = int(self._host_process.match.group(2))
+        self._CloseProcess()
         raise Exception('Failed to forward port %d to %d' % (device_port,
                                                              host_port))
       elif index == 2:
-        logging.error(process.before)
-        process.close()
+        logging.error(self._host_process.before)
+        self._CloseProcess()
         raise Exception('Unexpected EOF while trying to forward ports %s' %
                         port_pairs)
       elif index == 3:
-        logging.error(process.before)
-        process.close()
+        logging.error(self._host_process.before)
+        self._CloseProcess()
         raise Exception('Timeout while trying to forward ports %s' % port_pairs)
 
-    self._process = process
-
-  def _KillForwardersUsingDevicePort(self, device_port):
-    """Check if the device port is in use and if it is try to terminate the
-       forwarder process (if any) that may already be forwarding it"""
-    processes = self._adb.ProcessesUsingDevicePort(device_port)
-    for pid, name in processes:
-      if name == 'forwarder':
-        logging.warning(
-            'Killing forwarder process with pid %d using device_port %d' % (
-                 pid, device_port))
-        self._adb.RunShellCommand('kill %d' % pid)
-      else:
-        logging.error(
-             'Not killing process with pid %d (%s) using device_port %d' % (
-                 pid, name, device_port))
+  def _KillHostForwarderBlocking(self, host_pattern, timeout_sec):
+    """Kills any existing host forwarders using the provided pattern.
+
+    Note that this waits until the process terminates.
+    """
+    cmd_helper.RunCmd(['pkill', '-f', host_pattern])
+    elapsed = 0
+    wait_period = 0.1
+    while not cmd_helper.RunCmd(['pgrep', '-f', host_pattern]) and (
+        elapsed < timeout_sec):
+      time.sleep(wait_period)
+      elapsed += wait_period
+    if elapsed >= timeout_sec:
+        raise Exception('Timed out while killing ' + host_pattern)
+
+  def _KillDeviceForwarderBlocking(self, timeout_sec):
+    """Kills any existing device forwarders.
+
+    Note that this waits until the process terminates.
+    """
+    processes_killed = self._adb.KillAllBlocking(
+        'device_forwarder', timeout_sec)
+    if not processes_killed:
+      pids = self._adb.ExtractPid('device_forwarder')
+      if pids:
+        raise Exception('Timed out while killing device_forwarder')
+
+  def _CloseProcess(self):
+    if self._host_process:
+      self._host_process.close()
+    if self._device_process:
+      self._device_process.close()
+    if self._adb_forward_process:
+      self._adb_forward_process.close()
+    self._host_process = None
+    self._device_process = None
+    self._adb_forward_process = None
 
   def DevicePortForHostPort(self, host_port):
     """Get the device port that corresponds to a given host port."""
@@ -116,6 +194,4 @@ class Forwarder(object):
 
   def Close(self):
     """Terminate the forwarder process."""
-    if self._process:
-      self._process.close()
-      self._process = None
+    self._CloseProcess()