Chrome updater test framework

pyautolib/fetch_prebuilt_pyauto.py

 #!/usr/bin/env python
 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Fetch prebuilt binaries to run PyAuto.
 
 Sets up Chrome and PyAuto binaries using prebuilt binaries from the
 continuous build archives.  Works on mac, win, linux (32 & 64 bit).
 
@@ skipping 10 matching lines @@
 import glob
 import httplib
 import optparse
 import os
 import platform
 import shutil
 import sys
 import urllib
 import urllib2
 import urlparse
+import socket
+import stat
 
 import pyauto_utils
 
+            
+class FetchPrebuilt(object):
+  """Util class to fetch prebuilt binaries to run PyAuto."""  
+  
+  def __init__(self, url, outdir, platform):  
+    self._url = url
+    if not self.DoesUrlExist(self._url):
+      raise httplib.error('%s is not a valid URL.' % self._url)
+    self._outdir = outdir    
+    self._platform = ((lambda p: p and p or pyauto_utils.GetCurrentPlatform())
+                      (platform))    
+    self._InitZipName()    
+    self._InitDownloadUrls()
 
-class FetchPrebuilt(object):
-  """Util class to fetch prebuilt binaries to run PyAuto."""
-
-  def _ParseArgs(self):
-    parser = optparse.OptionParser()
-    parser.add_option(
-        '-d', '--outdir', type='string', default=None,
-        help='Directory in which to setup. This is typically the directory '
-        'where the binaries would go when compiled from source.')
-    parser.add_option(
-        '-p', '--platform', type='string',
-        default=pyauto_utils.GetCurrentPlatform(),
-        help='Platform. Valid options: win, mac, linux32, linux64. '
-        'Default: current platform (%s)' % pyauto_utils.GetCurrentPlatform())
-    parser.add_option(
-        '-l', '--latest', action='store_true', default=False,
-        help='Download the latest chromium build from commondatastorage. '
-        '[default=False]')
-    self._options, self._args = parser.parse_args()
-    if self._options.latest:
-      self._url = self._GetLastestDownloadURL(self._options.platform)
-    elif not self._args:
-      print >>sys.stderr, 'Need download url'
-      sys.exit(2)
-    else:
-      self._url = self._args[0]
-    if not self._options.outdir:
-      print >>sys.stderr, 'Need output directory: -d/--outdir'
-      sys.exit(1)
-    self._outdir = self._options.outdir
-    # Chromium continuous build archive has a non-standard format.
-    if 'index.html?path=' in self._url:
-      self._url = self._url.replace('index.html?path=', '')
-    self._url = self._url.rstrip('/')
-    # Determine name of zip.
-    if not self._options.platform.startswith('linux'):
-      self._chrome_zip_name = 'chrome-%s' % {'mac': 'mac',
-                                             'win': 'win32'
-                                            }[self._options.platform]
+  def _InitZipName(self):
+    """Determines the name of the zip file based on platform."""
+    if not self._platform.startswith('linux'):
+      self._chrome_zip_name = 'chrome-%s' % { 'mac' : 'mac',
+                                              'win' : 'win32'
+                                            }[self._platform]
     else:
       linux_32_names = ['linux', 'lucid32bit']
       linux_64_names = ['linux64', 'lucid64bit']
       linux_names = {'linux': linux_32_names + linux_64_names,
                      'linux32': linux_32_names,
                      'linux64': linux_64_names
-                    }[self._options.platform]
+                    }[self._platform]
       for name in linux_names:
         zip_name = 'chrome-' + name
-        if self._DoesURLExist('%s/%s.zip' % (self._url, zip_name)):
+        if self.DoesUrlExist('%s/%s.zip' % (self._url, zip_name)):
           self._chrome_zip_name = zip_name
           break
       else:
-        raise RuntimeError('Could not find chrome zip at ' + self._url)
+        raise RuntimeError('Could not find %s at ' % zip_name + self._url)
 
-    # Setup urls to download.
+  def _InitDownloadUrls(self):
+    """Generates URLs for files that are to be downloaded."""
     self._chrome_zip_url = '%s/%s.zip' % (self._url, self._chrome_zip_name)
     self._remoting_zip_url = self._url + '/' + 'remoting-webapp.zip'
     chrome_test_url = '%s/%s.test' % (self._url, self._chrome_zip_name)
     self._pyautolib_py_url = '%s/pyautolib.py' % chrome_test_url
-    if self._options.platform == 'win':
+    if self._platform == 'win':
       self._pyautolib_so_name = '_pyautolib.pyd'
       self._chromedriver_name = 'chromedriver.exe'
     else:
       self._pyautolib_so_name = '_pyautolib.so'
       self._chromedriver_name = 'chromedriver'
     self._pyautolib_so_url = chrome_test_url + '/' + self._pyautolib_so_name
     self._chromedriver_url = chrome_test_url + '/' + self._chromedriver_name
 
-  def _GetLastestDownloadURL(self, os_platform):
+  @staticmethod
+  def GetLastestDownloadURL(os_platform):
     os_type = {'win': 'Win',
                'mac': 'Mac',
                'linux': 'Linux',
                'linux32': 'Linux',
                'linux64': 'Linux_x64'}[os_platform]
     if os_type == 'Linux' and platform.architecture()[0] == '64bit':
       os_type = 'Linux_x64'
     last_change_url = ('http://commondatastorage.googleapis.com/'
                        'chromium-browser-continuous/%s/LAST_CHANGE' % os_type)
     response = urllib2.urlopen(last_change_url)
     last_change = response.read()
     if not last_change:
       print >>sys.stderr, ('Unable to get latest from %s' % last_change_url)
       sys.exit(2)
     last_change_url = ('http://commondatastorage.googleapis.com/'
                        'chromium-browser-continuous/%s/%s' % (os_type,
                                                               last_change))
     return last_change_url
 
-  def _DoesURLExist(self, url):
+  def DoesUrlExist(self, url):
     """Determines whether a resource exists at the given URL."""
     parsed = urlparse.urlparse(url)
-    conn = httplib.HTTPConnection(parsed.netloc)
-    conn.request('HEAD', parsed.path)
-    response = conn.getresponse()
-    if response.status == 302:  # Redirect; follow it.
-      return self._DoesURLExist(response.getheader('location'))
+    try:
+      conn = httplib.HTTPConnection(parsed.netloc)
+      conn.request('HEAD', parsed.path)
+      response = conn.getresponse()
+    except (socket.gaierror, socket.error), err:
+      print 'FetchPrebuilt.DoesUrlExist: %s' % err
+      return False
+    # Follow both permanent (301) and temporary (302) redirects.
+    if response.status == 302 or response.status == 301:  
+      return self.DoesUrlExist(response.getheader('location'))
     return response.status == 200
 
-  def Cleanup(self):
+  def _Cleanup(self):
     """Remove old binaries, if any."""
-    pass
+    lst_dirs = ['PepperFlash', 'remoting', 'locales', 'resources', 'installer']

[kkania, 2012/05/29 18:23:14]

rename this var

[nkang, 2012/05/29 19:29:57]

I'm assuming you were talking about 'files', so I renamed it to 'paths'.

+    files = [os.path.join(self._outdir, name) for name in lst_dirs]
+    for f in files:
+      if os.path.exists(f):
+        try:
+          shutil.rmtree(f, onerror=self._OnError)
+        except:
+          raise RuntimeError('Could not delete %s, access violation' % f)
 
-  def Run(self):
-    self._ParseArgs()
+  def _OnError(self, func, path, exc_info):

[kkania, 2012/05/29 18:23:14]

how about move this func into _Cleanup?

[nkang, 2012/05/29 19:29:57]

Wasn't sure how the style guide felt about nested functions, so that's why I
played it safe. But since its okay to do so, I went ahead and moved 'OnError'
into _Cleanup.

+    """Callback for shutil.rmtree."""    
+    if not os.access(path, os.W_OK):      
+      os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)      
+      func(path)    
+    else:
+      raise OSError('%s is not a read-only file, access denied.' % path)
+
+  def Run(self):    
     if not os.path.isdir(self._outdir):
       os.makedirs(self._outdir)
-    get_remoting = self._DoesURLExist(self._remoting_zip_url)
-
+    else:      
+      print 'Cleaning up %s' % self._outdir
+      self._Cleanup()
+    get_remoting = self.DoesUrlExist(self._remoting_zip_url)    
     # Fetch chrome & pyauto binaries
     print 'Fetching', self._chrome_zip_url
     chrome_zip = urllib.urlretrieve(self._chrome_zip_url)[0]
 
     if get_remoting:
       print 'Fetching', self._remoting_zip_url
       remoting_zip = urllib.urlretrieve(self._remoting_zip_url)[0]
     else:
       print 'Warning: %s does not exist.' % self._remoting_zip_url
 
     print 'Fetching', self._pyautolib_py_url
     pyautolib_py = urllib.urlretrieve(self._pyautolib_py_url)[0]
 
     print 'Fetching', self._pyautolib_so_url
     pyautolib_so = urllib.urlretrieve(self._pyautolib_so_url)[0]
 
     print 'Fetching', self._chromedriver_url
     chromedriver = urllib.urlretrieve(self._chromedriver_url)[0]
 
     chrome_unzip_dir = os.path.join(self._outdir, self._chrome_zip_name)
     if os.path.exists(chrome_unzip_dir):
       print 'Cleaning', chrome_unzip_dir
       pyauto_utils.RemovePath(chrome_unzip_dir)
     print 'Unzipping'
-    pyauto_utils.UnzipFilenameToDir(chrome_zip, self._outdir)
+    try:
+      pyauto_utils.UnzipFilenameToDir(chrome_zip, self._outdir)
+    except:

[kkania, 2012/05/29 18:23:14]

why did you change this? I think it was better to catch the particular
exceptions than to catch every exception.

[nkang, 2012/05/29 19:29:57]

I changed it because we have to import 'zipfile' just so we can catch the
BadZipfile exception. Since we are raising a RuntimeError exception now, I
didn't think it mattered, plus one less import that way. But since you want it
to be a particular exception, I went ahead and changed it back to
zipfile.BadZipfile exception. Also changed the try/catch block in _Cleanup
method. That method also had a generic 'catch' block, which was also raising a
RuntimeException. I changed it, so we only catch OSError/IOError exceptions.

+      raise RuntimeError('Specified zip file does not exist or is corrupted.')
     if get_remoting:
-      pyauto_utils.UnzipFilenameToDir(remoting_zip, self._outdir)
+      pyauto_utils.UnzipFilenameToDir(remoting_zip, self._outdir)       
       shutil.move(self._outdir + '/remoting-webapp',
                   self._outdir + '/remoting/remoting.webapp')
-
-    # Copy over the binaries to outdir
+    # Copy over the binaries to destination directory.
     items_to_copy = {
       pyautolib_py: os.path.join(self._outdir, 'pyautolib.py'),
       pyautolib_so: os.path.join(self._outdir, self._pyautolib_so_name),
       chromedriver: os.path.join(self._outdir, self._chromedriver_name)
     }
     unzip_dir_contents = glob.glob(os.path.join(chrome_unzip_dir, '*'))
     for item in unzip_dir_contents:
       name = os.path.basename(item)
       items_to_copy[item] = os.path.join(self._outdir, name)
 
     for src, dest in items_to_copy.iteritems():
       pyauto_utils.RemovePath(dest)
       print '%s ==> %s' % (os.path.basename(src), dest)
       shutil.move(src, dest)
     pyauto_utils.RemovePath(chrome_unzip_dir)
 
     # Final setup (if any)
     # Set executable bit on chromedriver binary.
-    if not self._options.platform == 'win':
+    if not self._platform == 'win':
       os.chmod(items_to_copy[chromedriver], 0700)
 
     # Create symlink to .framework on Mac
-    if self._options.platform == 'mac':
+    if self._platform == 'mac':
       mac_app_name = os.path.basename([x for x in unzip_dir_contents
                                        if x.endswith('.app')][0])
       os.chdir(self._outdir)
       framework = glob.glob(os.path.join(
           mac_app_name, 'Contents', 'Versions', '*', '*.framework'))[0]
       print framework
       dest = os.path.basename(framework)
       os.path.lexists(dest) and os.remove(dest)
       print 'Creating symlink "%s"' % dest
       os.symlink(framework, dest)
 
     print 'Prepared binaries in "%s"' % self._outdir
     return 0
 
 
-if __name__ == '__main__':
-  sys.exit(FetchPrebuilt().Run())
+class Main:
+
+  def __init__(self):
+    self._ParseArgs()
+    FetchPrebuilt(self._url, self._outdir, self._platform).Run()  
+
+  def _ParseArgs(self):
+    parser = optparse.OptionParser()
+    parser.add_option(
+        '-d', '--outdir', type='string', default=None,
+        help='Directory in which to setup. This is typically the directory '
+        'where the binaries would go when compiled from source.')
+    parser.add_option(
+        '-p', '--platform', type='string',
+        default=pyauto_utils.GetCurrentPlatform(),
+        help='Platform. Valid options: win, mac, linux32, linux64. '
+        'Default: current platform (%s)' % pyauto_utils.GetCurrentPlatform())
+    parser.add_option(
+        '-l', '--latest', action='store_true', default=False,
+        help='Download the latest chromium build from commondatastorage. '
+        '[default=False]')
+    options, args = parser.parse_args()
+    if not options.outdir:
+      print >>sys.stderr, 'Need output directory: -d/--outdir'
+      sys.exit(1)
+    if options.latest:
+      self._url = FetchPrebuilt.GetLastestDownloadURL(options.platform)
+    elif not args:
+      print >>sys.stderr, 'Need download url'
+      sys.exit(2)
+    else:
+      self._url = args[0]
+
+    self._platform = options.platform
+    self._outdir = options.outdir
+    # Chromium continuous build archive has a non-standard format.
+    if 'index.html?path=' in self._url:
+      self._url = self._url.replace('index.html?path=', '')
+    self._url = self._url.rstrip('/')
+    
+
+if __name__ == '__main__':  
+  Main()