Add tool for bots to download a web directory.

scripts/slave/download_from_url.py

+#!/usr/bin/python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Download all files and directories from a URL.
+
+Provided a URL, this script will download files and folders with 'href'
+attribute as the name and the URL as the base path.
+
+This was only tested on Apache web directory.
+"""
+
+import optparse
+import os
+import re
+import sys
+import urllib
+import urllib2
+
+
+def IsFile(url):
+  """Check if URL is a file type.
+
+  If 'Etag' is in the header response, it must be a file type.
+  """
+  try:
+    response = urllib2.urlopen(url)
+    if 'Etag' in response.info():
+      return True
+  except urllib2.URLError:
+    pass
+  return False
+  
+  
+def Download(base_url, target):
+  """Recursively download files and directories.
+
+  Returns:
+    True if succeeded downloading all items, false otherwise.
+  """
+  # Get list of files in this url path.
+  try:
+    response = urllib2.urlopen(base_url)
+  except urllib2.URLError:
+    print 'URL does not exist : ' + base_url
+    return False
+  html = response.read()
+  item_list = re.findall(r'href="([^"]*)"', html)
+
+  # Download all files.
+  for item in item_list:
+    if item.startswith('/'):
+      continue
+    url = base_url + item
+    if item.endswith('/'):
+      dest = os.path.join(target, urllib.unquote(item))
+      if not os.path.isdir(dest):
+        os.makedirs(dest)
+        os.chmod(dest, 0755)
+      if not Download(url, dest):
+        return False
+    elif IsFile(url):
+      dest = os.path.join(target, urllib.unquote(item))
+      print 'Downloading %s to %s' % (url, dest)
+      urllib.urlretrieve(url, dest)
+      os.chmod(dest, 0755)
+  return True
+
+
+def main():
+  """Download files from |url| to |target_dir|."""
+  option_parser = optparse.OptionParser()
+
+  option_parser.add_option('--url',
+                           help='Base URL to the files.')
+  target_dir = os.getcwd()
+  option_parser.add_option('--target-dir', default=target_dir,
+                           help='Target path to put the downloaded files.')
+  options, _ = option_parser.parse_args()
+
+  if not options.url:
+    print 'No url provided.'
+    return 1
+  base_url = options.url
+  if not base_url.endswith('/'):
+    base_url += '/'
+
+  if not os.path.isdir(options.target_dir):
+    os.makedirs(options.target_dir)
+  os.chmod(options.target_dir, 0755)
+
+  success = Download(base_url, options.target_dir)
+  if success:
+    return 0
+  return 1
+
+
+if '__main__' == __name__:
+  sys.exit(main())