Moving mozrunner and simplejson to a toplevel location.

src/third_party/pylib/mozrunner/__init__.py

Index: src/third_party/pylib/mozrunner/__init__.py
===================================================================
--- src/third_party/pylib/mozrunner/__init__.py	(revision 9275)
+++ src/third_party/pylib/mozrunner/__init__.py	(working copy)
@@ -1,639 +0,0 @@
-# ***** BEGIN LICENSE BLOCK *****
-# Version: MPL 1.1/GPL 2.0/LGPL 2.1
-#
-# The contents of this file are subject to the Mozilla Public License Version
-# 1.1 (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-# http://www.mozilla.org/MPL/
-#
-# Software distributed under the License is distributed on an "AS IS" basis,
-# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
-# for the specific language governing rights and limitations under the
-# License.
-#
-# The Original Code is Mozilla Corporation Code.
-#
-# The Initial Developer of the Original Code is
-# Mikeal Rogers.
-# Portions created by the Initial Developer are Copyright (C) 2008-2009
-# the Initial Developer. All Rights Reserved.
-#
-# Contributor(s):
-#  Mikeal Rogers <mikeal.rogers@gmail.com>
-#  Clint Talbert <ctalbert@mozilla.com>
-#  Henrik Skupin <hskupin@mozilla.com>
-#
-# Alternatively, the contents of this file may be used under the terms of
-# either the GNU General Public License Version 2 or later (the "GPL"), or
-# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
-# in which case the provisions of the GPL or the LGPL are applicable instead
-# of those above. If you wish to allow use of your version of this file only
-# under the terms of either the GPL or the LGPL, and not to allow others to
-# use your version of this file under the terms of the MPL, indicate your
-# decision by deleting the provisions above and replace them with the notice
-# and other provisions required by the GPL or the LGPL. If you do not delete
-# the provisions above, a recipient may use your version of this file under
-# the terms of any one of the MPL, the GPL or the LGPL.
-#
-# ***** END LICENSE BLOCK *****
-
-import os
-import sys
-import copy
-import tempfile
-import signal
-import commands
-import zipfile
-import optparse
-import killableprocess
-import subprocess
-import platform
-
-from distutils import dir_util
-from time import sleep
-from xml.dom import minidom
-
-# conditional (version-dependent) imports
-try:
-    import simplejson
-except ImportError:
-    import json as simplejson
-
-import logging
-logger = logging.getLogger(__name__)
-
-# Use dir_util for copy/rm operations because shutil is all kinds of broken
-copytree = dir_util.copy_tree
-rmtree = dir_util.remove_tree
-
-def findInPath(fileName, path=os.environ['PATH']):
-    dirs = path.split(os.pathsep)
-    for dir in dirs:
-        if os.path.isfile(os.path.join(dir, fileName)):
-            return os.path.join(dir, fileName)
-        if os.name == 'nt' or sys.platform == 'cygwin':
-            if os.path.isfile(os.path.join(dir, fileName + ".exe")):
-                return os.path.join(dir, fileName + ".exe")
-    return None
-
-stdout = sys.stdout
-stderr = sys.stderr
-stdin = sys.stdin
-
-def run_command(cmd, env=None, **kwargs):
-    """Run the given command in killable process."""
-    killable_kwargs = {'stdout':stdout ,'stderr':stderr, 'stdin':stdin}
-    killable_kwargs.update(kwargs)
-
-    if sys.platform != "win32":
-        return killableprocess.Popen(cmd, preexec_fn=lambda : os.setpgid(0, 0),
-                                     env=env, **killable_kwargs)
-    else:
-        return killableprocess.Popen(cmd, env=env, **killable_kwargs)
-
-def getoutput(l):
-    tmp = tempfile.mktemp()
-    x = open(tmp, 'w')
-    subprocess.call(l, stdout=x, stderr=x)
-    x.close(); x = open(tmp, 'r')
-    r = x.read() ; x.close()
-    os.remove(tmp)
-    return r
-
-def get_pids(name, minimun_pid=0):
-    """Get all the pids matching name, exclude any pids below minimum_pid."""
-    if os.name == 'nt' or sys.platform == 'cygwin':
-        import wpk
-
-        pids = wpk.get_pids(name)
-
-    else:
-        # get_pids_cmd = ['ps', 'ax']
-        # h = killableprocess.runCommand(get_pids_cmd, stdout=subprocess.PIPE, universal_newlines=True)
-        # h.wait(group=False)
-        # data = h.stdout.readlines()
-        data = getoutput(['ps', 'ax']).splitlines()
-        pids = [int(line.split()[0]) for line in data if line.find(name) is not -1]
-
-    matching_pids = [m for m in pids if m > minimun_pid]
-    return matching_pids
-
-def kill_process_by_name(name):
-    """Find and kill all processes containing a certain name"""
-
-    pids = get_pids(name)
-
-    if os.name == 'nt' or sys.platform == 'cygwin':
-        for p in pids:
-            import wpk
-
-            wpk.kill_pid(p)
-
-    else:
-        for pid in pids:
-            try:
-                os.kill(pid, signal.SIGTERM)
-            except OSError: pass
-            sleep(.5)
-            if len(get_pids(name)) is not 0:
-                try:
-                    os.kill(pid, signal.SIGKILL)
-                except OSError: pass
-                sleep(.5)
-                if len(get_pids(name)) is not 0:
-                    logger.error('Could not kill process')
-
-def makedirs(name):
-
-    head, tail = os.path.split(name)
-    if not tail:
-        head, tail = os.path.split(head)
-    if head and tail and not os.path.exists(head):
-        try:
-            makedirs(head)
-        except OSError, e:
-            pass
-        if tail == os.curdir:           # xxx/newdir/. exists if xxx/newdir exists
-            return
-    try:
-        os.mkdir(name)
-    except:
-        pass
-
-class Profile(object):
-    """Handles all operations regarding profile. Created new profiles, installs extensions,
-    sets preferences and handles cleanup."""
-
-    def __init__(self, binary=None, profile=None, addons=None,
-                 preferences=None):
-
-        self.binary = binary
-
-        self.create_new = not(bool(profile))
-        if profile:
-            self.profile = profile
-        else:
-            self.profile = self.create_new_profile(self.binary)
-
-        self.addons_installed = []
-        self.addons = addons or []
-
-        ### set preferences from class preferences
-        preferences = preferences or {}
-        if hasattr(self.__class__, 'preferences'):
-            self.preferences = self.__class__.preferences.copy()
-        else:
-            self.preferences = {}
-        self.preferences.update(preferences)
-
-        for addon in self.addons:
-            self.install_addon(addon)
-
-        self.set_preferences(self.preferences)
-
-    def create_new_profile(self, binary):
-        """Create a new clean profile in tmp which is a simple empty folder"""
-        profile = tempfile.mkdtemp(suffix='.mozrunner')
-        return profile
-
-    ### methods related to addons
-
-    @classmethod
-    def addon_id(self, addon_path):
-        """
-        return the id for a given addon, or None if not found
-        - addon_path : path to the addon directory
-        """
-        
-        def find_id(desc):
-            """finds the addon id give its description"""
-            
-            addon_id = None
-            for elem in desc:
-                apps = elem.getElementsByTagName('em:targetApplication')
-                if apps:
-                    for app in apps:
-                        # remove targetApplication nodes, they contain id's we aren't interested in
-                        elem.removeChild(app)
-                    if elem.getElementsByTagName('em:id'):
-                        addon_id = str(elem.getElementsByTagName('em:id')[0].firstChild.data)
-                    elif elem.hasAttribute('em:id'):
-                        addon_id = str(elem.getAttribute('em:id'))
-            return addon_id
-
-        doc = minidom.parse(os.path.join(addon_path, 'install.rdf')) 
-
-        for tag in 'Description', 'RDF:Description':
-            desc = doc.getElementsByTagName(tag)
-            addon_id = find_id(desc)
-            if addon_id:
-                return addon_id
-
-
-    def install_addon(self, path):
-        """Installs the given addon or directory of addons in the profile."""
-        
-        # if the addon is a directory, install all addons in it
-        addons = [path]
-        if not path.endswith('.xpi') and not os.path.exists(os.path.join(path, 'install.rdf')):
-            addons = [os.path.join(path, x) for x in os.listdir(path)]
-
-        # install each addon
-        for addon in addons:
-
-            # if the addon is an .xpi, uncompress it to a temporary directory
-            if addon.endswith('.xpi'):
-                tmpdir = tempfile.mkdtemp(suffix = "." + os.path.split(addon)[-1])
-                compressed_file = zipfile.ZipFile(addon, "r")
-                for name in compressed_file.namelist():
-                    if name.endswith('/'):
-                        makedirs(os.path.join(tmpdir, name))
-                    else:
-                        if not os.path.isdir(os.path.dirname(os.path.join(tmpdir, name))):
-                            makedirs(os.path.dirname(os.path.join(tmpdir, name)))
-                        data = compressed_file.read(name)
-                        f = open(os.path.join(tmpdir, name), 'wb')
-                        f.write(data) ; f.close()
-                addon = tmpdir
-
-            # determine the addon id
-            addon_id = Profile.addon_id(addon)
-            assert addon_id is not None, "The addon id could not be found: %s" % addon
-
-            # copy the addon to the profile
-            addon_path = os.path.join(self.profile, 'extensions', addon_id)
-            copytree(addon, addon_path, preserve_symlinks=1)
-            self.addons_installed.append(addon_path)
-
-    def clean_addons(self):
-        """Cleans up addons in the profile."""
-        for addon in self.addons_installed:
-            if os.path.isdir(addon):
-                rmtree(addon)
-
-    ### methods related to preferences
-
-    def set_preferences(self, preferences):
-        """Adds preferences dict to profile preferences"""
-
-        prefs_file = os.path.join(self.profile, 'user.js')
-
-        # Ensure that the file exists first otherwise create an empty file
-        if os.path.isfile(prefs_file):
-            f = open(prefs_file, 'a+')
-        else:
-            f = open(prefs_file, 'w')
-
-        f.write('\n#MozRunner Prefs Start\n')
-
-        pref_lines = ['user_pref(%s, %s);' %
-                      (simplejson.dumps(k), simplejson.dumps(v) ) for k, v in
-                       preferences.items()]
-        for line in pref_lines:
-            f.write(line+'\n')
-        f.write('#MozRunner Prefs End\n')
-        f.flush() ; f.close()
-
-    def clean_preferences(self):
-        """Removed preferences added by mozrunner."""
-        lines = open(os.path.join(self.profile, 'user.js'), 'r').read().splitlines()
-        s = lines.index('#MozRunner Prefs Start') ; e = lines.index('#MozRunner Prefs End')
-        cleaned_prefs = '\n'.join(lines[:s] + lines[e+1:])
-        f = open(os.path.join(self.profile, 'user.js'), 'w')
-        f.write(cleaned_prefs) ; f.flush() ; f.close()
-
-    ### cleanup 
-
-    def cleanup(self):
-        """Cleanup operations on the profile."""
-        if self.create_new:
-            rmtree(self.profile)
-        else:
-            self.clean_preferences()
-            self.clean_addons()
-
-class FirefoxProfile(Profile):
-    """Specialized Profile subclass for Firefox"""
-    preferences = {# Don't automatically update the application
-                   'app.update.enabled' : False,
-                   # Don't restore the last open set of tabs if the browser has crashed
-                   'browser.sessionstore.resume_from_crash': False,
-                   # Don't check for the default web browser
-                   'browser.shell.checkDefaultBrowser' : False,
-                   # Don't warn on exit when multiple tabs are open
-                   'browser.tabs.warnOnClose' : False,
-                   # Don't warn when exiting the browser
-                   'browser.warnOnQuit': False,
-                   # Only install add-ons from the profile and the app folder
-                   'extensions.enabledScopes' : 5,
-                   # Dont' run the add-on compatibility check during start-up
-                   'extensions.showMismatchUI' : False,
-                   # Don't automatically update add-ons
-                   'extensions.update.enabled'    : False,
-                   # Don't open a dialog to show available add-on updates
-                   'extensions.update.notifyUser' : False,
-                   }
-
-    @property
-    def names(self):
-        if sys.platform == 'darwin':
-            return ['firefox', 'minefield', 'shiretoko']
-        if (sys.platform == 'linux2') or (sys.platform in ('sunos5', 'solaris')):
-            return ['firefox', 'mozilla-firefox', 'iceweasel']
-        if os.name == 'nt' or sys.platform == 'cygwin':
-            return ['firefox']
-
-class ThunderbirdProfile(Profile):
-    preferences = {'extensions.update.enabled'    : False,
-                   'extensions.update.notifyUser' : False,
-                   'browser.shell.checkDefaultBrowser' : False,
-                   'browser.tabs.warnOnClose' : False,
-                   'browser.warnOnQuit': False,
-                   'browser.sessionstore.resume_from_crash': False,
-                   }
-    names = ["thunderbird", "shredder"]
-
-
-class Runner(object):
-    """Handles all running operations. Finds bins, runs and kills the process."""
-
-    def __init__(self, binary=None, profile=None, cmdargs=[], env=None,
-                 aggressively_kill=['crashreporter'], kp_kwargs={}):
-        if binary is None:
-            self.binary = self.find_binary()
-        elif sys.platform == 'darwin' and binary.find('Contents/MacOS/') == -1:
-            self.binary = os.path.join(binary, 'Contents/MacOS/%s-bin' % self.names[0])
-        else:
-            self.binary = binary
-
-        if not os.path.exists(self.binary):
-            raise Exception("Binary path does not exist "+self.binary)
-
-        if sys.platform == 'linux2' and self.binary.endswith('-bin'):
-            dirname = os.path.dirname(self.binary)
-            if os.environ.get('LD_LIBRARY_PATH', None):
-                os.environ['LD_LIBRARY_PATH'] = '%s:%s' % (os.environ['LD_LIBRARY_PATH'], dirname)
-            else:
-                os.environ['LD_LIBRARY_PATH'] = dirname
-
-        self.profile = profile
-
-        self.cmdargs = cmdargs
-        if env is None:
-            self.env = copy.copy(os.environ)
-            self.env.update({'MOZ_NO_REMOTE':"1",})
-        else:
-            self.env = env
-        self.aggressively_kill = aggressively_kill
-        self.kp_kwargs = kp_kwargs
-
-    def find_binary(self):
-        """Finds the binary for self.names if one was not provided."""
-        binary = None
-        if sys.platform in ('linux2', 'sunos5', 'solaris'):
-            for name in reversed(self.names):
-                binary = findInPath(name)
-        elif os.name == 'nt' or sys.platform == 'cygwin':
-
-            # find the default executable from the windows registry
-            try:
-                # assumes self.app_name is defined, as it should be for
-                # implementors
-                import _winreg
-                app_key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, r"Software\Mozilla\Mozilla %s" % self.app_name)
-                version, _type = _winreg.QueryValueEx(app_key, "CurrentVersion")
-                version_key = _winreg.OpenKey(app_key, version + r"\Main")
-                path, _ = _winreg.QueryValueEx(version_key, "PathToExe")
-                return path
-            except: # XXX not sure what type of exception this should be
-                pass
-
-            # search for the binary in the path            
-            for name in reversed(self.names):
-                binary = findInPath(name)
-                if sys.platform == 'cygwin':
-                    program_files = os.environ['PROGRAMFILES']
-                else:
-                    program_files = os.environ['ProgramFiles']
-
-                if binary is None:
-                    for bin in [(program_files, 'Mozilla Firefox', 'firefox.exe'),
-                                (os.environ.get("ProgramFiles(x86)"),'Mozilla Firefox', 'firefox.exe'),
-                                (program_files,'Minefield', 'firefox.exe'),
-                                (os.environ.get("ProgramFiles(x86)"),'Minefield', 'firefox.exe')
-                                ]:
-                        path = os.path.join(*bin)
-                        if os.path.isfile(path):
-                            binary = path
-                            break
-        elif sys.platform == 'darwin':
-            for name in reversed(self.names):
-                appdir = os.path.join('Applications', name.capitalize()+'.app')
-                if os.path.isdir(os.path.join(os.path.expanduser('~/'), appdir)):
-                    binary = os.path.join(os.path.expanduser('~/'), appdir,
-                                          'Contents/MacOS/'+name+'-bin')
-                elif os.path.isdir('/'+appdir):
-                    binary = os.path.join("/"+appdir, 'Contents/MacOS/'+name+'-bin')
-
-                if binary is not None:
-                    if not os.path.isfile(binary):
-                        binary = binary.replace(name+'-bin', 'firefox-bin')
-                    if not os.path.isfile(binary):
-                        binary = None
-        if binary is None:
-            raise Exception('Mozrunner could not locate your binary, you will need to set it.')
-        return binary
-
-    @property
-    def command(self):
-        """Returns the command list to run."""
-        cmd = [self.binary, '-profile', self.profile.profile]
-        # On i386 OS X machines, i386+x86_64 universal binaries need to be told
-        # to run as i386 binaries.  If we're not running a i386+x86_64 universal
-        # binary, then this command modification is harmless.
-        if sys.platform == 'darwin':
-            if hasattr(platform, 'architecture') and platform.architecture()[0] == '32bit':
-                cmd = ['arch', '-i386'] + cmd
-        return cmd
-
-    def get_repositoryInfo(self):
-        """Read repository information from application.ini and platform.ini."""
-        import ConfigParser
-
-        config = ConfigParser.RawConfigParser()
-        dirname = os.path.dirname(self.binary)
-        repository = { }
-
-        for entry in [['application', 'App'], ['platform', 'Build']]:
-            (file, section) = entry
-            config.read(os.path.join(dirname, '%s.ini' % file))
-
-            for entry in [['SourceRepository', 'repository'], ['SourceStamp', 'changeset']]:
-                (key, id) = entry
-
-                try:
-                    repository['%s_%s' % (file, id)] = config.get(section, key);
-                except:
-                    repository['%s_%s' % (file, id)] = None
-
-        return repository
-
-    def start(self):
-        """Run self.command in the proper environment."""
-        if self.profile is None:
-            self.profile = self.profile_class()
-        self.process_handler = run_command(self.command+self.cmdargs, self.env, **self.kp_kwargs)
-
-    def wait(self, timeout=None):
-        """Wait for the browser to exit."""
-        self.process_handler.wait(timeout=timeout)
-
-        if sys.platform != 'win32':
-            for name in self.names:
-                for pid in get_pids(name, self.process_handler.pid):
-                    self.process_handler.pid = pid
-                    self.process_handler.wait(timeout=timeout)
-
-    def kill(self, kill_signal=signal.SIGTERM):
-        """Kill the browser"""
-        if sys.platform != 'win32':
-            self.process_handler.kill()
-            for name in self.names:
-                for pid in get_pids(name, self.process_handler.pid):
-                    self.process_handler.pid = pid
-                    self.process_handler.kill()
-        else:
-            try:
-                self.process_handler.kill(group=True)
-            except Exception, e:
-                logger.error('Cannot kill process, '+type(e).__name__+' '+e.message)
-
-        for name in self.aggressively_kill:
-            kill_process_by_name(name)
-
-    def stop(self):
-        self.kill()
-
-class FirefoxRunner(Runner):
-    """Specialized Runner subclass for running Firefox."""
-
-    app_name = 'Firefox'
-    profile_class = FirefoxProfile
-
-    @property
-    def names(self):
-        if sys.platform == 'darwin':
-            return ['firefox', 'minefield', 'shiretoko']
-        if (sys.platform == 'linux2') or (sys.platform in ('sunos5', 'solaris')):
-            return ['firefox', 'mozilla-firefox', 'iceweasel']
-        if os.name == 'nt' or sys.platform == 'cygwin':
-            return ['firefox']
-
-class ThunderbirdRunner(Runner):
-    """Specialized Runner subclass for running Thunderbird"""
-
-    app_name = 'Thunderbird'
-    profile_class = ThunderbirdProfile
-
-    names = ["thunderbird", "shredder"]
-
-class CLI(object):
-    """Command line interface."""
-
-    runner_class = FirefoxRunner
-    profile_class = FirefoxProfile
-    module = "mozrunner"
-
-    parser_options = {("-b", "--binary",): dict(dest="binary", help="Binary path.",
-                                                metavar=None, default=None),
-                      ('-p', "--profile",): dict(dest="profile", help="Profile path.",
-                                                 metavar=None, default=None),
-                      ('-a', "--addons",): dict(dest="addons", 
-                                                help="Addons paths to install.",
-                                                metavar=None, default=None),
-                      ("--info",): dict(dest="info", default=False,
-                                        action="store_true",
-                                        help="Print module information")
-                     }
-
-    def __init__(self):
-        """ Setup command line parser and parse arguments """
-        self.metadata = self.get_metadata_from_egg()
-        self.parser = optparse.OptionParser(version="%prog " + self.metadata["Version"])
-        for names, opts in self.parser_options.items():
-            self.parser.add_option(*names, **opts)
-        (self.options, self.args) = self.parser.parse_args()
-
-        if self.options.info:
-            self.print_metadata()
-            sys.exit(0)
-            
-        # XXX should use action='append' instead of rolling our own
-        try:
-            self.addons = self.options.addons.split(',')
-        except:
-            self.addons = []
-            
-    def get_metadata_from_egg(self):
-        import pkg_resources
-        ret = {}
-        dist = pkg_resources.get_distribution(self.module)
-        if dist.has_metadata("PKG-INFO"):
-            for line in dist.get_metadata_lines("PKG-INFO"):
-                key, value = line.split(':', 1)
-                ret[key] = value
-        if dist.has_metadata("requires.txt"):
-            ret["Dependencies"] = "\n" + dist.get_metadata("requires.txt")    
-        return ret
-        
-    def print_metadata(self, data=("Name", "Version", "Summary", "Home-page", 
-                                   "Author", "Author-email", "License", "Platform", "Dependencies")):
-        for key in data:
-            if key in self.metadata:
-                print key + ": " + self.metadata[key]
-
-    def create_runner(self):
-        """ Get the runner object """
-        runner = self.get_runner(binary=self.options.binary)
-        profile = self.get_profile(binary=runner.binary,
-                                   profile=self.options.profile,
-                                   addons=self.addons)
-        runner.profile = profile
-        return runner
-
-    def get_runner(self, binary=None, profile=None):
-        """Returns the runner instance for the given command line binary argument
-        the profile instance returned from self.get_profile()."""
-        return self.runner_class(binary, profile)
-
-    def get_profile(self, binary=None, profile=None, addons=None, preferences=None):
-        """Returns the profile instance for the given command line arguments."""
-        addons = addons or []
-        preferences = preferences or {}
-        return self.profile_class(binary, profile, addons, preferences)
-
-    def run(self):
-        runner = self.create_runner()
-        self.start(runner)
-        runner.profile.cleanup()
-
-    def start(self, runner):
-        """Starts the runner and waits for Firefox to exitor Keyboard Interrupt.
-        Shoule be overwritten to provide custom running of the runner instance."""
-        runner.start()
-        print 'Started:', ' '.join(runner.command)
-        try:
-            runner.wait()
-        except KeyboardInterrupt:
-            runner.stop()
-
-
-def cli():
-    CLI().run()
-
-def print_addon_ids(args=sys.argv[1:]):
-    """print addon ids for testing"""
-    for arg in args:
-        print Profile.addon_id(arg)
-    
-