Adding cython v0.20.2 in third-party.

third_party/cython/src/pyximport/pyximport.py

Index: third_party/cython/src/pyximport/pyximport.py
diff --git a/third_party/cython/src/pyximport/pyximport.py b/third_party/cython/src/pyximport/pyximport.py
new file mode 100644
index 0000000000000000000000000000000000000000..4fd7fe90e70b060d19a2675399d6267ba4ac66d2
--- /dev/null
+++ b/third_party/cython/src/pyximport/pyximport.py
@@ -0,0 +1,559 @@
+"""
+Import hooks; when installed with the install() function, these hooks 
+allow importing .pyx files as if they were Python modules.
+
+If you want the hook installed every time you run Python
+you can add it to your Python version by adding these lines to
+sitecustomize.py (which you can create from scratch in site-packages 
+if it doesn't exist there or somewhere else on your python path)::
+
+    import pyximport
+    pyximport.install()
+
+For instance on the Mac with a non-system Python 2.3, you could create
+sitecustomize.py with only those two lines at
+/usr/local/lib/python2.3/site-packages/sitecustomize.py .
+
+A custom distutils.core.Extension instance and setup() args
+(Distribution) for for the build can be defined by a <modulename>.pyxbld
+file like:
+
+# examplemod.pyxbld
+def make_ext(modname, pyxfilename):
+    from distutils.extension import Extension
+    return Extension(name = modname,
+                     sources=[pyxfilename, 'hello.c'],
+                     include_dirs=['/myinclude'] )
+def make_setup_args():
+    return dict(script_args=["--compiler=mingw32"])
+
+Extra dependencies can be defined by a <modulename>.pyxdep .
+See README.
+
+Since Cython 0.11, the :mod:`pyximport` module also has experimental
+compilation support for normal Python modules.  This allows you to
+automatically run Cython on every .pyx and .py module that Python
+imports, including parts of the standard library and installed
+packages.  Cython will still fail to compile a lot of Python modules,
+in which case the import mechanism will fall back to loading the
+Python source modules instead.  The .py import mechanism is installed
+like this::
+
+    pyximport.install(pyimport = True)
+
+Running this module as a top-level script will run a test and then print
+the documentation.
+
+This code is based on the Py2.3+ import protocol as described in PEP 302.
+"""
+
+import sys
+import os
+import glob
+import imp
+
+mod_name = "pyximport"
+
+assert sys.hexversion >= 0x2030000, "need Python 2.3 or later"
+
+PYX_EXT = ".pyx"
+PYXDEP_EXT = ".pyxdep"
+PYXBLD_EXT = ".pyxbld"
+
+DEBUG_IMPORT = False
+
+def _print(message, args):
+    if args:
+        message = message % args
+    print(message)
+
+def _debug(message, *args):
+    if DEBUG_IMPORT:
+        _print(message, args)
+
+def _info(message, *args):
+    _print(message, args)
+
+# Performance problem: for every PYX file that is imported, we will 
+# invoke the whole distutils infrastructure even if the module is 
+# already built. It might be more efficient to only do it when the 
+# mod time of the .pyx is newer than the mod time of the .so but
+# the question is how to get distutils to tell me the name of the .so
+# before it builds it. Maybe it is easy...but maybe the peformance
+# issue isn't real.
+def _load_pyrex(name, filename):
+    "Load a pyrex file given a name and filename."
+
+def get_distutils_extension(modname, pyxfilename, language_level=None):
+#    try:
+#        import hashlib
+#    except ImportError:
+#        import md5 as hashlib
+#    extra = "_" + hashlib.md5(open(pyxfilename).read()).hexdigest()  
+#    modname = modname + extra
+    extension_mod,setup_args = handle_special_build(modname, pyxfilename)
+    if not extension_mod:
+        if not isinstance(pyxfilename, str):
+            # distutils is stupid in Py2 and requires exactly 'str'
+            # => encode accidentally coerced unicode strings back to str
+            pyxfilename = pyxfilename.encode(sys.getfilesystemencoding())
+        from distutils.extension import Extension
+        extension_mod = Extension(name = modname, sources=[pyxfilename])
+        if language_level is not None:
+            extension_mod.cython_directives = {'language_level': language_level}
+    return extension_mod,setup_args
+
+def handle_special_build(modname, pyxfilename):
+    special_build = os.path.splitext(pyxfilename)[0] + PYXBLD_EXT
+    ext = None
+    setup_args={}
+    if os.path.exists(special_build): 
+        # globls = {}
+        # locs = {}
+        # execfile(special_build, globls, locs)
+        # ext = locs["make_ext"](modname, pyxfilename)
+        mod = imp.load_source("XXXX", special_build, open(special_build))
+        make_ext = getattr(mod,'make_ext',None)
+        if make_ext:
+            ext = make_ext(modname, pyxfilename)
+            assert ext and ext.sources, ("make_ext in %s did not return Extension" 
+                                         % special_build)
+        make_setup_args = getattr(mod,'make_setup_args',None)
+        if make_setup_args:
+            setup_args = make_setup_args()
+            assert isinstance(setup_args,dict), ("make_setup_args in %s did not return a dict" 
+                                         % special_build)
+        assert set or setup_args, ("neither make_ext nor make_setup_args %s" 
+                                         % special_build)
+        ext.sources = [os.path.join(os.path.dirname(special_build), source) 
+                       for source in ext.sources]
+    return ext, setup_args
+
+def handle_dependencies(pyxfilename):
+    testing = '_test_files' in globals()
+    dependfile = os.path.splitext(pyxfilename)[0] + PYXDEP_EXT
+
+    # by default let distutils decide whether to rebuild on its own
+    # (it has a better idea of what the output file will be)
+
+    # but we know more about dependencies so force a rebuild if 
+    # some of the dependencies are newer than the pyxfile.
+    if os.path.exists(dependfile):
+        depends = open(dependfile).readlines()
+        depends = [depend.strip() for depend in depends]
+
+        # gather dependencies in the "files" variable
+        # the dependency file is itself a dependency
+        files = [dependfile]
+        for depend in depends:
+            fullpath = os.path.join(os.path.dirname(dependfile),
+                                    depend) 
+            files.extend(glob.glob(fullpath))
+
+        # only for unit testing to see we did the right thing
+        if testing:
+            _test_files[:] = []  #$pycheck_no
+
+        # if any file that the pyxfile depends upon is newer than
+        # the pyx file, 'touch' the pyx file so that distutils will
+        # be tricked into rebuilding it.
+        for file in files:
+            from distutils.dep_util import newer
+            if newer(file, pyxfilename):
+                _debug("Rebuilding %s because of %s", pyxfilename, file)
+                filetime = os.path.getmtime(file)
+                os.utime(pyxfilename, (filetime, filetime))
+                if testing:
+                    _test_files.append(file)
+
+def build_module(name, pyxfilename, pyxbuild_dir=None, inplace=False, language_level=None):
+    assert os.path.exists(pyxfilename), (
+        "Path does not exist: %s" % pyxfilename)
+    handle_dependencies(pyxfilename)
+
+    extension_mod,setup_args = get_distutils_extension(name, pyxfilename, language_level)
+    build_in_temp=pyxargs.build_in_temp
+    sargs=pyxargs.setup_args.copy()
+    sargs.update(setup_args)
+    build_in_temp=sargs.pop('build_in_temp',build_in_temp)
+
+    import pyxbuild
+    so_path = pyxbuild.pyx_to_dll(pyxfilename, extension_mod,
+                                  build_in_temp=build_in_temp,
+                                  pyxbuild_dir=pyxbuild_dir,
+                                  setup_args=sargs,
+                                  inplace=inplace,
+                                  reload_support=pyxargs.reload_support)
+    assert os.path.exists(so_path), "Cannot find: %s" % so_path
+    
+    junkpath = os.path.join(os.path.dirname(so_path), name+"_*") #very dangerous with --inplace ? yes, indeed, trying to eat my files ;)
+    junkstuff = glob.glob(junkpath)
+    for path in junkstuff:
+        if path!=so_path:
+            try:
+                os.remove(path)
+            except IOError:
+                _info("Couldn't remove %s", path)
+
+    return so_path
+
+def load_module(name, pyxfilename, pyxbuild_dir=None, is_package=False,
+                build_inplace=False, language_level=None, so_path=None):
+    try:
+        if so_path is None:
+            if is_package:
+                module_name = name + '.__init__'
+            else:
+                module_name = name
+            so_path = build_module(module_name, pyxfilename, pyxbuild_dir,
+                                   inplace=build_inplace, language_level=language_level)
+        mod = imp.load_dynamic(name, so_path)
+        if is_package and not hasattr(mod, '__path__'):
+            mod.__path__ = [os.path.dirname(so_path)]
+        assert mod.__file__ == so_path, (mod.__file__, so_path)
+    except Exception:
+        if pyxargs.load_py_module_on_import_failure and pyxfilename.endswith('.py'):
+            # try to fall back to normal import
+            mod = imp.load_source(name, pyxfilename)
+            assert mod.__file__ in (pyxfilename, pyxfilename+'c', pyxfilename+'o'), (mod.__file__, pyxfilename)
+        else:
+            import traceback
+            raise ImportError("Building module %s failed: %s" %
+                              (name,
+                               traceback.format_exception_only(*sys.exc_info()[:2]))), None, sys.exc_info()[2]
+    return mod
+
+
+# import hooks
+
+class PyxImporter(object):
+    """A meta-path importer for .pyx files.
+    """
+    def __init__(self, extension=PYX_EXT, pyxbuild_dir=None, inplace=False,
+                 language_level=None):
+        self.extension = extension
+        self.pyxbuild_dir = pyxbuild_dir
+        self.inplace = inplace
+        self.language_level = language_level
+
+    def find_module(self, fullname, package_path=None):
+        if fullname in sys.modules  and  not pyxargs.reload_support:
+            return None  # only here when reload() 
+        try:
+            fp, pathname, (ext,mode,ty) = imp.find_module(fullname,package_path)
+            if fp: fp.close()  # Python should offer a Default-Loader to avoid this double find/open!
+            if pathname and ty == imp.PKG_DIRECTORY:
+                pkg_file = os.path.join(pathname, '__init__'+self.extension)
+                if os.path.isfile(pkg_file):
+                    return PyxLoader(fullname, pathname,
+                        init_path=pkg_file,
+                        pyxbuild_dir=self.pyxbuild_dir,
+                        inplace=self.inplace,
+                        language_level=self.language_level)
+            if pathname and pathname.endswith(self.extension):
+                return PyxLoader(fullname, pathname,
+                                 pyxbuild_dir=self.pyxbuild_dir,
+                                 inplace=self.inplace,
+                                 language_level=self.language_level)
+            if ty != imp.C_EXTENSION: # only when an extension, check if we have a .pyx next!
+                return None
+
+            # find .pyx fast, when .so/.pyd exist --inplace
+            pyxpath = os.path.splitext(pathname)[0]+self.extension
+            if os.path.isfile(pyxpath):
+                return PyxLoader(fullname, pyxpath,
+                                 pyxbuild_dir=self.pyxbuild_dir,
+                                 inplace=self.inplace,
+                                 language_level=self.language_level)
+
+            # .so/.pyd's on PATH should not be remote from .pyx's
+            # think no need to implement PyxArgs.importer_search_remote here?
+
+        except ImportError:
+            pass
+
+        # searching sys.path ...
+
+        #if DEBUG_IMPORT:  print "SEARCHING", fullname, package_path
+        if '.' in fullname: # only when package_path anyway?
+            mod_parts = fullname.split('.')
+            module_name = mod_parts[-1]
+        else:
+            module_name = fullname
+        pyx_module_name = module_name + self.extension
+        # this may work, but it returns the file content, not its path
+        #import pkgutil
+        #pyx_source = pkgutil.get_data(package, pyx_module_name)
+
+        if package_path:
+            paths = package_path
+        else:
+            paths = sys.path
+        join_path = os.path.join
+        is_file = os.path.isfile
+        is_abs = os.path.isabs
+        abspath = os.path.abspath
+        #is_dir = os.path.isdir
+        sep = os.path.sep
+        for path in paths:
+            if not path:
+                path = os.getcwd()
+            elif not is_abs(path):
+                path = abspath(path)
+            if is_file(path+sep+pyx_module_name):
+                return PyxLoader(fullname, join_path(path, pyx_module_name),
+                                 pyxbuild_dir=self.pyxbuild_dir,
+                                 inplace=self.inplace,
+                                 language_level=self.language_level)
+
+        # not found, normal package, not a .pyx file, none of our business
+        _debug("%s not found" % fullname)
+        return None
+
+class PyImporter(PyxImporter):
+    """A meta-path importer for normal .py files.
+    """
+    def __init__(self, pyxbuild_dir=None, inplace=False, language_level=None):
+        if language_level is None:
+            language_level = sys.version_info[0]
+        self.super = super(PyImporter, self)
+        self.super.__init__(extension='.py', pyxbuild_dir=pyxbuild_dir, inplace=inplace,
+                            language_level=language_level)
+        self.uncompilable_modules = {}
+        self.blocked_modules = ['Cython', 'pyxbuild', 'pyximport.pyxbuild',
+                                'distutils.extension', 'distutils.sysconfig']
+
+    def find_module(self, fullname, package_path=None):
+        if fullname in sys.modules:
+            return None
+        if fullname.startswith('Cython.'):
+            return None
+        if fullname in self.blocked_modules:
+            # prevent infinite recursion
+            return None
+        if _lib_loader.knows(fullname):
+            return _lib_loader
+        _debug("trying import of module '%s'", fullname)
+        if fullname in self.uncompilable_modules:
+            path, last_modified = self.uncompilable_modules[fullname]
+            try:
+                new_last_modified = os.stat(path).st_mtime
+                if new_last_modified > last_modified:
+                    # import would fail again
+                    return None
+            except OSError:
+                # module is no longer where we found it, retry the import
+                pass
+
+        self.blocked_modules.append(fullname)
+        try:
+            importer = self.super.find_module(fullname, package_path)
+            if importer is not None:
+                if importer.init_path:
+                    path = importer.init_path
+                    real_name = fullname + '.__init__'
+                else:
+                    path = importer.path
+                    real_name = fullname
+                _debug("importer found path %s for module %s", path, real_name)
+                try:
+                    so_path = build_module(
+                        real_name, path,
+                        pyxbuild_dir=self.pyxbuild_dir,
+                        language_level=self.language_level,
+                        inplace=self.inplace)
+                    _lib_loader.add_lib(fullname, path, so_path,
+                                        is_package=bool(importer.init_path))
+                    return _lib_loader
+                except Exception:
+                    if DEBUG_IMPORT:
+                        import traceback
+                        traceback.print_exc()
+                    # build failed, not a compilable Python module
+                    try:
+                        last_modified = os.stat(path).st_mtime
+                    except OSError:
+                        last_modified = 0
+                    self.uncompilable_modules[fullname] = (path, last_modified)
+                    importer = None
+        finally:
+            self.blocked_modules.pop()
+        return importer
+
+class LibLoader(object):
+    def __init__(self):
+        self._libs = {}
+
+    def load_module(self, fullname):
+        try:
+            source_path, so_path, is_package = self._libs[fullname]
+        except KeyError:
+            raise ValueError("invalid module %s" % fullname)
+        _debug("Loading shared library module '%s' from %s", fullname, so_path)
+        return load_module(fullname, source_path, so_path=so_path, is_package=is_package)
+
+    def add_lib(self, fullname, path, so_path, is_package):
+        self._libs[fullname] = (path, so_path, is_package)
+
+    def knows(self, fullname):
+        return fullname in self._libs
+
+_lib_loader = LibLoader()
+
+class PyxLoader(object):
+    def __init__(self, fullname, path, init_path=None, pyxbuild_dir=None,
+                 inplace=False, language_level=None):
+        _debug("PyxLoader created for loading %s from %s (init path: %s)",
+               fullname, path, init_path)
+        self.fullname = fullname
+        self.path, self.init_path = path, init_path
+        self.pyxbuild_dir = pyxbuild_dir
+        self.inplace = inplace
+        self.language_level = language_level
+
+    def load_module(self, fullname):
+        assert self.fullname == fullname, (
+            "invalid module, expected %s, got %s" % (
+            self.fullname, fullname))
+        if self.init_path:
+            # package
+            #print "PACKAGE", fullname
+            module = load_module(fullname, self.init_path,
+                                 self.pyxbuild_dir, is_package=True,
+                                 build_inplace=self.inplace,
+                                 language_level=self.language_level)
+            module.__path__ = [self.path]
+        else:
+            #print "MODULE", fullname
+            module = load_module(fullname, self.path,
+                                 self.pyxbuild_dir,
+                                 build_inplace=self.inplace,
+                                 language_level=self.language_level)
+        return module
+
+
+#install args
+class PyxArgs(object):
+    build_dir=True
+    build_in_temp=True
+    setup_args={}   #None
+
+##pyxargs=None   
+
+def _have_importers():
+    has_py_importer = False
+    has_pyx_importer = False
+    for importer in sys.meta_path:
+        if isinstance(importer, PyxImporter):
+            if isinstance(importer, PyImporter):
+                has_py_importer = True
+            else:
+                has_pyx_importer = True
+
+    return has_py_importer, has_pyx_importer
+
+def install(pyximport=True, pyimport=False, build_dir=None, build_in_temp=True,
+            setup_args={}, reload_support=False,
+            load_py_module_on_import_failure=False, inplace=False,
+            language_level=None):
+    """Main entry point. Call this to install the .pyx import hook in
+    your meta-path for a single Python process.  If you want it to be
+    installed whenever you use Python, add it to your sitecustomize
+    (as described above).
+
+    You can pass ``pyimport=True`` to also install the .py import hook
+    in your meta-path.  Note, however, that it is highly experimental,
+    will not work for most .py files, and will therefore only slow
+    down your imports.  Use at your own risk.
+
+    By default, compiled modules will end up in a ``.pyxbld``
+    directory in the user's home directory.  Passing a different path
+    as ``build_dir`` will override this.
+
+    ``build_in_temp=False`` will produce the C files locally. Working
+    with complex dependencies and debugging becomes more easy. This
+    can principally interfere with existing files of the same name.
+    build_in_temp can be overriden by <modulename>.pyxbld/make_setup_args()
+    by a dict item of 'build_in_temp'
+
+    ``setup_args``: dict of arguments for Distribution - see
+    distutils.core.setup() . They are extended/overriden by those of
+    <modulename>.pyxbld/make_setup_args()
+
+    ``reload_support``:  Enables support for dynamic
+    reload(<pyxmodulename>), e.g. after a change in the Cython code.
+    Additional files <so_path>.reloadNN may arise on that account, when
+    the previously loaded module file cannot be overwritten.
+
+    ``load_py_module_on_import_failure``: If the compilation of a .py
+    file succeeds, but the subsequent import fails for some reason,
+    retry the import with the normal .py module instead of the
+    compiled module.  Note that this may lead to unpredictable results
+    for modules that change the system state during their import, as
+    the second import will rerun these modifications in whatever state
+    the system was left after the import of the compiled module
+    failed.
+
+    ``inplace``: Install the compiled module next to the source file.
+
+    ``language_level``: The source language level to use: 2 or 3.
+    The default is to use the language level of the current Python
+    runtime for .py files and Py2 for .pyx files.
+    """
+    if not build_dir:
+        build_dir = os.path.join(os.path.expanduser('~'), '.pyxbld')
+        
+    global pyxargs
+    pyxargs = PyxArgs()  #$pycheck_no
+    pyxargs.build_dir = build_dir
+    pyxargs.build_in_temp = build_in_temp
+    pyxargs.setup_args = (setup_args or {}).copy()
+    pyxargs.reload_support = reload_support
+    pyxargs.load_py_module_on_import_failure = load_py_module_on_import_failure
+
+    has_py_importer, has_pyx_importer = _have_importers()
+    py_importer, pyx_importer = None, None
+
+    if pyimport and not has_py_importer:
+        py_importer = PyImporter(pyxbuild_dir=build_dir, inplace=inplace,
+                                 language_level=language_level)
+        # make sure we import Cython before we install the import hook
+        import Cython.Compiler.Main, Cython.Compiler.Pipeline, Cython.Compiler.Optimize
+        sys.meta_path.insert(0, py_importer)
+
+    if pyximport and not has_pyx_importer:
+        pyx_importer = PyxImporter(pyxbuild_dir=build_dir, inplace=inplace,
+                                   language_level=language_level)
+        sys.meta_path.append(pyx_importer)
+
+    return py_importer, pyx_importer
+
+def uninstall(py_importer, pyx_importer):
+    """
+    Uninstall an import hook.
+    """
+    try:
+        sys.meta_path.remove(py_importer)
+    except ValueError:
+        pass
+
+    try:
+        sys.meta_path.remove(pyx_importer)
+    except ValueError:
+        pass
+
+# MAIN
+
+def show_docs():
+    import __main__
+    __main__.__name__ = mod_name
+    for name in dir(__main__):
+        item = getattr(__main__, name)
+        try:
+            setattr(item, "__module__", mod_name)
+        except (AttributeError, TypeError):
+            pass
+    help(__main__)
+
+if __name__ == '__main__':
+    show_docs()