Adding cython v0.20.2 in third-party.

third_party/cython/src/Cython/Build/Dependencies.py

+import cython
+from Cython import __version__
+
+import re, os, sys, time
+try:
+    from glob import iglob
+except ImportError:
+    # Py2.4
+    from glob import glob as iglob
+
+try:
+    import gzip
+    gzip_open = gzip.open
+    gzip_ext = '.gz'
+except ImportError:
+    gzip_open = open
+    gzip_ext = ''
+import shutil
+import subprocess
+
+try:
+    import hashlib
+except ImportError:
+    import md5 as hashlib
+
+try:
+    from io import open as io_open
+except ImportError:
+    from codecs import open as io_open
+
+try:
+    from os.path import relpath as _relpath
+except ImportError:
+    # Py<2.6
+    def _relpath(path, start=os.path.curdir):
+        if not path:
+            raise ValueError("no path specified")
+        start_list = os.path.abspath(start).split(os.path.sep)
+        path_list = os.path.abspath(path).split(os.path.sep)
+        i = len(os.path.commonprefix([start_list, path_list]))
+        rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
+        if not rel_list:
+            return os.path.curdir
+        return os.path.join(*rel_list)
+
+
+from distutils.extension import Extension
+
+from Cython import Utils
+from Cython.Utils import cached_function, cached_method, path_exists, find_root_package_dir
+from Cython.Compiler.Main import Context, CompilationOptions, default_options
+
+join_path = cached_function(os.path.join)
+
+if sys.version_info[0] < 3:
+    # stupid Py2 distutils enforces str type in list of sources
+    _fs_encoding = sys.getfilesystemencoding()
+    if _fs_encoding is None:
+        _fs_encoding = sys.getdefaultencoding()
+    def encode_filename_in_py2(filename):
+        if isinstance(filename, unicode):
+            return filename.encode(_fs_encoding)
+        return filename
+else:
+    def encode_filename_in_py2(filename):
+        return filename
+    basestring = str
+
+def extended_iglob(pattern):
+    if '**/' in pattern:
+        seen = set()
+        first, rest = pattern.split('**/', 1)
+        if first:
+            first = iglob(first+'/')
+        else:
+            first = ['']
+        for root in first:
+            for path in extended_iglob(join_path(root, rest)):
+                if path not in seen:
+                    seen.add(path)
+                    yield path
+            for path in extended_iglob(join_path(root, '*', '**/' + rest)):
+                if path not in seen:
+                    seen.add(path)
+                    yield path
+    else:
+        for path in iglob(pattern):
+            yield path
+
+@cached_function
+def file_hash(filename):
+    path = os.path.normpath(filename.encode("UTF-8"))
+    m = hashlib.md5(str(len(path)) + ":")
+    m.update(path)
+    f = open(filename, 'rb')
+    try:
+        data = f.read(65000)
+        while data:
+            m.update(data)
+            data = f.read(65000)
+    finally:
+        f.close()
+    return m.hexdigest()
+
+def parse_list(s):
+    """
+    >>> parse_list("a b c")
+    ['a', 'b', 'c']
+    >>> parse_list("[a, b, c]")
+    ['a', 'b', 'c']
+    >>> parse_list('a " " b')
+    ['a', ' ', 'b']
+    >>> parse_list('[a, ",a", "a,", ",", ]')
+    ['a', ',a', 'a,', ',']
+    """
+    if s[0] == '[' and s[-1] == ']':
+        s = s[1:-1]
+        delimiter = ','
+    else:
+        delimiter = ' '
+    s, literals = strip_string_literals(s)
+    def unquote(literal):
+        literal = literal.strip()
+        if literal[0] in "'\"":
+            return literals[literal[1:-1]]
+        else:
+            return literal
+    return [unquote(item) for item in s.split(delimiter) if item.strip()]
+
+transitive_str = object()
+transitive_list = object()
+
+distutils_settings = {
+    'name':                 str,
+    'sources':              list,
+    'define_macros':        list,
+    'undef_macros':         list,
+    'libraries':            transitive_list,
+    'library_dirs':         transitive_list,
+    'runtime_library_dirs': transitive_list,
+    'include_dirs':         transitive_list,
+    'extra_objects':        list,
+    'extra_compile_args':   transitive_list,
+    'extra_link_args':      transitive_list,
+    'export_symbols':       list,
+    'depends':              transitive_list,
+    'language':             transitive_str,
+}
+
+@cython.locals(start=long, end=long)
+def line_iter(source):
+    if isinstance(source, basestring):
+        start = 0
+        while True:
+            end = source.find('\n', start)
+            if end == -1:
+                yield source[start:]
+                return
+            yield source[start:end]
+            start = end+1
+    else:
+        for line in source:
+            yield line
+
+class DistutilsInfo(object):
+
+    def __init__(self, source=None, exn=None):
+        self.values = {}
+        if source is not None:
+            for line in line_iter(source):
+                line = line.strip()
+                if line != '' and line[0] != '#':
+                    break
+                line = line[1:].strip()
+                if line[:10] == 'distutils:':
+                    line = line[10:]
+                    ix = line.index('=')
+                    key = str(line[:ix].strip())
+                    value = line[ix+1:].strip()
+                    type = distutils_settings[key]
+                    if type in (list, transitive_list):
+                        value = parse_list(value)
+                        if key == 'define_macros':
+                            value = [tuple(macro.split('=')) for macro in value]
+                    self.values[key] = value
+        elif exn is not None:
+            for key in distutils_settings:
+                if key in ('name', 'sources'):
+                    continue
+                value = getattr(exn, key, None)
+                if value:
+                    self.values[key] = value
+
+    def merge(self, other):
+        if other is None:
+            return self
+        for key, value in other.values.items():
+            type = distutils_settings[key]
+            if type is transitive_str and key not in self.values:
+                self.values[key] = value
+            elif type is transitive_list:
+                if key in self.values:
+                    all = self.values[key]
+                    for v in value:
+                        if v not in all:
+                            all.append(v)
+                else:
+                    self.values[key] = value
+        return self
+
+    def subs(self, aliases):
+        if aliases is None:
+            return self
+        resolved = DistutilsInfo()
+        for key, value in self.values.items():
+            type = distutils_settings[key]
+            if type in [list, transitive_list]:
+                new_value_list = []
+                for v in value:
+                    if v in aliases:
+                        v = aliases[v]
+                    if isinstance(v, list):
+                        new_value_list += v
+                    else:
+                        new_value_list.append(v)
+                value = new_value_list
+            else:
+                if value in aliases:
+                    value = aliases[value]
+            resolved.values[key] = value
+        return resolved
+
+    def apply(self, extension):
+        for key, value in self.values.items():
+            type = distutils_settings[key]
+            if type in [list, transitive_list]:
+                getattr(extension, key).extend(value)
+            else:
+                setattr(extension, key, value)
+
+@cython.locals(start=long, q=long, single_q=long, double_q=long, hash_mark=long,
+               end=long, k=long, counter=long, quote_len=long)
+def strip_string_literals(code, prefix='__Pyx_L'):
+    """
+    Normalizes every string literal to be of the form '__Pyx_Lxxx',
+    returning the normalized code and a mapping of labels to
+    string literals.
+    """
+    new_code = []
+    literals = {}
+    counter = 0
+    start = q = 0
+    in_quote = False
+    hash_mark = single_q = double_q = -1
+    code_len = len(code)
+
+    while True:
+        if hash_mark < q:
+            hash_mark = code.find('#', q)
+        if single_q < q:
+            single_q = code.find("'", q)
+        if double_q < q:
+            double_q = code.find('"', q)
+        q = min(single_q, double_q)
+        if q == -1: q = max(single_q, double_q)
+
+        # We're done.
+        if q == -1 and hash_mark == -1:
+            new_code.append(code[start:])
+            break
+
+        # Try to close the quote.
+        elif in_quote:
+            if code[q-1] == u'\\':
+                k = 2
+                while q >= k and code[q-k] == u'\\':
+                    k += 1
+                if k % 2 == 0:
+                    q += 1
+                    continue
+            if code[q] == quote_type and (quote_len == 1 or (code_len > q + 2 and quote_type == code[q+1] == code[q+2])):
+                counter += 1
+                label = "%s%s_" % (prefix, counter)
+                literals[label] = code[start+quote_len:q]
+                full_quote = code[q:q+quote_len]
+                new_code.append(full_quote)
+                new_code.append(label)
+                new_code.append(full_quote)
+                q += quote_len
+                in_quote = False
+                start = q
+            else:
+                q += 1
+
+        # Process comment.
+        elif -1 != hash_mark and (hash_mark < q or q == -1):
+            new_code.append(code[start:hash_mark+1])
+            end = code.find('\n', hash_mark)
+            counter += 1
+            label = "%s%s_" % (prefix, counter)
+            if end == -1:
+                end_or_none = None
+            else:
+                end_or_none = end
+            literals[label] = code[hash_mark+1:end_or_none]
+            new_code.append(label)
+            if end == -1:
+                break
+            start = q = end
+
+        # Open the quote.
+        else:
+            if code_len >= q+3 and (code[q] == code[q+1] == code[q+2]):
+                quote_len = 3
+            else:
+                quote_len = 1
+            in_quote = True
+            quote_type = code[q]
+            new_code.append(code[start:q])
+            start = q
+            q += quote_len
+
+    return "".join(new_code), literals
+
+
+dependancy_regex = re.compile(r"(?:^from +([0-9a-zA-Z_.]+) +cimport)|"
+                              r"(?:^cimport +([0-9a-zA-Z_.]+)\b)|"
+                              r"(?:^cdef +extern +from +['\"]([^'\"]+)['\"])|"
+                              r"(?:^include +['\"]([^'\"]+)['\"])", re.M)
+
+def normalize_existing(base_path, rel_paths):
+    return normalize_existing0(os.path.dirname(base_path), tuple(set(rel_paths)))
+
+@cached_function
+def normalize_existing0(base_dir, rel_paths):
+    normalized = []
+    for rel in rel_paths:
+        path = join_path(base_dir, rel)
+        if path_exists(path):
+            normalized.append(os.path.normpath(path))
+        else:
+            normalized.append(rel)
+    return normalized
+
+def resolve_depends(depends, include_dirs):
+    include_dirs = tuple(include_dirs)
+    resolved = []
+    for depend in depends:
+        path = resolve_depend(depend, include_dirs)
+        if path is not None:
+            resolved.append(path)
+    return resolved
+
+@cached_function
+def resolve_depend(depend, include_dirs):
+    if depend[0] == '<' and depend[-1] == '>':
+        return None
+    for dir in include_dirs:
+        path = join_path(dir, depend)
+        if path_exists(path):
+            return os.path.normpath(path)
+    return None
+
+@cached_function
+def package(filename):
+    dir = os.path.dirname(os.path.abspath(str(filename)))
+    if dir != filename and path_exists(join_path(dir, '__init__.py')):
+        return package(dir) + (os.path.basename(dir),)
+    else:
+        return ()
+
+@cached_function
+def fully_qualified_name(filename):
+    module = os.path.splitext(os.path.basename(filename))[0]
+    return '.'.join(package(filename) + (module,))
+
+
+@cached_function
+def parse_dependencies(source_filename):
+    # Actual parsing is way to slow, so we use regular expressions.
+    # The only catch is that we must strip comments and string
+    # literals ahead of time.
+    fh = Utils.open_source_file(source_filename, "rU", error_handling='ignore')
+    try:
+        source = fh.read()
+    finally:
+        fh.close()
+    distutils_info = DistutilsInfo(source)
+    source, literals = strip_string_literals(source)
+    source = source.replace('\\\n', ' ').replace('\t', ' ')
+
+    # TODO: pure mode
+    cimports = []
+    includes = []
+    externs  = []
+    for m in dependancy_regex.finditer(source):
+        cimport_from, cimport, extern, include = m.groups()
+        if cimport_from:
+            cimports.append(cimport_from)
+        elif cimport:
+            cimports.append(cimport)
+        elif extern:
+            externs.append(literals[extern])
+        else:
+            includes.append(literals[include])
+    return cimports, includes, externs, distutils_info
+
+
+class DependencyTree(object):
+
+    def __init__(self, context, quiet=False):
+        self.context = context
+        self.quiet = quiet
+        self._transitive_cache = {}
+
+    def parse_dependencies(self, source_filename):
+        return parse_dependencies(source_filename)
+
+    @cached_method
+    def included_files(self, filename):
+        # This is messy because included files are textually included, resolving
+        # cimports (but not includes) relative to the including file.
+        all = set()
+        for include in self.parse_dependencies(filename)[1]:
+            include_path = join_path(os.path.dirname(filename), include)
+            if not path_exists(include_path):
+                include_path = self.context.find_include_file(include, None)
+            if include_path:
+                if '.' + os.path.sep in include_path:
+                    include_path = os.path.normpath(include_path)
+                all.add(include_path)
+                all.update(self.included_files(include_path))
+            elif not self.quiet:
+                print("Unable to locate '%s' referenced from '%s'" % (filename, include))
+        return all
+
+    @cached_method
+    def cimports_and_externs(self, filename):
+        # This is really ugly. Nested cimports are resolved with respect to the
+        # includer, but includes are resolved with respect to the includee.
+        cimports, includes, externs = self.parse_dependencies(filename)[:3]
+        cimports = set(cimports)
+        externs = set(externs)
+        for include in self.included_files(filename):
+            included_cimports, included_externs = self.cimports_and_externs(include)
+            cimports.update(included_cimports)
+            externs.update(included_externs)
+        return tuple(cimports), normalize_existing(filename, externs)
+
+    def cimports(self, filename):
+        return self.cimports_and_externs(filename)[0]
+
+    def package(self, filename):
+        return package(filename)
+
+    def fully_qualified_name(self, filename):
+        return fully_qualified_name(filename)
+
+    @cached_method
+    def find_pxd(self, module, filename=None):
+        is_relative = module[0] == '.'
+        if is_relative and not filename:
+            raise NotImplementedError("New relative imports.")
+        if filename is not None:
+            module_path = module.split('.')
+            if is_relative:
+                module_path.pop(0)  # just explicitly relative
+            package_path = list(self.package(filename))
+            while module_path and not module_path[0]:
+                try:
+                    package_path.pop()
+                except IndexError:
+                    return None   # FIXME: error?
+                module_path.pop(0)
+            relative = '.'.join(package_path + module_path)
+            pxd = self.context.find_pxd_file(relative, None)
+            if pxd:
+                return pxd
+        if is_relative:
+            return None   # FIXME: error?
+        return self.context.find_pxd_file(module, None)
+
+    @cached_method
+    def cimported_files(self, filename):
+        if filename[-4:] == '.pyx' and path_exists(filename[:-4] + '.pxd'):
+            pxd_list = [filename[:-4] + '.pxd']
+        else:
+            pxd_list = []
+        for module in self.cimports(filename):
+            if module[:7] == 'cython.' or module == 'cython':
+                continue
+            pxd_file = self.find_pxd(module, filename)
+            if pxd_file is not None:
+                pxd_list.append(pxd_file)
+            elif not self.quiet:
+                print("missing cimport in module '%s': %s" % (module, filename))
+        return tuple(pxd_list)
+
+    @cached_method
+    def immediate_dependencies(self, filename):
+        all = set([filename])
+        all.update(self.cimported_files(filename))
+        all.update(self.included_files(filename))
+        return all
+
+    def all_dependencies(self, filename):
+        return self.transitive_merge(filename, self.immediate_dependencies, set.union)
+
+    @cached_method
+    def timestamp(self, filename):
+        return os.path.getmtime(filename)
+
+    def extract_timestamp(self, filename):
+        return self.timestamp(filename), filename
+
+    def newest_dependency(self, filename):
+        return max([self.extract_timestamp(f) for f in self.all_dependencies(filename)])
+
+    def transitive_fingerprint(self, filename, extra=None):
+        try:
+            m = hashlib.md5(__version__)
+            m.update(file_hash(filename))
+            for x in sorted(self.all_dependencies(filename)):
+                if os.path.splitext(x)[1] not in ('.c', '.cpp', '.h'):
+                    m.update(file_hash(x))
+            if extra is not None:
+                m.update(str(extra))
+            return m.hexdigest()
+        except IOError:
+            return None
+
+    def distutils_info0(self, filename):
+        info = self.parse_dependencies(filename)[3]
+        externs = self.cimports_and_externs(filename)[1]
+        if externs:
+            if 'depends' in info.values:
+                info.values['depends'] = list(set(info.values['depends']).union(externs))
+            else:
+                info.values['depends'] = list(externs)
+        return info
+
+    def distutils_info(self, filename, aliases=None, base=None):
+        return (self.transitive_merge(filename, self.distutils_info0, DistutilsInfo.merge)
+            .subs(aliases)
+            .merge(base))
+
+    def transitive_merge(self, node, extract, merge):
+        try:
+            seen = self._transitive_cache[extract, merge]
+        except KeyError:
+            seen = self._transitive_cache[extract, merge] = {}
+        return self.transitive_merge_helper(
+            node, extract, merge, seen, {}, self.cimported_files)[0]
+
+    def transitive_merge_helper(self, node, extract, merge, seen, stack, outgoing):
+        if node in seen:
+            return seen[node], None
+        deps = extract(node)
+        if node in stack:
+            return deps, node
+        try:
+            stack[node] = len(stack)
+            loop = None
+            for next in outgoing(node):
+                sub_deps, sub_loop = self.transitive_merge_helper(next, extract, merge, seen, stack, outgoing)
+                if sub_loop is not None:
+                    if loop is not None and stack[loop] < stack[sub_loop]:
+                        pass
+                    else:
+                        loop = sub_loop
+                deps = merge(deps, sub_deps)
+            if loop == node:
+                loop = None
+            if loop is None:
+                seen[node] = deps
+            return deps, loop
+        finally:
+            del stack[node]
+
+_dep_tree = None
+def create_dependency_tree(ctx=None, quiet=False):
+    global _dep_tree
+    if _dep_tree is None:
+        if ctx is None:
+            ctx = Context(["."], CompilationOptions(default_options))
+        _dep_tree = DependencyTree(ctx, quiet=quiet)
+    return _dep_tree
+
+# This may be useful for advanced users?
+def create_extension_list(patterns, exclude=[], ctx=None, aliases=None, quiet=False, exclude_failures=False):
+    if not isinstance(patterns, (list, tuple)):
+        patterns = [patterns]
+    explicit_modules = set([m.name for m in patterns if isinstance(m, Extension)])
+    seen = set()
+    deps = create_dependency_tree(ctx, quiet=quiet)
+    to_exclude = set()
+    if not isinstance(exclude, list):
+        exclude = [exclude]
+    for pattern in exclude:
+        to_exclude.update(map(os.path.abspath, extended_iglob(pattern)))
+    module_list = []
+    for pattern in patterns:
+        if isinstance(pattern, str):
+            filepattern = pattern
+            template = None
+            name = '*'
+            base = None
+            exn_type = Extension
+        elif isinstance(pattern, Extension):
+            filepattern = pattern.sources[0]
+            if os.path.splitext(filepattern)[1] not in ('.py', '.pyx'):
+                # ignore non-cython modules
+                module_list.append(pattern)
+                continue
+            template = pattern
+            name = template.name
+            base = DistutilsInfo(exn=template)
+            exn_type = template.__class__
+        else:
+            raise TypeError(pattern)
+        for file in extended_iglob(filepattern):
+            if os.path.abspath(file) in to_exclude:
+                continue
+            pkg = deps.package(file)
+            if '*' in name:
+                module_name = deps.fully_qualified_name(file)
+                if module_name in explicit_modules:
+                    continue
+            else:
+                module_name = name
+            if module_name not in seen:
+                try:
+                    kwds = deps.distutils_info(file, aliases, base).values
+                except Exception:
+                    if exclude_failures:
+                        continue
+                    raise
+                if base is not None:
+                    for key, value in base.values.items():
+                        if key not in kwds:
+                            kwds[key] = value
+                sources = [file]
+                if template is not None:
+                    sources += template.sources[1:]
+                if 'sources' in kwds:
+                    # allow users to add .c files etc.
+                    for source in kwds['sources']:
+                        source = encode_filename_in_py2(source)
+                        if source not in sources:
+                            sources.append(source)
+                    del kwds['sources']
+                if 'depends' in kwds:
+                    depends = resolve_depends(kwds['depends'], (kwds.get('include_dirs') or []) + [find_root_package_dir(file)])
+                    if template is not None:
+                        # Always include everything from the template.
+                        depends = list(set(template.depends).union(set(depends)))
+                    kwds['depends'] = depends
+                module_list.append(exn_type(
+                        name=module_name,
+                        sources=sources,
+                        **kwds))
+                m = module_list[-1]
+                seen.add(name)
+    return module_list
+
+# This is the user-exposed entry point.
+def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, force=False,
+              exclude_failures=False, **options):
+    """
+    Compile a set of source modules into C/C++ files and return a list of distutils
+    Extension objects for them.
+
+    As module list, pass either a glob pattern, a list of glob patterns or a list of
+    Extension objects.  The latter allows you to configure the extensions separately
+    through the normal distutils options.
+
+    When using glob patterns, you can exclude certain module names explicitly
+    by passing them into the 'exclude' option.
+
+    For parallel compilation, set the 'nthreads' option to the number of
+    concurrent builds.
+
+    For a broad 'try to compile' mode that ignores compilation failures and
+    simply excludes the failed extensions, pass 'exclude_failures=True'. Note
+    that this only really makes sense for compiling .py files which can also
+    be used without compilation.
+
+    Additional compilation options can be passed as keyword arguments.
+    """
+    if 'include_path' not in options:
+        options['include_path'] = ['.']
+    if 'common_utility_include_dir' in options:
+        if options.get('cache'):
+            raise NotImplementedError("common_utility_include_dir does not yet work with caching")
+        if not os.path.exists(options['common_utility_include_dir']):
+            os.makedirs(options['common_utility_include_dir'])
+    c_options = CompilationOptions(**options)
+    cpp_options = CompilationOptions(**options); cpp_options.cplus = True
+    ctx = c_options.create_context()
+    options = c_options
+    module_list = create_extension_list(
+        module_list,
+        exclude=exclude,
+        ctx=ctx,
+        quiet=quiet,
+        exclude_failures=exclude_failures,
+        aliases=aliases)
+    deps = create_dependency_tree(ctx, quiet=quiet)
+    build_dir = getattr(options, 'build_dir', None)
+    modules_by_cfile = {}
+    to_compile = []
+    for m in module_list:
+        if build_dir:
+            root = os.path.realpath(os.path.abspath(find_root_package_dir(m.sources[0])))
+            def copy_to_build_dir(filepath, root=root):
+                filepath_abs = os.path.realpath(os.path.abspath(filepath))
+                if os.path.isabs(filepath):
+                    filepath = filepath_abs
+                if filepath_abs.startswith(root):
+                    mod_dir = os.path.join(build_dir,
+                            os.path.dirname(_relpath(filepath, root)))
+                    if not os.path.isdir(mod_dir):
+                        os.makedirs(mod_dir)
+                    shutil.copy(filepath, mod_dir)
+            for dep in m.depends:
+                copy_to_build_dir(dep)
+
+        new_sources = []
+        for source in m.sources:
+            base, ext = os.path.splitext(source)
+            if ext in ('.pyx', '.py'):
+                if m.language == 'c++':
+                    c_file = base + '.cpp'
+                    options = cpp_options
+                else:
+                    c_file = base + '.c'
+                    options = c_options
+
+                # setup for out of place build directory if enabled
+                if build_dir:
+                    c_file = os.path.join(build_dir, c_file)
+                    dir = os.path.dirname(c_file)
+                    if not os.path.isdir(dir):
+                        os.makedirs(dir)
+
+                if os.path.exists(c_file):
+                    c_timestamp = os.path.getmtime(c_file)
+                else:
+                    c_timestamp = -1
+
+                # Priority goes first to modified files, second to direct
+                # dependents, and finally to indirect dependents.
+                if c_timestamp < deps.timestamp(source):
+                    dep_timestamp, dep = deps.timestamp(source), source
+                    priority = 0
+                else:
+                    dep_timestamp, dep = deps.newest_dependency(source)
+                    priority = 2 - (dep in deps.immediate_dependencies(source))
+                if force or c_timestamp < dep_timestamp:
+                    if not quiet:
+                        if source == dep:
+                            print("Compiling %s because it changed." % source)
+                        else:
+                            print("Compiling %s because it depends on %s." % (source, dep))
+                    if not force and hasattr(options, 'cache'):
+                        extra = m.language
+                        fingerprint = deps.transitive_fingerprint(source, extra)
+                    else:
+                        fingerprint = None
+                    to_compile.append((priority, source, c_file, fingerprint, quiet,
+                                       options, not exclude_failures))
+                new_sources.append(c_file)
+                if c_file not in modules_by_cfile:
+                    modules_by_cfile[c_file] = [m]
+                else:
+                    modules_by_cfile[c_file].append(m)
+            else:
+                new_sources.append(source)
+                if build_dir:
+                    copy_to_build_dir(source)
+        m.sources = new_sources
+    if hasattr(options, 'cache'):
+        if not os.path.exists(options.cache):
+            os.makedirs(options.cache)
+    to_compile.sort()
+    if nthreads:
+        # Requires multiprocessing (or Python >= 2.6)
+        try:
+            import multiprocessing
+            pool = multiprocessing.Pool(nthreads)
+        except (ImportError, OSError):
+            print("multiprocessing required for parallel cythonization")
+            nthreads = 0
+        else:
+            pool.map(cythonize_one_helper, to_compile)
+    if not nthreads:
+        for args in to_compile:
+            cythonize_one(*args[1:])
+    if exclude_failures:
+        failed_modules = set()
+        for c_file, modules in modules_by_cfile.iteritems():
+            if not os.path.exists(c_file):
+                failed_modules.update(modules)
+            elif os.path.getsize(c_file) < 200:
+                f = io_open(c_file, 'r', encoding='iso8859-1')
+                try:
+                    if f.read(len('#error ')) == '#error ':
+                        # dead compilation result
+                        failed_modules.update(modules)
+                finally:
+                    f.close()
+        if failed_modules:
+            for module in failed_modules:
+                module_list.remove(module)
+            print("Failed compilations: %s" % ', '.join(sorted([
+                module.name for module in failed_modules])))
+    if hasattr(options, 'cache'):
+        cleanup_cache(options.cache, getattr(options, 'cache_size', 1024 * 1024 * 100))
+    # cythonize() is often followed by the (non-Python-buffered)
+    # compiler output, flush now to avoid interleaving output.
+    sys.stdout.flush()
+    return module_list
+
+
+if os.environ.get('XML_RESULTS'):
+    compile_result_dir = os.environ['XML_RESULTS']
+    def record_results(func):
+        def with_record(*args):
+            t = time.time()
+            success = True
+            try:
+                try:
+                    func(*args)
+                except:
+                    success = False
+            finally:
+                t = time.time() - t
+                module = fully_qualified_name(args[0])
+                name = "cythonize." + module
+                failures = 1 - success
+                if success:
+                    failure_item = ""
+                else:
+                    failure_item = "failure"
+                output = open(os.path.join(compile_result_dir, name + ".xml"), "w")
+                output.write("""
+                    <?xml version="1.0" ?>
+                    <testsuite name="%(name)s" errors="0" failures="%(failures)s" tests="1" time="%(t)s">
+                    <testcase classname="%(name)s" name="cythonize">
+                    %(failure_item)s
+                    </testcase>
+                    </testsuite>
+                """.strip() % locals())
+                output.close()
+        return with_record
+else:
+    record_results = lambda x: x
+
+# TODO: Share context? Issue: pyx processing leaks into pxd module
+@record_results
+def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_failure=True):
+    from Cython.Compiler.Main import compile, default_options
+    from Cython.Compiler.Errors import CompileError, PyrexError
+
+    if fingerprint:
+        if not os.path.exists(options.cache):
+            try:
+                os.mkdir(options.cache)
+            except:
+                if not os.path.exists(options.cache):
+                    raise
+        # Cython-generated c files are highly compressible.
+        # (E.g. a compression ratio of about 10 for Sage).
+        fingerprint_file = join_path(
+            options.cache, "%s-%s%s" % (os.path.basename(c_file), fingerprint, gzip_ext))
+        if os.path.exists(fingerprint_file):
+            if not quiet:
+                print("Found compiled %s in cache" % pyx_file)
+            os.utime(fingerprint_file, None)
+            g = gzip_open(fingerprint_file, 'rb')
+            try:
+                f = open(c_file, 'wb')
+                try:
+                    shutil.copyfileobj(g, f)
+                finally:
+                    f.close()
+            finally:
+                g.close()
+            return
+    if not quiet:
+        print("Cythonizing %s" % pyx_file)
+    if options is None:
+        options = CompilationOptions(default_options)
+    options.output_file = c_file
+
+    any_failures = 0
+    try:
+        result = compile([pyx_file], options)
+        if result.num_errors > 0:
+            any_failures = 1
+    except (EnvironmentError, PyrexError), e:
+        sys.stderr.write('%s\n' % e)
+        any_failures = 1
+        # XXX
+        import traceback
+        traceback.print_exc()
+    except Exception:
+        if raise_on_failure:
+            raise
+        import traceback
+        traceback.print_exc()
+        any_failures = 1
+    if any_failures:
+        if raise_on_failure:
+            raise CompileError(None, pyx_file)
+        elif os.path.exists(c_file):
+            os.remove(c_file)
+    elif fingerprint:
+        f = open(c_file, 'rb')
+        try:
+            g = gzip_open(fingerprint_file, 'wb')
+            try:
+                shutil.copyfileobj(f, g)
+            finally:
+                g.close()
+        finally:
+            f.close()
+
+def cythonize_one_helper(m):
+    import traceback
+    try:
+        return cythonize_one(*m[1:])
+    except Exception:
+        traceback.print_exc()
+        raise
+
+def cleanup_cache(cache, target_size, ratio=.85):
+    try:
+        p = subprocess.Popen(['du', '-s', '-k', os.path.abspath(cache)], stdout=subprocess.PIPE)
+        res = p.wait()
+        if res == 0:
+            total_size = 1024 * int(p.stdout.read().strip().split()[0])
+            if total_size < target_size:
+                return
+    except (OSError, ValueError):
+        pass
+    total_size = 0
+    all = []
+    for file in os.listdir(cache):
+        path = join_path(cache, file)
+        s = os.stat(path)
+        total_size += s.st_size
+        all.append((s.st_atime, s.st_size, path))
+    if total_size > target_size:
+        for time, size, file in reversed(sorted(all)):
+            os.unlink(file)
+            total_size -= size
+            if total_size < target_size * ratio:
+                break