Adding cython v0.20.2 in third-party.

third_party/cython/src/Cython/Compiler/Main.py

+#
+#   Cython Top Level
+#
+
+import os, sys, re, codecs
+if sys.version_info[:2] < (2, 4):
+    sys.stderr.write("Sorry, Cython requires Python 2.4 or later\n")
+    sys.exit(1)
+
+import Errors
+# Do not import Parsing here, import it when needed, because Parsing imports
+# Nodes, which globally needs debug command line options initialized to set a
+# conditional metaclass. These options are processed by CmdLine called from
+# main() in this file.
+# import Parsing
+import Version
+from Scanning import PyrexScanner, FileSourceDescriptor
+from Errors import PyrexError, CompileError, error, warning
+from Symtab import ModuleScope
+from Cython import Utils
+import Options
+
+module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$")
+
+verbose = 0
+
+class CompilationData(object):
+    #  Bundles the information that is passed from transform to transform.
+    #  (For now, this is only)
+
+    #  While Context contains every pxd ever loaded, path information etc.,
+    #  this only contains the data related to a single compilation pass
+    #
+    #  pyx                   ModuleNode              Main code tree of this compilation.
+    #  pxds                  {string : ModuleNode}   Trees for the pxds used in the pyx.
+    #  codewriter            CCodeWriter             Where to output final code.
+    #  options               CompilationOptions
+    #  result                CompilationResult
+    pass
+
+class Context(object):
+    #  This class encapsulates the context needed for compiling
+    #  one or more Cython implementation files along with their
+    #  associated and imported declaration files. It includes
+    #  the root of the module import namespace and the list
+    #  of directories to search for include files.
+    #
+    #  modules               {string : ModuleScope}
+    #  include_directories   [string]
+    #  future_directives     [object]
+    #  language_level        int     currently 2 or 3 for Python 2/3
+
+    cython_scope = None
+
+    def __init__(self, include_directories, compiler_directives, cpp=False,
+                 language_level=2, options=None, create_testscope=True):
+        # cython_scope is a hack, set to False by subclasses, in order to break
+        # an infinite loop.
+        # Better code organization would fix it.
+
+        import Builtin, CythonScope
+        self.modules = {"__builtin__" : Builtin.builtin_scope}
+        self.cython_scope = CythonScope.create_cython_scope(self)
+        self.modules["cython"] = self.cython_scope
+        self.include_directories = include_directories
+        self.future_directives = set()
+        self.compiler_directives = compiler_directives
+        self.cpp = cpp
+        self.options = options
+
+        self.pxds = {} # full name -> node tree
+
+        standard_include_path = os.path.abspath(os.path.normpath(
+            os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes')))
+        self.include_directories = include_directories + [standard_include_path]
+
+        self.set_language_level(language_level)
+
+        self.gdb_debug_outputwriter = None
+
+    def set_language_level(self, level):
+        self.language_level = level
+        if level >= 3:
+            from Future import print_function, unicode_literals, absolute_import
+            self.future_directives.update([print_function, unicode_literals, absolute_import])
+            self.modules['builtins'] = self.modules['__builtin__']
+
+    # pipeline creation functions can now be found in Pipeline.py
+
+    def process_pxd(self, source_desc, scope, module_name):
+        import Pipeline
+        if isinstance(source_desc, FileSourceDescriptor) and source_desc._file_type == 'pyx':
+            source = CompilationSource(source_desc, module_name, os.getcwd())
+            result_sink = create_default_resultobj(source, self.options)
+            pipeline = Pipeline.create_pyx_as_pxd_pipeline(self, result_sink)
+            result = Pipeline.run_pipeline(pipeline, source)
+        else:
+            pipeline = Pipeline.create_pxd_pipeline(self, scope, module_name)
+            result = Pipeline.run_pipeline(pipeline, source_desc)
+        return result
+
+    def nonfatal_error(self, exc):
+        return Errors.report_error(exc)
+
+    def find_module(self, module_name,
+            relative_to = None, pos = None, need_pxd = 1, check_module_name = True):
+        # Finds and returns the module scope corresponding to
+        # the given relative or absolute module name. If this
+        # is the first time the module has been requested, finds
+        # the corresponding .pxd file and process it.
+        # If relative_to is not None, it must be a module scope,
+        # and the module will first be searched for relative to
+        # that module, provided its name is not a dotted name.
+        debug_find_module = 0
+        if debug_find_module:
+            print("Context.find_module: module_name = %s, relative_to = %s, pos = %s, need_pxd = %s" % (
+                    module_name, relative_to, pos, need_pxd))
+
+        scope = None
+        pxd_pathname = None
+        if check_module_name and not module_name_pattern.match(module_name):
+            if pos is None:
+                pos = (module_name, 0, 0)
+            raise CompileError(pos,
+                "'%s' is not a valid module name" % module_name)
+        if "." not in module_name and relative_to:
+            if debug_find_module:
+                print("...trying relative import")
+            scope = relative_to.lookup_submodule(module_name)
+            if not scope:
+                qualified_name = relative_to.qualify_name(module_name)
+                pxd_pathname = self.find_pxd_file(qualified_name, pos)
+                if pxd_pathname:
+                    scope = relative_to.find_submodule(module_name)
+        if not scope:
+            if debug_find_module:
+                print("...trying absolute import")
+            scope = self
+            for name in module_name.split("."):
+                scope = scope.find_submodule(name)
+        if debug_find_module:
+            print("...scope =", scope)
+        if not scope.pxd_file_loaded:
+            if debug_find_module:
+                print("...pxd not loaded")
+            scope.pxd_file_loaded = 1
+            if not pxd_pathname:
+                if debug_find_module:
+                    print("...looking for pxd file")
+                pxd_pathname = self.find_pxd_file(module_name, pos)
+                if debug_find_module:
+                    print("......found ", pxd_pathname)
+                if not pxd_pathname and need_pxd:
+                    package_pathname = self.search_include_directories(module_name, ".py", pos)
+                    if package_pathname and package_pathname.endswith('__init__.py'):
+                        pass
+                    else:
+                        error(pos, "'%s.pxd' not found" % module_name)
+            if pxd_pathname:
+                try:
+                    if debug_find_module:
+                        print("Context.find_module: Parsing %s" % pxd_pathname)
+                    rel_path = module_name.replace('.', os.sep) + os.path.splitext(pxd_pathname)[1]
+                    if not pxd_pathname.endswith(rel_path):
+                        rel_path = pxd_pathname # safety measure to prevent printing incorrect paths
+                    source_desc = FileSourceDescriptor(pxd_pathname, rel_path)
+                    err, result = self.process_pxd(source_desc, scope, module_name)
+                    if err:
+                        raise err
+                    (pxd_codenodes, pxd_scope) = result
+                    self.pxds[module_name] = (pxd_codenodes, pxd_scope)
+                except CompileError:
+                    pass
+        return scope
+
+    def find_pxd_file(self, qualified_name, pos):
+        # Search include path for the .pxd file corresponding to the
+        # given fully-qualified module name.
+        # Will find either a dotted filename or a file in a
+        # package directory. If a source file position is given,
+        # the directory containing the source file is searched first
+        # for a dotted filename, and its containing package root
+        # directory is searched first for a non-dotted filename.
+        pxd = self.search_include_directories(qualified_name, ".pxd", pos, sys_path=True)
+        if pxd is None: # XXX Keep this until Includes/Deprecated is removed
+            if (qualified_name.startswith('python') or
+                qualified_name in ('stdlib', 'stdio', 'stl')):
+                standard_include_path = os.path.abspath(os.path.normpath(
+                        os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes')))
+                deprecated_include_path = os.path.join(standard_include_path, 'Deprecated')
+                self.include_directories.append(deprecated_include_path)
+                try:
+                    pxd = self.search_include_directories(qualified_name, ".pxd", pos)
+                finally:
+                    self.include_directories.pop()
+                if pxd:
+                    name = qualified_name
+                    if name.startswith('python'):
+                        warning(pos, "'%s' is deprecated, use 'cpython'" % name, 1)
+                    elif name in ('stdlib', 'stdio'):
+                        warning(pos, "'%s' is deprecated, use 'libc.%s'" % (name, name), 1)
+                    elif name in ('stl'):
+                        warning(pos, "'%s' is deprecated, use 'libcpp.*.*'" % name, 1)
+        if pxd is None and Options.cimport_from_pyx:
+            return self.find_pyx_file(qualified_name, pos)
+        return pxd
+
+    def find_pyx_file(self, qualified_name, pos):
+        # Search include path for the .pyx file corresponding to the
+        # given fully-qualified module name, as for find_pxd_file().
+        return self.search_include_directories(qualified_name, ".pyx", pos)
+
+    def find_include_file(self, filename, pos):
+        # Search list of include directories for filename.
+        # Reports an error and returns None if not found.
+        path = self.search_include_directories(filename, "", pos,
+                                               include=True)
+        if not path:
+            error(pos, "'%s' not found" % filename)
+        return path
+
+    def search_include_directories(self, qualified_name, suffix, pos,
+                                   include=False, sys_path=False):
+        return Utils.search_include_directories(
+            tuple(self.include_directories), qualified_name, suffix, pos, include, sys_path)
+
+    def find_root_package_dir(self, file_path):
+        return Utils.find_root_package_dir(file_path)
+
+    def check_package_dir(self, dir, package_names):
+        return Utils.check_package_dir(dir, tuple(package_names))
+
+    def c_file_out_of_date(self, source_path):
+        c_path = Utils.replace_suffix(source_path, ".c")
+        if not os.path.exists(c_path):
+            return 1
+        c_time = Utils.modification_time(c_path)
+        if Utils.file_newer_than(source_path, c_time):
+            return 1
+        pos = [source_path]
+        pxd_path = Utils.replace_suffix(source_path, ".pxd")
+        if os.path.exists(pxd_path) and Utils.file_newer_than(pxd_path, c_time):
+            return 1
+        for kind, name in self.read_dependency_file(source_path):
+            if kind == "cimport":
+                dep_path = self.find_pxd_file(name, pos)
+            elif kind == "include":
+                dep_path = self.search_include_directories(name, pos)
+            else:
+                continue
+            if dep_path and Utils.file_newer_than(dep_path, c_time):
+                return 1
+        return 0
+
+    def find_cimported_module_names(self, source_path):
+        return [ name for kind, name in self.read_dependency_file(source_path)
+                 if kind == "cimport" ]
+
+    def is_package_dir(self, dir_path):
+        return Utils.is_package_dir(dir_path)
+
+    def read_dependency_file(self, source_path):
+        dep_path = Utils.replace_suffix(source_path, ".dep")
+        if os.path.exists(dep_path):
+            f = open(dep_path, "rU")
+            chunks = [ line.strip().split(" ", 1)
+                       for line in f.readlines()
+                       if " " in line.strip() ]
+            f.close()
+            return chunks
+        else:
+            return ()
+
+    def lookup_submodule(self, name):
+        # Look up a top-level module. Returns None if not found.
+        return self.modules.get(name, None)
+
+    def find_submodule(self, name):
+        # Find a top-level module, creating a new one if needed.
+        scope = self.lookup_submodule(name)
+        if not scope:
+            scope = ModuleScope(name,
+                parent_module = None, context = self)
+            self.modules[name] = scope
+        return scope
+
+    def parse(self, source_desc, scope, pxd, full_module_name):
+        if not isinstance(source_desc, FileSourceDescriptor):
+            raise RuntimeError("Only file sources for code supported")
+        source_filename = source_desc.filename
+        scope.cpp = self.cpp
+        # Parse the given source file and return a parse tree.
+        num_errors = Errors.num_errors
+        try:
+            f = Utils.open_source_file(source_filename, "rU")
+            try:
+                import Parsing
+                s = PyrexScanner(f, source_desc, source_encoding = f.encoding,
+                                 scope = scope, context = self)
+                tree = Parsing.p_module(s, pxd, full_module_name)
+            finally:
+                f.close()
+        except UnicodeDecodeError, e:
+            #import traceback
+            #traceback.print_exc()
+            line = 1
+            column = 0
+            msg = e.args[-1]
+            position = e.args[2]
+            encoding = e.args[0]
+
+            f = open(source_filename, "rb")
+            try:
+                byte_data = f.read()
+            finally:
+                f.close()
+
+            # FIXME: make this at least a little less inefficient
+            for idx, c in enumerate(byte_data):
+                if c in (ord('\n'), '\n'):
+                    line += 1
+                    column = 0
+                if idx == position:
+                    break
+
+                column += 1
+
+            error((source_desc, line, column),
+                  "Decoding error, missing or incorrect coding=<encoding-name> "
+                  "at top of source (cannot decode with encoding %r: %s)" % (encoding, msg))
+
+        if Errors.num_errors > num_errors:
+            raise CompileError()
+        return tree
+
+    def extract_module_name(self, path, options):
+        # Find fully_qualified module name from the full pathname
+        # of a source file.
+        dir, filename = os.path.split(path)
+        module_name, _ = os.path.splitext(filename)
+        if "." in module_name:
+            return module_name
+        names = [module_name]
+        while self.is_package_dir(dir):
+            parent, package_name = os.path.split(dir)
+            if parent == dir:
+                break
+            names.append(package_name)
+            dir = parent
+        names.reverse()
+        return ".".join(names)
+
+    def setup_errors(self, options, result):
+        Errors.reset() # clear any remaining error state
+        if options.use_listing_file:
+            result.listing_file = Utils.replace_suffix(source, ".lis")
+            path = result.listing_file
+        else:
+            path = None
+        Errors.open_listing_file(path=path,
+                                 echo_to_stderr=options.errors_to_stderr)
+
+    def teardown_errors(self, err, options, result):
+        source_desc = result.compilation_source.source_desc
+        if not isinstance(source_desc, FileSourceDescriptor):
+            raise RuntimeError("Only file sources for code supported")
+        Errors.close_listing_file()
+        result.num_errors = Errors.num_errors
+        if result.num_errors > 0:
+            err = True
+        if err and result.c_file:
+            try:
+                Utils.castrate_file(result.c_file, os.stat(source_desc.filename))
+            except EnvironmentError:
+                pass
+            result.c_file = None
+
+def create_default_resultobj(compilation_source, options):
+    result = CompilationResult()
+    result.main_source_file = compilation_source.source_desc.filename
+    result.compilation_source = compilation_source
+    source_desc = compilation_source.source_desc
+    if options.output_file:
+        result.c_file = os.path.join(compilation_source.cwd, options.output_file)
+    else:
+        if options.cplus:
+            c_suffix = ".cpp"
+        else:
+            c_suffix = ".c"
+        result.c_file = Utils.replace_suffix(source_desc.filename, c_suffix)
+    return result
+
+def run_pipeline(source, options, full_module_name=None, context=None):
+    import Pipeline
+
+    source_ext = os.path.splitext(source)[1]
+    options.configure_language_defaults(source_ext[1:]) # py/pyx
+    if context is None:
+        context = options.create_context()
+
+    # Set up source object
+    cwd = os.getcwd()
+    abs_path = os.path.abspath(source)
+    full_module_name = full_module_name or context.extract_module_name(source, options)
+
+    if options.relative_path_in_code_position_comments:
+        rel_path = full_module_name.replace('.', os.sep) + source_ext
+        if not abs_path.endswith(rel_path):
+            rel_path = source # safety measure to prevent printing incorrect paths
+    else:
+        rel_path = abs_path
+    source_desc = FileSourceDescriptor(abs_path, rel_path)
+    source = CompilationSource(source_desc, full_module_name, cwd)
+
+    # Set up result object
+    result = create_default_resultobj(source, options)
+
+    if options.annotate is None:
+        # By default, decide based on whether an html file already exists.
+        html_filename = os.path.splitext(result.c_file)[0] + ".html"
+        if os.path.exists(html_filename):
+            line = codecs.open(html_filename, "r", encoding="UTF-8").readline()
+            if line.startswith(u'<!-- Generated by Cython'):
+                options.annotate = True
+
+    # Get pipeline
+    if source_ext.lower() == '.py' or not source_ext:
+        pipeline = Pipeline.create_py_pipeline(context, options, result)
+    else:
+        pipeline = Pipeline.create_pyx_pipeline(context, options, result)
+
+    context.setup_errors(options, result)
+    err, enddata = Pipeline.run_pipeline(pipeline, source)
+    context.teardown_errors(err, options, result)
+    return result
+
+
+#------------------------------------------------------------------------
+#
+#  Main Python entry points
+#
+#------------------------------------------------------------------------
+
+class CompilationSource(object):
+    """
+    Contains the data necesarry to start up a compilation pipeline for
+    a single compilation unit.
+    """
+    def __init__(self, source_desc, full_module_name, cwd):
+        self.source_desc = source_desc
+        self.full_module_name = full_module_name
+        self.cwd = cwd
+
+class CompilationOptions(object):
+    """
+    Options to the Cython compiler:
+
+    show_version      boolean   Display version number
+    use_listing_file  boolean   Generate a .lis file
+    errors_to_stderr  boolean   Echo errors to stderr when using .lis
+    include_path      [string]  Directories to search for include files
+    output_file       string    Name of generated .c file
+    generate_pxi      boolean   Generate .pxi file for public declarations
+    capi_reexport_cincludes  
+                      boolean   Add cincluded headers to any auto-generated 
+                                header files.
+    timestamps        boolean   Only compile changed source files.
+    verbose           boolean   Always print source names being compiled
+    compiler_directives  dict      Overrides for pragma options (see Options.py)
+    evaluate_tree_assertions boolean  Test support: evaluate parse tree assertions
+    language_level    integer   The Python language level: 2 or 3
+
+    cplus             boolean   Compile as c++ code
+    """
+
+    def __init__(self, defaults = None, **kw):
+        self.include_path = []
+        if defaults:
+            if isinstance(defaults, CompilationOptions):
+                defaults = defaults.__dict__
+        else:
+            defaults = default_options
+
+        options = dict(defaults)
+        options.update(kw)
+
+        directives = dict(options['compiler_directives']) # copy mutable field
+        options['compiler_directives'] = directives
+        if 'language_level' in directives and 'language_level' not in kw:
+            options['language_level'] = int(directives['language_level'])
+        if 'cache' in options:
+            if options['cache'] is True:
+                options['cache'] = os.path.expanduser("~/.cycache")
+            elif options['cache'] in (False, None):
+                del options['cache']
+
+        self.__dict__.update(options)
+
+    def configure_language_defaults(self, source_extension):
+        if source_extension == 'py':
+            if self.compiler_directives.get('binding') is None:
+                self.compiler_directives['binding'] = True
+
+    def create_context(self):
+        return Context(self.include_path, self.compiler_directives,
+                       self.cplus, self.language_level, options=self)
+
+
+class CompilationResult(object):
+    """
+    Results from the Cython compiler:
+
+    c_file           string or None   The generated C source file
+    h_file           string or None   The generated C header file
+    i_file           string or None   The generated .pxi file
+    api_file         string or None   The generated C API .h file
+    listing_file     string or None   File of error messages
+    object_file      string or None   Result of compiling the C file
+    extension_file   string or None   Result of linking the object file
+    num_errors       integer          Number of compilation errors
+    compilation_source CompilationSource
+    """
+
+    def __init__(self):
+        self.c_file = None
+        self.h_file = None
+        self.i_file = None
+        self.api_file = None
+        self.listing_file = None
+        self.object_file = None
+        self.extension_file = None
+        self.main_source_file = None
+
+
+class CompilationResultSet(dict):
+    """
+    Results from compiling multiple Pyrex source files. A mapping
+    from source file paths to CompilationResult instances. Also
+    has the following attributes:
+
+    num_errors   integer   Total number of compilation errors
+    """
+
+    num_errors = 0
+
+    def add(self, source, result):
+        self[source] = result
+        self.num_errors += result.num_errors
+
+
+def compile_single(source, options, full_module_name = None):
+    """
+    compile_single(source, options, full_module_name)
+
+    Compile the given Pyrex implementation file and return a CompilationResult.
+    Always compiles a single file; does not perform timestamp checking or
+    recursion.
+    """
+    return run_pipeline(source, options, full_module_name)
+
+
+def compile_multiple(sources, options):
+    """
+    compile_multiple(sources, options)
+
+    Compiles the given sequence of Pyrex implementation files and returns
+    a CompilationResultSet. Performs timestamp checking and/or recursion
+    if these are specified in the options.
+    """
+    # run_pipeline creates the context
+    # context = options.create_context()
+    sources = [os.path.abspath(source) for source in sources]
+    processed = set()
+    results = CompilationResultSet()
+    timestamps = options.timestamps
+    verbose = options.verbose
+    context = None
+    for source in sources:
+        if source not in processed:
+            if context is None:
+                context = options.create_context()
+            if not timestamps or context.c_file_out_of_date(source):
+                if verbose:
+                    sys.stderr.write("Compiling %s\n" % source)
+
+                result = run_pipeline(source, options, context=context)
+                results.add(source, result)
+                # Compiling multiple sources in one context doesn't quite
+                # work properly yet.
+                context = None
+            processed.add(source)
+    return results
+
+def compile(source, options = None, full_module_name = None, **kwds):
+    """
+    compile(source [, options], [, <option> = <value>]...)
+
+    Compile one or more Pyrex implementation files, with optional timestamp
+    checking and recursing on dependecies. The source argument may be a string
+    or a sequence of strings If it is a string and no recursion or timestamp
+    checking is requested, a CompilationResult is returned, otherwise a
+    CompilationResultSet is returned.
+    """
+    options = CompilationOptions(defaults = options, **kwds)
+    if isinstance(source, basestring) and not options.timestamps:
+        return compile_single(source, options, full_module_name)
+    else:
+        return compile_multiple(source, options)
+
+#------------------------------------------------------------------------
+#
+#  Main command-line entry point
+#
+#------------------------------------------------------------------------
+def setuptools_main():
+    return main(command_line = 1)
+
+def main(command_line = 0):
+    args = sys.argv[1:]
+    any_failures = 0
+    if command_line:
+        from CmdLine import parse_command_line
+        options, sources = parse_command_line(args)
+    else:
+        options = CompilationOptions(default_options)
+        sources = args
+
+    if options.show_version:
+        sys.stderr.write("Cython version %s\n" % Version.version)
+    if options.working_path!="":
+        os.chdir(options.working_path)
+    try:
+        result = compile(sources, options)
+        if result.num_errors > 0:
+            any_failures = 1
+    except (EnvironmentError, PyrexError), e:
+        sys.stderr.write(str(e) + '\n')
+        any_failures = 1
+    if any_failures:
+        sys.exit(1)
+
+
+
+#------------------------------------------------------------------------
+#
+#  Set the default options depending on the platform
+#
+#------------------------------------------------------------------------
+
+default_options = dict(
+    show_version = 0,
+    use_listing_file = 0,
+    errors_to_stderr = 1,
+    cplus = 0,
+    output_file = None,
+    annotate = None,
+    generate_pxi = 0,
+    capi_reexport_cincludes = 0,
+    working_path = "",
+    timestamps = None,
+    verbose = 0,
+    quiet = 0,
+    compiler_directives = {},
+    evaluate_tree_assertions = False,
+    emit_linenums = False,
+    relative_path_in_code_position_comments = True,
+    c_line_in_traceback = True,
+    language_level = 2,
+    gdb_debug = False,
+    compile_time_env = None,
+    common_utility_include_dir = None,
+)