| Index: third_party/cython/src/Cython/Compiler/ParseTreeTransforms.py
|
| diff --git a/third_party/cython/src/Cython/Compiler/ParseTreeTransforms.py b/third_party/cython/src/Cython/Compiler/ParseTreeTransforms.py
|
| new file mode 100644
|
| index 0000000000000000000000000000000000000000..62c869b49d20b916d2a2fb089dacf0f236a79f02
|
| --- /dev/null
|
| +++ b/third_party/cython/src/Cython/Compiler/ParseTreeTransforms.py
|
| @@ -0,0 +1,3005 @@
|
| +import cython
|
| +cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
|
| + Options=object, UtilNodes=object, LetNode=object,
|
| + LetRefNode=object, TreeFragment=object, EncodedString=object,
|
| + error=object, warning=object, copy=object)
|
| +
|
| +import PyrexTypes
|
| +import Naming
|
| +import ExprNodes
|
| +import Nodes
|
| +import Options
|
| +import Builtin
|
| +
|
| +from Cython.Compiler.Visitor import VisitorTransform, TreeVisitor
|
| +from Cython.Compiler.Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
|
| +from Cython.Compiler.UtilNodes import LetNode, LetRefNode, ResultRefNode
|
| +from Cython.Compiler.TreeFragment import TreeFragment
|
| +from Cython.Compiler.StringEncoding import EncodedString
|
| +from Cython.Compiler.Errors import error, warning, CompileError, InternalError
|
| +from Cython.Compiler.Code import UtilityCode
|
| +
|
| +import copy
|
| +
|
| +
|
| +class NameNodeCollector(TreeVisitor):
|
| + """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
|
| + attribute.
|
| + """
|
| + def __init__(self):
|
| + super(NameNodeCollector, self).__init__()
|
| + self.name_nodes = []
|
| +
|
| + def visit_NameNode(self, node):
|
| + self.name_nodes.append(node)
|
| +
|
| + def visit_Node(self, node):
|
| + self._visitchildren(node, None)
|
| +
|
| +
|
| +class SkipDeclarations(object):
|
| + """
|
| + Variable and function declarations can often have a deep tree structure,
|
| + and yet most transformations don't need to descend to this depth.
|
| +
|
| + Declaration nodes are removed after AnalyseDeclarationsTransform, so there
|
| + is no need to use this for transformations after that point.
|
| + """
|
| + def visit_CTypeDefNode(self, node):
|
| + return node
|
| +
|
| + def visit_CVarDefNode(self, node):
|
| + return node
|
| +
|
| + def visit_CDeclaratorNode(self, node):
|
| + return node
|
| +
|
| + def visit_CBaseTypeNode(self, node):
|
| + return node
|
| +
|
| + def visit_CEnumDefNode(self, node):
|
| + return node
|
| +
|
| + def visit_CStructOrUnionDefNode(self, node):
|
| + return node
|
| +
|
| +class NormalizeTree(CythonTransform):
|
| + """
|
| + This transform fixes up a few things after parsing
|
| + in order to make the parse tree more suitable for
|
| + transforms.
|
| +
|
| + a) After parsing, blocks with only one statement will
|
| + be represented by that statement, not by a StatListNode.
|
| + When doing transforms this is annoying and inconsistent,
|
| + as one cannot in general remove a statement in a consistent
|
| + way and so on. This transform wraps any single statements
|
| + in a StatListNode containing a single statement.
|
| +
|
| + b) The PassStatNode is a noop and serves no purpose beyond
|
| + plugging such one-statement blocks; i.e., once parsed a
|
| +` "pass" can just as well be represented using an empty
|
| + StatListNode. This means less special cases to worry about
|
| + in subsequent transforms (one always checks to see if a
|
| + StatListNode has no children to see if the block is empty).
|
| + """
|
| +
|
| + def __init__(self, context):
|
| + super(NormalizeTree, self).__init__(context)
|
| + self.is_in_statlist = False
|
| + self.is_in_expr = False
|
| +
|
| + def visit_ExprNode(self, node):
|
| + stacktmp = self.is_in_expr
|
| + self.is_in_expr = True
|
| + self.visitchildren(node)
|
| + self.is_in_expr = stacktmp
|
| + return node
|
| +
|
| + def visit_StatNode(self, node, is_listcontainer=False):
|
| + stacktmp = self.is_in_statlist
|
| + self.is_in_statlist = is_listcontainer
|
| + self.visitchildren(node)
|
| + self.is_in_statlist = stacktmp
|
| + if not self.is_in_statlist and not self.is_in_expr:
|
| + return Nodes.StatListNode(pos=node.pos, stats=[node])
|
| + else:
|
| + return node
|
| +
|
| + def visit_StatListNode(self, node):
|
| + self.is_in_statlist = True
|
| + self.visitchildren(node)
|
| + self.is_in_statlist = False
|
| + return node
|
| +
|
| + def visit_ParallelAssignmentNode(self, node):
|
| + return self.visit_StatNode(node, True)
|
| +
|
| + def visit_CEnumDefNode(self, node):
|
| + return self.visit_StatNode(node, True)
|
| +
|
| + def visit_CStructOrUnionDefNode(self, node):
|
| + return self.visit_StatNode(node, True)
|
| +
|
| + def visit_PassStatNode(self, node):
|
| + """Eliminate PassStatNode"""
|
| + if not self.is_in_statlist:
|
| + return Nodes.StatListNode(pos=node.pos, stats=[])
|
| + else:
|
| + return []
|
| +
|
| + def visit_ExprStatNode(self, node):
|
| + """Eliminate useless string literals"""
|
| + if node.expr.is_string_literal:
|
| + return self.visit_PassStatNode(node)
|
| + else:
|
| + return self.visit_StatNode(node)
|
| +
|
| + def visit_CDeclaratorNode(self, node):
|
| + return node
|
| +
|
| +
|
| +class PostParseError(CompileError): pass
|
| +
|
| +# error strings checked by unit tests, so define them
|
| +ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
|
| +ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
|
| +ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
|
| +class PostParse(ScopeTrackingTransform):
|
| + """
|
| + Basic interpretation of the parse tree, as well as validity
|
| + checking that can be done on a very basic level on the parse
|
| + tree (while still not being a problem with the basic syntax,
|
| + as such).
|
| +
|
| + Specifically:
|
| + - Default values to cdef assignments are turned into single
|
| + assignments following the declaration (everywhere but in class
|
| + bodies, where they raise a compile error)
|
| +
|
| + - Interpret some node structures into Python runtime values.
|
| + Some nodes take compile-time arguments (currently:
|
| + TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
|
| + which should be interpreted. This happens in a general way
|
| + and other steps should be taken to ensure validity.
|
| +
|
| + Type arguments cannot be interpreted in this way.
|
| +
|
| + - For __cythonbufferdefaults__ the arguments are checked for
|
| + validity.
|
| +
|
| + TemplatedTypeNode has its directives interpreted:
|
| + Any first positional argument goes into the "dtype" attribute,
|
| + any "ndim" keyword argument goes into the "ndim" attribute and
|
| + so on. Also it is checked that the directive combination is valid.
|
| + - __cythonbufferdefaults__ attributes are parsed and put into the
|
| + type information.
|
| +
|
| + Note: Currently Parsing.py does a lot of interpretation and
|
| + reorganization that can be refactored into this transform
|
| + if a more pure Abstract Syntax Tree is wanted.
|
| + """
|
| +
|
| + def __init__(self, context):
|
| + super(PostParse, self).__init__(context)
|
| + self.specialattribute_handlers = {
|
| + '__cythonbufferdefaults__' : self.handle_bufferdefaults
|
| + }
|
| +
|
| + def visit_ModuleNode(self, node):
|
| + self.lambda_counter = 1
|
| + self.genexpr_counter = 1
|
| + return super(PostParse, self).visit_ModuleNode(node)
|
| +
|
| + def visit_LambdaNode(self, node):
|
| + # unpack a lambda expression into the corresponding DefNode
|
| + lambda_id = self.lambda_counter
|
| + self.lambda_counter += 1
|
| + node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
|
| + collector = YieldNodeCollector()
|
| + collector.visitchildren(node.result_expr)
|
| + if collector.yields or isinstance(node.result_expr, ExprNodes.YieldExprNode):
|
| + body = Nodes.ExprStatNode(
|
| + node.result_expr.pos, expr=node.result_expr)
|
| + else:
|
| + body = Nodes.ReturnStatNode(
|
| + node.result_expr.pos, value=node.result_expr)
|
| + node.def_node = Nodes.DefNode(
|
| + node.pos, name=node.name, lambda_name=node.lambda_name,
|
| + args=node.args, star_arg=node.star_arg,
|
| + starstar_arg=node.starstar_arg,
|
| + body=body, doc=None)
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_GeneratorExpressionNode(self, node):
|
| + # unpack a generator expression into the corresponding DefNode
|
| + genexpr_id = self.genexpr_counter
|
| + self.genexpr_counter += 1
|
| + node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)
|
| +
|
| + node.def_node = Nodes.DefNode(node.pos, name=node.name,
|
| + doc=None,
|
| + args=[], star_arg=None,
|
| + starstar_arg=None,
|
| + body=node.loop)
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + # cdef variables
|
| + def handle_bufferdefaults(self, decl):
|
| + if not isinstance(decl.default, ExprNodes.DictNode):
|
| + raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
|
| + self.scope_node.buffer_defaults_node = decl.default
|
| + self.scope_node.buffer_defaults_pos = decl.pos
|
| +
|
| + def visit_CVarDefNode(self, node):
|
| + # This assumes only plain names and pointers are assignable on
|
| + # declaration. Also, it makes use of the fact that a cdef decl
|
| + # must appear before the first use, so we don't have to deal with
|
| + # "i = 3; cdef int i = i" and can simply move the nodes around.
|
| + try:
|
| + self.visitchildren(node)
|
| + stats = [node]
|
| + newdecls = []
|
| + for decl in node.declarators:
|
| + declbase = decl
|
| + while isinstance(declbase, Nodes.CPtrDeclaratorNode):
|
| + declbase = declbase.base
|
| + if isinstance(declbase, Nodes.CNameDeclaratorNode):
|
| + if declbase.default is not None:
|
| + if self.scope_type in ('cclass', 'pyclass', 'struct'):
|
| + if isinstance(self.scope_node, Nodes.CClassDefNode):
|
| + handler = self.specialattribute_handlers.get(decl.name)
|
| + if handler:
|
| + if decl is not declbase:
|
| + raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
|
| + handler(decl)
|
| + continue # Remove declaration
|
| + raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
|
| + first_assignment = self.scope_type != 'module'
|
| + stats.append(Nodes.SingleAssignmentNode(node.pos,
|
| + lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
|
| + rhs=declbase.default, first=first_assignment))
|
| + declbase.default = None
|
| + newdecls.append(decl)
|
| + node.declarators = newdecls
|
| + return stats
|
| + except PostParseError, e:
|
| + # An error in a cdef clause is ok, simply remove the declaration
|
| + # and try to move on to report more errors
|
| + self.context.nonfatal_error(e)
|
| + return None
|
| +
|
| + # Split parallel assignments (a,b = b,a) into separate partial
|
| + # assignments that are executed rhs-first using temps. This
|
| + # restructuring must be applied before type analysis so that known
|
| + # types on rhs and lhs can be matched directly. It is required in
|
| + # the case that the types cannot be coerced to a Python type in
|
| + # order to assign from a tuple.
|
| +
|
| + def visit_SingleAssignmentNode(self, node):
|
| + self.visitchildren(node)
|
| + return self._visit_assignment_node(node, [node.lhs, node.rhs])
|
| +
|
| + def visit_CascadedAssignmentNode(self, node):
|
| + self.visitchildren(node)
|
| + return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
|
| +
|
| + def _visit_assignment_node(self, node, expr_list):
|
| + """Flatten parallel assignments into separate single
|
| + assignments or cascaded assignments.
|
| + """
|
| + if sum([ 1 for expr in expr_list
|
| + if expr.is_sequence_constructor or expr.is_string_literal ]) < 2:
|
| + # no parallel assignments => nothing to do
|
| + return node
|
| +
|
| + expr_list_list = []
|
| + flatten_parallel_assignments(expr_list, expr_list_list)
|
| + temp_refs = []
|
| + eliminate_rhs_duplicates(expr_list_list, temp_refs)
|
| +
|
| + nodes = []
|
| + for expr_list in expr_list_list:
|
| + lhs_list = expr_list[:-1]
|
| + rhs = expr_list[-1]
|
| + if len(lhs_list) == 1:
|
| + node = Nodes.SingleAssignmentNode(rhs.pos,
|
| + lhs = lhs_list[0], rhs = rhs)
|
| + else:
|
| + node = Nodes.CascadedAssignmentNode(rhs.pos,
|
| + lhs_list = lhs_list, rhs = rhs)
|
| + nodes.append(node)
|
| +
|
| + if len(nodes) == 1:
|
| + assign_node = nodes[0]
|
| + else:
|
| + assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
|
| +
|
| + if temp_refs:
|
| + duplicates_and_temps = [ (temp.expression, temp)
|
| + for temp in temp_refs ]
|
| + sort_common_subsequences(duplicates_and_temps)
|
| + for _, temp_ref in duplicates_and_temps[::-1]:
|
| + assign_node = LetNode(temp_ref, assign_node)
|
| +
|
| + return assign_node
|
| +
|
| + def _flatten_sequence(self, seq, result):
|
| + for arg in seq.args:
|
| + if arg.is_sequence_constructor:
|
| + self._flatten_sequence(arg, result)
|
| + else:
|
| + result.append(arg)
|
| + return result
|
| +
|
| + def visit_DelStatNode(self, node):
|
| + self.visitchildren(node)
|
| + node.args = self._flatten_sequence(node, [])
|
| + return node
|
| +
|
| + def visit_ExceptClauseNode(self, node):
|
| + if node.is_except_as:
|
| + # except-as must delete NameNode target at the end
|
| + del_target = Nodes.DelStatNode(
|
| + node.pos,
|
| + args=[ExprNodes.NameNode(
|
| + node.target.pos, name=node.target.name)],
|
| + ignore_nonexisting=True)
|
| + node.body = Nodes.StatListNode(
|
| + node.pos,
|
| + stats=[Nodes.TryFinallyStatNode(
|
| + node.pos,
|
| + body=node.body,
|
| + finally_clause=Nodes.StatListNode(
|
| + node.pos,
|
| + stats=[del_target]))])
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| +
|
| +def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
|
| + """Replace rhs items by LetRefNodes if they appear more than once.
|
| + Creates a sequence of LetRefNodes that set up the required temps
|
| + and appends them to ref_node_sequence. The input list is modified
|
| + in-place.
|
| + """
|
| + seen_nodes = set()
|
| + ref_nodes = {}
|
| + def find_duplicates(node):
|
| + if node.is_literal or node.is_name:
|
| + # no need to replace those; can't include attributes here
|
| + # as their access is not necessarily side-effect free
|
| + return
|
| + if node in seen_nodes:
|
| + if node not in ref_nodes:
|
| + ref_node = LetRefNode(node)
|
| + ref_nodes[node] = ref_node
|
| + ref_node_sequence.append(ref_node)
|
| + else:
|
| + seen_nodes.add(node)
|
| + if node.is_sequence_constructor:
|
| + for item in node.args:
|
| + find_duplicates(item)
|
| +
|
| + for expr_list in expr_list_list:
|
| + rhs = expr_list[-1]
|
| + find_duplicates(rhs)
|
| + if not ref_nodes:
|
| + return
|
| +
|
| + def substitute_nodes(node):
|
| + if node in ref_nodes:
|
| + return ref_nodes[node]
|
| + elif node.is_sequence_constructor:
|
| + node.args = list(map(substitute_nodes, node.args))
|
| + return node
|
| +
|
| + # replace nodes inside of the common subexpressions
|
| + for node in ref_nodes:
|
| + if node.is_sequence_constructor:
|
| + node.args = list(map(substitute_nodes, node.args))
|
| +
|
| + # replace common subexpressions on all rhs items
|
| + for expr_list in expr_list_list:
|
| + expr_list[-1] = substitute_nodes(expr_list[-1])
|
| +
|
| +def sort_common_subsequences(items):
|
| + """Sort items/subsequences so that all items and subsequences that
|
| + an item contains appear before the item itself. This is needed
|
| + because each rhs item must only be evaluated once, so its value
|
| + must be evaluated first and then reused when packing sequences
|
| + that contain it.
|
| +
|
| + This implies a partial order, and the sort must be stable to
|
| + preserve the original order as much as possible, so we use a
|
| + simple insertion sort (which is very fast for short sequences, the
|
| + normal case in practice).
|
| + """
|
| + def contains(seq, x):
|
| + for item in seq:
|
| + if item is x:
|
| + return True
|
| + elif item.is_sequence_constructor and contains(item.args, x):
|
| + return True
|
| + return False
|
| + def lower_than(a,b):
|
| + return b.is_sequence_constructor and contains(b.args, a)
|
| +
|
| + for pos, item in enumerate(items):
|
| + key = item[1] # the ResultRefNode which has already been injected into the sequences
|
| + new_pos = pos
|
| + for i in xrange(pos-1, -1, -1):
|
| + if lower_than(key, items[i][0]):
|
| + new_pos = i
|
| + if new_pos != pos:
|
| + for i in xrange(pos, new_pos, -1):
|
| + items[i] = items[i-1]
|
| + items[new_pos] = item
|
| +
|
| +def unpack_string_to_character_literals(literal):
|
| + chars = []
|
| + pos = literal.pos
|
| + stype = literal.__class__
|
| + sval = literal.value
|
| + sval_type = sval.__class__
|
| + for char in sval:
|
| + cval = sval_type(char)
|
| + chars.append(stype(pos, value=cval, constant_result=cval))
|
| + return chars
|
| +
|
| +def flatten_parallel_assignments(input, output):
|
| + # The input is a list of expression nodes, representing the LHSs
|
| + # and RHS of one (possibly cascaded) assignment statement. For
|
| + # sequence constructors, rearranges the matching parts of both
|
| + # sides into a list of equivalent assignments between the
|
| + # individual elements. This transformation is applied
|
| + # recursively, so that nested structures get matched as well.
|
| + rhs = input[-1]
|
| + if (not (rhs.is_sequence_constructor or isinstance(rhs, ExprNodes.UnicodeNode))
|
| + or not sum([lhs.is_sequence_constructor for lhs in input[:-1]])):
|
| + output.append(input)
|
| + return
|
| +
|
| + complete_assignments = []
|
| +
|
| + if rhs.is_sequence_constructor:
|
| + rhs_args = rhs.args
|
| + elif rhs.is_string_literal:
|
| + rhs_args = unpack_string_to_character_literals(rhs)
|
| +
|
| + rhs_size = len(rhs_args)
|
| + lhs_targets = [ [] for _ in xrange(rhs_size) ]
|
| + starred_assignments = []
|
| + for lhs in input[:-1]:
|
| + if not lhs.is_sequence_constructor:
|
| + if lhs.is_starred:
|
| + error(lhs.pos, "starred assignment target must be in a list or tuple")
|
| + complete_assignments.append(lhs)
|
| + continue
|
| + lhs_size = len(lhs.args)
|
| + starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
|
| + if starred_targets > 1:
|
| + error(lhs.pos, "more than 1 starred expression in assignment")
|
| + output.append([lhs,rhs])
|
| + continue
|
| + elif lhs_size - starred_targets > rhs_size:
|
| + error(lhs.pos, "need more than %d value%s to unpack"
|
| + % (rhs_size, (rhs_size != 1) and 's' or ''))
|
| + output.append([lhs,rhs])
|
| + continue
|
| + elif starred_targets:
|
| + map_starred_assignment(lhs_targets, starred_assignments,
|
| + lhs.args, rhs_args)
|
| + elif lhs_size < rhs_size:
|
| + error(lhs.pos, "too many values to unpack (expected %d, got %d)"
|
| + % (lhs_size, rhs_size))
|
| + output.append([lhs,rhs])
|
| + continue
|
| + else:
|
| + for targets, expr in zip(lhs_targets, lhs.args):
|
| + targets.append(expr)
|
| +
|
| + if complete_assignments:
|
| + complete_assignments.append(rhs)
|
| + output.append(complete_assignments)
|
| +
|
| + # recursively flatten partial assignments
|
| + for cascade, rhs in zip(lhs_targets, rhs_args):
|
| + if cascade:
|
| + cascade.append(rhs)
|
| + flatten_parallel_assignments(cascade, output)
|
| +
|
| + # recursively flatten starred assignments
|
| + for cascade in starred_assignments:
|
| + if cascade[0].is_sequence_constructor:
|
| + flatten_parallel_assignments(cascade, output)
|
| + else:
|
| + output.append(cascade)
|
| +
|
| +def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
|
| + # Appends the fixed-position LHS targets to the target list that
|
| + # appear left and right of the starred argument.
|
| + #
|
| + # The starred_assignments list receives a new tuple
|
| + # (lhs_target, rhs_values_list) that maps the remaining arguments
|
| + # (those that match the starred target) to a list.
|
| +
|
| + # left side of the starred target
|
| + for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
|
| + if expr.is_starred:
|
| + starred = i
|
| + lhs_remaining = len(lhs_args) - i - 1
|
| + break
|
| + targets.append(expr)
|
| + else:
|
| + raise InternalError("no starred arg found when splitting starred assignment")
|
| +
|
| + # right side of the starred target
|
| + for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
|
| + lhs_args[starred + 1:])):
|
| + targets.append(expr)
|
| +
|
| + # the starred target itself, must be assigned a (potentially empty) list
|
| + target = lhs_args[starred].target # unpack starred node
|
| + starred_rhs = rhs_args[starred:]
|
| + if lhs_remaining:
|
| + starred_rhs = starred_rhs[:-lhs_remaining]
|
| + if starred_rhs:
|
| + pos = starred_rhs[0].pos
|
| + else:
|
| + pos = target.pos
|
| + starred_assignments.append([
|
| + target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
|
| +
|
| +
|
| +class PxdPostParse(CythonTransform, SkipDeclarations):
|
| + """
|
| + Basic interpretation/validity checking that should only be
|
| + done on pxd trees.
|
| +
|
| + A lot of this checking currently happens in the parser; but
|
| + what is listed below happens here.
|
| +
|
| + - "def" functions are let through only if they fill the
|
| + getbuffer/releasebuffer slots
|
| +
|
| + - cdef functions are let through only if they are on the
|
| + top level and are declared "inline"
|
| + """
|
| + ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
|
| + ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
|
| +
|
| + def __call__(self, node):
|
| + self.scope_type = 'pxd'
|
| + return super(PxdPostParse, self).__call__(node)
|
| +
|
| + def visit_CClassDefNode(self, node):
|
| + old = self.scope_type
|
| + self.scope_type = 'cclass'
|
| + self.visitchildren(node)
|
| + self.scope_type = old
|
| + return node
|
| +
|
| + def visit_FuncDefNode(self, node):
|
| + # FuncDefNode always come with an implementation (without
|
| + # an imp they are CVarDefNodes..)
|
| + err = self.ERR_INLINE_ONLY
|
| +
|
| + if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
|
| + and node.name in ('__getbuffer__', '__releasebuffer__')):
|
| + err = None # allow these slots
|
| +
|
| + if isinstance(node, Nodes.CFuncDefNode):
|
| + if (u'inline' in node.modifiers and
|
| + self.scope_type in ('pxd', 'cclass')):
|
| + node.inline_in_pxd = True
|
| + if node.visibility != 'private':
|
| + err = self.ERR_NOGO_WITH_INLINE % node.visibility
|
| + elif node.api:
|
| + err = self.ERR_NOGO_WITH_INLINE % 'api'
|
| + else:
|
| + err = None # allow inline function
|
| + else:
|
| + err = self.ERR_INLINE_ONLY
|
| +
|
| + if err:
|
| + self.context.nonfatal_error(PostParseError(node.pos, err))
|
| + return None
|
| + else:
|
| + return node
|
| +
|
| +class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
|
| + """
|
| + After parsing, directives can be stored in a number of places:
|
| + - #cython-comments at the top of the file (stored in ModuleNode)
|
| + - Command-line arguments overriding these
|
| + - @cython.directivename decorators
|
| + - with cython.directivename: statements
|
| +
|
| + This transform is responsible for interpreting these various sources
|
| + and store the directive in two ways:
|
| + - Set the directives attribute of the ModuleNode for global directives.
|
| + - Use a CompilerDirectivesNode to override directives for a subtree.
|
| +
|
| + (The first one is primarily to not have to modify with the tree
|
| + structure, so that ModuleNode stay on top.)
|
| +
|
| + The directives are stored in dictionaries from name to value in effect.
|
| + Each such dictionary is always filled in for all possible directives,
|
| + using default values where no value is given by the user.
|
| +
|
| + The available directives are controlled in Options.py.
|
| +
|
| + Note that we have to run this prior to analysis, and so some minor
|
| + duplication of functionality has to occur: We manually track cimports
|
| + and which names the "cython" module may have been imported to.
|
| + """
|
| + unop_method_nodes = {
|
| + 'typeof': ExprNodes.TypeofNode,
|
| +
|
| + 'operator.address': ExprNodes.AmpersandNode,
|
| + 'operator.dereference': ExprNodes.DereferenceNode,
|
| + 'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
|
| + 'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
|
| + 'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
|
| + 'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
|
| +
|
| + # For backwards compatability.
|
| + 'address': ExprNodes.AmpersandNode,
|
| + }
|
| +
|
| + binop_method_nodes = {
|
| + 'operator.comma' : ExprNodes.c_binop_constructor(','),
|
| + }
|
| +
|
| + special_methods = set(['declare', 'union', 'struct', 'typedef',
|
| + 'sizeof', 'cast', 'pointer', 'compiled',
|
| + 'NULL', 'fused_type', 'parallel'])
|
| + special_methods.update(unop_method_nodes.keys())
|
| +
|
| + valid_parallel_directives = set([
|
| + "parallel",
|
| + "prange",
|
| + "threadid",
|
| +# "threadsavailable",
|
| + ])
|
| +
|
| + def __init__(self, context, compilation_directive_defaults):
|
| + super(InterpretCompilerDirectives, self).__init__(context)
|
| + self.compilation_directive_defaults = {}
|
| + for key, value in compilation_directive_defaults.items():
|
| + self.compilation_directive_defaults[unicode(key)] = copy.deepcopy(value)
|
| + self.cython_module_names = set()
|
| + self.directive_names = {}
|
| + self.parallel_directives = {}
|
| +
|
| + def check_directive_scope(self, pos, directive, scope):
|
| + legal_scopes = Options.directive_scopes.get(directive, None)
|
| + if legal_scopes and scope not in legal_scopes:
|
| + self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
|
| + 'is not allowed in %s scope' % (directive, scope)))
|
| + return False
|
| + else:
|
| + if (directive not in Options.directive_defaults
|
| + and directive not in Options.directive_types):
|
| + error(pos, "Invalid directive: '%s'." % (directive,))
|
| + return True
|
| +
|
| + # Set up processing and handle the cython: comments.
|
| + def visit_ModuleNode(self, node):
|
| + for key, value in node.directive_comments.items():
|
| + if not self.check_directive_scope(node.pos, key, 'module'):
|
| + self.wrong_scope_error(node.pos, key, 'module')
|
| + del node.directive_comments[key]
|
| +
|
| + self.module_scope = node.scope
|
| +
|
| + directives = copy.deepcopy(Options.directive_defaults)
|
| + directives.update(copy.deepcopy(self.compilation_directive_defaults))
|
| + directives.update(node.directive_comments)
|
| + self.directives = directives
|
| + node.directives = directives
|
| + node.parallel_directives = self.parallel_directives
|
| + self.visitchildren(node)
|
| + node.cython_module_names = self.cython_module_names
|
| + return node
|
| +
|
| + # The following four functions track imports and cimports that
|
| + # begin with "cython"
|
| + def is_cython_directive(self, name):
|
| + return (name in Options.directive_types or
|
| + name in self.special_methods or
|
| + PyrexTypes.parse_basic_type(name))
|
| +
|
| + def is_parallel_directive(self, full_name, pos):
|
| + """
|
| + Checks to see if fullname (e.g. cython.parallel.prange) is a valid
|
| + parallel directive. If it is a star import it also updates the
|
| + parallel_directives.
|
| + """
|
| + result = (full_name + ".").startswith("cython.parallel.")
|
| +
|
| + if result:
|
| + directive = full_name.split('.')
|
| + if full_name == u"cython.parallel":
|
| + self.parallel_directives[u"parallel"] = u"cython.parallel"
|
| + elif full_name == u"cython.parallel.*":
|
| + for name in self.valid_parallel_directives:
|
| + self.parallel_directives[name] = u"cython.parallel.%s" % name
|
| + elif (len(directive) != 3 or
|
| + directive[-1] not in self.valid_parallel_directives):
|
| + error(pos, "No such directive: %s" % full_name)
|
| +
|
| + self.module_scope.use_utility_code(
|
| + UtilityCode.load_cached("InitThreads", "ModuleSetupCode.c"))
|
| +
|
| + return result
|
| +
|
| + def visit_CImportStatNode(self, node):
|
| + if node.module_name == u"cython":
|
| + self.cython_module_names.add(node.as_name or u"cython")
|
| + elif node.module_name.startswith(u"cython."):
|
| + if node.module_name.startswith(u"cython.parallel."):
|
| + error(node.pos, node.module_name + " is not a module")
|
| + if node.module_name == u"cython.parallel":
|
| + if node.as_name and node.as_name != u"cython":
|
| + self.parallel_directives[node.as_name] = node.module_name
|
| + else:
|
| + self.cython_module_names.add(u"cython")
|
| + self.parallel_directives[
|
| + u"cython.parallel"] = node.module_name
|
| + self.module_scope.use_utility_code(
|
| + UtilityCode.load_cached("InitThreads", "ModuleSetupCode.c"))
|
| + elif node.as_name:
|
| + self.directive_names[node.as_name] = node.module_name[7:]
|
| + else:
|
| + self.cython_module_names.add(u"cython")
|
| + # if this cimport was a compiler directive, we don't
|
| + # want to leave the cimport node sitting in the tree
|
| + return None
|
| + return node
|
| +
|
| + def visit_FromCImportStatNode(self, node):
|
| + if (node.module_name == u"cython") or \
|
| + node.module_name.startswith(u"cython."):
|
| + submodule = (node.module_name + u".")[7:]
|
| + newimp = []
|
| +
|
| + for pos, name, as_name, kind in node.imported_names:
|
| + full_name = submodule + name
|
| + qualified_name = u"cython." + full_name
|
| +
|
| + if self.is_parallel_directive(qualified_name, node.pos):
|
| + # from cython cimport parallel, or
|
| + # from cython.parallel cimport parallel, prange, ...
|
| + self.parallel_directives[as_name or name] = qualified_name
|
| + elif self.is_cython_directive(full_name):
|
| + if as_name is None:
|
| + as_name = full_name
|
| +
|
| + self.directive_names[as_name] = full_name
|
| + if kind is not None:
|
| + self.context.nonfatal_error(PostParseError(pos,
|
| + "Compiler directive imports must be plain imports"))
|
| + else:
|
| + newimp.append((pos, name, as_name, kind))
|
| +
|
| + if not newimp:
|
| + return None
|
| +
|
| + node.imported_names = newimp
|
| + return node
|
| +
|
| + def visit_FromImportStatNode(self, node):
|
| + if (node.module.module_name.value == u"cython") or \
|
| + node.module.module_name.value.startswith(u"cython."):
|
| + submodule = (node.module.module_name.value + u".")[7:]
|
| + newimp = []
|
| + for name, name_node in node.items:
|
| + full_name = submodule + name
|
| + qualified_name = u"cython." + full_name
|
| + if self.is_parallel_directive(qualified_name, node.pos):
|
| + self.parallel_directives[name_node.name] = qualified_name
|
| + elif self.is_cython_directive(full_name):
|
| + self.directive_names[name_node.name] = full_name
|
| + else:
|
| + newimp.append((name, name_node))
|
| + if not newimp:
|
| + return None
|
| + node.items = newimp
|
| + return node
|
| +
|
| + def visit_SingleAssignmentNode(self, node):
|
| + if isinstance(node.rhs, ExprNodes.ImportNode):
|
| + module_name = node.rhs.module_name.value
|
| + is_parallel = (module_name + u".").startswith(u"cython.parallel.")
|
| +
|
| + if module_name != u"cython" and not is_parallel:
|
| + return node
|
| +
|
| + module_name = node.rhs.module_name.value
|
| + as_name = node.lhs.name
|
| +
|
| + node = Nodes.CImportStatNode(node.pos,
|
| + module_name = module_name,
|
| + as_name = as_name)
|
| + node = self.visit_CImportStatNode(node)
|
| + else:
|
| + self.visitchildren(node)
|
| +
|
| + return node
|
| +
|
| + def visit_NameNode(self, node):
|
| + if node.name in self.cython_module_names:
|
| + node.is_cython_module = True
|
| + else:
|
| + node.cython_attribute = self.directive_names.get(node.name)
|
| + return node
|
| +
|
| + def try_to_parse_directives(self, node):
|
| + # If node is the contents of an directive (in a with statement or
|
| + # decorator), returns a list of (directivename, value) pairs.
|
| + # Otherwise, returns None
|
| + if isinstance(node, ExprNodes.CallNode):
|
| + self.visit(node.function)
|
| + optname = node.function.as_cython_attribute()
|
| + if optname:
|
| + directivetype = Options.directive_types.get(optname)
|
| + if directivetype:
|
| + args, kwds = node.explicit_args_kwds()
|
| + directives = []
|
| + key_value_pairs = []
|
| + if kwds is not None and directivetype is not dict:
|
| + for keyvalue in kwds.key_value_pairs:
|
| + key, value = keyvalue
|
| + sub_optname = "%s.%s" % (optname, key.value)
|
| + if Options.directive_types.get(sub_optname):
|
| + directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
|
| + else:
|
| + key_value_pairs.append(keyvalue)
|
| + if not key_value_pairs:
|
| + kwds = None
|
| + else:
|
| + kwds.key_value_pairs = key_value_pairs
|
| + if directives and not kwds and not args:
|
| + return directives
|
| + directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
|
| + return directives
|
| + elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
|
| + self.visit(node)
|
| + optname = node.as_cython_attribute()
|
| + if optname:
|
| + directivetype = Options.directive_types.get(optname)
|
| + if directivetype is bool:
|
| + return [(optname, True)]
|
| + elif directivetype is None:
|
| + return [(optname, None)]
|
| + else:
|
| + raise PostParseError(
|
| + node.pos, "The '%s' directive should be used as a function call." % optname)
|
| + return None
|
| +
|
| + def try_to_parse_directive(self, optname, args, kwds, pos):
|
| + directivetype = Options.directive_types.get(optname)
|
| + if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
|
| + return optname, Options.directive_defaults[optname]
|
| + elif directivetype is bool:
|
| + if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
|
| + raise PostParseError(pos,
|
| + 'The %s directive takes one compile-time boolean argument' % optname)
|
| + return (optname, args[0].value)
|
| + elif directivetype is int:
|
| + if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.IntNode):
|
| + raise PostParseError(pos,
|
| + 'The %s directive takes one compile-time integer argument' % optname)
|
| + return (optname, int(args[0].value))
|
| + elif directivetype is str:
|
| + if kwds is not None or len(args) != 1 or not isinstance(
|
| + args[0], (ExprNodes.StringNode, ExprNodes.UnicodeNode)):
|
| + raise PostParseError(pos,
|
| + 'The %s directive takes one compile-time string argument' % optname)
|
| + return (optname, str(args[0].value))
|
| + elif directivetype is type:
|
| + if kwds is not None or len(args) != 1:
|
| + raise PostParseError(pos,
|
| + 'The %s directive takes one type argument' % optname)
|
| + return (optname, args[0])
|
| + elif directivetype is dict:
|
| + if len(args) != 0:
|
| + raise PostParseError(pos,
|
| + 'The %s directive takes no prepositional arguments' % optname)
|
| + return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
|
| + elif directivetype is list:
|
| + if kwds and len(kwds) != 0:
|
| + raise PostParseError(pos,
|
| + 'The %s directive takes no keyword arguments' % optname)
|
| + return optname, [ str(arg.value) for arg in args ]
|
| + elif callable(directivetype):
|
| + if kwds is not None or len(args) != 1 or not isinstance(
|
| + args[0], (ExprNodes.StringNode, ExprNodes.UnicodeNode)):
|
| + raise PostParseError(pos,
|
| + 'The %s directive takes one compile-time string argument' % optname)
|
| + return (optname, directivetype(optname, str(args[0].value)))
|
| + else:
|
| + assert False
|
| +
|
| + def visit_with_directives(self, body, directives):
|
| + olddirectives = self.directives
|
| + newdirectives = copy.copy(olddirectives)
|
| + newdirectives.update(directives)
|
| + self.directives = newdirectives
|
| + assert isinstance(body, Nodes.StatListNode), body
|
| + retbody = self.visit_Node(body)
|
| + directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
|
| + directives=newdirectives)
|
| + self.directives = olddirectives
|
| + return directive
|
| +
|
| + # Handle decorators
|
| + def visit_FuncDefNode(self, node):
|
| + directives = self._extract_directives(node, 'function')
|
| + if not directives:
|
| + return self.visit_Node(node)
|
| + body = Nodes.StatListNode(node.pos, stats=[node])
|
| + return self.visit_with_directives(body, directives)
|
| +
|
| + def visit_CVarDefNode(self, node):
|
| + directives = self._extract_directives(node, 'function')
|
| + if not directives:
|
| + return node
|
| + for name, value in directives.iteritems():
|
| + if name == 'locals':
|
| + node.directive_locals = value
|
| + elif name != 'final':
|
| + self.context.nonfatal_error(PostParseError(
|
| + node.pos,
|
| + "Cdef functions can only take cython.locals() "
|
| + "or final decorators, got %s." % name))
|
| + body = Nodes.StatListNode(node.pos, stats=[node])
|
| + return self.visit_with_directives(body, directives)
|
| +
|
| + def visit_CClassDefNode(self, node):
|
| + directives = self._extract_directives(node, 'cclass')
|
| + if not directives:
|
| + return self.visit_Node(node)
|
| + body = Nodes.StatListNode(node.pos, stats=[node])
|
| + return self.visit_with_directives(body, directives)
|
| +
|
| + def visit_PyClassDefNode(self, node):
|
| + directives = self._extract_directives(node, 'class')
|
| + if not directives:
|
| + return self.visit_Node(node)
|
| + body = Nodes.StatListNode(node.pos, stats=[node])
|
| + return self.visit_with_directives(body, directives)
|
| +
|
| + def _extract_directives(self, node, scope_name):
|
| + if not node.decorators:
|
| + return {}
|
| + # Split the decorators into two lists -- real decorators and directives
|
| + directives = []
|
| + realdecs = []
|
| + for dec in node.decorators:
|
| + new_directives = self.try_to_parse_directives(dec.decorator)
|
| + if new_directives is not None:
|
| + for directive in new_directives:
|
| + if self.check_directive_scope(node.pos, directive[0], scope_name):
|
| + directives.append(directive)
|
| + else:
|
| + realdecs.append(dec)
|
| + if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode)):
|
| + raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
|
| + else:
|
| + node.decorators = realdecs
|
| + # merge or override repeated directives
|
| + optdict = {}
|
| + directives.reverse() # Decorators coming first take precedence
|
| + for directive in directives:
|
| + name, value = directive
|
| + if name in optdict:
|
| + old_value = optdict[name]
|
| + # keywords and arg lists can be merged, everything
|
| + # else overrides completely
|
| + if isinstance(old_value, dict):
|
| + old_value.update(value)
|
| + elif isinstance(old_value, list):
|
| + old_value.extend(value)
|
| + else:
|
| + optdict[name] = value
|
| + else:
|
| + optdict[name] = value
|
| + return optdict
|
| +
|
| + # Handle with statements
|
| + def visit_WithStatNode(self, node):
|
| + directive_dict = {}
|
| + for directive in self.try_to_parse_directives(node.manager) or []:
|
| + if directive is not None:
|
| + if node.target is not None:
|
| + self.context.nonfatal_error(
|
| + PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
|
| + else:
|
| + name, value = directive
|
| + if name in ('nogil', 'gil'):
|
| + # special case: in pure mode, "with nogil" spells "with cython.nogil"
|
| + node = Nodes.GILStatNode(node.pos, state = name, body = node.body)
|
| + return self.visit_Node(node)
|
| + if self.check_directive_scope(node.pos, name, 'with statement'):
|
| + directive_dict[name] = value
|
| + if directive_dict:
|
| + return self.visit_with_directives(node.body, directive_dict)
|
| + return self.visit_Node(node)
|
| +
|
| +
|
| +class ParallelRangeTransform(CythonTransform, SkipDeclarations):
|
| + """
|
| + Transform cython.parallel stuff. The parallel_directives come from the
|
| + module node, set there by InterpretCompilerDirectives.
|
| +
|
| + x = cython.parallel.threadavailable() -> ParallelThreadAvailableNode
|
| + with nogil, cython.parallel.parallel(): -> ParallelWithBlockNode
|
| + print cython.parallel.threadid() -> ParallelThreadIdNode
|
| + for i in cython.parallel.prange(...): -> ParallelRangeNode
|
| + ...
|
| + """
|
| +
|
| + # a list of names, maps 'cython.parallel.prange' in the code to
|
| + # ['cython', 'parallel', 'prange']
|
| + parallel_directive = None
|
| +
|
| + # Indicates whether a namenode in an expression is the cython module
|
| + namenode_is_cython_module = False
|
| +
|
| + # Keep track of whether we are the context manager of a 'with' statement
|
| + in_context_manager_section = False
|
| +
|
| + # One of 'prange' or 'with parallel'. This is used to disallow closely
|
| + # nested 'with parallel:' blocks
|
| + state = None
|
| +
|
| + directive_to_node = {
|
| + u"cython.parallel.parallel": Nodes.ParallelWithBlockNode,
|
| + # u"cython.parallel.threadsavailable": ExprNodes.ParallelThreadsAvailableNode,
|
| + u"cython.parallel.threadid": ExprNodes.ParallelThreadIdNode,
|
| + u"cython.parallel.prange": Nodes.ParallelRangeNode,
|
| + }
|
| +
|
| + def node_is_parallel_directive(self, node):
|
| + return node.name in self.parallel_directives or node.is_cython_module
|
| +
|
| + def get_directive_class_node(self, node):
|
| + """
|
| + Figure out which parallel directive was used and return the associated
|
| + Node class.
|
| +
|
| + E.g. for a cython.parallel.prange() call we return ParallelRangeNode
|
| + """
|
| + if self.namenode_is_cython_module:
|
| + directive = '.'.join(self.parallel_directive)
|
| + else:
|
| + directive = self.parallel_directives[self.parallel_directive[0]]
|
| + directive = '%s.%s' % (directive,
|
| + '.'.join(self.parallel_directive[1:]))
|
| + directive = directive.rstrip('.')
|
| +
|
| + cls = self.directive_to_node.get(directive)
|
| + if cls is None and not (self.namenode_is_cython_module and
|
| + self.parallel_directive[0] != 'parallel'):
|
| + error(node.pos, "Invalid directive: %s" % directive)
|
| +
|
| + self.namenode_is_cython_module = False
|
| + self.parallel_directive = None
|
| +
|
| + return cls
|
| +
|
| + def visit_ModuleNode(self, node):
|
| + """
|
| + If any parallel directives were imported, copy them over and visit
|
| + the AST
|
| + """
|
| + if node.parallel_directives:
|
| + self.parallel_directives = node.parallel_directives
|
| + return self.visit_Node(node)
|
| +
|
| + # No parallel directives were imported, so they can't be used :)
|
| + return node
|
| +
|
| + def visit_NameNode(self, node):
|
| + if self.node_is_parallel_directive(node):
|
| + self.parallel_directive = [node.name]
|
| + self.namenode_is_cython_module = node.is_cython_module
|
| + return node
|
| +
|
| + def visit_AttributeNode(self, node):
|
| + self.visitchildren(node)
|
| + if self.parallel_directive:
|
| + self.parallel_directive.append(node.attribute)
|
| + return node
|
| +
|
| + def visit_CallNode(self, node):
|
| + self.visit(node.function)
|
| + if not self.parallel_directive:
|
| + return node
|
| +
|
| + # We are a parallel directive, replace this node with the
|
| + # corresponding ParallelSomethingSomething node
|
| +
|
| + if isinstance(node, ExprNodes.GeneralCallNode):
|
| + args = node.positional_args.args
|
| + kwargs = node.keyword_args
|
| + else:
|
| + args = node.args
|
| + kwargs = {}
|
| +
|
| + parallel_directive_class = self.get_directive_class_node(node)
|
| + if parallel_directive_class:
|
| + # Note: in case of a parallel() the body is set by
|
| + # visit_WithStatNode
|
| + node = parallel_directive_class(node.pos, args=args, kwargs=kwargs)
|
| +
|
| + return node
|
| +
|
| + def visit_WithStatNode(self, node):
|
| + "Rewrite with cython.parallel.parallel() blocks"
|
| + newnode = self.visit(node.manager)
|
| +
|
| + if isinstance(newnode, Nodes.ParallelWithBlockNode):
|
| + if self.state == 'parallel with':
|
| + error(node.manager.pos,
|
| + "Nested parallel with blocks are disallowed")
|
| +
|
| + self.state = 'parallel with'
|
| + body = self.visit(node.body)
|
| + self.state = None
|
| +
|
| + newnode.body = body
|
| + return newnode
|
| + elif self.parallel_directive:
|
| + parallel_directive_class = self.get_directive_class_node(node)
|
| +
|
| + if not parallel_directive_class:
|
| + # There was an error, stop here and now
|
| + return None
|
| +
|
| + if parallel_directive_class is Nodes.ParallelWithBlockNode:
|
| + error(node.pos, "The parallel directive must be called")
|
| + return None
|
| +
|
| + node.body = self.visit(node.body)
|
| + return node
|
| +
|
| + def visit_ForInStatNode(self, node):
|
| + "Rewrite 'for i in cython.parallel.prange(...):'"
|
| + self.visit(node.iterator)
|
| + self.visit(node.target)
|
| +
|
| + in_prange = isinstance(node.iterator.sequence,
|
| + Nodes.ParallelRangeNode)
|
| + previous_state = self.state
|
| +
|
| + if in_prange:
|
| + # This will replace the entire ForInStatNode, so copy the
|
| + # attributes
|
| + parallel_range_node = node.iterator.sequence
|
| +
|
| + parallel_range_node.target = node.target
|
| + parallel_range_node.body = node.body
|
| + parallel_range_node.else_clause = node.else_clause
|
| +
|
| + node = parallel_range_node
|
| +
|
| + if not isinstance(node.target, ExprNodes.NameNode):
|
| + error(node.target.pos,
|
| + "Can only iterate over an iteration variable")
|
| +
|
| + self.state = 'prange'
|
| +
|
| + self.visit(node.body)
|
| + self.state = previous_state
|
| + self.visit(node.else_clause)
|
| + return node
|
| +
|
| + def visit(self, node):
|
| + "Visit a node that may be None"
|
| + if node is not None:
|
| + return super(ParallelRangeTransform, self).visit(node)
|
| +
|
| +
|
| +class WithTransform(CythonTransform, SkipDeclarations):
|
| + def visit_WithStatNode(self, node):
|
| + self.visitchildren(node, 'body')
|
| + pos = node.pos
|
| + body, target, manager = node.body, node.target, node.manager
|
| + node.enter_call = ExprNodes.SimpleCallNode(
|
| + pos, function=ExprNodes.AttributeNode(
|
| + pos, obj=ExprNodes.CloneNode(manager),
|
| + attribute=EncodedString('__enter__'),
|
| + is_special_lookup=True),
|
| + args=[],
|
| + is_temp=True)
|
| + if target is not None:
|
| + body = Nodes.StatListNode(
|
| + pos, stats = [
|
| + Nodes.WithTargetAssignmentStatNode(
|
| + pos, lhs = target,
|
| + rhs = ResultRefNode(node.enter_call),
|
| + orig_rhs = node.enter_call),
|
| + body])
|
| +
|
| + excinfo_target = ExprNodes.TupleNode(pos, slow=True, args=[
|
| + ExprNodes.ExcValueNode(pos) for _ in range(3)])
|
| + except_clause = Nodes.ExceptClauseNode(
|
| + pos, body=Nodes.IfStatNode(
|
| + pos, if_clauses=[
|
| + Nodes.IfClauseNode(
|
| + pos, condition=ExprNodes.NotNode(
|
| + pos, operand=ExprNodes.WithExitCallNode(
|
| + pos, with_stat=node,
|
| + test_if_run=False,
|
| + args=excinfo_target)),
|
| + body=Nodes.ReraiseStatNode(pos),
|
| + ),
|
| + ],
|
| + else_clause=None),
|
| + pattern=None,
|
| + target=None,
|
| + excinfo_target=excinfo_target,
|
| + )
|
| +
|
| + node.body = Nodes.TryFinallyStatNode(
|
| + pos, body=Nodes.TryExceptStatNode(
|
| + pos, body=body,
|
| + except_clauses=[except_clause],
|
| + else_clause=None,
|
| + ),
|
| + finally_clause=Nodes.ExprStatNode(
|
| + pos, expr=ExprNodes.WithExitCallNode(
|
| + pos, with_stat=node,
|
| + test_if_run=True,
|
| + args=ExprNodes.TupleNode(
|
| + pos, args=[ExprNodes.NoneNode(pos) for _ in range(3)]
|
| + ))),
|
| + handle_error_case=False,
|
| + )
|
| + return node
|
| +
|
| + def visit_ExprNode(self, node):
|
| + # With statements are never inside expressions.
|
| + return node
|
| +
|
| +
|
| +class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations):
|
| + """Originally, this was the only place where decorators were
|
| + transformed into the corresponding calling code. Now, this is
|
| + done directly in DefNode and PyClassDefNode to avoid reassignments
|
| + to the function/class name - except for cdef class methods. For
|
| + those, the reassignment is required as methods are originally
|
| + defined in the PyMethodDef struct.
|
| +
|
| + The IndirectionNode allows DefNode to override the decorator
|
| + """
|
| +
|
| + def visit_DefNode(self, func_node):
|
| + scope_type = self.scope_type
|
| + func_node = self.visit_FuncDefNode(func_node)
|
| + if scope_type != 'cclass' or not func_node.decorators:
|
| + return func_node
|
| + return self.handle_decorators(func_node, func_node.decorators,
|
| + func_node.name)
|
| +
|
| + def handle_decorators(self, node, decorators, name):
|
| + decorator_result = ExprNodes.NameNode(node.pos, name = name)
|
| + for decorator in decorators[::-1]:
|
| + decorator_result = ExprNodes.SimpleCallNode(
|
| + decorator.pos,
|
| + function = decorator.decorator,
|
| + args = [decorator_result])
|
| +
|
| + name_node = ExprNodes.NameNode(node.pos, name = name)
|
| + reassignment = Nodes.SingleAssignmentNode(
|
| + node.pos,
|
| + lhs = name_node,
|
| + rhs = decorator_result)
|
| +
|
| + reassignment = Nodes.IndirectionNode([reassignment])
|
| + node.decorator_indirection = reassignment
|
| + return [node, reassignment]
|
| +
|
| +class CnameDirectivesTransform(CythonTransform, SkipDeclarations):
|
| + """
|
| + Only part of the CythonUtilityCode pipeline. Must be run before
|
| + DecoratorTransform in case this is a decorator for a cdef class.
|
| + It filters out @cname('my_cname') decorators and rewrites them to
|
| + CnameDecoratorNodes.
|
| + """
|
| +
|
| + def handle_function(self, node):
|
| + if not getattr(node, 'decorators', None):
|
| + return self.visit_Node(node)
|
| +
|
| + for i, decorator in enumerate(node.decorators):
|
| + decorator = decorator.decorator
|
| +
|
| + if (isinstance(decorator, ExprNodes.CallNode) and
|
| + decorator.function.is_name and
|
| + decorator.function.name == 'cname'):
|
| + args, kwargs = decorator.explicit_args_kwds()
|
| +
|
| + if kwargs:
|
| + raise AssertionError(
|
| + "cname decorator does not take keyword arguments")
|
| +
|
| + if len(args) != 1:
|
| + raise AssertionError(
|
| + "cname decorator takes exactly one argument")
|
| +
|
| + if not (args[0].is_literal and
|
| + args[0].type == Builtin.str_type):
|
| + raise AssertionError(
|
| + "argument to cname decorator must be a string literal")
|
| +
|
| + cname = args[0].compile_time_value(None).decode('UTF-8')
|
| + del node.decorators[i]
|
| + node = Nodes.CnameDecoratorNode(pos=node.pos, node=node,
|
| + cname=cname)
|
| + break
|
| +
|
| + return self.visit_Node(node)
|
| +
|
| + visit_FuncDefNode = handle_function
|
| + visit_CClassDefNode = handle_function
|
| + visit_CEnumDefNode = handle_function
|
| + visit_CStructOrUnionDefNode = handle_function
|
| +
|
| +
|
| +class ForwardDeclareTypes(CythonTransform):
|
| +
|
| + def visit_CompilerDirectivesNode(self, node):
|
| + env = self.module_scope
|
| + old = env.directives
|
| + env.directives = node.directives
|
| + self.visitchildren(node)
|
| + env.directives = old
|
| + return node
|
| +
|
| + def visit_ModuleNode(self, node):
|
| + self.module_scope = node.scope
|
| + self.module_scope.directives = node.directives
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_CDefExternNode(self, node):
|
| + old_cinclude_flag = self.module_scope.in_cinclude
|
| + self.module_scope.in_cinclude = 1
|
| + self.visitchildren(node)
|
| + self.module_scope.in_cinclude = old_cinclude_flag
|
| + return node
|
| +
|
| + def visit_CEnumDefNode(self, node):
|
| + node.declare(self.module_scope)
|
| + return node
|
| +
|
| + def visit_CStructOrUnionDefNode(self, node):
|
| + if node.name not in self.module_scope.entries:
|
| + node.declare(self.module_scope)
|
| + return node
|
| +
|
| + def visit_CClassDefNode(self, node):
|
| + if node.class_name not in self.module_scope.entries:
|
| + node.declare(self.module_scope)
|
| + return node
|
| +
|
| +
|
| +class AnalyseDeclarationsTransform(EnvTransform):
|
| +
|
| + basic_property = TreeFragment(u"""
|
| +property NAME:
|
| + def __get__(self):
|
| + return ATTR
|
| + def __set__(self, value):
|
| + ATTR = value
|
| + """, level='c_class', pipeline=[NormalizeTree(None)])
|
| + basic_pyobject_property = TreeFragment(u"""
|
| +property NAME:
|
| + def __get__(self):
|
| + return ATTR
|
| + def __set__(self, value):
|
| + ATTR = value
|
| + def __del__(self):
|
| + ATTR = None
|
| + """, level='c_class', pipeline=[NormalizeTree(None)])
|
| + basic_property_ro = TreeFragment(u"""
|
| +property NAME:
|
| + def __get__(self):
|
| + return ATTR
|
| + """, level='c_class', pipeline=[NormalizeTree(None)])
|
| +
|
| + struct_or_union_wrapper = TreeFragment(u"""
|
| +cdef class NAME:
|
| + cdef TYPE value
|
| + def __init__(self, MEMBER=None):
|
| + cdef int count
|
| + count = 0
|
| + INIT_ASSIGNMENTS
|
| + if IS_UNION and count > 1:
|
| + raise ValueError, "At most one union member should be specified."
|
| + def __str__(self):
|
| + return STR_FORMAT % MEMBER_TUPLE
|
| + def __repr__(self):
|
| + return REPR_FORMAT % MEMBER_TUPLE
|
| + """, pipeline=[NormalizeTree(None)])
|
| +
|
| + init_assignment = TreeFragment(u"""
|
| +if VALUE is not None:
|
| + ATTR = VALUE
|
| + count += 1
|
| + """, pipeline=[NormalizeTree(None)])
|
| +
|
| + fused_function = None
|
| + in_lambda = 0
|
| +
|
| + def __call__(self, root):
|
| + # needed to determine if a cdef var is declared after it's used.
|
| + self.seen_vars_stack = []
|
| + self.fused_error_funcs = set()
|
| + super_class = super(AnalyseDeclarationsTransform, self)
|
| + self._super_visit_FuncDefNode = super_class.visit_FuncDefNode
|
| + return super_class.__call__(root)
|
| +
|
| + def visit_NameNode(self, node):
|
| + self.seen_vars_stack[-1].add(node.name)
|
| + return node
|
| +
|
| + def visit_ModuleNode(self, node):
|
| + self.seen_vars_stack.append(set())
|
| + node.analyse_declarations(self.current_env())
|
| + self.visitchildren(node)
|
| + self.seen_vars_stack.pop()
|
| + return node
|
| +
|
| + def visit_LambdaNode(self, node):
|
| + self.in_lambda += 1
|
| + node.analyse_declarations(self.current_env())
|
| + self.visitchildren(node)
|
| + self.in_lambda -= 1
|
| + return node
|
| +
|
| + def visit_CClassDefNode(self, node):
|
| + node = self.visit_ClassDefNode(node)
|
| + if node.scope and node.scope.implemented:
|
| + stats = []
|
| + for entry in node.scope.var_entries:
|
| + if entry.needs_property:
|
| + property = self.create_Property(entry)
|
| + property.analyse_declarations(node.scope)
|
| + self.visit(property)
|
| + stats.append(property)
|
| + if stats:
|
| + node.body.stats += stats
|
| + return node
|
| +
|
| + def _handle_fused_def_decorators(self, old_decorators, env, node):
|
| + """
|
| + Create function calls to the decorators and reassignments to
|
| + the function.
|
| + """
|
| + # Delete staticmethod and classmethod decorators, this is
|
| + # handled directly by the fused function object.
|
| + decorators = []
|
| + for decorator in old_decorators:
|
| + func = decorator.decorator
|
| + if (not func.is_name or
|
| + func.name not in ('staticmethod', 'classmethod') or
|
| + env.lookup_here(func.name)):
|
| + # not a static or classmethod
|
| + decorators.append(decorator)
|
| +
|
| + if decorators:
|
| + transform = DecoratorTransform(self.context)
|
| + def_node = node.node
|
| + _, reassignments = transform.handle_decorators(
|
| + def_node, decorators, def_node.name)
|
| + reassignments.analyse_declarations(env)
|
| + node = [node, reassignments]
|
| +
|
| + return node
|
| +
|
| + def _handle_def(self, decorators, env, node):
|
| + "Handle def or cpdef fused functions"
|
| + # Create PyCFunction nodes for each specialization
|
| + node.stats.insert(0, node.py_func)
|
| + node.py_func = self.visit(node.py_func)
|
| + node.update_fused_defnode_entry(env)
|
| + pycfunc = ExprNodes.PyCFunctionNode.from_defnode(node.py_func,
|
| + True)
|
| + pycfunc = ExprNodes.ProxyNode(pycfunc.coerce_to_temp(env))
|
| + node.resulting_fused_function = pycfunc
|
| + # Create assignment node for our def function
|
| + node.fused_func_assignment = self._create_assignment(
|
| + node.py_func, ExprNodes.CloneNode(pycfunc), env)
|
| +
|
| + if decorators:
|
| + node = self._handle_fused_def_decorators(decorators, env, node)
|
| +
|
| + return node
|
| +
|
| + def _create_fused_function(self, env, node):
|
| + "Create a fused function for a DefNode with fused arguments"
|
| + from Cython.Compiler import FusedNode
|
| +
|
| + if self.fused_function or self.in_lambda:
|
| + if self.fused_function not in self.fused_error_funcs:
|
| + if self.in_lambda:
|
| + error(node.pos, "Fused lambdas not allowed")
|
| + else:
|
| + error(node.pos, "Cannot nest fused functions")
|
| +
|
| + self.fused_error_funcs.add(self.fused_function)
|
| +
|
| + node.body = Nodes.PassStatNode(node.pos)
|
| + for arg in node.args:
|
| + if arg.type.is_fused:
|
| + arg.type = arg.type.get_fused_types()[0]
|
| +
|
| + return node
|
| +
|
| + decorators = getattr(node, 'decorators', None)
|
| + node = FusedNode.FusedCFuncDefNode(node, env)
|
| + self.fused_function = node
|
| + self.visitchildren(node)
|
| + self.fused_function = None
|
| + if node.py_func:
|
| + node = self._handle_def(decorators, env, node)
|
| +
|
| + return node
|
| +
|
| + def _handle_nogil_cleanup(self, lenv, node):
|
| + "Handle cleanup for 'with gil' blocks in nogil functions."
|
| + if lenv.nogil and lenv.has_with_gil_block:
|
| + # Acquire the GIL for cleanup in 'nogil' functions, by wrapping
|
| + # the entire function body in try/finally.
|
| + # The corresponding release will be taken care of by
|
| + # Nodes.FuncDefNode.generate_function_definitions()
|
| + node.body = Nodes.NogilTryFinallyStatNode(
|
| + node.body.pos,
|
| + body=node.body,
|
| + finally_clause=Nodes.EnsureGILNode(node.body.pos))
|
| +
|
| + def _handle_fused(self, node):
|
| + if node.is_generator and node.has_fused_arguments:
|
| + node.has_fused_arguments = False
|
| + error(node.pos, "Fused generators not supported")
|
| + node.gbody = Nodes.StatListNode(node.pos,
|
| + stats=[],
|
| + body=Nodes.PassStatNode(node.pos))
|
| +
|
| + return node.has_fused_arguments
|
| +
|
| + def visit_FuncDefNode(self, node):
|
| + """
|
| + Analyse a function and its body, as that hasn't happend yet. Also
|
| + analyse the directive_locals set by @cython.locals(). Then, if we are
|
| + a function with fused arguments, replace the function (after it has
|
| + declared itself in the symbol table!) with a FusedCFuncDefNode, and
|
| + analyse its children (which are in turn normal functions). If we're a
|
| + normal function, just analyse the body of the function.
|
| + """
|
| + env = self.current_env()
|
| +
|
| + self.seen_vars_stack.append(set())
|
| + lenv = node.local_scope
|
| + node.declare_arguments(lenv)
|
| +
|
| + for var, type_node in node.directive_locals.items():
|
| + if not lenv.lookup_here(var): # don't redeclare args
|
| + type = type_node.analyse_as_type(lenv)
|
| + if type:
|
| + lenv.declare_var(var, type, type_node.pos)
|
| + else:
|
| + error(type_node.pos, "Not a type")
|
| +
|
| + if self._handle_fused(node):
|
| + node = self._create_fused_function(env, node)
|
| + else:
|
| + node.body.analyse_declarations(lenv)
|
| + self._handle_nogil_cleanup(lenv, node)
|
| + self._super_visit_FuncDefNode(node)
|
| +
|
| + self.seen_vars_stack.pop()
|
| + return node
|
| +
|
| + def visit_DefNode(self, node):
|
| + node = self.visit_FuncDefNode(node)
|
| + env = self.current_env()
|
| + if (not isinstance(node, Nodes.DefNode) or
|
| + node.fused_py_func or node.is_generator_body or
|
| + not node.needs_assignment_synthesis(env)):
|
| + return node
|
| + return [node, self._synthesize_assignment(node, env)]
|
| +
|
| + def visit_GeneratorBodyDefNode(self, node):
|
| + return self.visit_FuncDefNode(node)
|
| +
|
| + def _synthesize_assignment(self, node, env):
|
| + # Synthesize assignment node and put it right after defnode
|
| + genv = env
|
| + while genv.is_py_class_scope or genv.is_c_class_scope:
|
| + genv = genv.outer_scope
|
| +
|
| + if genv.is_closure_scope:
|
| + rhs = node.py_cfunc_node = ExprNodes.InnerFunctionNode(
|
| + node.pos, def_node=node,
|
| + pymethdef_cname=node.entry.pymethdef_cname,
|
| + code_object=ExprNodes.CodeObjectNode(node))
|
| + else:
|
| + binding = self.current_directives.get('binding')
|
| + rhs = ExprNodes.PyCFunctionNode.from_defnode(node, binding)
|
| +
|
| + if env.is_py_class_scope:
|
| + rhs.binding = True
|
| +
|
| + node.is_cyfunction = rhs.binding
|
| + return self._create_assignment(node, rhs, env)
|
| +
|
| + def _create_assignment(self, def_node, rhs, env):
|
| + if def_node.decorators:
|
| + for decorator in def_node.decorators[::-1]:
|
| + rhs = ExprNodes.SimpleCallNode(
|
| + decorator.pos,
|
| + function = decorator.decorator,
|
| + args = [rhs])
|
| + def_node.decorators = None
|
| +
|
| + assmt = Nodes.SingleAssignmentNode(
|
| + def_node.pos,
|
| + lhs=ExprNodes.NameNode(def_node.pos, name=def_node.name),
|
| + rhs=rhs)
|
| + assmt.analyse_declarations(env)
|
| + return assmt
|
| +
|
| + def visit_ScopedExprNode(self, node):
|
| + env = self.current_env()
|
| + node.analyse_declarations(env)
|
| + # the node may or may not have a local scope
|
| + if node.has_local_scope:
|
| + self.seen_vars_stack.append(set(self.seen_vars_stack[-1]))
|
| + self.enter_scope(node, node.expr_scope)
|
| + node.analyse_scoped_declarations(node.expr_scope)
|
| + self.visitchildren(node)
|
| + self.exit_scope()
|
| + self.seen_vars_stack.pop()
|
| + else:
|
| + node.analyse_scoped_declarations(env)
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_TempResultFromStatNode(self, node):
|
| + self.visitchildren(node)
|
| + node.analyse_declarations(self.current_env())
|
| + return node
|
| +
|
| + def visit_CppClassNode(self, node):
|
| + if node.visibility == 'extern':
|
| + return None
|
| + else:
|
| + return self.visit_ClassDefNode(node)
|
| +
|
| + def visit_CStructOrUnionDefNode(self, node):
|
| + # Create a wrapper node if needed.
|
| + # We want to use the struct type information (so it can't happen
|
| + # before this phase) but also create new objects to be declared
|
| + # (so it can't happen later).
|
| + # Note that we don't return the original node, as it is
|
| + # never used after this phase.
|
| + if True: # private (default)
|
| + return None
|
| +
|
| + self_value = ExprNodes.AttributeNode(
|
| + pos = node.pos,
|
| + obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
|
| + attribute = EncodedString(u"value"))
|
| + var_entries = node.entry.type.scope.var_entries
|
| + attributes = []
|
| + for entry in var_entries:
|
| + attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
|
| + obj = self_value,
|
| + attribute = entry.name))
|
| + # __init__ assignments
|
| + init_assignments = []
|
| + for entry, attr in zip(var_entries, attributes):
|
| + # TODO: branch on visibility
|
| + init_assignments.append(self.init_assignment.substitute({
|
| + u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
|
| + u"ATTR": attr,
|
| + }, pos = entry.pos))
|
| +
|
| + # create the class
|
| + str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
|
| + wrapper_class = self.struct_or_union_wrapper.substitute({
|
| + u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
|
| + u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
|
| + u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
|
| + u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
|
| + u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
|
| + }, pos = node.pos).stats[0]
|
| + wrapper_class.class_name = node.name
|
| + wrapper_class.shadow = True
|
| + class_body = wrapper_class.body.stats
|
| +
|
| + # fix value type
|
| + assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
|
| + class_body[0].base_type.name = node.name
|
| +
|
| + # fix __init__ arguments
|
| + init_method = class_body[1]
|
| + assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
|
| + arg_template = init_method.args[1]
|
| + if not node.entry.type.is_struct:
|
| + arg_template.kw_only = True
|
| + del init_method.args[1]
|
| + for entry, attr in zip(var_entries, attributes):
|
| + arg = copy.deepcopy(arg_template)
|
| + arg.declarator.name = entry.name
|
| + init_method.args.append(arg)
|
| +
|
| + # setters/getters
|
| + for entry, attr in zip(var_entries, attributes):
|
| + # TODO: branch on visibility
|
| + if entry.type.is_pyobject:
|
| + template = self.basic_pyobject_property
|
| + else:
|
| + template = self.basic_property
|
| + property = template.substitute({
|
| + u"ATTR": attr,
|
| + }, pos = entry.pos).stats[0]
|
| + property.name = entry.name
|
| + wrapper_class.body.stats.append(property)
|
| +
|
| + wrapper_class.analyse_declarations(self.current_env())
|
| + return self.visit_CClassDefNode(wrapper_class)
|
| +
|
| + # Some nodes are no longer needed after declaration
|
| + # analysis and can be dropped. The analysis was performed
|
| + # on these nodes in a seperate recursive process from the
|
| + # enclosing function or module, so we can simply drop them.
|
| + def visit_CDeclaratorNode(self, node):
|
| + # necessary to ensure that all CNameDeclaratorNodes are visited.
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_CTypeDefNode(self, node):
|
| + return node
|
| +
|
| + def visit_CBaseTypeNode(self, node):
|
| + return None
|
| +
|
| + def visit_CEnumDefNode(self, node):
|
| + if node.visibility == 'public':
|
| + return node
|
| + else:
|
| + return None
|
| +
|
| + def visit_CNameDeclaratorNode(self, node):
|
| + if node.name in self.seen_vars_stack[-1]:
|
| + entry = self.current_env().lookup(node.name)
|
| + if (entry is None or entry.visibility != 'extern'
|
| + and not entry.scope.is_c_class_scope):
|
| + warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_CVarDefNode(self, node):
|
| + # to ensure all CNameDeclaratorNodes are visited.
|
| + self.visitchildren(node)
|
| + return None
|
| +
|
| + def visit_CnameDecoratorNode(self, node):
|
| + child_node = self.visit(node.node)
|
| + if not child_node:
|
| + return None
|
| + if type(child_node) is list: # Assignment synthesized
|
| + node.child_node = child_node[0]
|
| + return [node] + child_node[1:]
|
| + node.node = child_node
|
| + return node
|
| +
|
| + def create_Property(self, entry):
|
| + if entry.visibility == 'public':
|
| + if entry.type.is_pyobject:
|
| + template = self.basic_pyobject_property
|
| + else:
|
| + template = self.basic_property
|
| + elif entry.visibility == 'readonly':
|
| + template = self.basic_property_ro
|
| + property = template.substitute({
|
| + u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
|
| + obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
|
| + attribute=entry.name),
|
| + }, pos=entry.pos).stats[0]
|
| + property.name = entry.name
|
| + property.doc = entry.doc
|
| + return property
|
| +
|
| +
|
| +class CalculateQualifiedNamesTransform(EnvTransform):
|
| + """
|
| + Calculate and store the '__qualname__' and the global
|
| + module name on some nodes.
|
| + """
|
| + def visit_ModuleNode(self, node):
|
| + self.module_name = self.global_scope().qualified_name
|
| + self.qualified_name = []
|
| + _super = super(CalculateQualifiedNamesTransform, self)
|
| + self._super_visit_FuncDefNode = _super.visit_FuncDefNode
|
| + self._super_visit_ClassDefNode = _super.visit_ClassDefNode
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def _set_qualname(self, node, name=None):
|
| + if name:
|
| + qualname = self.qualified_name[:]
|
| + qualname.append(name)
|
| + else:
|
| + qualname = self.qualified_name
|
| + node.qualname = EncodedString('.'.join(qualname))
|
| + node.module_name = self.module_name
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def _append_entry(self, entry):
|
| + if entry.is_pyglobal and not entry.is_pyclass_attr:
|
| + self.qualified_name = [entry.name]
|
| + else:
|
| + self.qualified_name.append(entry.name)
|
| +
|
| + def visit_ClassNode(self, node):
|
| + return self._set_qualname(node, node.name)
|
| +
|
| + def visit_PyClassNamespaceNode(self, node):
|
| + # class name was already added by parent node
|
| + return self._set_qualname(node)
|
| +
|
| + def visit_PyCFunctionNode(self, node):
|
| + return self._set_qualname(node, node.def_node.name)
|
| +
|
| + def visit_FuncDefNode(self, node):
|
| + orig_qualified_name = self.qualified_name[:]
|
| + if getattr(node, 'name', None) == '<lambda>':
|
| + self.qualified_name.append('<lambda>')
|
| + else:
|
| + self._append_entry(node.entry)
|
| + self.qualified_name.append('<locals>')
|
| + self._super_visit_FuncDefNode(node)
|
| + self.qualified_name = orig_qualified_name
|
| + return node
|
| +
|
| + def visit_ClassDefNode(self, node):
|
| + orig_qualified_name = self.qualified_name[:]
|
| + entry = (getattr(node, 'entry', None) or # PyClass
|
| + self.current_env().lookup_here(node.name)) # CClass
|
| + self._append_entry(entry)
|
| + self._super_visit_ClassDefNode(node)
|
| + self.qualified_name = orig_qualified_name
|
| + return node
|
| +
|
| +
|
| +class AnalyseExpressionsTransform(CythonTransform):
|
| +
|
| + def visit_ModuleNode(self, node):
|
| + node.scope.infer_types()
|
| + node.body = node.body.analyse_expressions(node.scope)
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_FuncDefNode(self, node):
|
| + node.local_scope.infer_types()
|
| + node.body = node.body.analyse_expressions(node.local_scope)
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_ScopedExprNode(self, node):
|
| + if node.has_local_scope:
|
| + node.expr_scope.infer_types()
|
| + node = node.analyse_scoped_expressions(node.expr_scope)
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_IndexNode(self, node):
|
| + """
|
| + Replace index nodes used to specialize cdef functions with fused
|
| + argument types with the Attribute- or NameNode referring to the
|
| + function. We then need to copy over the specialization properties to
|
| + the attribute or name node.
|
| +
|
| + Because the indexing might be a Python indexing operation on a fused
|
| + function, or (usually) a Cython indexing operation, we need to
|
| + re-analyse the types.
|
| + """
|
| + self.visit_Node(node)
|
| +
|
| + if node.is_fused_index and not node.type.is_error:
|
| + node = node.base
|
| + elif node.memslice_ellipsis_noop:
|
| + # memoryviewslice[...] expression, drop the IndexNode
|
| + node = node.base
|
| +
|
| + return node
|
| +
|
| +
|
| +class FindInvalidUseOfFusedTypes(CythonTransform):
|
| +
|
| + def visit_FuncDefNode(self, node):
|
| + # Errors related to use in functions with fused args will already
|
| + # have been detected
|
| + if not node.has_fused_arguments:
|
| + if not node.is_generator_body and node.return_type.is_fused:
|
| + error(node.pos, "Return type is not specified as argument type")
|
| + else:
|
| + self.visitchildren(node)
|
| +
|
| + return node
|
| +
|
| + def visit_ExprNode(self, node):
|
| + if node.type and node.type.is_fused:
|
| + error(node.pos, "Invalid use of fused types, type cannot be specialized")
|
| + else:
|
| + self.visitchildren(node)
|
| +
|
| + return node
|
| +
|
| +
|
| +class ExpandInplaceOperators(EnvTransform):
|
| +
|
| + def visit_InPlaceAssignmentNode(self, node):
|
| + lhs = node.lhs
|
| + rhs = node.rhs
|
| + if lhs.type.is_cpp_class:
|
| + # No getting around this exact operator here.
|
| + return node
|
| + if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
|
| + # There is code to handle this case.
|
| + return node
|
| +
|
| + env = self.current_env()
|
| + def side_effect_free_reference(node, setting=False):
|
| + if isinstance(node, ExprNodes.NameNode):
|
| + return node, []
|
| + elif node.type.is_pyobject and not setting:
|
| + node = LetRefNode(node)
|
| + return node, [node]
|
| + elif isinstance(node, ExprNodes.IndexNode):
|
| + if node.is_buffer_access:
|
| + raise ValueError("Buffer access")
|
| + base, temps = side_effect_free_reference(node.base)
|
| + index = LetRefNode(node.index)
|
| + return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
|
| + elif isinstance(node, ExprNodes.AttributeNode):
|
| + obj, temps = side_effect_free_reference(node.obj)
|
| + return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
|
| + else:
|
| + node = LetRefNode(node)
|
| + return node, [node]
|
| + try:
|
| + lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
|
| + except ValueError:
|
| + return node
|
| + dup = lhs.__class__(**lhs.__dict__)
|
| + binop = ExprNodes.binop_node(node.pos,
|
| + operator = node.operator,
|
| + operand1 = dup,
|
| + operand2 = rhs,
|
| + inplace=True)
|
| + # Manually analyse types for new node.
|
| + lhs.analyse_target_types(env)
|
| + dup.analyse_types(env)
|
| + binop.analyse_operation(env)
|
| + node = Nodes.SingleAssignmentNode(
|
| + node.pos,
|
| + lhs = lhs,
|
| + rhs=binop.coerce_to(lhs.type, env))
|
| + # Use LetRefNode to avoid side effects.
|
| + let_ref_nodes.reverse()
|
| + for t in let_ref_nodes:
|
| + node = LetNode(t, node)
|
| + return node
|
| +
|
| + def visit_ExprNode(self, node):
|
| + # In-place assignments can't happen within an expression.
|
| + return node
|
| +
|
| +class AdjustDefByDirectives(CythonTransform, SkipDeclarations):
|
| + """
|
| + Adjust function and class definitions by the decorator directives:
|
| +
|
| + @cython.cfunc
|
| + @cython.cclass
|
| + @cython.ccall
|
| + """
|
| +
|
| + def visit_ModuleNode(self, node):
|
| + self.directives = node.directives
|
| + self.in_py_class = False
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_CompilerDirectivesNode(self, node):
|
| + old_directives = self.directives
|
| + self.directives = node.directives
|
| + self.visitchildren(node)
|
| + self.directives = old_directives
|
| + return node
|
| +
|
| + def visit_DefNode(self, node):
|
| + if 'ccall' in self.directives:
|
| + node = node.as_cfunction(overridable=True, returns=self.directives.get('returns'))
|
| + return self.visit(node)
|
| + if 'cfunc' in self.directives:
|
| + if self.in_py_class:
|
| + error(node.pos, "cfunc directive is not allowed here")
|
| + else:
|
| + node = node.as_cfunction(overridable=False, returns=self.directives.get('returns'))
|
| + return self.visit(node)
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_PyClassDefNode(self, node):
|
| + if 'cclass' in self.directives:
|
| + node = node.as_cclass()
|
| + return self.visit(node)
|
| + else:
|
| + old_in_pyclass = self.in_py_class
|
| + self.in_py_class = True
|
| + self.visitchildren(node)
|
| + self.in_py_class = old_in_pyclass
|
| + return node
|
| +
|
| + def visit_CClassDefNode(self, node):
|
| + old_in_pyclass = self.in_py_class
|
| + self.in_py_class = False
|
| + self.visitchildren(node)
|
| + self.in_py_class = old_in_pyclass
|
| + return node
|
| +
|
| +
|
| +class AlignFunctionDefinitions(CythonTransform):
|
| + """
|
| + This class takes the signatures from a .pxd file and applies them to
|
| + the def methods in a .py file.
|
| + """
|
| +
|
| + def visit_ModuleNode(self, node):
|
| + self.scope = node.scope
|
| + self.directives = node.directives
|
| + self.imported_names = set() # hack, see visit_FromImportStatNode()
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_PyClassDefNode(self, node):
|
| + pxd_def = self.scope.lookup(node.name)
|
| + if pxd_def:
|
| + if pxd_def.is_cclass:
|
| + return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
|
| + elif not pxd_def.scope or not pxd_def.scope.is_builtin_scope:
|
| + error(node.pos, "'%s' redeclared" % node.name)
|
| + if pxd_def.pos:
|
| + error(pxd_def.pos, "previous declaration here")
|
| + return None
|
| + return node
|
| +
|
| + def visit_CClassDefNode(self, node, pxd_def=None):
|
| + if pxd_def is None:
|
| + pxd_def = self.scope.lookup(node.class_name)
|
| + if pxd_def:
|
| + outer_scope = self.scope
|
| + self.scope = pxd_def.type.scope
|
| + self.visitchildren(node)
|
| + if pxd_def:
|
| + self.scope = outer_scope
|
| + return node
|
| +
|
| + def visit_DefNode(self, node):
|
| + pxd_def = self.scope.lookup(node.name)
|
| + if pxd_def and (not pxd_def.scope or not pxd_def.scope.is_builtin_scope):
|
| + if not pxd_def.is_cfunction:
|
| + error(node.pos, "'%s' redeclared" % node.name)
|
| + if pxd_def.pos:
|
| + error(pxd_def.pos, "previous declaration here")
|
| + return None
|
| + node = node.as_cfunction(pxd_def)
|
| + elif (self.scope.is_module_scope and self.directives['auto_cpdef']
|
| + and not node.name in self.imported_names
|
| + and node.is_cdef_func_compatible()):
|
| + # FIXME: cpdef-ing should be done in analyse_declarations()
|
| + node = node.as_cfunction(scope=self.scope)
|
| + # Enable this when nested cdef functions are allowed.
|
| + # self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_FromImportStatNode(self, node):
|
| + # hack to prevent conditional import fallback functions from
|
| + # being cdpef-ed (global Python variables currently conflict
|
| + # with imports)
|
| + if self.scope.is_module_scope:
|
| + for name, _ in node.items:
|
| + self.imported_names.add(name)
|
| + return node
|
| +
|
| + def visit_ExprNode(self, node):
|
| + # ignore lambdas and everything else that appears in expressions
|
| + return node
|
| +
|
| +
|
| +class RemoveUnreachableCode(CythonTransform):
|
| + def visit_StatListNode(self, node):
|
| + if not self.current_directives['remove_unreachable']:
|
| + return node
|
| + self.visitchildren(node)
|
| + for idx, stat in enumerate(node.stats):
|
| + idx += 1
|
| + if stat.is_terminator:
|
| + if idx < len(node.stats):
|
| + if self.current_directives['warn.unreachable']:
|
| + warning(node.stats[idx].pos, "Unreachable code", 2)
|
| + node.stats = node.stats[:idx]
|
| + node.is_terminator = True
|
| + break
|
| + return node
|
| +
|
| + def visit_IfClauseNode(self, node):
|
| + self.visitchildren(node)
|
| + if node.body.is_terminator:
|
| + node.is_terminator = True
|
| + return node
|
| +
|
| + def visit_IfStatNode(self, node):
|
| + self.visitchildren(node)
|
| + if node.else_clause and node.else_clause.is_terminator:
|
| + for clause in node.if_clauses:
|
| + if not clause.is_terminator:
|
| + break
|
| + else:
|
| + node.is_terminator = True
|
| + return node
|
| +
|
| + def visit_TryExceptStatNode(self, node):
|
| + self.visitchildren(node)
|
| + if node.body.is_terminator and node.else_clause:
|
| + if self.current_directives['warn.unreachable']:
|
| + warning(node.else_clause.pos, "Unreachable code", 2)
|
| + node.else_clause = None
|
| + return node
|
| +
|
| +
|
| +class YieldNodeCollector(TreeVisitor):
|
| +
|
| + def __init__(self):
|
| + super(YieldNodeCollector, self).__init__()
|
| + self.yields = []
|
| + self.returns = []
|
| + self.has_return_value = False
|
| +
|
| + def visit_Node(self, node):
|
| + self.visitchildren(node)
|
| +
|
| + def visit_YieldExprNode(self, node):
|
| + self.yields.append(node)
|
| + self.visitchildren(node)
|
| +
|
| + def visit_ReturnStatNode(self, node):
|
| + self.visitchildren(node)
|
| + if node.value:
|
| + self.has_return_value = True
|
| + self.returns.append(node)
|
| +
|
| + def visit_ClassDefNode(self, node):
|
| + pass
|
| +
|
| + def visit_FuncDefNode(self, node):
|
| + pass
|
| +
|
| + def visit_LambdaNode(self, node):
|
| + pass
|
| +
|
| + def visit_GeneratorExpressionNode(self, node):
|
| + pass
|
| +
|
| +
|
| +class MarkClosureVisitor(CythonTransform):
|
| +
|
| + def visit_ModuleNode(self, node):
|
| + self.needs_closure = False
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_FuncDefNode(self, node):
|
| + self.needs_closure = False
|
| + self.visitchildren(node)
|
| + node.needs_closure = self.needs_closure
|
| + self.needs_closure = True
|
| +
|
| + collector = YieldNodeCollector()
|
| + collector.visitchildren(node)
|
| +
|
| + if collector.yields:
|
| + if isinstance(node, Nodes.CFuncDefNode):
|
| + # Will report error later
|
| + return node
|
| + for i, yield_expr in enumerate(collector.yields):
|
| + yield_expr.label_num = i + 1 # no enumerate start arg in Py2.4
|
| + for retnode in collector.returns:
|
| + retnode.in_generator = True
|
| +
|
| + gbody = Nodes.GeneratorBodyDefNode(
|
| + pos=node.pos, name=node.name, body=node.body)
|
| + generator = Nodes.GeneratorDefNode(
|
| + pos=node.pos, name=node.name, args=node.args,
|
| + star_arg=node.star_arg, starstar_arg=node.starstar_arg,
|
| + doc=node.doc, decorators=node.decorators,
|
| + gbody=gbody, lambda_name=node.lambda_name)
|
| + return generator
|
| + return node
|
| +
|
| + def visit_CFuncDefNode(self, node):
|
| + self.visit_FuncDefNode(node)
|
| + if node.needs_closure:
|
| + error(node.pos, "closures inside cdef functions not yet supported")
|
| + return node
|
| +
|
| + def visit_LambdaNode(self, node):
|
| + self.needs_closure = False
|
| + self.visitchildren(node)
|
| + node.needs_closure = self.needs_closure
|
| + self.needs_closure = True
|
| + return node
|
| +
|
| + def visit_ClassDefNode(self, node):
|
| + self.visitchildren(node)
|
| + self.needs_closure = True
|
| + return node
|
| +
|
| +class CreateClosureClasses(CythonTransform):
|
| + # Output closure classes in module scope for all functions
|
| + # that really need it.
|
| +
|
| + def __init__(self, context):
|
| + super(CreateClosureClasses, self).__init__(context)
|
| + self.path = []
|
| + self.in_lambda = False
|
| +
|
| + def visit_ModuleNode(self, node):
|
| + self.module_scope = node.scope
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def find_entries_used_in_closures(self, node):
|
| + from_closure = []
|
| + in_closure = []
|
| + for name, entry in node.local_scope.entries.items():
|
| + if entry.from_closure:
|
| + from_closure.append((name, entry))
|
| + elif entry.in_closure:
|
| + in_closure.append((name, entry))
|
| + return from_closure, in_closure
|
| +
|
| + def create_class_from_scope(self, node, target_module_scope, inner_node=None):
|
| + # move local variables into closure
|
| + if node.is_generator:
|
| + for entry in node.local_scope.entries.values():
|
| + if not entry.from_closure:
|
| + entry.in_closure = True
|
| +
|
| + from_closure, in_closure = self.find_entries_used_in_closures(node)
|
| + in_closure.sort()
|
| +
|
| + # Now from the begining
|
| + node.needs_closure = False
|
| + node.needs_outer_scope = False
|
| +
|
| + func_scope = node.local_scope
|
| + cscope = node.entry.scope
|
| + while cscope.is_py_class_scope or cscope.is_c_class_scope:
|
| + cscope = cscope.outer_scope
|
| +
|
| + if not from_closure and (self.path or inner_node):
|
| + if not inner_node:
|
| + if not node.py_cfunc_node:
|
| + raise InternalError("DefNode does not have assignment node")
|
| + inner_node = node.py_cfunc_node
|
| + inner_node.needs_self_code = False
|
| + node.needs_outer_scope = False
|
| +
|
| + if node.is_generator:
|
| + pass
|
| + elif not in_closure and not from_closure:
|
| + return
|
| + elif not in_closure:
|
| + func_scope.is_passthrough = True
|
| + func_scope.scope_class = cscope.scope_class
|
| + node.needs_outer_scope = True
|
| + return
|
| +
|
| + as_name = '%s_%s' % (
|
| + target_module_scope.next_id(Naming.closure_class_prefix),
|
| + node.entry.cname)
|
| +
|
| + entry = target_module_scope.declare_c_class(
|
| + name=as_name, pos=node.pos, defining=True,
|
| + implementing=True)
|
| + entry.type.is_final_type = True
|
| +
|
| + func_scope.scope_class = entry
|
| + class_scope = entry.type.scope
|
| + class_scope.is_internal = True
|
| + if Options.closure_freelist_size:
|
| + class_scope.directives['freelist'] = Options.closure_freelist_size
|
| +
|
| + if from_closure:
|
| + assert cscope.is_closure_scope
|
| + class_scope.declare_var(pos=node.pos,
|
| + name=Naming.outer_scope_cname,
|
| + cname=Naming.outer_scope_cname,
|
| + type=cscope.scope_class.type,
|
| + is_cdef=True)
|
| + node.needs_outer_scope = True
|
| + for name, entry in in_closure:
|
| + closure_entry = class_scope.declare_var(pos=entry.pos,
|
| + name=entry.name,
|
| + cname=entry.cname,
|
| + type=entry.type,
|
| + is_cdef=True)
|
| + if entry.is_declared_generic:
|
| + closure_entry.is_declared_generic = 1
|
| + node.needs_closure = True
|
| + # Do it here because other classes are already checked
|
| + target_module_scope.check_c_class(func_scope.scope_class)
|
| +
|
| + def visit_LambdaNode(self, node):
|
| + if not isinstance(node.def_node, Nodes.DefNode):
|
| + # fused function, an error has been previously issued
|
| + return node
|
| +
|
| + was_in_lambda = self.in_lambda
|
| + self.in_lambda = True
|
| + self.create_class_from_scope(node.def_node, self.module_scope, node)
|
| + self.visitchildren(node)
|
| + self.in_lambda = was_in_lambda
|
| + return node
|
| +
|
| + def visit_FuncDefNode(self, node):
|
| + if self.in_lambda:
|
| + self.visitchildren(node)
|
| + return node
|
| + if node.needs_closure or self.path:
|
| + self.create_class_from_scope(node, self.module_scope)
|
| + self.path.append(node)
|
| + self.visitchildren(node)
|
| + self.path.pop()
|
| + return node
|
| +
|
| + def visit_GeneratorBodyDefNode(self, node):
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_CFuncDefNode(self, node):
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| +
|
| +class GilCheck(VisitorTransform):
|
| + """
|
| + Call `node.gil_check(env)` on each node to make sure we hold the
|
| + GIL when we need it. Raise an error when on Python operations
|
| + inside a `nogil` environment.
|
| +
|
| + Additionally, raise exceptions for closely nested with gil or with nogil
|
| + statements. The latter would abort Python.
|
| + """
|
| +
|
| + def __call__(self, root):
|
| + self.env_stack = [root.scope]
|
| + self.nogil = False
|
| +
|
| + # True for 'cdef func() nogil:' functions, as the GIL may be held while
|
| + # calling this function (thus contained 'nogil' blocks may be valid).
|
| + self.nogil_declarator_only = False
|
| + return super(GilCheck, self).__call__(root)
|
| +
|
| + def visit_FuncDefNode(self, node):
|
| + self.env_stack.append(node.local_scope)
|
| + was_nogil = self.nogil
|
| + self.nogil = node.local_scope.nogil
|
| +
|
| + if self.nogil:
|
| + self.nogil_declarator_only = True
|
| +
|
| + if self.nogil and node.nogil_check:
|
| + node.nogil_check(node.local_scope)
|
| +
|
| + self.visitchildren(node)
|
| +
|
| + # This cannot be nested, so it doesn't need backup/restore
|
| + self.nogil_declarator_only = False
|
| +
|
| + self.env_stack.pop()
|
| + self.nogil = was_nogil
|
| + return node
|
| +
|
| + def visit_GILStatNode(self, node):
|
| + if self.nogil and node.nogil_check:
|
| + node.nogil_check()
|
| +
|
| + was_nogil = self.nogil
|
| + self.nogil = (node.state == 'nogil')
|
| +
|
| + if was_nogil == self.nogil and not self.nogil_declarator_only:
|
| + if not was_nogil:
|
| + error(node.pos, "Trying to acquire the GIL while it is "
|
| + "already held.")
|
| + else:
|
| + error(node.pos, "Trying to release the GIL while it was "
|
| + "previously released.")
|
| +
|
| + if isinstance(node.finally_clause, Nodes.StatListNode):
|
| + # The finally clause of the GILStatNode is a GILExitNode,
|
| + # which is wrapped in a StatListNode. Just unpack that.
|
| + node.finally_clause, = node.finally_clause.stats
|
| +
|
| + self.visitchildren(node)
|
| + self.nogil = was_nogil
|
| + return node
|
| +
|
| + def visit_ParallelRangeNode(self, node):
|
| + if node.nogil:
|
| + node.nogil = False
|
| + node = Nodes.GILStatNode(node.pos, state='nogil', body=node)
|
| + return self.visit_GILStatNode(node)
|
| +
|
| + if not self.nogil:
|
| + error(node.pos, "prange() can only be used without the GIL")
|
| + # Forget about any GIL-related errors that may occur in the body
|
| + return None
|
| +
|
| + node.nogil_check(self.env_stack[-1])
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_ParallelWithBlockNode(self, node):
|
| + if not self.nogil:
|
| + error(node.pos, "The parallel section may only be used without "
|
| + "the GIL")
|
| + return None
|
| +
|
| + if node.nogil_check:
|
| + # It does not currently implement this, but test for it anyway to
|
| + # avoid potential future surprises
|
| + node.nogil_check(self.env_stack[-1])
|
| +
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_TryFinallyStatNode(self, node):
|
| + """
|
| + Take care of try/finally statements in nogil code sections.
|
| + """
|
| + if not self.nogil or isinstance(node, Nodes.GILStatNode):
|
| + return self.visit_Node(node)
|
| +
|
| + node.nogil_check = None
|
| + node.is_try_finally_in_nogil = True
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_Node(self, node):
|
| + if self.env_stack and self.nogil and node.nogil_check:
|
| + node.nogil_check(self.env_stack[-1])
|
| + self.visitchildren(node)
|
| + node.in_nogil_context = self.nogil
|
| + return node
|
| +
|
| +
|
| +class TransformBuiltinMethods(EnvTransform):
|
| +
|
| + def visit_SingleAssignmentNode(self, node):
|
| + if node.declaration_only:
|
| + return None
|
| + else:
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def visit_AttributeNode(self, node):
|
| + self.visitchildren(node)
|
| + return self.visit_cython_attribute(node)
|
| +
|
| + def visit_NameNode(self, node):
|
| + return self.visit_cython_attribute(node)
|
| +
|
| + def visit_cython_attribute(self, node):
|
| + attribute = node.as_cython_attribute()
|
| + if attribute:
|
| + if attribute == u'compiled':
|
| + node = ExprNodes.BoolNode(node.pos, value=True)
|
| + elif attribute == u'__version__':
|
| + import Cython
|
| + node = ExprNodes.StringNode(node.pos, value=EncodedString(Cython.__version__))
|
| + elif attribute == u'NULL':
|
| + node = ExprNodes.NullNode(node.pos)
|
| + elif attribute in (u'set', u'frozenset'):
|
| + node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
|
| + entry=self.current_env().builtin_scope().lookup_here(attribute))
|
| + elif PyrexTypes.parse_basic_type(attribute):
|
| + pass
|
| + elif self.context.cython_scope.lookup_qualified_name(attribute):
|
| + pass
|
| + else:
|
| + error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
|
| + return node
|
| +
|
| + def visit_ExecStatNode(self, node):
|
| + lenv = self.current_env()
|
| + self.visitchildren(node)
|
| + if len(node.args) == 1:
|
| + node.args.append(ExprNodes.GlobalsExprNode(node.pos))
|
| + if not lenv.is_module_scope:
|
| + node.args.append(
|
| + ExprNodes.LocalsExprNode(
|
| + node.pos, self.current_scope_node(), lenv))
|
| + return node
|
| +
|
| + def _inject_locals(self, node, func_name):
|
| + # locals()/dir()/vars() builtins
|
| + lenv = self.current_env()
|
| + entry = lenv.lookup_here(func_name)
|
| + if entry:
|
| + # not the builtin
|
| + return node
|
| + pos = node.pos
|
| + if func_name in ('locals', 'vars'):
|
| + if func_name == 'locals' and len(node.args) > 0:
|
| + error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d"
|
| + % len(node.args))
|
| + return node
|
| + elif func_name == 'vars':
|
| + if len(node.args) > 1:
|
| + error(self.pos, "Builtin 'vars()' called with wrong number of args, expected 0-1, got %d"
|
| + % len(node.args))
|
| + if len(node.args) > 0:
|
| + return node # nothing to do
|
| + return ExprNodes.LocalsExprNode(pos, self.current_scope_node(), lenv)
|
| + else: # dir()
|
| + if len(node.args) > 1:
|
| + error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d"
|
| + % len(node.args))
|
| + if len(node.args) > 0:
|
| + # optimised in Builtin.py
|
| + return node
|
| + if lenv.is_py_class_scope or lenv.is_module_scope:
|
| + if lenv.is_py_class_scope:
|
| + pyclass = self.current_scope_node()
|
| + locals_dict = ExprNodes.CloneNode(pyclass.dict)
|
| + else:
|
| + locals_dict = ExprNodes.GlobalsExprNode(pos)
|
| + return ExprNodes.SortedDictKeysNode(locals_dict)
|
| + local_names = [ var.name for var in lenv.entries.values() if var.name ]
|
| + items = [ ExprNodes.IdentifierStringNode(pos, value=var)
|
| + for var in local_names ]
|
| + return ExprNodes.ListNode(pos, args=items)
|
| +
|
| + def visit_PrimaryCmpNode(self, node):
|
| + # special case: for in/not-in test, we do not need to sort locals()
|
| + self.visitchildren(node)
|
| + if node.operator in 'not_in': # in/not_in
|
| + if isinstance(node.operand2, ExprNodes.SortedDictKeysNode):
|
| + arg = node.operand2.arg
|
| + if isinstance(arg, ExprNodes.NoneCheckNode):
|
| + arg = arg.arg
|
| + node.operand2 = arg
|
| + return node
|
| +
|
| + def visit_CascadedCmpNode(self, node):
|
| + return self.visit_PrimaryCmpNode(node)
|
| +
|
| + def _inject_eval(self, node, func_name):
|
| + lenv = self.current_env()
|
| + entry = lenv.lookup_here(func_name)
|
| + if entry or len(node.args) != 1:
|
| + return node
|
| + # Inject globals and locals
|
| + node.args.append(ExprNodes.GlobalsExprNode(node.pos))
|
| + if not lenv.is_module_scope:
|
| + node.args.append(
|
| + ExprNodes.LocalsExprNode(
|
| + node.pos, self.current_scope_node(), lenv))
|
| + return node
|
| +
|
| + def _inject_super(self, node, func_name):
|
| + lenv = self.current_env()
|
| + entry = lenv.lookup_here(func_name)
|
| + if entry or node.args:
|
| + return node
|
| + # Inject no-args super
|
| + def_node = self.current_scope_node()
|
| + if (not isinstance(def_node, Nodes.DefNode) or not def_node.args or
|
| + len(self.env_stack) < 2):
|
| + return node
|
| + class_node, class_scope = self.env_stack[-2]
|
| + if class_scope.is_py_class_scope:
|
| + def_node.requires_classobj = True
|
| + class_node.class_cell.is_active = True
|
| + node.args = [
|
| + ExprNodes.ClassCellNode(
|
| + node.pos, is_generator=def_node.is_generator),
|
| + ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
|
| + ]
|
| + elif class_scope.is_c_class_scope:
|
| + node.args = [
|
| + ExprNodes.NameNode(
|
| + node.pos, name=class_node.scope.name,
|
| + entry=class_node.entry),
|
| + ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
|
| + ]
|
| + return node
|
| +
|
| + def visit_SimpleCallNode(self, node):
|
| + # cython.foo
|
| + function = node.function.as_cython_attribute()
|
| + if function:
|
| + if function in InterpretCompilerDirectives.unop_method_nodes:
|
| + if len(node.args) != 1:
|
| + error(node.function.pos, u"%s() takes exactly one argument" % function)
|
| + else:
|
| + node = InterpretCompilerDirectives.unop_method_nodes[function](node.function.pos, operand=node.args[0])
|
| + elif function in InterpretCompilerDirectives.binop_method_nodes:
|
| + if len(node.args) != 2:
|
| + error(node.function.pos, u"%s() takes exactly two arguments" % function)
|
| + else:
|
| + node = InterpretCompilerDirectives.binop_method_nodes[function](node.function.pos, operand1=node.args[0], operand2=node.args[1])
|
| + elif function == u'cast':
|
| + if len(node.args) != 2:
|
| + error(node.function.pos, u"cast() takes exactly two arguments")
|
| + else:
|
| + type = node.args[0].analyse_as_type(self.current_env())
|
| + if type:
|
| + node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
|
| + else:
|
| + error(node.args[0].pos, "Not a type")
|
| + elif function == u'sizeof':
|
| + if len(node.args) != 1:
|
| + error(node.function.pos, u"sizeof() takes exactly one argument")
|
| + else:
|
| + type = node.args[0].analyse_as_type(self.current_env())
|
| + if type:
|
| + node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
|
| + else:
|
| + node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
|
| + elif function == 'cmod':
|
| + if len(node.args) != 2:
|
| + error(node.function.pos, u"cmod() takes exactly two arguments")
|
| + else:
|
| + node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
|
| + node.cdivision = True
|
| + elif function == 'cdiv':
|
| + if len(node.args) != 2:
|
| + error(node.function.pos, u"cdiv() takes exactly two arguments")
|
| + else:
|
| + node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
|
| + node.cdivision = True
|
| + elif function == u'set':
|
| + node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
|
| + elif self.context.cython_scope.lookup_qualified_name(function):
|
| + pass
|
| + else:
|
| + error(node.function.pos,
|
| + u"'%s' not a valid cython language construct" % function)
|
| +
|
| + self.visitchildren(node)
|
| +
|
| + if isinstance(node, ExprNodes.SimpleCallNode) and node.function.is_name:
|
| + func_name = node.function.name
|
| + if func_name in ('dir', 'locals', 'vars'):
|
| + return self._inject_locals(node, func_name)
|
| + if func_name == 'eval':
|
| + return self._inject_eval(node, func_name)
|
| + if func_name == 'super':
|
| + return self._inject_super(node, func_name)
|
| + return node
|
| +
|
| +
|
| +class ReplaceFusedTypeChecks(VisitorTransform):
|
| + """
|
| + This is not a transform in the pipeline. It is invoked on the specific
|
| + versions of a cdef function with fused argument types. It filters out any
|
| + type branches that don't match. e.g.
|
| +
|
| + if fused_t is mytype:
|
| + ...
|
| + elif fused_t in other_fused_type:
|
| + ...
|
| + """
|
| + def __init__(self, local_scope):
|
| + super(ReplaceFusedTypeChecks, self).__init__()
|
| + self.local_scope = local_scope
|
| + # defer the import until now to avoid circular import time dependencies
|
| + from Cython.Compiler import Optimize
|
| + self.transform = Optimize.ConstantFolding(reevaluate=True)
|
| +
|
| + def visit_IfStatNode(self, node):
|
| + """
|
| + Filters out any if clauses with false compile time type check
|
| + expression.
|
| + """
|
| + self.visitchildren(node)
|
| + return self.transform(node)
|
| +
|
| + def visit_PrimaryCmpNode(self, node):
|
| + type1 = node.operand1.analyse_as_type(self.local_scope)
|
| + type2 = node.operand2.analyse_as_type(self.local_scope)
|
| +
|
| + if type1 and type2:
|
| + false_node = ExprNodes.BoolNode(node.pos, value=False)
|
| + true_node = ExprNodes.BoolNode(node.pos, value=True)
|
| +
|
| + type1 = self.specialize_type(type1, node.operand1.pos)
|
| + op = node.operator
|
| +
|
| + if op in ('is', 'is_not', '==', '!='):
|
| + type2 = self.specialize_type(type2, node.operand2.pos)
|
| +
|
| + is_same = type1.same_as(type2)
|
| + eq = op in ('is', '==')
|
| +
|
| + if (is_same and eq) or (not is_same and not eq):
|
| + return true_node
|
| +
|
| + elif op in ('in', 'not_in'):
|
| + # We have to do an instance check directly, as operand2
|
| + # needs to be a fused type and not a type with a subtype
|
| + # that is fused. First unpack the typedef
|
| + if isinstance(type2, PyrexTypes.CTypedefType):
|
| + type2 = type2.typedef_base_type
|
| +
|
| + if type1.is_fused:
|
| + error(node.operand1.pos, "Type is fused")
|
| + elif not type2.is_fused:
|
| + error(node.operand2.pos,
|
| + "Can only use 'in' or 'not in' on a fused type")
|
| + else:
|
| + types = PyrexTypes.get_specialized_types(type2)
|
| +
|
| + for specialized_type in types:
|
| + if type1.same_as(specialized_type):
|
| + if op == 'in':
|
| + return true_node
|
| + else:
|
| + return false_node
|
| +
|
| + if op == 'not_in':
|
| + return true_node
|
| +
|
| + return false_node
|
| +
|
| + return node
|
| +
|
| + def specialize_type(self, type, pos):
|
| + try:
|
| + return type.specialize(self.local_scope.fused_to_specific)
|
| + except KeyError:
|
| + error(pos, "Type is not specific")
|
| + return type
|
| +
|
| + def visit_Node(self, node):
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| +
|
| +class DebugTransform(CythonTransform):
|
| + """
|
| + Write debug information for this Cython module.
|
| + """
|
| +
|
| + def __init__(self, context, options, result):
|
| + super(DebugTransform, self).__init__(context)
|
| + self.visited = set()
|
| + # our treebuilder and debug output writer
|
| + # (see Cython.Debugger.debug_output.CythonDebugWriter)
|
| + self.tb = self.context.gdb_debug_outputwriter
|
| + #self.c_output_file = options.output_file
|
| + self.c_output_file = result.c_file
|
| +
|
| + # Closure support, basically treat nested functions as if the AST were
|
| + # never nested
|
| + self.nested_funcdefs = []
|
| +
|
| + # tells visit_NameNode whether it should register step-into functions
|
| + self.register_stepinto = False
|
| +
|
| + def visit_ModuleNode(self, node):
|
| + self.tb.module_name = node.full_module_name
|
| + attrs = dict(
|
| + module_name=node.full_module_name,
|
| + filename=node.pos[0].filename,
|
| + c_filename=self.c_output_file)
|
| +
|
| + self.tb.start('Module', attrs)
|
| +
|
| + # serialize functions
|
| + self.tb.start('Functions')
|
| + # First, serialize functions normally...
|
| + self.visitchildren(node)
|
| +
|
| + # ... then, serialize nested functions
|
| + for nested_funcdef in self.nested_funcdefs:
|
| + self.visit_FuncDefNode(nested_funcdef)
|
| +
|
| + self.register_stepinto = True
|
| + self.serialize_modulenode_as_function(node)
|
| + self.register_stepinto = False
|
| + self.tb.end('Functions')
|
| +
|
| + # 2.3 compatibility. Serialize global variables
|
| + self.tb.start('Globals')
|
| + entries = {}
|
| +
|
| + for k, v in node.scope.entries.iteritems():
|
| + if (v.qualified_name not in self.visited and not
|
| + v.name.startswith('__pyx_') and not
|
| + v.type.is_cfunction and not
|
| + v.type.is_extension_type):
|
| + entries[k]= v
|
| +
|
| + self.serialize_local_variables(entries)
|
| + self.tb.end('Globals')
|
| + # self.tb.end('Module') # end Module after the line number mapping in
|
| + # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
|
| + return node
|
| +
|
| + def visit_FuncDefNode(self, node):
|
| + self.visited.add(node.local_scope.qualified_name)
|
| +
|
| + if getattr(node, 'is_wrapper', False):
|
| + return node
|
| +
|
| + if self.register_stepinto:
|
| + self.nested_funcdefs.append(node)
|
| + return node
|
| +
|
| + # node.entry.visibility = 'extern'
|
| + if node.py_func is None:
|
| + pf_cname = ''
|
| + else:
|
| + pf_cname = node.py_func.entry.func_cname
|
| +
|
| + attrs = dict(
|
| + name=node.entry.name or getattr(node, 'name', '<unknown>'),
|
| + cname=node.entry.func_cname,
|
| + pf_cname=pf_cname,
|
| + qualified_name=node.local_scope.qualified_name,
|
| + lineno=str(node.pos[1]))
|
| +
|
| + self.tb.start('Function', attrs=attrs)
|
| +
|
| + self.tb.start('Locals')
|
| + self.serialize_local_variables(node.local_scope.entries)
|
| + self.tb.end('Locals')
|
| +
|
| + self.tb.start('Arguments')
|
| + for arg in node.local_scope.arg_entries:
|
| + self.tb.start(arg.name)
|
| + self.tb.end(arg.name)
|
| + self.tb.end('Arguments')
|
| +
|
| + self.tb.start('StepIntoFunctions')
|
| + self.register_stepinto = True
|
| + self.visitchildren(node)
|
| + self.register_stepinto = False
|
| + self.tb.end('StepIntoFunctions')
|
| + self.tb.end('Function')
|
| +
|
| + return node
|
| +
|
| + def visit_NameNode(self, node):
|
| + if (self.register_stepinto and
|
| + node.type.is_cfunction and
|
| + getattr(node, 'is_called', False) and
|
| + node.entry.func_cname is not None):
|
| + # don't check node.entry.in_cinclude, as 'cdef extern: ...'
|
| + # declared functions are not 'in_cinclude'.
|
| + # This means we will list called 'cdef' functions as
|
| + # "step into functions", but this is not an issue as they will be
|
| + # recognized as Cython functions anyway.
|
| + attrs = dict(name=node.entry.func_cname)
|
| + self.tb.start('StepIntoFunction', attrs=attrs)
|
| + self.tb.end('StepIntoFunction')
|
| +
|
| + self.visitchildren(node)
|
| + return node
|
| +
|
| + def serialize_modulenode_as_function(self, node):
|
| + """
|
| + Serialize the module-level code as a function so the debugger will know
|
| + it's a "relevant frame" and it will know where to set the breakpoint
|
| + for 'break modulename'.
|
| + """
|
| + name = node.full_module_name.rpartition('.')[-1]
|
| +
|
| + cname_py2 = 'init' + name
|
| + cname_py3 = 'PyInit_' + name
|
| +
|
| + py2_attrs = dict(
|
| + name=name,
|
| + cname=cname_py2,
|
| + pf_cname='',
|
| + # Ignore the qualified_name, breakpoints should be set using
|
| + # `cy break modulename:lineno` for module-level breakpoints.
|
| + qualified_name='',
|
| + lineno='1',
|
| + is_initmodule_function="True",
|
| + )
|
| +
|
| + py3_attrs = dict(py2_attrs, cname=cname_py3)
|
| +
|
| + self._serialize_modulenode_as_function(node, py2_attrs)
|
| + self._serialize_modulenode_as_function(node, py3_attrs)
|
| +
|
| + def _serialize_modulenode_as_function(self, node, attrs):
|
| + self.tb.start('Function', attrs=attrs)
|
| +
|
| + self.tb.start('Locals')
|
| + self.serialize_local_variables(node.scope.entries)
|
| + self.tb.end('Locals')
|
| +
|
| + self.tb.start('Arguments')
|
| + self.tb.end('Arguments')
|
| +
|
| + self.tb.start('StepIntoFunctions')
|
| + self.register_stepinto = True
|
| + self.visitchildren(node)
|
| + self.register_stepinto = False
|
| + self.tb.end('StepIntoFunctions')
|
| +
|
| + self.tb.end('Function')
|
| +
|
| + def serialize_local_variables(self, entries):
|
| + for entry in entries.values():
|
| + if not entry.cname:
|
| + # not a local variable
|
| + continue
|
| + if entry.type.is_pyobject:
|
| + vartype = 'PythonObject'
|
| + else:
|
| + vartype = 'CObject'
|
| +
|
| + if entry.from_closure:
|
| + # We're dealing with a closure where a variable from an outer
|
| + # scope is accessed, get it from the scope object.
|
| + cname = '%s->%s' % (Naming.cur_scope_cname,
|
| + entry.outer_entry.cname)
|
| +
|
| + qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
|
| + entry.scope.name,
|
| + entry.name)
|
| + elif entry.in_closure:
|
| + cname = '%s->%s' % (Naming.cur_scope_cname,
|
| + entry.cname)
|
| + qname = entry.qualified_name
|
| + else:
|
| + cname = entry.cname
|
| + qname = entry.qualified_name
|
| +
|
| + if not entry.pos:
|
| + # this happens for variables that are not in the user's code,
|
| + # e.g. for the global __builtins__, __doc__, etc. We can just
|
| + # set the lineno to 0 for those.
|
| + lineno = '0'
|
| + else:
|
| + lineno = str(entry.pos[1])
|
| +
|
| + attrs = dict(
|
| + name=entry.name,
|
| + cname=cname,
|
| + qualified_name=qname,
|
| + type=vartype,
|
| + lineno=lineno)
|
| +
|
| + self.tb.start('LocalVar', attrs)
|
| + self.tb.end('LocalVar')
|
|
|