Merge pull request #36 from vitek/_control_flow

Merged in Vitja's control flow analysis implementation.

Cython/Compiler/Buffer.py

@@ -1140,7 +1140,7 @@ static CYTHON_INLINE void __Pyx_ZeroBuffer(Py_buffer* buf) {
 }
 
 static CYTHON_INLINE int __Pyx_GetBufferAndValidate(Py_buffer* buf, PyObject* obj, __Pyx_TypeInfo* dtype, int flags, int nd, int cast, __Pyx_BufFmt_StackElem* stack) {
-  if (obj == Py_None) {
+  if (obj == Py_None || obj == NULL) {
     __Pyx_ZeroBuffer(buf);
     return 0;
   }

Cython/Compiler/CmdLine.py

@@ -39,6 +39,7 @@ Options:
   -3                             Compile based on Python-3 syntax and code semantics.
   --fast-fail                    Abort the compilation on the first error
   --warning-error, -Werror       Make all warnings into errors
+  --warning-extra, -Wextra       Enable extra warnings
   -X, --directive <name>=<value>[,<name=value,...] Overrides a compiler directive
 """
 
@@ -132,6 +133,8 @@ def parse_command_line(args):
                 Options.fast_fail = True
             elif option in ('-Werror', '--warning-errors'):
                 Options.warning_errors = True
+            elif option in ('-Wextra', '--warning-extra'):
+                options.compiler_directives.update(Options.extra_warnings)
             elif option == "--disable-function-redefinition":
                 Options.disable_function_redefinition = True
             elif option == "--directive" or option.startswith('-X'):

Cython/Compiler/Code.py

@@ -1187,6 +1187,8 @@ class CCodeWriter(object):
                 entry.cname, dll_linkage = dll_linkage))
         if entry.init is not None:
             self.put_safe(" = %s" % entry.type.literal_code(entry.init))
+        elif entry.type.is_pyobject:
+            self.put(" = NULL");
         self.putln(";")
 
     def put_temp_declarations(self, func_context):
@@ -1290,10 +1292,7 @@ class CCodeWriter(object):
 
     def put_var_decref(self, entry):
         if entry.type.is_pyobject:
-            if entry.init_to_none is False:  # FIXME: 0 and False are treated differently???
             self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
-            else:
-                self.putln("__Pyx_DECREF(%s);" % self.entry_as_pyobject(entry))
 
     def put_var_decref_clear(self, entry):
         if entry.type.is_pyobject:
@@ -1420,6 +1419,19 @@ class CCodeWriter(object):
 #        return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos)))  # TODO this path is almost _never_ taken, yet this macro makes is slower!
         return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))
 
+    def put_error_if_unbound(self, pos, entry):
+        import ExprNodes
+        if entry.from_closure:
+            func = '__Pyx_RaiseClosureNameError'
+            self.globalstate.use_utility_code(
+                ExprNodes.raise_closure_name_error_utility_code)
+        else:
+            func = '__Pyx_RaiseUnboundLocalError'
+            self.globalstate.use_utility_code(
+                ExprNodes.raise_unbound_local_error_utility_code)
+        self.put('if (unlikely(!%s)) { %s("%s"); %s }' % (
+            entry.cname, func, entry.name, self.error_goto(pos)))
+
     def set_error_info(self, pos):
         self.funcstate.should_declare_error_indicator = True
         if self.c_line_in_traceback:

Cython/Compiler/ExprNodes.py

@@ -1267,14 +1267,20 @@ class NameNode(AtomicExprNode):
     #  name            string    Python name of the variable
     #  entry           Entry     Symbol table entry
     #  type_entry      Entry     For extension type names, the original type entry
+    #  cf_is_null      boolean   Is uninitialized before this node
+    #  cf_maybe_null   boolean   Maybe uninitialized before this node
+    #  allow_null      boolean   Don't raise UnboundLocalError
 
     is_name = True
     is_cython_module = False
     cython_attribute = None
-    lhs_of_first_assignment = False
+    lhs_of_first_assignment = False # TODO: remove me
     is_used_as_rvalue = 0
     entry = None
     type_entry = None
+    cf_maybe_null = True
+    cf_is_null = False
+    allow_null = False
 
     def create_analysed_rvalue(pos, env, entry):
         node = NameNode(pos)
@@ -1550,15 +1556,11 @@ class NameNode(AtomicExprNode):
                 code.error_goto_if_null(self.result(), self.pos)))
             code.put_gotref(self.py_result())
 
-        elif entry.is_local and False:
-            # control flow not good enough yet
-            assigned = entry.scope.control_flow.get_state((entry.name, 'initialized'), self.pos)
-            if assigned is False:
-                error(self.pos, "local variable '%s' referenced before assignment" % entry.name)
-            elif not Options.init_local_none and assigned is None:
-                code.putln('if (%s == 0) { PyErr_SetString(PyExc_UnboundLocalError, "%s"); %s }' %
-                           (entry.cname, entry.name, code.error_goto(self.pos)))
-                entry.scope.control_flow.set_state(self.pos, (entry.name, 'initialized'), True)
+        elif entry.is_local or entry.in_closure or entry.from_closure:
+            if entry.type.is_pyobject:
+                if (self.cf_maybe_null or self.cf_is_null) \
+                       and not self.allow_null:
+                    code.put_error_if_unbound(self.pos, entry)
 
     def generate_assignment_code(self, rhs, code):
         #print "NameNode.generate_assignment_code:", self.name ###
@@ -1627,14 +1629,17 @@ class NameNode(AtomicExprNode):
                 if self.use_managed_ref:
                     rhs.make_owned_reference(code)
                     is_external_ref = entry.is_cglobal or self.entry.in_closure or self.entry.from_closure
-                    if not self.lhs_of_first_assignment:
                     if is_external_ref:
+                        if not self.cf_is_null:
+                            if self.cf_maybe_null:
+                                code.put_xgotref(self.py_result())
+                            else:
                                 code.put_gotref(self.py_result())
-                        if entry.is_local and not Options.init_local_none:
-                            initialized = entry.scope.control_flow.get_state((entry.name, 'initialized'), self.pos)
-                            if initialized is True:
+                    if entry.is_cglobal:
                         code.put_decref(self.result(), self.ctype())
-                            elif initialized is None:
+                    else:
+                        if not self.cf_is_null:
+                            if self.cf_maybe_null:
                                 code.put_xdecref(self.result(), self.ctype())
                             else:
                                 code.put_decref(self.result(), self.ctype())
@@ -1686,8 +1691,11 @@ class NameNode(AtomicExprNode):
                     Naming.module_cname,
                     self.entry.name))
         elif self.entry.type.is_pyobject:
-            # Fake it until we can do it for real...
-            self.generate_assignment_code(NoneNode(self.pos), code)
+            if not self.cf_is_null:
+                if self.cf_maybe_null:
+                    code.put_error_if_unbound(self.pos, self.entry)
+                code.put_decref(self.result(), self.ctype())
+                code.putln('%s = NULL;' % self.result())
         else:
             error(self.pos, "Deletion of C names not supported")
 
@@ -4485,8 +4493,6 @@ class ScopedExprNode(ExprNode):
             generate_inner_evaluation_code(code)
             code.putln('} /* exit inner scope */')
             return
-        for entry in py_entries:
-            code.put_init_var_to_py_none(entry)
 
         # must free all local Python references at each exit point
         old_loop_labels = tuple(code.new_loop_labels())
@@ -4732,11 +4738,13 @@ class DictNode(ExprNode):
     #  Dictionary constructor.
     #
     #  key_value_pairs     [DictItemNode]
+    #  exclude_null_values [boolean]          Do not add NULL values to dict
     #
     # obj_conversion_errors    [PyrexError]   used internally
 
     subexprs = ['key_value_pairs']
     is_temp = 1
+    exclude_null_values = False
     type = dict_type
 
     obj_conversion_errors = []
@@ -4824,11 +4832,15 @@ class DictNode(ExprNode):
         for item in self.key_value_pairs:
             item.generate_evaluation_code(code)
             if self.type.is_pyobject:
+                if self.exclude_null_values:
+                    code.putln('if (%s) {' % item.value.py_result())
                 code.put_error_if_neg(self.pos,
                     "PyDict_SetItem(%s, %s, %s)" % (
                         self.result(),
                         item.key.py_result(),
                         item.value.py_result()))
+                if self.exclude_null_values:
+                    code.putln('}')
             else:
                 code.putln("%s.%s = %s;" % (
                         self.result(),
@@ -8280,6 +8292,26 @@ static CYTHON_INLINE void __Pyx_RaiseNoneNotIterableError(void) {
 }
 ''')
 
+raise_unbound_local_error_utility_code = UtilityCode(
+proto = """
+static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname);
+""",
+impl = """
+static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname) {
+    PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname);
+}
+""")
+
+raise_closure_name_error_utility_code = UtilityCode(
+proto = """
+static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname);
+""",
+impl = """
+static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) {
+    PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname);
+}
+""")
+
 #------------------------------------------------------------------------------------
 
 getitem_dict_utility_code = UtilityCode(

Cython/Compiler/FlowControl.pxd

+cimport cython
+
+cdef class ControlBlock:
+     cdef public set children
+     cdef public set parents
+     cdef public set positions
+     cdef public list stats
+     cdef public dict gen
+     cdef public set bounded
+     cdef public dict input
+     cdef public dict output
+
+     # Big integer it bitsets
+     cdef public object i_input
+     cdef public object i_output
+     cdef public object i_gen
+     cdef public object i_kill
+     cdef public object i_state
+
+     cpdef bint empty(self)
+     cpdef detach(self)
+     cpdef add_child(self, block)
+
+cdef class ExitBlock(ControlBlock):
+     cpdef bint empty(self)
+
+cdef class NameAssignment:
+    cdef public bint is_arg
+    cdef public object lhs
+    cdef public object rhs
+    cdef public object entry
+    cdef public object pos
+    cdef public set refs
+    cdef public object bit
+
+cdef class AssignmentList:
+    cdef public object bit
+    cdef public object mask
+    cdef public list stats
+
+cdef class ControlFlow:
+     cdef public set blocks
+     cdef public set entries
+     cdef public list loops
+     cdef public list exceptions
+
+     cdef public ControlBlock entry_point
+     cdef public ExitBlock exit_point
+     cdef public ControlBlock block
+
+     cdef public dict assmts
+
+     cpdef newblock(self, parent=*)
+     cpdef nextblock(self, parent=*)
+     cpdef bint is_tracked(self, entry)
+     cpdef mark_position(self, node)
+     cpdef mark_assignment(self, lhs, rhs, entry=*)
+     cpdef mark_argument(self, lhs, rhs, entry)
+     cpdef mark_deletion(self, node, entry)
+     cpdef mark_reference(self, node, entry)
+     cpdef normalize(self)
+
+     @cython.locals(offset=object, assmts=AssignmentList,
+                    block=ControlBlock)
+     cpdef initialize(self)
+
+     @cython.locals(assmts=AssignmentList, assmt=NameAssignment)
+     cpdef set map_one(self, istate, entry)
+
+     @cython.locals(block=ControlBlock, parent=ControlBlock)
+     cdef reaching_definitions(self)
+
+cdef class Uninitialized:
+     pass
+
+@cython.locals(dirty=bint, block=ControlBlock, parent=ControlBlock)
+cdef check_definitions(ControlFlow flow, dict compiler_directives)

Cython/Compiler/FlowControl.py

+
+import cython
+cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
+               Options=object, UtilNodes=object, ModuleNode=object,
+               LetNode=object, LetRefNode=object, TreeFragment=object,
+               TemplateTransform=object, EncodedString=object,
+               error=object, warning=object, copy=object)
+
+import Builtin
+import ExprNodes
+import Nodes
+from PyrexTypes import py_object_type, unspecified_type
+
+from Visitor import TreeVisitor, CythonTransform
+from Errors import error, warning, CompileError, InternalError
+
+from cython import set
+
+class TypedExprNode(ExprNodes.ExprNode):
+    # Used for declaring assignments of a specified type whithout a known entry.
+    def __init__(self, type):
+        self.type = type
+
+object_expr = TypedExprNode(py_object_type)
+
+class ControlBlock(object):
+    """Control flow graph node. Sequence of assignments and name references.
+
+       children  set of children nodes
+       parents   set of parent nodes
+       positions set of position markers
+
+       stats     list of block statements
+       gen       dict of assignments generated by this block
+       bounded   set  of entries that are definitely bounded in this block
+
+       Example:
+
+        a = 1
+        b = a + c # 'c' is already bounded or exception here
+
+        stats = [Assignment(a), NameReference(a), NameReference(c),
+                     Assignment(b)]
+        gen = {Entry(a): Assignment(a), Entry(b): Assignment(b)}
+        bounded = set([Entry(a), Entry(c)])
+
+    """
+
+    def __init__(self):
+        self.children = set()
+        self.parents = set()
+        self.positions = set()
+
+        self.stats = []
+        self.gen = {}
+        self.bounded = set()
+
+        self.i_input = 0
+        self.i_output = 0
+        self.i_gen = 0
+        self.i_kill = 0
+        self.i_state = 0
+
+    def empty(self):
+        return (not self.stats and not self.positions)
+
+    def detach(self):
+        """Detach block from parents and children."""
+        for child in self.children:
+            child.parents.remove(self)
+        for parent in self.parents:
+            parent.children.remove(self)
+        self.parents.clear()
+        self.children.clear()
+
+    def add_child(self, block):
+        self.children.add(block)
+        block.parents.add(self)
+
+
+class ExitBlock(ControlBlock):
+    """Non-empty exit point block."""
+
+    def empty(self):
+        return False
+
+
+class AssignmentList:
+    def __init__(self):
+        self.stats = []
+
+
+class ControlFlow(object):
+    """Control-flow graph.
+
+       entry_point ControlBlock entry point for this graph
+       exit_point  ControlBlock normal exit point
+       block       ControlBlock current block
+       blocks      set    children nodes
+       entries     set    tracked entries
+       loops       list   stack for loop descriptors
+       exceptions  list   stack for exception descriptors
+
+    """
+
+    def __init__(self):
+        self.blocks = set()
+        self.entries = set()
+        self.loops = []
+        self.exceptions = []
+
+        self.entry_point = ControlBlock()
+        self.exit_point = ExitBlock()
+        self.blocks.add(self.exit_point)
+        self.block = self.entry_point
+
+    def newblock(self, parent=None):
+        """Create floating block linked to `parent` if given.
+
+           NOTE: Block is NOT added to self.blocks
+        """
+        block = ControlBlock()
+        self.blocks.add(block)
+        if parent:
+            parent.add_child(block)
+        return block
+
+    def nextblock(self, parent=None):
+        """Create block children block linked to current or `parent` if given.
+
+           NOTE: Block is added to self.blocks
+        """
+        block = ControlBlock()
+        self.blocks.add(block)
+        if parent:
+            parent.add_child(block)
+        elif self.block:
+            self.block.add_child(block)
+        self.block = block
+        return self.block
+
+    def is_tracked(self, entry):
+        if entry.is_anonymous:
+            return False
+        if entry.type.is_array or entry.type.is_struct_or_union:
+            return False
+        return (entry.is_local or entry.is_pyclass_attr or entry.is_arg or
+                entry.from_closure or entry.in_closure)
+
+    def mark_position(self, node):
+        """Mark position, will be used to draw graph nodes."""
+        if self.block:
+            self.block.positions.add(node.pos[:2])
+
+    def mark_assignment(self, lhs, rhs, entry=None):
+        if self.block:
+            if entry is None:
+                entry = lhs.entry
+            if not self.is_tracked(entry):
+                return
+            assignment = NameAssignment(lhs, rhs, entry)
+            self.block.stats.append(assignment)
+            self.block.gen[entry] = assignment
+            self.entries.add(entry)
+
+    def mark_argument(self, lhs, rhs, entry):
+        if self.block and self.is_tracked(entry):
+            assignment = Argument(lhs, rhs, entry)
+            self.block.stats.append(assignment)
+            self.block.gen[entry] = assignment
+            self.entries.add(entry)
+
+    def mark_deletion(self, node, entry):
+        if self.block and self.is_tracked(entry):
+            assignment = NameAssignment(node, None, entry)
+            self.block.stats.append(assignment)
+            self.block.gen[entry] = Uninitialized
+            self.entries.add(entry)
+
+    def mark_reference(self, node, entry):
+        if self.block and self.is_tracked(entry):
+            self.block.stats.append(NameReference(node, entry))
+            # Local variable is definitely bound after this reference
+            if not node.allow_null:
+                self.block.bounded.add(entry)
+            self.entries.add(entry)
+
+    def normalize(self):
+        """Delete unreachable and orphan blocks."""
+        queue = set([self.entry_point])
+        visited = set()
+        while queue:
+            root = queue.pop()
+            visited.add(root)
+            for child in root.children:
+                if child not in visited:
+                    queue.add(child)
+        unreachable = self.blocks - visited
+        for block in unreachable:
+            block.detach()
+        visited.remove(self.entry_point)
+        for block in visited:
+            if block.empty():
+                for parent in block.parents: # Re-parent
+                    for child in block.children:
+                        parent.add_child(child)
+                block.detach()
+                unreachable.add(block)
+        self.blocks -= unreachable
+
+    def initialize(self):
+        """Set initial state, map assignments to bits."""
+        self.assmts = {}
+
+        offset = 0
+        for entry in self.entries:
+            assmts = AssignmentList()
+            assmts.bit = 1 << offset
+            assmts.mask = assmts.bit
+            self.assmts[entry] = assmts
+            offset += 1
+
+        for block in self.blocks:
+            for stat in block.stats:
+                if isinstance(stat, NameAssignment):
+                    stat.bit = 1 << offset
+                    assmts = self.assmts[stat.entry]
+                    assmts.stats.append(stat)
+                    assmts.mask |= stat.bit
+                    offset += 1
+
+        for block in self.blocks:
+            for entry, stat in block.gen.items():
+                assmts = self.assmts[entry]
+                if stat is Uninitialized:
+                    block.i_gen |= assmts.bit
+                else:
+                    block.i_gen |= stat.bit
+                block.i_kill |= assmts.mask
+            block.i_output = block.i_gen
+            for entry in block.bounded:
+                block.i_kill |= self.assmts[entry].bit
+
+        for assmts in self.assmts.itervalues():
+            self.entry_point.i_gen |= assmts.bit
+        self.entry_point.i_output = self.entry_point.i_gen
+
+    def map_one(self, istate, entry):
+        ret = set()
+        assmts = self.assmts[entry]
+        if istate & assmts.bit:
+            ret.add(Uninitialized)
+        for assmt in assmts.stats:
+            if istate & assmt.bit:
+                ret.add(assmt)
+        return ret
+
+    def reaching_definitions(self):
+        """Per-block reaching definitions analysis."""
+        dirty = True
+        while dirty:
+            dirty = False
+            for block in self.blocks:
+                i_input = 0
+                for parent in block.parents:
+                    i_input |= parent.i_output
+                i_output = (i_input & ~block.i_kill) | block.i_gen
+                if i_output != block.i_output:
+                    dirty = True
+                block.i_input = i_input
+                block.i_output = i_output
+
+
+class LoopDescr(object):
+    def __init__(self, next_block, loop_block):
+        self.next_block = next_block
+        self.loop_block = loop_block
+        self.exceptions = []
+
+
+class ExceptionDescr(object):
+    """Exception handling helper.
+
+    entry_point   ControlBlock Exception handling entry point
+    finally_enter ControlBlock Normal finally clause entry point
+    finally_exit  ControlBlock Normal finally clause exit point
+    """
+
+    def __init__(self, entry_point, finally_enter=None, finally_exit=None):
+        self.entry_point = entry_point
+        self.finally_enter = finally_enter
+        self.finally_exit = finally_exit
+
+class NameAssignment(object):
+    def __init__(self, lhs, rhs, entry):
+        if lhs.cf_state is None:
+            lhs.cf_state = set()
+        self.lhs = lhs
+        self.rhs = rhs
+        self.entry = entry
+        self.pos = lhs.pos
+        self.refs = set()
+        self.is_arg = False
+
+    def __repr__(self):
+        return '%s(entry=%r)' % (self.__class__.__name__, self.entry)
+
+class Argument(NameAssignment):
+    def __init__(self, lhs, rhs, entry):
+        NameAssignment.__init__(self, lhs, rhs, entry)
+        self.is_arg = True
+
+class Uninitialized(object):
+    pass
+
+class NameReference(object):
+    def __init__(self, node, entry):
+        if node.cf_state is None:
+            node.cf_state = set()
+        self.node = node
+        self.entry = entry
+        self.pos = node.pos
+
+    def __repr__(self):
+        return '%s(entry=%r)' % (self.__class__.__name__, self.entry)
+
+
+class GVContext(object):
+    """Graphviz subgraph object."""
+
+    def __init__(self):
+        self.blockids = {}
+        self.nextid = 0
+        self.children = []
+        self.sources = {}
+
+    def add(self, child):
+        self.children.append(child)
+
+    def nodeid(self, block):
+        if block not in self.blockids:
+            self.blockids[block] = 'block%d' % self.nextid
+            self.nextid += 1
+        return self.blockids[block]
+
+    def extract_sources(self, block):
+        if not block.positions:
+            return ''
+        start = min(block.positions)
+        stop = max(block.positions)
+        srcdescr = start[0]
+        if not srcdescr in self.sources:
+            self.sources[srcdescr] = list(srcdescr.get_lines())
+        lines = self.sources[srcdescr]
+        return '\\n'.join([l.strip() for l in lines[start[1] - 1:stop[1]]])
+
+    def render(self, fp, name, annotate_defs=False):
+        """Render graphviz dot graph"""
+        fp.write('digraph %s {\n' % name)
+        fp.write(' node [shape=box];\n')
+        for child in self.children:
+            child.render(fp, self, annotate_defs)
+        fp.write('}\n')
+
+    def escape(self, text):
+        return text.replace('"', '\\"').replace('\n', '\\n')
+
+
+class GV(object):
+    """Graphviz DOT renderer."""
+
+    def __init__(self, name, flow):
+        self.name = name
+        self.flow = flow
+
+    def render(self, fp, ctx, annotate_defs=False):
+        fp.write(' subgraph %s {\n' % self.name)
+        for block in self.flow.blocks:
+            label = ctx.extract_sources(block)
+            if annotate_defs:
+                for stat in block.stats:
+                    if isinstance(stat, NameAssignment):
+                        label += '\n %s [definition]' % stat.entry.name
+                    elif isinstance(stat, NameReference):
+                        if stat.entry:
+                            label += '\n %s [reference]' % stat.entry.name
+            if not label:
+                label = 'empty'
+            pid = ctx.nodeid(block)
+            fp.write('  %s [label="%s"];\n' % (pid, ctx.escape(label)))
+        for block in self.flow.blocks:
+            pid = ctx.nodeid(block)
+            for child in block.children:
+                fp.write('  %s -> %s;\n' % (pid, ctx.nodeid(child)))
+        fp.write(' }\n')
+
+
+class MessageCollection:
+    """Collect error/warnings messages first then sort"""
+    def __init__(self):
+        self.messages = []
+
+    def error(self, pos, message):
+        self.messages.append((pos, True, message))
+
+    def warning(self, pos, message):
+        self.messages.append((pos, False, message))
+
+    def report(self):
+        self.messages.sort()
+        for pos, is_error, message in self.messages:
+            if is_error:
+                error(pos, message)
+            else:
+                warning(pos, message, 2)
+
+
+def check_definitions(flow, compiler_directives):
+    flow.initialize()
+    flow.reaching_definitions()
+
+    # Track down state
+    assignments = set()
+    # Node to entry map
+    references = {}
+    assmt_nodes = set()
+
+    for block in flow.blocks:
+        i_state = block.i_input
+        for stat in block.stats:
+            i_assmts = flow.assmts[stat.entry]
+            state = flow.map_one(i_state, stat.entry)
+            if isinstance(stat, NameAssignment):
+                stat.lhs.cf_state.update(state)
+                assmt_nodes.add(stat.lhs)
+                i_state = i_state & ~i_assmts.mask
+                if stat.rhs:
+                    i_state |= stat.bit
+                else:
+                    i_state |= i_assmts.bit
+                assignments.add(stat)
+                stat.entry.cf_assignments.append(stat)
+            elif isinstance(stat, NameReference):
+                references[stat.node] = stat.entry
+                stat.entry.cf_references.append(stat)
+                stat.node.cf_state.update(state)
+                if not stat.node.allow_null:
+                    i_state &= ~i_assmts.bit
+                state.discard(Uninitialized)
+                for assmt in state:
+                    assmt.refs.add(stat)
+
+    # Check variable usage
+    warn_maybe_uninitialized = compiler_directives['warn.maybe_uninitialized']
+    warn_unused_result = compiler_directives['warn.unused_result']
+    warn_unused = compiler_directives['warn.unused']
+    warn_unused_arg = compiler_directives['warn.unused_arg']
+
+    messages = MessageCollection()
+
+    # assignment hints
+    for node in assmt_nodes:
+        if Uninitialized in node.cf_state:
+            node.cf_maybe_null = True
+            if len(node.cf_state) == 1:
+                node.cf_is_null = True
+            else:
+                node.cf_is_null = False
+        else:
+            node.cf_is_null = False
+            node.cf_maybe_null = False
+
+    # Find uninitialized references and cf-hints
+    for node, entry in references.iteritems():
+        if Uninitialized in node.cf_state:
+            node.cf_maybe_null = True
+            if not entry.from_closure and len(node.cf_state) == 1:
+                node.cf_is_null = True
+            if node.allow_null or entry.from_closure:
+                pass # Can be uninitialized here
+            elif node.cf_is_null:
+                if entry.type.is_pyobject or entry.type.is_unspecified:
+                    messages.error(
+                        node.pos,
+                        "local variable '%s' referenced before assignment"
+                        % entry.name)
+                else:
+                    messages.warning(
+                        node.pos,
+                        "local variable '%s' referenced before assignment"
+                        % entry.name)
+            elif warn_maybe_uninitialized:
+                messages.warning(
+                    node.pos,
+                    "local variable '%s' might be referenced before assignment"
+                    % entry.name)
+        else:
+            node.cf_is_null = False
+            node.cf_maybe_null = False
+
+    # Unused result
+    for assmt in assignments:
+        if not assmt.refs and not assmt.entry.is_pyclass_attr \
+               and not assmt.entry.in_closure:
+            if assmt.entry.cf_references and warn_unused_result:
+                if assmt.is_arg:
+                    messages.warning(assmt.pos, "Unused argument value '%s'" % assmt.entry.name)
+                else:
+                    messages.warning(assmt.pos, "Unused result in '%s'" % assmt.entry.name)
+            assmt.lhs.cf_used = False
+
+    # Unused entries
+    for entry in flow.entries:
+        if not entry.cf_references and not entry.is_pyclass_attr and not entry.in_closure:
+            # TODO: starred args entries are not marked with is_arg flag
+            for assmt in entry.cf_assignments:
+                if assmt.is_arg:
+                    is_arg = True
+                    break
+            else:
+                is_arg = False
+            if is_arg:
+                if warn_unused_arg:
+                    messages.warning(entry.pos, "Unused argument '%s'" % entry.name)
+                # TODO: handle unused arguments
+                entry.cf_used = True
+            else:
+                if warn_unused:
+                    messages.warning(entry.pos, "Unused entry '%s'" % entry.name)
+                entry.cf_used = False
+
+    messages.report()
+
+
+class AssignmentCollector(TreeVisitor):
+    def __init__(self):
+        super(AssignmentCollector, self).__init__()
+        self.assignments = []
+
+    def visit_Node(self):
+        self.visitchildren(self)
+
+    def visit_SingleAssignmentNode(self, node):
+        self.assignments.append((node.lhs, node.rhs))
+
+    def visit_CascadedAssignmentNode(self, node):
+        for lhs in node.lhs_list:
+            self.assignments.append((lhs, node.rhs))
+
+
+class CreateControlFlowGraph(CythonTransform):
+    """Create NameNode use and assignment graph."""
+
+    def visit_ModuleNode(self, node):
+        self.gv_ctx = GVContext()
+
+        self.env_stack = []
+        self.env = node.scope
+        self.stack = []
+        self.flow = ControlFlow()
+        self.visitchildren(node)
+
+        dot_output = self.current_directives['control_flow.dot_output']
+        if dot_output:
+            annotate_defs = self.current_directives['control_flow.dot_annotate_defs']
+            fp = open(dot_output, 'wt')
+            try:
+                self.gv_ctx.render(fp, 'module', annotate_defs=annotate_defs)
+            finally:
+                fp.close()
+        return node
+
+    def visit_FuncDefNode(self, node):
+        self.env_stack.append(self.env)
+        self.env = node.local_scope
+        self.stack.append(self.flow)
+        self.flow = ControlFlow()
+
+        self.mark_position(node)
+        # Function body block
+        self.flow.nextblock()
+
+        if node.star_arg:
+            self.flow.mark_argument(node.star_arg,
+                                    TypedExprNode(Builtin.tuple_type),
+                                    node.star_arg.entry)
+        if node.starstar_arg:
+            self.flow.mark_argument(node.starstar_arg,
+                                    TypedExprNode(Builtin.dict_type),
+                                    node.starstar_arg.entry)
+        self.visitchildren(node)
+        # Workaround for generators
+        if node.is_generator:
+            self.visit(node.gbody.body)
+
+        # Exit point
+        if self.flow.block:
+            self.flow.block.add_child(self.flow.exit_point)
+
+        # Cleanup graph
+        self.flow.normalize()
+        check_definitions(self.flow, self.current_directives)
+        self.flow.blocks.add(self.flow.entry_point)
+
+        self.gv_ctx.add(GV(node.local_scope.name, self.flow))
+
+        self.flow = self.stack.pop()
+        self.env = self.env_stack.pop()
+        return node
+
+    def visit_DefNode(self, node):
+        ## XXX: no target name node here
+        node.used = True
+        self.flow.mark_assignment(node, object_expr, self.env.lookup(node.name))
+        return self.visit_FuncDefNode(node)
+
+    def visit_GeneratorBodyDefNode(self, node):
+        return node
+
+    def visit_CTypeDefNode(self, node):
+        return node
+
+    def mark_assignment(self, lhs, rhs=None):
+        if not self.flow.block:
+            return
+        if self.flow.exceptions:
+            exc_descr = self.flow.exceptions[-1]
+            self.flow.block.add_child(exc_descr.entry_point)
+            self.flow.nextblock()
+
+        if isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode)):
+            self.visit(lhs)
+            return
+
+        if not rhs:
+            rhs = object_expr
+        if lhs.is_name:
+            if lhs.entry is None:
+                # TODO: This shouldn't happen...
+                return
+            self.flow.mark_assignment(lhs, rhs)
+        elif isinstance(lhs, ExprNodes.SequenceNode):
+            for arg in lhs.args:
+                self.mark_assignment(arg)
+        else:
+            # Could use this info to infer cdef class attributes...
+            pass
+
+        if self.flow.exceptions:
+            exc_descr = self.flow.exceptions[-1]
+            self.flow.block.add_child(exc_descr.entry_point)
+            self.flow.nextblock()
+
+    def mark_position(self, node):
+        """Mark position if DOT output is enabled."""
+        if self.current_directives['control_flow.dot_output']:
+            self.flow.mark_position(node)
+
+    def visit_FromImportStatNode(self, node):
+        for name, target in node.items:
+            if name != "*":
+                self.mark_assignment(target)
+        self.visitchildren(node)
+        return node
+
+    def visit_AssignmentNode(self, node):
+        raise InternalError, "Unhandled assignment node"
+
+    def visit_SingleAssignmentNode(self, node):
+        self.visit(node.rhs)
+        self.mark_assignment(node.lhs, node.rhs)
+        return node
+
+    def visit_CascadedAssignmentNode(self, node):
+        self.visit(node.rhs)
+        for lhs in node.lhs_list:
+            self.mark_assignment(lhs, node.rhs)
+        return node
+
+    def visit_ParallelAssignmentNode(self, node):
+        collector = AssignmentCollector()
+        collector.visitchildren(node)
+        for lhs, rhs in collector.assignments:
+            self.visit(rhs)
+        for lhs, rhs in collector.assignments:
+            self.mark_assignment(lhs, rhs)
+        return node
+
+    def visit_InPlaceAssignmentNode(self, node):
+        self.visitchildren(node)
+        self.mark_assignment(node.lhs, node.create_binop_node())
+        return node
+
+    def visit_DelStatNode(self, node):
+        for arg in node.args:
+            if arg.is_name:
+                entry = arg.entry or self.env.lookup(arg.name)
+                if entry.in_closure or entry.from_closure:
+                    error(arg.pos, "can not delete variable '%s' referenced in nested scope" % entry.name)
+                # Mark reference
+                self.visit(arg)
+                self.flow.mark_deletion(arg, entry)
+        return node
+
+    def visit_CArgDeclNode(self, node):
+        entry = self.env.lookup(node.name)
+        self.flow.mark_argument(node, TypedExprNode(entry.type), entry)
+        return node
+
+    def visit_NameNode(self, node):
+        if self.flow.block:
+            entry = node.entry or self.env.lookup(node.name)
+            if entry:
+                self.flow.mark_reference(node, entry)
+        return node
+
+    def visit_StatListNode(self, node):
+        if self.flow.block:
+            for stat in node.stats:
+                self.visit(stat)
+                if not self.flow.block:
+                    stat.is_terminator = True
+                    break
+        return node
+
+    def visit_Node(self, node):
+        self.visitchildren(node)
+        self.mark_position(node)
+        return node
+
+    def visit_IfStatNode(self, node):
+        next_block = self.flow.newblock()
+        parent = self.flow.block
+        # If clauses
+        for clause in node.if_clauses:
+            parent = self.flow.nextblock(parent)
+            self.visit(clause.condition)
+            self.flow.nextblock()
+            self.visit(clause.body)
+            if self.flow.block:
+                self.flow.block.add_child(next_block)
+        # Else clause
+        if node.else_clause:
+            self.flow.nextblock(parent=parent)
+            self.visit(node.else_clause)
+            if self.flow.block:
+                self.flow.block.add_child(next_block)
+        else:
+            parent.add_child(next_block)
+
+        if next_block.parents:
+            self.flow.block = next_block
+        else:
+            self.flow.block = None
+        return node
+
+    def visit_WhileStatNode(self, node):
+        condition_block = self.flow.nextblock()
+        next_block = self.flow.newblock()
+        # Condition block
+        self.flow.loops.append(LoopDescr(next_block, condition_block))
+        self.visit(node.condition)
+        # Body block
+        self.flow.nextblock()
+        self.visit(node.body)
+        self.flow.loops.pop()
+        # Loop it
+        if self.flow.block:
+            self.flow.block.add_child(condition_block)
+            self.flow.block.add_child(next_block)
+        # Else clause
+        if node.else_clause:
+            self.flow.nextblock(parent=condition_block)
+            self.visit(node.else_clause)
+            if self.flow.block:
+                self.flow.block.add_child(next_block)
+        else:
+            condition_block.add_child(next_block)
+
+        if next_block.parents:
+            self.flow.block = next_block
+        else:
+            self.flow.block = None
+        return node
+
+    def visit_ForInStatNode(self, node):
+        condition_block = self.flow.nextblock()
+        next_block = self.flow.newblock()
+        # Condition with iterator
+        self.flow.loops.append(LoopDescr(next_block, condition_block))
+        self.visit(node.iterator)
+        # Target assignment
+        self.flow.nextblock()
+        self.mark_assignment(node.target)
+        # Body block
+        self.flow.nextblock()
+        self.visit(node.body)
+        self.flow.loops.pop()
+        # Loop it
+        if self.flow.block:
+            self.flow.block.add_child(condition_block)
+        # Else clause
+        if node.else_clause:
+            self.flow.nextblock(parent=condition_block)
+            self.visit(node.else_clause)
+            if self.flow.block:
+                self.flow.block.add_child(next_block)
+        else:
+            condition_block.add_child(next_block)
+
+        if next_block.parents:
+            self.flow.block = next_block
+        else:
+            self.flow.block = None
+        return node
+
+    def visit_ForFromStatNode(self, node):
+        condition_block = self.flow.nextblock()
+        next_block = self.flow.newblock()
+        # Condition with iterator
+        self.flow.loops.append(LoopDescr(next_block, condition_block))
+        self.visit(node.bound1)
+        self.visit(node.bound2)
+        if node.step:
+            self.visit(node.step)
+        # Target assignment
+        self.flow.nextblock()
+        self.mark_assignment(node.target)
+
+        # Body block
+        self.flow.nextblock()
+        self.visit(node.body)
+        self.flow.loops.pop()
+        # Loop it
+        if self.flow.block:
+            self.flow.block.add_child(condition_block)
+        # Else clause
+        if node.else_clause:
+            self.flow.nextblock(parent=condition_block)
+            self.visit(node.else_clause)
+            if self.flow.block:
+                self.flow.block.add_child(next_block)
+        else:
+            condition_block.add_child(next_block)
+
+        if next_block.parents:
+            self.flow.block = next_block
+        else:
+            self.flow.block = None
+        return node
+
+    def visit_LoopNode(self, node):
+        raise InternalError, "Generic loops are not supported"
+
+    def visit_WithTargetAssignmentStatNode(self, node):
+        self.mark_assignment(node.lhs)
+        return node
+
+    def visit_WithStatNode(self, node):
+        self.visit(node.manager)
+        self.visit(node.body)
+        return node
+
+    def visit_TryExceptStatNode(self, node):
+        # After exception handling
+        next_block = self.flow.newblock()
+        # Body block
+        self.flow.newblock()
+        # Exception entry point
+        entry_point = self.flow.newblock()
+        self.flow.exceptions.append(ExceptionDescr(entry_point))
+        self.flow.nextblock()
+        ## XXX: links to exception handling point should be added by
+        ## XXX: children nodes
+        self.flow.block.add_child(entry_point)
+        self.visit(node.body)
+        self.flow.exceptions.pop()
+
+        # After exception
+        if self.flow.block:
+            if node.else_clause:
+                self.flow.nextblock()
+                self.visit(node.else_clause)
+            if self.flow.block:
+                self.flow.block.add_child(next_block)
+
+        for clause in node.except_clauses:
+            self.flow.block = entry_point
+            if clause.pattern:
+                for pattern in clause.pattern:
+                    self.visit(pattern)
+            else:
+                # TODO: handle * pattern
+                pass
+            if clause.target:
+                self.mark_assignment(clause.target)
+            entry_point = self.flow.newblock(parent=self.flow.block)
+            self.flow.nextblock()
+            self.visit(clause.body)
+            if self.flow.block:
+                self.flow.block.add_child(next_block)
+
+        if self.flow.exceptions:
+            entry_point.add_child(self.flow.exceptions[-1].entry_point)
+
+        if next_block.parents:
+            self.flow.block = next_block
+        else:
+            self.flow.block = None
+        return node
+
+    def visit_TryFinallyStatNode(self, node):
+        body_block = self.flow.nextblock()
+
+        # Exception entry point
+        entry_point = self.flow.newblock()
+        self.flow.block = entry_point
+        self.visit(node.finally_clause)
+
+        if self.flow.block and self.flow.exceptions:
+            self.flow.block.add_child(self.flow.exceptions[-1].entry_point)
+
+        # Normal execution
+        finally_enter = self.flow.newblock()
+        self.flow.block = finally_enter
+        self.visit(node.finally_clause)
+        finally_exit = self.flow.block
+
+        descr = ExceptionDescr(entry_point, finally_enter, finally_exit)
+        self.flow.exceptions.append(descr)
+        if self.flow.loops:
+            self.flow.loops[-1].exceptions.append(descr)
+        self.flow.block = body_block
+        ## XXX: Is it still required
+        body_block.add_child(entry_point)
+        self.visit(node.body)
+        self.flow.exceptions.pop()
+        if self.flow.loops:
+            self.flow.loops[-1].exceptions.pop()
+
+        if self.flow.block:
+            self.flow.block.add_child(finally_enter)
+            if finally_exit:
+                self.flow.block = self.flow.nextblock(parent=finally_exit)
+            else:
+                self.flow.block = None
+        return node
+
+    def visit_RaiseStatNode(self, node):
+        self.mark_position(node)
+        if self.flow.exceptions:
+            self.flow.block.add_child(self.flow.exceptions[-1].entry_point)
+        self.flow.block = None
+        return node
+
+    def visit_ReraiseStatNode(self, node):
+        self.mark_position(node)
+        if self.flow.exceptions:
+            self.flow.block.add_child(self.flow.exceptions[-1].entry_point)
+        self.flow.block = None
+        return node
+
+    def visit_ReturnStatNode(self, node):
+        self.mark_position(node)
+        self.visitchildren(node)
+
+        for exception in self.flow.exceptions[::-1]:
+            if exception.finally_enter:
+                self.flow.block.add_child(exception.finally_enter)
+                if exception.finally_exit:
+                    exception.finally_exit.add_child(self.flow.exit_point)
+                break
+        else:
+            if self.flow.block:
+                self.flow.block.add_child(self.flow.exit_point)
+        self.flow.block = None
+        return node
+
+    def visit_BreakStatNode(self, node):
+        if not self.flow.loops:
+            #error(node.pos, "break statement not inside loop")
+            return node
+        loop = self.flow.loops[-1]
+        self.mark_position(node)
+        for exception in loop.exceptions[::-1]:
+            if exception.finally_enter:
+                self.flow.block.add_child(exception.finally_enter)
+                if exception.finally_exit:
+                    exception.finally_exit.add_child(loop.next_block)
+                break
+        else:
+            self.flow.block.add_child(loop.next_block)
+        self.flow.block = None
+        return node
+
+    def visit_ContinueStatNode(self, node):
+        if not self.flow.loops:
+            #error(node.pos, "continue statement not inside loop")
+            return node
+        loop = self.flow.loops[-1]
+        self.mark_position(node)
+        for exception in loop.exceptions[::-1]:
+            if exception.finally_enter:
+                self.flow.block.add_child(exception.finally_enter)
+                if exception.finally_exit:
+                    exception.finally_exit.add_child(loop.loop_block)
+                break
+        else:
+            self.flow.block.add_child(loop.loop_block)
+        self.flow.block = None
+        return node
+
+    def visit_ComprehensionNode(self, node):
+        if node.expr_scope:
+            self.env_stack.append(self.env)
+            self.env = node.expr_scope
+        # Skip append node here
+        self.visit(node.target)
+        self.visit(node.loop)
+        if node.expr_scope:
+            self.env = self.env_stack.pop()
+        return node
+
+    def visit_ScopedExprNode(self, node):
+        if node.expr_scope:
+            self.env_stack.append(self.env)
+            self.env = node.expr_scope
+        self.visitchildren(node)
+        if node.expr_scope:
+            self.env = self.env_stack.pop()
+        return node
+
+    def visit_PyClassDefNode(self, node):
+        self.flow.mark_assignment(node.target,
+                                  object_expr, self.env.lookup(node.name))
+        # TODO: add negative attribute list to "visitchildren"?
+        self.visitchildren(node, attrs=['dict', 'metaclass', 'mkw', 'bases', 'classobj'])
+        self.env_stack.append(self.env)
+        self.env = node.scope
+        self.flow.nextblock()
+        self.visitchildren(node, attrs=['body'])
+        self.flow.nextblock()
+        self.env = self.env_stack.pop()
+        return node

Cython/Compiler/Main.py

@@ -110,6 +110,7 @@ class Context(object):
         from TypeInference import MarkAssignments, MarkOverflowingArithmetic
         from ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions
         from ParseTreeTransforms import RemoveUnreachableCode, GilCheck
+        from FlowControl import CreateControlFlowGraph
         from AnalysedTreeTransforms import AutoTestDictTransform
         from AutoDocTransforms import EmbedSignature
         from Optimize import FlattenInListTransform, SwitchTransform, IterationTransform
@@ -149,9 +150,11 @@ class Context(object):
             AutoTestDictTransform(self),
             EmbedSignature(self),
             EarlyReplaceBuiltinCalls(self),  ## Necessary?
+            TransformBuiltinMethods(self),  ## Necessary?
+            CreateControlFlowGraph(self),
+            RemoveUnreachableCode(self),
             MarkAssignments(self),
             MarkOverflowingArithmetic(self),
-            TransformBuiltinMethods(self),  ## Necessary?
             IntroduceBufferAuxiliaryVars(self),
             _check_c_declarations,
             AnalyseExpressionsTransform(self),

Cython/Compiler/Nodes.py

@@ -136,6 +136,9 @@ class Node(object):
     # can either contain a single node or a list of nodes. See Visitor.py.
     child_attrs = None
 
+    cf_state = None
+
+
     def __init__(self, pos, **kw):
         self.pos = pos
         self.__dict__.update(kw)
@@ -1177,6 +1180,8 @@ class FuncDefNode(StatNode, BlockNode):
     #  needs_closure   boolean        Whether or not this function has inner functions/classes/yield
     #  needs_outer_scope boolean      Whether or not this function requires outer scope
     #  directive_locals { string : NameNode } locals defined by cython.locals(...)
+    # star_arg      PyArgDeclNode or None  * argument
+    # starstar_arg  PyArgDeclNode or None  ** argument
 
     py_func = None
     assmt = None
@@ -1185,6 +1190,8 @@ class FuncDefNode(StatNode, BlockNode):
     is_generator = False
     is_generator_body = False
     modifiers = []
+    star_arg = None
+    starstar_arg = None
 
     def analyse_default_values(self, env):
         genv = env.global_scope()
@@ -1409,10 +1416,6 @@ class FuncDefNode(StatNode, BlockNode):
             if entry.type.is_pyobject:
                 if (acquire_gil or entry.assignments) and not entry.in_closure:
                     code.put_var_incref(entry)
-        # ----- Initialise local variables
-        for entry in lenv.var_entries:
-            if entry.type.is_pyobject and entry.init_to_none and entry.used:
-                code.put_init_var_to_py_none(entry)
         # ----- Initialise local buffer auxiliary variables
         for entry in lenv.var_entries + lenv.arg_entries:
             if entry.type.is_buffer and entry.buffer_aux.buffer_info_var.used:
@@ -1940,8 +1943,6 @@ class DefNode(FuncDefNode):
     # lambda_name   string                 the internal name of a lambda 'function'
     # decorators    [DecoratorNode]        list of decorators
     # args          [CArgDeclNode]         formal arguments
-    # star_arg      PyArgDeclNode or None  * argument
-    # starstar_arg  PyArgDeclNode or None  ** argument
     # doc           EncodedString or None
     # body          StatListNode
     # return_type_annotation
@@ -1966,8 +1967,6 @@ class DefNode(FuncDefNode):
     entry = None
     acquire_gil = 0
     self_in_stararg = 0
-    star_arg = None
-    starstar_arg = None
     doc = None
 
     def __init__(self, pos, **kwds):
@@ -2261,7 +2260,6 @@ class DefNode(FuncDefNode):
                 arg.entry = env.declare_var(arg.name, arg.type, arg.pos)
                 if arg.type.is_pyobject:
                     arg.entry.init = "0"
-                arg.entry.init_to_none = 0
             else:
                 arg.entry = self.declare_argument(env, arg)
             arg.entry.used = 1
@@ -2282,7 +2280,6 @@ class DefNode(FuncDefNode):
             entry = env.declare_var(arg.name, type, arg.pos)
             entry.used = 1
             entry.init = "0"
-            entry.init_to_none = 0
             entry.xdecref_cleanup = 1
             arg.entry = entry
             env.control_flow.set_state((), (arg.name, 'initialized'), True)

Cython/Compiler/Optimize.py

@@ -3295,10 +3295,6 @@ class FinalOptimizePhase(Visitor.CythonTransform):
         if node.first:
             lhs = node.lhs
             lhs.lhs_of_first_assignment = True
-            if isinstance(lhs, ExprNodes.NameNode) and lhs.entry.type.is_pyobject:
-                # Have variable initialized to 0 rather than None
-                lhs.entry.init_to_none = False
-                lhs.entry.init = 0
         return node
 
     def visit_SimpleCallNode(self, node):

Cython/Compiler/Options.py

@@ -95,10 +95,19 @@ directive_defaults = {
     'warn': None,
     'warn.undeclared': False,
     'warn.unreachable': True,
+    'warn.maybe_uninitialized': False,
+    'warn.unreachable': True,
+    'warn.unused': False,
+    'warn.unused_arg': False,
+    'warn.unused_result': False,
 
 # remove unreachable code
     'remove_unreachable': True,
 
+# control flow debug directives
+    'control_flow.dot_output': "", # Graphviz output filename
+    'control_flow.dot_annotate_defs': False, # Annotate definitions
+
 # test support
     'test_assert_path_exists' : [],
     'test_fail_if_path_exists' : [],
@@ -107,6 +116,13 @@ directive_defaults = {
     'binding': False,
 }
 
+# Extra warning directives
+extra_warnings = {
+    'warn.maybe_uninitialized': True,
+    'warn.unreachable': True,
+    'warn.unused': True,
+}
+
 # Override types possibilities above, if needed
 directive_types = {
     'final' : bool,  # final cdef classes and methods

Cython/Compiler/ParseTreeTransforms.py

@@ -1540,8 +1540,6 @@ if VALUE is not None:
                 type_name = entry.type.module_name + '.' + type_name
             if entry.init is not None:
                 default_value = ' = ' + entry.init
-            elif entry.init_to_none:
-                default_value = ' = ' + repr(None)
             docstring = attr_name + ': ' + type_name + default_value
             property.doc = EncodedString(docstring)
         # ---------------------------------------
@@ -2203,9 +2201,9 @@ class TransformBuiltinMethods(EnvTransform):
                     return node # nothing to do
             items = [ ExprNodes.DictItemNode(pos,
                                              key=ExprNodes.StringNode(pos, value=var),
-                                             value=ExprNodes.NameNode(pos, name=var))
+                                             value=ExprNodes.NameNode(pos, name=var, allow_null=True))
                       for var in lenv.entries ]
-            return ExprNodes.DictNode(pos, key_value_pairs=items)
+            return ExprNodes.DictNode(pos, key_value_pairs=items, exclude_null_values=True)
         else: # dir()
             if len(node.args) > 1:
                 error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d"

Cython/Compiler/Symtab.py

@@ -96,7 +96,6 @@ class Entry(object):
     #                               holding its home namespace
     # pymethdef_cname  string     PyMethodDef structure
     # signature        Signature  Arg & return types for Python func
-    # init_to_none     boolean    True if initial value should be None
     # as_variable      Entry      Alternative interpretation of extension
     #                               type name or builtin C function as a variable
     # xdecref_cleanup  boolean    Use Py_XDECREF for error cleanup
@@ -157,7 +156,6 @@ class Entry(object):
     func_cname = None
     func_modifiers = []
     doc = None
-    init_to_none = 0
     as_variable = None
     xdecref_cleanup = 0
     in_cinclude = 0
@@ -186,6 +184,8 @@ class Entry(object):
         self.init = init
         self.overloaded_alternatives = []
         self.assignments = []
+        self.cf_assignments = []
+        self.cf_references = []
 
     def __repr__(self):
         return "Entry(name=%s, type=%s)" % (self.name, self.type)
@@ -1388,7 +1388,6 @@ class LocalScope(Scope):
                                   api=api, in_pxd=in_pxd, is_cdef=is_cdef)
         if type.is_pyobject and not Options.init_local_none:
             entry.init = "0"
-        entry.init_to_none = (type.is_pyobject or type.is_unspecified) and Options.init_local_none
         entry.is_local = 1
 
         entry.in_with_gil_block = self._in_with_gil_block

setup.py

@@ -101,6 +101,9 @@ def compile_cython_modules(profile=False, compile_more=False, cython_with_refnan
                         "Cython.Compiler.Visitor",
                         "Cython.Compiler.Code",
                         "Cython.Runtime.refnanny",]
+    if sys.version_info[:2] > (2,4):
+        compiled_modules += [
+            "Cython.Compiler.FlowControl",]
     if compile_more:
         compiled_modules.extend([
             "Cython.Compiler.ParseTreeTransforms",

tests/compile/ass2longlong.pyx

@@ -3,7 +3,7 @@
 cdef void spam():
     cdef long long L
     cdef unsigned long long U
-    cdef object x
+    cdef object x = object()
     L = x
     x = L
     U = x

tests/compile/builtinfuncs.pyx

 # mode: compile
 
 cdef int f() except -1:
-    cdef object x, y, z, w
+    cdef object x, y = 0, z = 0, w = 0
     cdef int i
     x = abs(y)
     delattr(x, 'spam')

tests/compile/extcmethcall.pyx

@@ -14,7 +14,7 @@ cdef class SuperSpam(Spam):
 
 cdef void tomato():
     cdef Spam spam
-    cdef SuperSpam superspam
+    cdef SuperSpam superspam = SuperSpam()
     spam = superspam
     spam.add_tons(42)
     superspam.add_tons(1764)

tests/compile/extcoerce.pyx

@@ -10,8 +10,8 @@ cdef class Swallow:
 
 def f(Grail g):
     cdef int i = 0
-    cdef Swallow s
-    cdef object x
+    cdef Swallow s = Swallow()
+    cdef object x = Grail()
     g = x
     x = g
     g = i

tests/compile/bufaccess_noassignT444.pyx → tests/errors/bufaccess_noassignT444.pyx

 # ticket: 444
-# mode: compile
+# mode: error
 
 def test():
      cdef object[int] not_assigned_to
      not_assigned_to[2] = 3
+
+_ERRORS = """
+6:20: local variable 'not_assigned_to' referenced before assignment
+"""

tests/errors/e_del.pyx

@@ -12,9 +12,18 @@ def f(a):
     del j # error: deletion of non-Python object
     del x[i] # error: deletion of non-Python object
     del s.m # error: deletion of non-Python object
+
+def outer(a):
+    def inner():
+        print a
+    del a
+    return inner()
+
+
 _ERRORS = u"""
-10:6: Cannot assign to or delete this
-11:45: Deletion of non-Python, non-C++ object
-13:6: Deletion of non-Python, non-C++ object
-14:6: Deletion of non-Python, non-C++ object
+10:9: Cannot assign to or delete this
+11:48: Deletion of non-Python, non-C++ object
+13:9: Deletion of non-Python, non-C++ object
+14:9: Deletion of non-Python, non-C++ object
+19:9: can not delete variable 'a' referenced in nested scope
 """

tests/errors/e_extweakref.pyx

@@ -10,7 +10,7 @@ cdef class E:
 	cdef readonly object __weakref__
 
 cdef void f():
-	cdef C c
+	cdef C c = C()
 	cdef object x
 	x = c.__weakref__
 	c.__weakref__ = x

tests/errors/nogil.pyx

@@ -18,7 +18,7 @@ cdef void r() nogil:
     q()
 
 cdef object m():
-    cdef object x, y, obj
+    cdef object x, y = 0, obj
     cdef int i, j, k
     global fred
     q()

tests/errors/w_uninitialized.pyx

+# cython: warn.maybe_uninitialized=True
+# mode: error
+# tag: werror
+
+def simple():
+    print a
+    a = 0
+
+def simple2(arg):
+    if arg > 0:
+        a = 1
+    return a
+
+def simple_pos(arg):
+    if arg > 0:
+        a = 1
+    else:
+        a = 0
+    return a
+
+def ifelif(c1, c2):
+    if c1 == 1:
+        if c2:
+            a = 1
+        else:
+            a = 2
+    elif c1 == 2:
+        a = 3
+    return a
+
+def nowimpossible(a):
+    if a:
+        b = 1
+    if a:
+        print b
+
+def fromclosure():
+    def bar():
+        print a
+    a = 1
+    return bar
+
+# Should work ok in both py2 and py3
+def list_comp(a):
+    return [i for i in a]
+
+def set_comp(a):
+    return set(i for i in a)
+
+def dict_comp(a):
+    return {i: j for i, j in a}
+
+# args and kwargs
+def generic_args_call(*args, **kwargs):
+    return args, kwargs
+
+def cascaded(x):
+    print a, b
+    a = b = x
+
+def from_import():
+    print bar
+    from foo import bar
+
+_ERRORS = """
+6:11: local variable 'a' referenced before assignment
+12:12: local variable 'a' might be referenced before assignment
+29:12: local variable 'a' might be referenced before assignment
+35:15: local variable 'b' might be referenced before assignment
+58:11: local variable 'a' referenced before assignment
+58:14: local variable 'b' referenced before assignment
+62:13: local variable 'bar' referenced before assignment
+"""

tests/errors/w_uninitialized_class.pyx

+# cython: warn.maybe_uninitialized=True
+# mode: error
+# tag: werror
+# class scope
+
+def foo(c):
+    class Foo(object):
+        if c > 0:
+            b = 1
+        print a, b
+        a = 1
+    return Foo
+
+_ERRORS = """
+10:15: local variable 'a' referenced before assignment
+10:18: local variable 'b' might be referenced before assignment
+"""

tests/errors/w_uninitialized_del.pyx

+# cython: warn.maybe_uninitialized=True
+# mode: error
+# tag: werror
+
+def foo(x):
+    a = 1
+    del a, b
+    b = 2
+    return a, b
+
+_ERRORS = """
+7:9: Deletion of non-Python, non-C++ object
+7:12: local variable 'b' referenced before assignment
+7:12: Deletion of non-Python, non-C++ object
+9:12: local variable 'a' referenced before assignment
+"""

tests/errors/w_uninitialized_exc.pyx

+# cython: warn.maybe_uninitialized=True
+# mode: error
+# tag: werror
+
+def exc_target():
+    try:
+        {}['foo']
+    except KeyError, e:
+        pass
+    except IndexError, i:
+        pass
+    return e, i
+
+def exc_body():
+    try:
+        a = 1
+    except Exception:
+        pass
+    return a
+
+def exc_else_pos():
+    try:
+        pass
+    except Exception, e:
+        pass
+    else:
+        e = 1
+    return e
+
+def exc_body_pos(d):
+    try:
+        a = d['foo']
+    except KeyError:
+        a = None
+    return a
+
+def exc_pos():
+    try:
+        a = 1
+    except Exception:
+        a = 1
+    return a
+
+def exc_finally():
+    try:
+        a = 1
+    finally:
+        pass
+    return a
+
+def exc_finally2():
+    try:
+        pass
+    finally:
+        a = 1
+    return a
+
+
+def exc_assmt_except(a):
+    try:
+        x = a
+    except:
+        return x
+
+def exc_assmt_finaly(a):
+    try:
+        x = a
+    except:
+        return x
+
+def raise_stat(a):
+    try:
+        if a < 0:
+            raise IndexError
+    except IndexError:
+        oops = 1
+    print oops
+
+def try_loop(args):
+    try:
+        x = 0
+        for i in args:
+            if i is 0:
+                continue
+            elif i is None:
+                break
+            elif i is False:
+                return
+            i()
+    except ValueError:
+        x = 1
+    finally:
+        return x
+
+def try_finally(a):
+    try:
+        for i in a:
+            if i > 0:
+                x = 1
+    finally:
+        return x
+
+def try_finally_nested(m):
+    try:
+        try:
+            try:
+                f = m()
+            except:
+                pass
+        finally:
+            pass
+    except:
+        print f
+
+_ERRORS = """
+12:12: local variable 'e' might be referenced before assignment
+12:15: local variable 'i' might be referenced before assignment
+19:12: local variable 'a' might be referenced before assignment
+63:16: local variable 'x' might be referenced before assignment
+69:16: local variable 'x' might be referenced before assignment
+77:14: local variable 'oops' might be referenced before assignment
+93:16: local variable 'x' might be referenced before assignment
+101:16: local variable 'x' might be referenced before assignment
+113:15: local variable 'f' might be referenced before assignment
+"""

tests/errors/w_uninitialized_for.pyx

+# cython: warn.maybe_uninitialized=True
+# mode: error
+# tag: werror
+
+def simple_for(n):
+    for i in n:
+        a = 1
+    return a
+
+def simple_for_break(n):
+    for i in n:
+        a = 1
+        break
+    return a
+
+def simple_for_pos(n):
+    for i in n:
+        a = 1
+    else:
+        a = 0
+    return a
+
+def simple_target(n):
+    for i in n:
+        pass
+    return i
+
+def simple_target_f(n):
+    for i in n:
+        i *= i
+    return i
+
+def simple_for_from(n):
+    for i from 0 <= i <= n:
+        x = i
+    else:
+        return x
+
+def for_continue(l):
+    for i in l:
+        if i > 0:
+            continue
+        x = i
+    print x
+
+def for_break(l):
+    for i in l:
+        if i > 0:
+            break
+        x = i
+    print x
+
+def for_finally_continue(f):
+    for i in f:
+        try:
+            x = i()
+        finally:
+            print x
+            continue
+
+def for_finally_break(f):
+    for i in f:
+        try:
+            x = i()
+        finally:
+            print x
+            break
+
+def for_finally_outer(p, f):
+    x = 1
+    try:
+        for i in f:
+            print x
+            x = i()
+            if x > 0:
+                continue
+            if x < 0:
+                break
+    finally:
+        del x
+
+
+_ERRORS = """
+8:12: local variable 'a' might be referenced before assignment
+14:12: local variable 'a' might be referenced before assignment
+26:12: local variable 'i' might be referenced before assignment
+31:12: local variable 'i' might be referenced before assignment
+37:16: local variable 'x' might be referenced before assignment
+44:11: local variable 'x' might be referenced before assignment
+51:11: local variable 'x' might be referenced before assignment
+58:19: local variable 'x' might be referenced before assignment
+66:19: local variable 'x' might be referenced before assignment
+"""

tests/errors/w_uninitialized_generators.pyx

+# cython: warn.maybe_uninitialized=True
+# mode: error
+# tag: werror
+
+def unbound_inside_generator(*args):
+    for i in args:
+        yield x
+        x = i + i
+
+_ERRORS = """
+7:15: local variable 'x' might be referenced before assignment
+"""

tests/errors/w_uninitialized_py2.pyx

+# cython: language_level=2, warn.maybe_uninitialized=True
+# mode: error
+# tag: werror
+
+def list_comp(a):
+    r = [i for i in a]
+    return i
+
+# dict comp is py3 feuture and don't leak here
+def dict_comp(a):
+    r = {i: j for i, j in a}
+    return i, j
+
+def dict_comp2(a):
+    r = {i: j for i, j in a}
+    print i, j
+    i, j = 0, 0
+
+
+_ERRORS = """
+7:12: local variable 'i' might be referenced before assignment
+12:12: undeclared name not builtin: i
+12:15: undeclared name not builtin: j
+16:11: local variable 'i' referenced before assignment
+16:14: local variable 'j' referenced before assignment
+"""

tests/errors/w_uninitialized_py3.pyx

+# cython: language_level=3, warn.maybe_uninitialized=True
+# mode: error
+# tag: werror
+
+def ref(obj):
+    pass
+
+def list_comp(a):
+    r = [i for i in a]
+    ref(i)
+    i = 0
+    return r
+
+def dict_comp(a):
+    r = {i: j for i, j in a}
+    ref(i)
+    i = 0
+    return r
+
+
+_ERRORS = """
+10:9: local variable 'i' referenced before assignment
+16:9: local variable 'i' referenced before assignment
+"""

tests/errors/w_uninitialized_while.pyx

+# cython: warn.maybe_uninitialized=True
+# mode: error
+# tag: werror
+
+def simple_while(n):
+    while n > 0:
+        n -= 1
+        a = 0
+    return a
+
+def simple_while_break(n):
+    while n > 0:
+        n -= 1
+        break
+    else:
+        a = 1
+    return a
+
+def simple_while_pos(n):
+    while n > 0:
+        n -= 1
+        a = 0
+    else:
+        a = 1
+    return a
+
+def while_finally_continue(p, f):
+    while p():
+        try:
+            x = f()
+        finally:
+            print x
+            continue
+
+def while_finally_break(p, f):
+    while p():
+        try:
+            x = f()
+        finally:
+            print x
+            break
+
+def while_finally_outer(p, f):
+    x = 1
+    try:
+        while p():
+            print x
+            x = f()
+            if x > 0:
+                continue
+            if x < 0:
+                break
+    finally:
+        del x
+
+
+_ERRORS = """
+9:12: local variable 'a' might be referenced before assignment
+17:12: local variable 'a' might be referenced before assignment
+32:19: local variable 'x' might be referenced before assignment
+40:19: local variable 'x' might be referenced before assignment
+"""

tests/errors/w_uninitialized_with.pyx

+# cython: warn.maybe_uninitialized=True
+# mode: error
+# tag: werror
+
+def with_no_target(m):
+    with m:
+        print a
+        a = 1
+
+def unbound_manager(m1):
+    with m2:
+        pass
+    m2 = m1
+
+def with_target(m):
+    with m as f:
+        print(f)
+
+def with_mgr(m):
+    try:
+        with m() as f:
+            pass
+    except:
+        print f
+
+_ERRORS = """
+7:15: local variable 'a' referenced before assignment
+11:11: local variable 'm2' referenced before assignment
+24:15: local variable 'f' might be referenced before assignment
+"""

tests/errors/w_unreachable_cf.pyx

+# mode: error
+# tag: werror, unreachable, control-flow
+
+def try_finally():
+    try:
+        return
+    finally:
+        return
+    print 'oops'
+
+def try_return():
+    try:
+        return
+    except:
+        return
+    print 'oops'
+
+def for_return(a):
+    for i in a:
+        return
+    else:
+        return
+    print 'oops'
+
+def while_return(a):
+    while a:
+        return
+    else:
+        return
+    print 'oops'
+
+def forfrom_return(a):
+    for i from 0 <= i <= a:
+        return
+    else:
+        return
+    print 'oops'
+
+_ERRORS = """
+9:4: Unreachable code
+16:4: Unreachable code
+23:4: Unreachable code
+30:4: Unreachable code
+37:4: Unreachable code
+"""

tests/errors/w_unused.pyx

+# cython: warn.unused=True, warn.unused_arg=True, warn.unused_result=True
+# mode: error
+# tag: werror
+
+def unused_variable():
+    a = 1
+
+def unused_cascade(arg):
+    a, b = arg.split()
+    return a
+
+def unused_arg(arg):
+    pass
+
+def unused_result():
+    r = 1 + 1
+    r = 2
+    return r
+
+def unused_nested():
+    def unused_one():
+        pass
+
+def unused_class():
+    class Unused:
+        pass
+
+# this should not generate warning
+def used(x, y):
+    x.y = 1
+    y[0] = 1
+    lambda x: x
+
+_ERRORS = """
+6:6: Unused entry 'a'
+9:9: Unused entry 'b'
+12:15: Unused argument 'arg'
+16:6: Unused result in 'r'
+21:4: Unused entry 'unused_one'
+25:4: Unused entry 'Unused'
+"""

tests/run/bufaccess.pyx

@@ -47,15 +47,6 @@ def nousage():
     """
     cdef object[int, ndim=2] buf
 
-def printbuf():
-    """
-    Just compilation.
-    """
-    cdef object[int, ndim=2] buf
-    print buf
-    return
-    buf[0,0] = 0
-
 @testcase
 def acquire_release(o1, o2):
     """
@@ -681,20 +672,20 @@ def mixed_get(object[int] buf, int unsafe_idx, int safe_idx):
 #
 # Coercions
 #
-@testcase
-def coercions(object[unsigned char] uc):
-    """
-TODO
-    """
-    print type(uc[0])
-    uc[0] = -1
-    print uc[0]
-    uc[0] = <int>3.14
-    print uc[0]
-
-    cdef char* ch = b"asfd"
-    cdef object[object] objbuf
-    objbuf[3] = ch
+## @testcase
+## def coercions(object[unsigned char] uc):
+##     """
+## TODO
+##     """
+##     print type(uc[0])
+##     uc[0] = -1
+##     print uc[0]
+##     uc[0] = <int>3.14
+##     print uc[0]
+
+##     cdef char* ch = b"asfd"
+##     cdef object[object] objbuf
+##     objbuf[3] = ch
 
 
 #

tests/run/delete.pyx

@@ -86,13 +86,13 @@ def del_local(a):
     >>> del_local(object())
     """
     del a
-    assert a is None # Until we have unbound locals...
+    assert 'a' not in locals()
 
 def del_seq(a, b, c):
     """
     >>> del_seq(1, 2, 3)
     """
     del a, (b, c)
-    assert a is None # Until we have unbound locals...
-    assert b is None # Until we have unbound locals...
-    assert c is None # Until we have unbound locals...
+    assert 'a' not in locals()
+    assert 'b' not in locals()
+    assert 'c' not in locals()

tests/run/locals_expressions_T430.pyx

@@ -8,7 +8,7 @@ __doc__ = u"""
 [('args', (2, 3)), ('kwds', {'k': 5}), ('x', 1), ('y', 'hi'), ('z', 5)]
 
 >>> sorted(get_locals_items_listcomp(1,2,3, k=5))
-[('args', (2, 3)), ('item', None), ('kwds', {'k': 5}), ('x', 1), ('y', 'hi'), ('z', 5)]
+[('args', (2, 3)), ('kwds', {'k': 5}), ('x', 1), ('y', 'hi'), ('z', 5)]
 """
 
 def get_locals(x, *args, **kwds):

tests/run/uninitialized.py

+# mode: run
+# tag: control-flow, uninitialized
+
+def conditional(cond):
+    """
+    >>> conditional(True)
+    []
+    >>> conditional(False)
+    Traceback (most recent call last):
+    ...
+    UnboundLocalError: local variable 'a' referenced before assignment
+    """
+    if cond:
+        a = []
+    return a
+
+def inside_loop(iter):
+    """
+    >>> inside_loop([1,2,3])
+    3
+    >>> inside_loop([])
+    Traceback (most recent call last):
+    ...
+    UnboundLocalError: local variable 'i' referenced before assignment
+    """
+    for i in iter:
+        pass
+    return i
+
+def try_except(cond):
+    """
+    >>> try_except(True)
+    []
+    >>> try_except(False)
+    Traceback (most recent call last):
+    ...
+    UnboundLocalError: local variable 'a' referenced before assignment
+    """
+    try:
+        if cond:
+            a = []
+        raise ValueError
+    except ValueError:
+        return a
+
+def try_finally(cond):
+    """
+    >>> try_finally(True)
+    []
+    >>> try_finally(False)
+    Traceback (most recent call last):
+    ...
+    UnboundLocalError: local variable 'a' referenced before assignment
+    """
+    try:
+        if cond:
+            a = []
+        raise ValueError
+    finally:
+        return a
+
+def deleted(cond):
+    """
+    >>> deleted(False)
+    {}
+    >>> deleted(True)
+    Traceback (most recent call last):
+    ...
+    UnboundLocalError: local variable 'a' referenced before assignment
+    """
+    a = {}
+    if cond:
+        del a
+    return a
+
+def test_nested(cond):
+    """
+    >>> test_nested(True)
+    >>> test_nested(False)
+    Traceback (most recent call last):
+    ...
+    UnboundLocalError: local variable 'a' referenced before assignment
+    """
+    if cond:
+        def a():
+            pass
+    return a()
+
+def test_outer(cond):
+    """
+    >>> test_outer(True)
+    {}
+    >>> test_outer(False)
+    Traceback (most recent call last):
+    ...
+    UnboundLocalError: local variable 'a' referenced before assignment
+    """
+    if cond:
+        a = {}
+    def inner():
+        return a
+    return a
+
+def test_inner(cond):
+    """
+    >>> test_inner(True)
+    {}
+    >>> test_inner(False)
+    Traceback (most recent call last):
+    ...
+    NameError: free variable 'a' referenced before assignment in enclosing scope
+    """
+    if cond:
+        a = {}
+    def inner():
+        return a
+    return inner()
+
+def test_class(cond):
+    """
+    >>> test_class(True)
+    1
+    >>> test_class(False)
+    Traceback (most recent call last):
+    ...
+    UnboundLocalError: local variable 'A' referenced before assignment
+    """
+    if cond:
+        class A:
+            x = 1
+    return A.x