Merge in fixes from master

Cython/Compiler/Buffer.py

@@ -754,7 +754,7 @@ def get_type_information_cname(code, dtype, maxdepth=None):
 def load_buffer_utility(util_code_name, **kwargs):
     return UtilityCode.load(util_code_name, "Buffer.c", **kwargs)
 
-context = dict(max_dims=Options.buffer_max_dims)
+context = dict(max_dims=str(Options.buffer_max_dims))
 buffer_struct_declare_code = load_buffer_utility("BufferStructDeclare",
                                                  context=context)
 

Cython/Compiler/Code.py

-# cython: language_level = 3, py2_import=True
+# cython: language_level = 2, py2_import=True
 #
 #   Code output module
 #

Cython/Compiler/CythonScope.py

@@ -9,6 +9,7 @@ import MemoryView
 
 class CythonScope(ModuleScope):
     is_cython_builtin = 1
+    _cythonscope_initialized = False
 
     def __init__(self, context):
         ModuleScope.__init__(self, u'cython', None, None)
@@ -32,11 +33,24 @@ class CythonScope(ModuleScope):
 
         return super(CythonScope, self).lookup_type(name)
 
+    def lookup(self, name):
+        entry = super(CythonScope, self).lookup(name)
+
+        if entry is None and not self._cythonscope_initialized:
+            self.load_cythonscope()
+            entry = super(CythonScope, self).lookup(name)
+
+        return entry
+
     def find_module(self, module_name, pos):
         error("cython.%s is not available" % module_name, pos)
 
     def find_submodule(self, module_name):
         entry = self.entries.get(module_name, None)
+        if not entry:
+            self.load_cythonscope()
+            entry = self.entries.get(module_name, None)
+
         if entry and entry.as_module:
             return entry.as_module
         else:
@@ -77,13 +91,15 @@ class CythonScope(ModuleScope):
             defining = 1,
             cname = 'PyObject_TypeCheck')
 
-#        self.test_cythonscope()
-
-    def test_cythonscope(self):
+    def load_cythonscope(self):
         """
         Creates some entries for testing purposes and entries for
         cython.array() and for cython.view.*.
         """
+        if self._cythonscope_initialized:
+            return
+
+        self._cythonscope_initialized = True
         cython_testscope_utility_code.declare_in_scope(
                                 self, cython_scope=self)
         cython_test_extclass_utility_code.declare_in_scope(

Cython/Compiler/ExprNodes.py

@@ -731,7 +731,7 @@ class ExprNode(Node):
         return self.result_in_temp()
 
     def may_be_none(self):
-        if not self.type.is_pyobject:
+        if self.type and not self.type.is_pyobject:
             return False
         if self.constant_result not in (not_a_constant, constant_value_not_set):
             return self.constant_result is not None
@@ -1564,7 +1564,7 @@ class NameNode(AtomicExprNode):
         return 1
 
     def may_be_none(self):
-        if self.type.is_pyobject and self.cf_state:
+        if self.cf_state and self.type and self.type.is_pyobject:
             # gard against infinite recursion on self-dependencies
             if getattr(self, '_none_checking', False):
                 # self-dependency - either this node receives a None
@@ -6983,6 +6983,16 @@ class NumBinopNode(BinopNode):
         else:
             return None
 
+    def may_be_none(self):
+        type1 = self.operand1.type
+        type2 = self.operand2.type
+        if type1 and type1.is_builtin_type and type2 and type2.is_builtin_type:
+            # XXX: I can't think of any case where a binary operation
+            # on builtin types evaluates to None - add a special case
+            # here if there is one.
+            return False
+        return super(NumBinopNode, self).may_be_none()
+
     def get_constant_c_result_code(self):
         value1 = self.operand1.get_constant_c_result_code()
         value2 = self.operand2.get_constant_c_result_code()

Cython/Compiler/FlowControl.py

@@ -530,6 +530,12 @@ def check_definitions(flow, compiler_directives):
 
     messages.report()
 
+    # Remove Uninitialized from cf_state
+    for node in assmt_nodes:
+        node.cf_state.discard(Uninitialized)
+    for node in references:
+        node.cf_state.discard(Uninitialized)
+
 
 class AssignmentCollector(TreeVisitor):
     def __init__(self):

Cython/Compiler/Main.py

@@ -65,12 +65,7 @@ class Context(object):
 
         import Builtin, CythonScope
         self.modules = {"__builtin__" : Builtin.builtin_scope}
-        #if self.cython_scope is None:
         self.cython_scope = CythonScope.create_cython_scope(self)
-
-        if create_testscope: # and not hasattr(self.cython_scope, 'viewscope'):
-            self.cython_scope.test_cythonscope()
-
         self.modules["cython"] = self.cython_scope
         self.include_directories = include_directories
         self.future_directives = set()

Cython/Compiler/MemoryView.py

@@ -685,6 +685,7 @@ def get_axes_specs(env, axes):
     '''
 
     cythonscope = env.global_scope().context.cython_scope
+    cythonscope.load_cythonscope()
     viewscope = cythonscope.viewscope
 
     access_specs = tuple([viewscope.lookup(name)

Cython/Compiler/Optimize.py

@@ -2359,6 +2359,28 @@ class OptimizeBuiltinCalls(Visitor.EnvTransform):
             may_return_none = True,
             utility_code = dict_getitem_default_utility_code)
 
+    Pyx_PyDict_SetDefault_func_type = PyrexTypes.CFuncType(
+        PyrexTypes.py_object_type, [
+            PyrexTypes.CFuncTypeArg("dict", PyrexTypes.py_object_type, None),
+            PyrexTypes.CFuncTypeArg("key", PyrexTypes.py_object_type, None),
+            PyrexTypes.CFuncTypeArg("default", PyrexTypes.py_object_type, None),
+            ])
+
+    def _handle_simple_method_dict_setdefault(self, node, args, is_unbound_method):
+        """Replace dict.setdefault() by calls to PyDict_GetItem() and PyDict_SetItem().
+        """
+        if len(args) == 2:
+            args.append(ExprNodes.NoneNode(node.pos))
+        elif len(args) != 3:
+            self._error_wrong_arg_count('dict.setdefault', node, args, "2 or 3")
+            return node
+
+        return self._substitute_method_call(
+            node, "__Pyx_PyDict_SetDefault", self.Pyx_PyDict_SetDefault_func_type,
+            'setdefault', is_unbound_method, args,
+            may_return_none = True,
+            utility_code = dict_setdefault_utility_code)
+
 
     ### unicode type methods
 
@@ -3103,6 +3125,45 @@ static PyObject* __Pyx_PyDict_GetItemDefault(PyObject* d, PyObject* key, PyObjec
 impl = ""
 )
 
+dict_setdefault_utility_code = UtilityCode(
+proto = """
+static PyObject *__Pyx_PyDict_SetDefault(PyObject *, PyObject *, PyObject *); /*proto*/
+""",
+impl = '''
+static PyObject *__Pyx_PyDict_SetDefault(PyObject *d, PyObject *key, PyObject *default_value) {
+    PyObject* value;
+#if PY_MAJOR_VERSION >= 3
+    value = PyDict_GetItemWithError(d, key);
+    if (unlikely(!value)) {
+        if (unlikely(PyErr_Occurred()))
+            return NULL;
+        if (unlikely(PyDict_SetItem(d, key, default_value) == -1))
+            return NULL;
+        value = default_value;
+    }
+    Py_INCREF(value);
+#else
+    if (PyString_CheckExact(key) || PyUnicode_CheckExact(key) || PyInt_CheckExact(key)) {
+        /* these presumably have safe hash functions */
+        value = PyDict_GetItem(d, key);
+        if (unlikely(!value)) {
+            if (unlikely(PyDict_SetItem(d, key, default_value) == -1))
+                return NULL;
+            value = default_value;
+        }
+        Py_INCREF(value);
+    } else {
+        PyObject *m;
+        m = __Pyx_GetAttrString(d, "setdefault");
+        if (!m) return NULL;
+        value = PyObject_CallFunctionObjArgs(m, key, default_value, NULL);
+        Py_DECREF(m);
+    }
+#endif
+    return value;
+}
+''')
+
 append_utility_code = UtilityCode(
 proto = """
 static CYTHON_INLINE PyObject* __Pyx_PyObject_Append(PyObject* L, PyObject* x) {

runtests.py

@@ -167,6 +167,13 @@ VER_DEP_MODULES = {
                                           'run.generators_py', # generators, with statement
                                           'run.pure_py', # decorators, with statement
                                           'run.purecdef',
+                                          'run.struct_conversion',
+                                          'run.cythonarray',
+                                          'run.memslice',
+                                          'run.numpy_memoryview',
+                                          'run.memoryviewattrs',
+                                          'run.memoryview',
+                                          'compile.memview_declaration',
                                           ]),
     (2,7) : (operator.lt, lambda x: x in ['run.withstat_py', # multi context with statement
                                           'run.yield_inside_lambda',

tests/bugs.txt

@@ -18,6 +18,7 @@ temp_sideeffects_T654
 class_scope_T671
 slice2_T636
 builtin_subtype_methods_T653
+default_args_T674
 
 # CPython regression tests that don't current work:
 pyregr.test_threadsignals

tests/run/cf_none.pyx

@@ -113,3 +113,10 @@ def self_dependency_none(int x):
     if x:
         a,b = b,a
     return b.get(2)
+
+@cython.test_fail_if_path_exists('//NoneCheckNode')
+def in_place_op():
+    vals = [0]
+    vals += [1]
+    for x in vals:
+        pass

tests/run/default_args_T674.py

+# mode: run
+# ticket: 674
+
+def test_inner(a):
+    """
+    >>> a = test_inner(1)
+    >>> b = test_inner(2)
+    >>> a()
+    1
+    >>> b()
+    2
+    """
+    def inner(b=a):
+        return b
+    return inner
+
+def test_lambda(n):
+    """
+    >>> [f() for f in test_lambda(3)]
+    [0, 1, 2]
+    """
+    return [lambda v=i: v for i in range(n)]

tests/run/dict_setdefault.py

+
+import cython
+
+class Unhashable(object):
+    def __hash__(self):
+        raise TypeError('I am not hashable')
+
+class Hashable(object):
+    def __hash__(self):
+        return 1
+    def __eq__(self, other):
+        return isinstance(other, Hashable)
+
+@cython.test_fail_if_path_exists('//AttributeNode')
+@cython.test_assert_path_exists('//PythonCapiCallNode')
+@cython.locals(d=dict)
+def setdefault1(d, key):
+    """
+    >>> d = {}
+    >>> setdefault1(d, 1)
+    >>> len(d)
+    1
+    >>> setdefault1(d, 1)
+    >>> len(d)
+    1
+    >>> d[1]
+    >>> setdefault1(d, Unhashable())
+    Traceback (most recent call last):
+    TypeError: I am not hashable
+    >>> len(d)
+    1
+    >>> h1 = setdefault1(d, Hashable())
+    >>> len(d)
+    2
+    >>> h2 = setdefault1(d, Hashable())
+    >>> len(d)
+    2
+    >>> d[Hashable()]
+    """
+    return d.setdefault(key)
+
+@cython.test_fail_if_path_exists('//AttributeNode')
+@cython.test_assert_path_exists('//PythonCapiCallNode')
+@cython.locals(d=dict)
+def setdefault2(d, key, value):
+    """
+    >>> d = {}
+    >>> setdefault2(d, 1, 2)
+    2
+    >>> len(d)
+    1
+    >>> setdefault2(d, 1, 2)
+    2
+    >>> len(d)
+    1
+    >>> l = setdefault2(d, 2, [])
+    >>> len(d)
+    2
+    >>> l.append(1)
+    >>> setdefault2(d, 2, [])
+    [1]
+    >>> len(d)
+    2
+    >>> setdefault2(d, Unhashable(), 1)
+    Traceback (most recent call last):
+    TypeError: I am not hashable
+    >>> h1 = setdefault2(d, Hashable(), 55)
+    >>> len(d)
+    3
+    >>> h2 = setdefault2(d, Hashable(), 66)
+    >>> len(d)
+    3
+    >>> d[Hashable()]
+    55
+    """
+    return d.setdefault(key, value)