Merge

Cython/Compiler/Lexicon.py

@@ -68,7 +68,9 @@ def make_lexicon():
     bra = Any("([{")
     ket = Any(")]}")
     punct = Any(":,;+-*/|&<>=.%`~^?")
-    diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=", "//")
+    diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "//",
+                    "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=", 
+                    "<<=", ">>=", "**=", "//=")
     spaces = Rep1(Any(" \t\f"))
     comment = Str("#") + Rep(AnyBut("\n"))
     escaped_newline = Str("\\\n")

Cython/Compiler/ModuleNode.py

@@ -230,6 +230,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         self.generate_typeobj_definitions(env, code)
         self.generate_method_table(env, code)
         self.generate_filename_init_prototype(code)
+        if env.has_import_star:
+            self.generate_import_star(env, code)
         self.generate_module_init_func(modules[:-1], env, code)
         code.mark_pos(None)
         self.generate_module_cleanup_func(env, code)
@@ -423,6 +425,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
 
         code.putln("#if PY_MAJOR_VERSION >= 3")
         code.putln("  #define PyBaseString_Type            PyUnicode_Type")
+        code.putln("  #define PyString_Type                PyBytes_Type")
         code.putln("  #define PyInt_Type                   PyLong_Type")
         code.putln("  #define PyInt_Check(op)              PyLong_Check(op)")
         code.putln("  #define PyInt_CheckExact(op)         PyLong_CheckExact(op)")
@@ -729,7 +732,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     if scope.defines_any(["__setitem__", "__delitem__"]):
                         self.generate_ass_subscript_function(scope, code)
                     if scope.defines_any(["__setslice__", "__delslice__"]):
-                        warning(self.pos, "__setslice__ and __delslice__ are not supported by Python 3")
+                        warning(self.pos, "__setslice__ and __delslice__ are not supported by Python 3", 1)
                         self.generate_ass_slice_function(scope, code)
                     if scope.defines_any(["__getattr__","__getattribute__"]):
                         self.generate_getattro_function(scope, code)
@@ -1429,6 +1432,66 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
     def generate_filename_init_prototype(self, code):
         code.putln("");
         code.putln("static void %s(void); /*proto*/" % Naming.fileinit_cname)
+        
+    def generate_import_star(self, env, code):
+        code.putln()
+        code.putln("char* %s_type_names[] = {" % Naming.import_star)
+        for name, entry in env.entries.items():
+            if entry.is_type:
+                code.putln('"%s",' % name)
+        code.putln("0")
+        code.putln("};")
+        code.putln()
+        code.putln("static int %s(PyObject *o, PyObject* py_name, char *name) {" % Naming.import_star_set)
+        code.putln("char** type_name = %s_type_names;" % Naming.import_star)
+        code.putln("while (*type_name) {")
+        code.putln("if (!strcmp(name, *type_name)) {")
+        code.putln('PyErr_Format(PyExc_TypeError, "Cannot overwrite C type %s", name);')
+        code.putln('goto bad;')
+        code.putln("}")
+        code.putln("type_name++;")
+        code.putln("}")
+        old_error_label = code.new_error_label()
+        code.putln("if (0);") # so the first one can be "else if"
+        for name, entry in env.entries.items():
+            if entry.is_cglobal and entry.used:
+                code.putln('else if (!strcmp(name, "%s")) {' % name)
+                if entry.type.is_pyobject:
+                    if entry.type.is_extension_type or entry.type.is_builtin_type:
+                        code.putln("if (!(%s)) %s;" % (
+                            entry.type.type_test_code("o"),
+                            code.error_goto(entry.pos)))
+                    code.put_var_decref(entry)
+                    code.putln("%s = %s;" % (
+                        entry.cname, 
+                        PyrexTypes.typecast(entry.type, py_object_type, "o")))
+                elif entry.type.from_py_function:
+                    rhs = "%s(o)" % entry.type.from_py_function
+                    if entry.type.is_enum:
+                        rhs = typecast(entry.type, c_long_type, rhs)
+                    code.putln("%s = %s; if (%s) %s;" % (
+                        entry.cname,
+                        rhs,
+                        entry.type.error_condition(entry.cname),
+                        code.error_goto(entry.pos)))
+                    code.putln("Py_DECREF(o);")
+                else:
+                    code.putln('PyErr_Format(PyExc_TypeError, "Cannot convert Python object %s to %s");' % (name, entry.type))
+                    code.putln(code.error_goto(entry.pos))
+                code.putln("}")
+        code.putln("else {")
+        code.putln("if (PyObject_SetAttr(%s, py_name, o) < 0) goto bad;" % Naming.module_cname)
+        code.putln("}")
+        code.putln("return 0;")
+        code.put_label(code.error_label)
+        # This helps locate the offending name.
+        code.putln('__Pyx_AddTraceback("%s");' % self.full_module_name);
+        code.error_label = old_error_label
+        code.putln("bad:")
+        code.putln("Py_DECREF(o);")
+        code.putln("return -1;")
+        code.putln("}")
+        code.putln(import_star_utility_code)
 
     def generate_module_init_func(self, imported_modules, env, code):
         code.putln("")
@@ -1543,6 +1606,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             "if (!%s) %s;" % (
                 env.module_cname,
                 code.error_goto(self.pos)));
+        code.putln(
+            "Py_INCREF(%s);" %
+                env.module_cname)
         code.putln(
             '%s = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME);' %
                 Naming.builtins_cname)
@@ -2016,3 +2082,94 @@ bad:
     return ret;
 }
 """]
+
+import_star_utility_code = """
+
+/* import_all_from is an unexposed function from ceval.c */
+
+static int
+__Pyx_import_all_from(PyObject *locals, PyObject *v)
+{
+	PyObject *all = PyObject_GetAttrString(v, "__all__");
+	PyObject *dict, *name, *value;
+	int skip_leading_underscores = 0;
+	int pos, err;
+
+	if (all == NULL) {
+		if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+			return -1; /* Unexpected error */
+		PyErr_Clear();
+		dict = PyObject_GetAttrString(v, "__dict__");
+		if (dict == NULL) {
+			if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+				return -1;
+			PyErr_SetString(PyExc_ImportError,
+			"from-import-* object has no __dict__ and no __all__");
+			return -1;
+		}
+		all = PyMapping_Keys(dict);
+		Py_DECREF(dict);
+		if (all == NULL)
+			return -1;
+		skip_leading_underscores = 1;
+	}
+
+	for (pos = 0, err = 0; ; pos++) {
+		name = PySequence_GetItem(all, pos);
+		if (name == NULL) {
+			if (!PyErr_ExceptionMatches(PyExc_IndexError))
+				err = -1;
+			else
+				PyErr_Clear();
+			break;
+		}
+		if (skip_leading_underscores &&
+		    PyString_Check(name) &&
+		    PyString_AS_STRING(name)[0] == '_')
+		{
+			Py_DECREF(name);
+			continue;
+		}
+		value = PyObject_GetAttr(v, name);
+		if (value == NULL)
+			err = -1;
+		else if (PyDict_CheckExact(locals))
+			err = PyDict_SetItem(locals, name, value);
+		else
+			err = PyObject_SetItem(locals, name, value);
+		Py_DECREF(name);
+		Py_XDECREF(value);
+		if (err != 0)
+			break;
+	}
+	Py_DECREF(all);
+	return err;
+}
+
+
+static int %s(PyObject* m) {
+
+    int i;
+    int ret = -1;
+    PyObject *locals = 0;
+    PyObject *list = 0;
+    PyObject *name;
+    PyObject *item;
+    
+    locals = PyDict_New();              if (!locals) goto bad;
+    if (__Pyx_import_all_from(locals, m) < 0) goto bad;
+    list = PyDict_Items(locals);        if (!list) goto bad;
+    
+    for(i=0; i<PyList_GET_SIZE(list); i++) {
+        name = PyTuple_GET_ITEM(PyList_GET_ITEM(list, i), 0);
+        item = PyTuple_GET_ITEM(PyList_GET_ITEM(list, i), 1);
+        if (%s(item, name, PyString_AsString(name)) < 0) goto bad;
+    }
+    ret = 0;
+    
+bad:
+    Py_XDECREF(locals);
+    Py_XDECREF(list);
+    return ret;
+}
+""" % ( Naming.import_star, Naming.import_star_set )

Cython/Compiler/Naming.py

@@ -67,6 +67,8 @@ print_function_kwargs   = pyrex_prefix + "print_kwargs"
 cleanup_cname    = pyrex_prefix + "module_cleanup"
 optional_args_cname = pyrex_prefix + "optional_args"
 no_opt_args      = pyrex_prefix + "no_opt_args"
+import_star      = pyrex_prefix + "import_star"
+import_star_set  = pyrex_prefix + "import_star_set"
 
 line_c_macro = "__LINE__"
 

Cython/Compiler/Nodes.py

@@ -2455,21 +2455,34 @@ class InPlaceAssignmentNode(AssignmentNode):
         self.rhs.generate_evaluation_code(code)
         self.dup.generate_subexpr_evaluation_code(code)
         self.dup.generate_result_code(code)
+        if self.operator == "**":
+            extra = ", Py_None"
+        else:
+            extra = ""
         if self.lhs.type.is_pyobject:
             code.putln(
-                "%s = %s(%s, %s); %s" % (
+                "%s = %s(%s, %s%s); %s" % (
                     self.result.result_code, 
                     self.py_operation_function(), 
                     self.dup.py_result(),
                     self.rhs.py_result(),
+                    extra,
                     code.error_goto_if_null(self.result.py_result(), self.pos)))
             self.result.generate_evaluation_code(code) # May be a type check...
             self.rhs.generate_disposal_code(code)
             self.dup.generate_disposal_code(code)
             self.lhs.generate_assignment_code(self.result, code)
         else: 
+            c_op = self.operator
+            if c_op == "//":
+                c_op = "/"
+            elif c_op == "**":
+                if self.lhs.type.is_int and self.rhs.type.is_int:
+                    error(self.pos, "** with two C int types is ambiguous")
+                else:
+                    error(self.pos, "No C inplace power operator")
             # have to do assignment directly to avoid side-effects
-            code.putln("%s %s= %s;" % (self.lhs.result_code, self.operator, self.rhs.result_code) )
+            code.putln("%s %s= %s;" % (self.lhs.result_code, c_op, self.rhs.result_code) )
             self.rhs.generate_disposal_code(code)
         if self.dup.is_temp:
             self.dup.generate_subexpr_disposal_code(code)
@@ -2499,6 +2512,10 @@ class InPlaceAssignmentNode(AssignmentNode):
         "*":		"PyNumber_InPlaceMultiply",
         "/":		"PyNumber_InPlaceDivide",
         "%":		"PyNumber_InPlaceRemainder",
+        "<<":		"PyNumber_InPlaceLshift",
+        ">>":		"PyNumber_InPlaceRshift",
+        "**":		"PyNumber_InPlacePower",
+        "//":		"PyNumber_InPlaceFloorDivide",
     }
 
     def annotate(self, code):
@@ -3663,10 +3680,14 @@ class FromCImportStatNode(StatNode):
         module_scope = env.find_module(self.module_name, self.pos)
         env.add_imported_module(module_scope)
         for pos, name, as_name in self.imported_names:
-            entry = module_scope.find(name, pos)
-            if entry:
-                local_name = as_name or name
-                env.add_imported_entry(local_name, entry, pos)
+            if name == "*":
+                for local_name, entry in module_scope.entries.items():
+                    env.add_imported_entry(local_name, entry, pos)
+            else:
+                entry = module_scope.find(name, pos)
+                if entry:
+                    local_name = as_name or name
+                    env.add_imported_entry(local_name, entry, pos)
 
     def analyse_expressions(self, env):
         pass
@@ -3682,12 +3703,21 @@ class FromImportStatNode(StatNode):
     #  items            [(string, NameNode)]
     #  interned_items   [(string, NameNode)]
     #  item             PyTempNode            used internally
+    #  import_star      boolean               used internally
 
     child_attrs = ["module"]
+    import_star = 0
     
     def analyse_declarations(self, env):
-        for _, target in self.items:
-            target.analyse_target_declaration(env)
+        for name, target in self.items:
+            if name == "*":
+                if not env.is_module_scope:
+                    error(self.pos, "import * only allowed at module level")
+                    return
+                env.has_import_star = 1
+                self.import_star = 1
+            else:
+                target.analyse_target_declaration(env)
     
     def analyse_expressions(self, env):
         import ExprNodes
@@ -3696,15 +3726,27 @@ class FromImportStatNode(StatNode):
         self.item.allocate_temp(env)
         self.interned_items = []
         for name, target in self.items:
-            self.interned_items.append(
-                (env.intern_identifier(name), target))
-            target.analyse_target_expression(env, None)
-            #target.release_target_temp(env) # was release_temp ?!?
+            if name == '*':
+                for _, entry in env.entries.items():
+                    if not entry.is_type and entry.type.is_extension_type:
+                        env.use_utility_code(ExprNodes.type_test_utility_code)
+                        break
+            else:
+                self.interned_items.append(
+                    (env.intern_identifier(name), target))
+                target.analyse_target_expression(env, None)
+                #target.release_target_temp(env) # was release_temp ?!?
         self.module.release_temp(env)
         self.item.release_temp(env)
     
     def generate_execution_code(self, code):
         self.module.generate_evaluation_code(code)
+        if self.import_star:
+            code.putln(
+                'if (%s(%s) < 0) %s;' % (
+                    Naming.import_star,
+                    self.module.py_result(),
+                    code.error_goto(self.pos)))
         for cname, target in self.interned_items:
             code.putln(
                 '%s = PyObject_GetAttr(%s, %s); %s' % (
@@ -4253,7 +4295,11 @@ static void __Pyx_AddTraceback(char *funcname) {
     if (!py_funcname) goto bad;
     py_globals = PyModule_GetDict(%(GLOBALS)s);
     if (!py_globals) goto bad;
+    #if PY_MAJOR_VERSION < 3
     empty_string = PyString_FromStringAndSize("", 0);
+    #else
+    empty_string = PyBytes_FromStringAndSize("", 0);
+    #endif
     if (!empty_string) goto bad;
     py_code = PyCode_New(
         0,            /*int argcount,*/
@@ -4366,17 +4412,18 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) {
             *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL);
         } else if (t->intern) {
             *t->p = PyString_InternFromString(t->s);
+        } else {
+            *t->p = PyString_FromStringAndSize(t->s, t->n - 1);
         }
         #else  /* Python 3+ has unicode identifiers */
         if (t->is_identifier || (t->is_unicode && t->intern)) {
             *t->p = PyUnicode_InternFromString(t->s);
         } else if (t->is_unicode) {
             *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1);
+        } else {
+            *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1);
         }
         #endif
-        else {
-            *t->p = PyString_FromStringAndSize(t->s, t->n - 1);
-        }
         if (!*t->p)
             return -1;
         ++t;

Cython/Compiler/Parsing.py

@@ -755,11 +755,11 @@ def p_expression_or_assignment(s):
         s.next()
         expr_list.append(p_expr(s))
     if len(expr_list) == 1:
-        if re.match("[+*/\%^\&|-]=", s.sy):
+        if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):
             lhs = expr_list[0]
             if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):
                 error(lhs.pos, "Illegal operand for inplace operation.")
-            operator = s.sy[0]
+            operator = s.sy[:-1]
             s.next()
             rhs = p_expr(s)
             return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)
@@ -944,8 +944,11 @@ def p_from_import_statement(s, first_statement = 0):
     else:
         s.error("Expected 'import' or 'cimport'")
     if s.sy == '*':
-        s.error("'import *' not supported")
-    imported_names = [p_imported_name(s)]
+#        s.error("'import *' not supported")
+        imported_names = [(s.position(), "*", None)]
+        s.next()
+    else:
+        imported_names = [p_imported_name(s)]
     while s.sy == ',':
         s.next()
         imported_names.append(p_imported_name(s))
@@ -1080,9 +1083,13 @@ def p_for_bounds(s):
         s.next()
         iterator = p_for_iterator(s)
         return { 'target': target, 'iterator': iterator }
-    elif s.sy == 'from':
-        s.next()
-        bound1 = p_bit_expr(s)
+    else:
+        if s.sy == 'from':
+            s.next()
+            bound1 = p_bit_expr(s)
+        else:
+            # Support shorter "for a <= x < b" syntax
+            bound1, target = target, None
         rel1 = p_for_from_relation(s)
         name2_pos = s.position()
         name2 = p_ident(s)
@@ -1090,12 +1097,15 @@ def p_for_bounds(s):
         rel2 = p_for_from_relation(s)
         bound2 = p_bit_expr(s)
         step = p_for_from_step(s)
-        if not target.is_name:
-            error(target.pos, 
-                "Target of for-from statement must be a variable name")
-        elif name2 != target.name:
-            error(name2_pos,
-                "Variable name in for-from range does not match target")
+        if target is None:
+            target = ExprNodes.NameNode(name2_pos, name = name2)
+        else:
+            if not target.is_name:
+                error(target.pos, 
+                    "Target of for-from statement must be a variable name")
+            elif name2 != target.name:
+                error(name2_pos,
+                    "Variable name in for-from range does not match target")
         if rel1[0] != rel2[0]:
             error(rel2_pos,
                 "Relation directions in for-from do not match")
@@ -1105,8 +1115,6 @@ def p_for_bounds(s):
                 'relation2': rel2,
                 'bound2': bound2,
                 'step': step }
-    else:
-        s.error("Expected 'in' or 'from'")
 
 def p_for_from_relation(s):
     if s.sy in inequality_relations:
@@ -1287,7 +1295,7 @@ def p_DEF_statement(s):
     return Nodes.PassStatNode(pos)
 
 def p_IF_statement(s, level, cdef_flag, visibility, api):
-    pos = s.position
+    pos = s.position()
     saved_eval = s.compile_time_eval
     current_eval = saved_eval
     denv = s.compile_time_env

Cython/Compiler/PyrexTypes.py

@@ -929,8 +929,8 @@ class CStringType:
     is_string = 1
     is_unicode = 0
     
-    to_py_function = "PyString_FromString"
-    from_py_function = "PyString_AsString"
+    to_py_function = "__Pyx_PyBytes_FromString"
+    from_py_function = "__Pyx_PyBytes_AsString"
     exception_value = "NULL"
 
     def literal_code(self, value):
@@ -1172,6 +1172,14 @@ def typecast(to_type, from_type, expr_code):
 type_conversion_predeclarations = """
 /* Type Conversion Predeclarations */
 
+#if PY_MAJOR_VERSION < 3
+#define __Pyx_PyBytes_FromString PyString_FromString
+#define __Pyx_PyBytes_AsString   PyString_AsString
+#else
+#define __Pyx_PyBytes_FromString PyBytes_FromString
+#define __Pyx_PyBytes_AsString   PyBytes_AsString
+#endif
+
 #define __Pyx_PyBool_FromLong(b) ((b) ? (Py_INCREF(Py_True), Py_True) : (Py_INCREF(Py_False), Py_False))
 static INLINE int __Pyx_PyObject_IsTrue(PyObject* x);
 static INLINE PY_LONG_LONG __pyx_PyInt_AsLongLong(PyObject* x);

Cython/Compiler/Symtab.py

@@ -693,8 +693,10 @@ class ModuleScope(Scope):
     # interned_nums        [int/long]         Interned numeric constants
     # all_pystring_entries [Entry]            Python string consts from all scopes
     # types_imported       {PyrexType : 1}    Set of types for which import code generated
+    # has_import_star      boolean            Module contains import *
     
     is_module_scope = 1
+    has_import_star = 0
 
     def __init__(self, name, parent_module, context):
         self.parent_module = parent_module
@@ -734,7 +736,10 @@ class ModuleScope(Scope):
     
     def declare_builtin(self, name, pos):
         if not hasattr(__builtin__, name):
-            if self.outer_scope is not None:
+            if self.has_import_star:
+                entry = self.declare_var(name, py_object_type, pos)
+                return entry
+            elif self.outer_scope is not None:
                 return self.outer_scope.declare_builtin(name, pos)
             else:
                 error(pos, "undeclared name not builtin: %s"%name)

runtests.py

@@ -17,20 +17,31 @@ CFLAGS = os.getenv('CFLAGS', '').split()
 
 
 class ErrorWriter(object):
-    match_error = re.compile('(?:.*:)?([-0-9]+):([-0-9]+):(.*)').match
+    match_error = re.compile('(warning:)?(?:.*:)?([-0-9]+):([-0-9]+):(.*)').match
     def __init__(self):
         self.output = []
         self.write = self.output.append
 
-    def geterrors(self):
+    def _collect(self, collect_errors, collect_warnings):
         s = ''.join(self.output)
-        errors = []
+        result = []
         for line in s.split('\n'):
             match = self.match_error(line)
             if match:
-                line, column, message = match.groups()
-                errors.append( "%d:%d:%s" % (int(line), int(column), message.strip()) )
-        return errors
+                is_warning, line, column, message = match.groups()
+                if (is_warning and collect_warnings) or \
+                        (not is_warning and collect_errors):
+                    result.append( "%d:%d:%s" % (int(line), int(column), message.strip()) )
+        return result
+
+    def geterrors(self):
+        return self._collect(True, False)
+
+    def getwarnings(self):
+        return self._collect(False, True)
+
+    def getall(self):
+        return self._collect(True, True)
 
 class TestBuilder(object):
     def __init__(self, rootdir, workdir, selectors, annotate, cleanup_workdir):
@@ -259,6 +270,8 @@ if __name__ == '__main__':
             CompilationOptions, \
             default_options as pyrex_default_options, \
             compile as cython_compile
+        from Cython.Compiler import Errors
+        Errors.LEVEL = 0 # show all warnings
 
     # RUN ALL TESTS!
     ROOTDIR = os.path.join(os.getcwd(), os.path.dirname(sys.argv[0]), 'tests')

tests/compile/forfromelse.pyx

@@ -5,3 +5,8 @@ cdef void spam():
     else:
         k = j
 
+    # new syntax
+    for 0 <= i < 10:
+        j = i
+    else:
+        j = k

tests/compile/indices.pyx

+cdef int* a
+cdef object x
+
+cdef int f(int i):
+    print i
+    return i
+
+x[f(1)] = 3
+a[f(1)] = 3
+
+x[f(2)] += 4
+a[f(2)] += 4
+
+print x[1]
+print a[1]
+
+x[<object>f(1)] = 15
\ No newline at end of file

tests/run/big_indices.pyx

+__doc__ = u"""
+    >>> test()
+    neg -1
+    pos 4294967294
+    neg
+    pos
+    neg
+    pos
+"""
+
+def test():
+    cdef long neg = -1
+    cdef unsigned long pos = -2 # will be a large positive number
+
+    print "neg", neg
+    print "pos", pos
+
+    D = { neg: 'neg', pos: 'pos' }
+
+    print D[<object>neg]
+    print D[<object>pos]
+
+    print D[neg]
+    print D[pos]
+    

tests/run/inplace.pyx

+__doc__ = u"""
+    >>> f(5, 7)
+    29509034655744
+
+    >>> g(13, 4)
+    32
+
+    >>> h(56, 7)
+    105.0
+"""
+
+def f(a,b):
+    a += b
+    a *= b
+    a **= b
+    return a
+
+def g(int a, int b):
+    a -= b
+    a /= b
+    a <<= b
+    return a
+
+def h(double a, double b):
+    a /= b
+    a += b
+    a *= b
+    return a