Merge branch 'master' of https://github.com/cython/cython

.hgtags

@@ -37,3 +37,4 @@ ef9d2c680684d0df7d81f529cda29e9e1741f575 cython-0.10.1
 7fa84cb6d3d75eb3d015aeeb60bf8b642171fe93 0.14.beta2
 7fa84cb6d3d75eb3d015aeeb60bf8b642171fe93 0.14.beta2
 8412b39fbc3eb709a543e2f1e95c0c8881ea9ed4 0.14.beta2
+a6b9f0a6d02d23fc3d3a9d0587867faa3afb2fcd 0.14.rc0

Cython/Build/Dependencies.py

 from glob import glob
 import re, os, sys
+from cython import set
 
-try:
-    set
-except NameError:
-    # Python 2.3
-    from sets import Set as set
 
 from distutils.extension import Extension
 
@@ -56,8 +52,8 @@ distutils_settings = {
     'runtime_library_dirs': transitive_list,
     'include_dirs':         transitive_list,
     'extra_objects':        list,
-    'extra_compile_args':   list,
-    'extra_link_args':      list,
+    'extra_compile_args':   transitive_list,
+    'extra_link_args':      transitive_list,
     'export_symbols':       list,
     'depends':              transitive_list,
     'language':             transitive_str,
@@ -97,7 +93,7 @@ class DistutilsInfo(object):
         elif exn is not None:
             for key in distutils_settings:
                 if key in ('name', 'sources'):
-                    pass
+                    continue
                 value = getattr(exn, key, None)
                 if value:
                     self.values[key] = value
@@ -168,7 +164,7 @@ def strip_string_literals(code, prefix='__Pyx_L'):
 
         # Try to close the quote.
         elif in_quote:
-            if code[q-1] == '\\':
+            if code[q-1] == '\\' and not raw:
                 k = 2
                 while q >= k and code[q-k] == '\\':
                     k += 1
@@ -177,7 +173,7 @@ def strip_string_literals(code, prefix='__Pyx_L'):
                     continue
             if code[q:q+len(in_quote)] == in_quote:
                 counter += 1
-                label = "%s%s" % (prefix, counter)
+                label = "%s%s_" % (prefix, counter)
                 literals[label] = code[start+len(in_quote):q]
                 new_code.append("%s%s%s" % (in_quote, label, in_quote))
                 q += len(in_quote)
@@ -193,7 +189,7 @@ def strip_string_literals(code, prefix='__Pyx_L'):
                 end = None
             new_code.append(code[start:hash_mark+1])
             counter += 1
-            label = "%s%s" % (prefix, counter)
+            label = "%s%s_" % (prefix, counter)
             literals[label] = code[hash_mark+1:end]
             new_code.append(label)
             if end is None:
@@ -208,11 +204,11 @@ def strip_string_literals(code, prefix='__Pyx_L'):
                 in_quote = code[q]*3
             else:
                 in_quote = code[q]
-            end = q
-            while end>0 and code[end-1] in 'rRbBuU':
-                if code[end-1] in 'rR':
+            end = marker = q
+            while marker > 0 and code[marker-1] in 'rRbBuU':
+                if code[marker-1] in 'rR':
                     raw = True
-                end -= 1
+                marker -= 1
             new_code.append(code[start:end])
             start = q
             q += len(in_quote)
@@ -314,8 +310,8 @@ class DependencyTree(object):
             self_pxd = []
         a = self.cimports(filename)
         b = filter(None, [self.find_pxd(m, filename) for m in self.cimports(filename)])
-        if len(a) != len(b):
-            print(filename)
+        if len(a) - int('cython' in a) != len(b):
+            print("missing cimport", filename)
             print("\n\t".join(a))
             print("\n\t".join(b))
         return tuple(self_pxd + filter(None, [self.find_pxd(m, filename) for m in self.cimports(filename)]))
@@ -390,9 +386,14 @@ def create_dependency_tree(ctx=None):
     return _dep_tree
 
 # This may be useful for advanced users?
-def create_extension_list(patterns, ctx=None, aliases=None):
+def create_extension_list(patterns, exclude=[], ctx=None, aliases=None):
     seen = set()
     deps = create_dependency_tree(ctx)
+    to_exclude = set()
+    if not isinstance(exclude, list):
+        exclude = [exclude]
+    for pattern in exclude:
+        to_exclude.update(glob(pattern))
     if not isinstance(patterns, list):
         patterns = [patterns]
     module_list = []
@@ -416,27 +417,39 @@ def create_extension_list(patterns, ctx=None, aliases=None):
         else:
             raise TypeError(pattern)
         for file in glob(filepattern):
+            if file in to_exclude:
+                continue
             pkg = deps.package(file)
             if '*' in name:
                 module_name = deps.fully_qualifeid_name(file)
             else:
                 module_name = name
             if module_name not in seen:
+                kwds = deps.distutils_info(file, aliases, base).values
+                if base is not None:
+                    for key, value in base.values.items():
+                        if key not in kwds:
+                            kwds[key] = value
                 module_list.append(exn_type(
                         name=module_name,
                         sources=[file],
-                        **deps.distutils_info(file, aliases, base).values))
+                        **kwds))
+                m = module_list[-1]
                 seen.add(name)
     return module_list
 
 # This is the user-exposed entry point.
-def cythonize(module_list, nthreads=0, aliases=None, **options):
+def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, **options):
     if 'include_path' not in options:
         options['include_path'] = ['.']
     c_options = CompilationOptions(**options)
     cpp_options = CompilationOptions(**options); cpp_options.cplus = True
     ctx = c_options.create_context()
-    module_list = create_extension_list(module_list, ctx=ctx, aliases=aliases)
+    module_list = create_extension_list(
+        module_list,
+        exclude=exclude,
+        ctx=ctx,
+        aliases=aliases)
     deps = create_dependency_tree(ctx)
     to_compile = []
     for m in module_list:
@@ -463,7 +476,11 @@ def cythonize(module_list, nthreads=0, aliases=None, **options):
                     dep_timestamp, dep = deps.newest_dependency(source)
                     priority = 2 - (dep in deps.immediate_dependencies(source))
                 if c_timestamp < dep_timestamp:
-                    print("Compiling %s because it depends on %s" % (source, dep))
+                    if not quiet:
+                        if source == dep:
+                            print("Compiling %s because it changed." % source)
+                        else:
+                            print("Compiling %s because it depends on %s." % (source, dep))
                     to_compile.append((priority, source, c_file, options))
                 new_sources.append(c_file)
             else:

Cython/Build/Inline.py

-#no doctest
-print "Warning: Using prototype cython.inline code..."
-
 import tempfile
 import sys, os, re, inspect
+from cython import set
 
 try:
     import hashlib
@@ -19,6 +17,16 @@ from Cython.Compiler.ParseTreeTransforms import CythonTransform, SkipDeclaration
 from Cython.Compiler.TreeFragment import parse_from_strings
 from Cython.Build.Dependencies import strip_string_literals, cythonize
 
+# A utility function to convert user-supplied ASCII strings to unicode.
+if sys.version_info[0] < 3:
+    def to_unicode(s):
+        if not isinstance(s, unicode):
+            return s.decode('ascii')
+        else:
+            return s
+else:
+    to_unicode = lambda x: x
+
 _code_cache = {}
 
 
@@ -30,11 +38,10 @@ class AllSymbols(CythonTransform, SkipDeclarations):
         self.names.add(node.name)
 
 def unbound_symbols(code, context=None):
+    code = to_unicode(code)
     if context is None:
         context = Context([], default_options)
     from Cython.Compiler.ParseTreeTransforms import AnalyseDeclarationsTransform
-    if isinstance(code, str):
-        code = code.decode('ascii')
     tree = parse_from_strings('(tree fragment)', code)
     for phase in context.create_pipeline(pxd=False):
         if phase is None:
@@ -51,7 +58,14 @@ def unbound_symbols(code, context=None):
             unbound.append(name)
     return unbound
 
-def get_type(arg, context=None):
+def unsafe_type(arg, context=None):
+    py_type = type(arg)
+    if py_type is int:
+        return 'long'
+    else:
+        return safe_type(arg, context)
+
+def safe_type(arg, context=None):
     py_type = type(arg)
     if py_type in [list, tuple, dict, str]:
         return py_type.__name__
@@ -61,8 +75,6 @@ def get_type(arg, context=None):
         return 'double'
     elif py_type is bool:
         return 'bint'
-    elif py_type is int:
-        return 'long'
     elif 'numpy' in sys.modules and isinstance(arg, sys.modules['numpy'].ndarray):
         return 'numpy.ndarray[numpy.%s_t, ndim=%s]' % (arg.dtype.name, arg.ndim)
     else:
@@ -77,15 +89,20 @@ def get_type(arg, context=None):
         return 'object'
 
 def cython_inline(code, 
-                  types='aggressive',
+                  get_type=unsafe_type,
                   lib_dir=os.path.expanduser('~/.cython/inline'),
                   cython_include_dirs=['.'],
+                  force=False,
+                  quiet=False,
                   locals=None,
                   globals=None,
                   **kwds):
+    if get_type is None:
+        get_type = lambda x: 'object'
+    code = to_unicode(code)
     code, literals = strip_string_literals(code)
     code = strip_common_indent(code)
-    ctx = Context(include_dirs, default_options)
+    ctx = Context(cython_include_dirs, default_options)
     if locals is None:
         locals = inspect.currentframe().f_back.f_back.f_locals
     if globals is None:
@@ -99,10 +116,11 @@ def cython_inline(code,
             elif symbol in globals:
                 kwds[symbol] = globals[symbol]
             else:
-                print "Couldn't find ", symbol
+                print("Couldn't find ", symbol)
     except AssertionError:
+        if not quiet:
             # Parsing from strings not fully supported (e.g. cimports).
-        print "Could not parse code as a string (to extract unbound symbols)."
+            print("Could not parse code as a string (to extract unbound symbols).")
     arg_names = kwds.keys()
     arg_names.sort()
     arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names])
@@ -113,8 +131,12 @@ def cython_inline(code,
             os.makedirs(lib_dir)
         if lib_dir not in sys.path:
             sys.path.append(lib_dir)
+        if force:
+            raise ImportError
+        else:
             __import__(module_name)
     except ImportError:
+        cflags = []
         c_include_dirs = []
         cimports = []
         qualified = re.compile(r'([.\w]+)[.]')
@@ -126,6 +148,7 @@ def cython_inline(code,
                 if m.groups()[0] == 'numpy':
                     import numpy
                     c_include_dirs.append(numpy.get_include())
+                    cflags.append('-Wno-unused')
         module_body, func_body = extract_func_code(code)
         params = ', '.join(['%s %s' % a for a in arg_sigs])
         module_code = """
@@ -141,10 +164,11 @@ def __invoke(%(params)s):
         extension = Extension(
             name = module_name,
             sources = [pyx_file],
-            include_dirs = c_include_dirs)
+            include_dirs = c_include_dirs,
+            extra_compile_args = cflags)
         build_extension = build_ext(Distribution())
         build_extension.finalize_options()
-        build_extension.extensions = cythonize([extension], ctx=ctx)
+        build_extension.extensions = cythonize([extension], ctx=ctx, quiet=quiet)
         build_extension.build_temp = os.path.dirname(pyx_file)
         build_extension.build_lib  = lib_dir
         build_extension.run()

Cython/Build/Tests/TestInline.py

+from Cython.Shadow import inline
+from Cython.Build.Inline import safe_type
+from Cython.TestUtils import CythonTest
+
+try:
+    import numpy
+    has_numpy = True
+except:
+    has_numpy = False
+
+test_kwds = dict(force=True, quiet=True)
+
+global_value = 100
+
+class TestInline(CythonTest):
+
+    def test_simple(self):
+        self.assertEquals(inline("return 1+2", **test_kwds), 3)
+
+    def test_types(self):
+        self.assertEquals(inline("""
+            cimport cython
+            return cython.typeof(a), cython.typeof(b)
+        """, a=1.0, b=[], **test_kwds), ('double', 'list object'))
+
+    def test_locals(self):
+        a = 1
+        b = 2
+        self.assertEquals(inline("return a+b", **test_kwds), 3)
+
+    def test_globals(self):
+        self.assertEquals(inline("return global_value + 1", **test_kwds), global_value + 1)
+
+    if has_numpy:
+    
+        def test_numpy(self):
+            import numpy
+            a = numpy.ndarray((10, 20))
+            a[0,0] = 10
+            self.assertEquals(safe_type(a), 'numpy.ndarray[numpy.float64_t, ndim=2]')
+            self.assertEquals(inline("return a[0,0]", a=a, **test_kwds), 10.0)

Cython/Build/Tests/TestStripLiterals.py

+from Cython.Build.Dependencies import strip_string_literals
+
+from Cython.TestUtils import CythonTest
+
+class TestStripLiterals(CythonTest):
+
+    def t(self, before, expected):
+        actual, literals = strip_string_literals(before, prefix="_L")
+        self.assertEquals(expected, actual)
+        for key, value in literals.items():
+            actual = actual.replace(key, value)
+        self.assertEquals(before, actual)
+
+    def test_empty(self):
+        self.t("", "")
+
+    def test_single_quote(self):
+        self.t("'x'", "'_L1_'")
+
+    def test_double_quote(self):
+        self.t('"x"', '"_L1_"')
+
+    def test_nested_quotes(self):
+        self.t(""" '"' "'" """, """ '_L1_' "_L2_" """)
+
+    def test_triple_quote(self):
+        self.t(" '''a\n''' ", " '''_L1_''' ")
+
+    def test_backslash(self):
+        self.t(r"'a\'b'", "'_L1_'")
+        self.t(r"'a\\'", "'_L1_'")
+        self.t(r"'a\\\'b'", "'_L1_'")
+
+    def test_unicode(self):
+        self.t("u'abc'", "u'_L1_'")
+
+    def test_raw(self):
+        self.t(r"r'abc\'", "r'_L1_'")
+
+    def test_raw_unicode(self):
+        self.t(r"ru'abc\'", "ru'_L1_'")
+
+    def test_comment(self):
+        self.t("abc # foo", "abc #_L1_")
+
+    def test_comment_and_quote(self):
+        self.t("abc # 'x'", "abc #_L1_")
+        self.t("'abc#'", "'_L1_'")
+
+    def test_include(self):
+        self.t("include 'a.pxi' # something here",
+               "include '_L1_' #_L2_")
+
+    def test_extern(self):
+        self.t("cdef extern from 'a.h': # comment",
+               "cdef extern from '_L1_': #_L2_")
+

Cython/Compiler/ExprNodes.py

@@ -4045,7 +4045,7 @@ class ListNode(SequenceNode):
             self.obj_conversion_errors = []
             if not self.type.subtype_of(dst_type):
                 error(self.pos, "Cannot coerce list to type '%s'" % dst_type)
-        elif dst_type.is_ptr:
+        elif dst_type.is_ptr and dst_type.base_type is not PyrexTypes.c_void_type:
             base_type = dst_type.base_type
             self.type = PyrexTypes.CArrayType(base_type, len(self.args))
             for i in range(len(self.original_args)):
@@ -4181,12 +4181,13 @@ class ScopedExprNode(ExprNode):
                 code.put_var_declaration(entry)
                 if entry.type.is_pyobject and entry.used:
                     py_entries.append(entry)
-                    code.put_init_var_to_py_none(entry)
         if not py_entries:
             # no local Python references => no cleanup required
             generate_inner_evaluation_code(code)
             code.putln('} /* exit inner scope */')
             return
+        for entry in py_entries:
+            code.put_init_var_to_py_none(entry)
 
         # must free all local Python references at each exit point
         old_loop_labels = tuple(code.new_loop_labels())

Cython/Compiler/ParseTreeTransforms.py

@@ -20,12 +20,6 @@ from Cython.Compiler.TreeFragment import TreeFragment, TemplateTransform
 from Cython.Compiler.StringEncoding import EncodedString
 from Cython.Compiler.Errors import error, warning, CompileError, InternalError
 
-
-try:
-    set
-except NameError:
-    from sets import Set as set
-
 import copy
 
 
@@ -1590,7 +1584,7 @@ class DebugTransform(CythonTransform):
     
     def __init__(self, context, options, result):
         super(DebugTransform, self).__init__(context)
-        self.visited = set()
+        self.visited = cython.set()
         # our treebuilder and debug output writer 
         # (see Cython.Debugger.debug_output.CythonDebugWriter)
         self.tb = self.context.gdb_debug_outputwriter

Cython/Compiler/PyrexTypes.py

@@ -635,6 +635,8 @@ class CNumericType(CType):
     
     is_numeric = 1
     default_value = "0"
+    has_attributes = True
+    scope = None
     
     sign_words = ("unsigned ", "", "signed ")
     
@@ -659,6 +661,23 @@ class CNumericType(CType):
             base_code = public_decl(type_name, dll_linkage)
         return self.base_declaration_code(base_code, entity_code)
                     
+    def attributes_known(self):
+        if self.scope is None:
+            import Symtab
+            self.scope = scope = Symtab.CClassScope(
+                    '',
+                    None,
+                    visibility="extern")
+            scope.parent_type = self
+            scope.directives = {}
+            entry = scope.declare_cfunction(
+                    "conjugate",
+                    CFuncType(self, [CFuncTypeArg("self", self, None)], nogil=True),
+                    pos=None,
+                    defining=1,
+                    cname=" ")
+        return True
+
 
 type_conversion_predeclarations = ""
 type_conversion_functions = ""
@@ -1080,7 +1099,7 @@ class CComplexType(CNumericType):
             scope.declare_var("imag", self.real_type, None, "imag", is_cdef=True)
             entry = scope.declare_cfunction(
                     "conjugate",
-                    CFuncType(self, [CFuncTypeArg("self", self, None)]),
+                    CFuncType(self, [CFuncTypeArg("self", self, None)], nogil=True),
                     pos=None,
                     defining=1,
                     cname="__Pyx_c_conj%s" % self.funcsuffix)

Cython/Shadow.py

@@ -188,6 +188,7 @@ except NameError: # Py3
 py_float = float
 py_complex = complex
 
+
 try:
     # Python 3
     from builtins import set, frozenset

Cython/__init__.py

-__version__ = "0.14.beta2"
+__version__ = "0.14.rc0"
 
 # Void cython.* directives (for case insensitive operating systems). 
 from Cython.Shadow import *

Demos/numeric_demo.pyx

-#
-#  This example demonstrates how to access the internals
-#  of a Numeric array object.
-#
-
-cdef extern from "Numeric/arrayobject.h":
-
-  struct PyArray_Descr:
-    int type_num, elsize
-    char type
-
-  ctypedef class Numeric.ArrayType [object PyArrayObject]:
-    cdef char *data
-    cdef int nd
-    cdef int *dimensions, *strides
-    cdef object base
-    cdef PyArray_Descr *descr
-    cdef int flags
-
-def print_2d_array(ArrayType a):
-  print "Type:", chr(a.descr.type)
-  if chr(a.descr.type) <> "f":
-    raise TypeError("Float array required")
-  if a.nd <> 2:
-    raise ValueError("2 dimensional array required")
-  cdef int nrows, ncols
-  cdef float *elems, x
-  nrows = a.dimensions[0]
-  ncols = a.dimensions[1]
-  elems = <float *>a.data
-  hyphen = "-"
-  divider = ("+" + 10 * hyphen) * ncols + "+"
-  print divider
-  for row in range(nrows):
-    for col in range(ncols):
-      x = elems[row * ncols + col]
-      print "| %8f" % x,
-    print "|"
-    print divider

Demos/numpy_demo.pyx

+cimport numpy
+import numpy
+
+def sum_of_squares(numpy.ndarray[double, ndim=1] arr):
+    cdef long N = arr.shape[0]
+    cdef double ss = 0
+    for i in range(N):
+        ss += arr[i]**2
+    return ss

Demos/setup.py

-import glob
+# Run as:
+#    python setup.py build_ext --inplace
 
 from distutils.core import setup
 from distutils.extension import Extension
-from Cython.Distutils import build_ext
+from Cython.Build import cythonize
 
+ext_modules = cythonize("*.pyx", exclude="numpy_*.pyx")
+
+# Only compile the following if numpy is installed.
 try:
     from numpy.distutils.misc_util import get_numpy_include_dirs
-    numpy_include_dirs = get_numpy_include_dirs()
-except:
-    numpy_include_dirs = []
-
-ext_modules=[ 
-    Extension("primes",       ["primes.pyx"]),
-    Extension("spam",         ["spam.pyx"]),
-]
-
-for file in glob.glob("*.pyx"):
-    if file != "numeric_demo.pyx":
-        ext_modules.append(Extension(file[:-4], [file], include_dirs = numpy_include_dirs))
+    numpy_demo = Extension("*",
+                           ["numpy_*.pyx"],
+                           include_dirs=get_numpy_include_dirs())
+    ext_modules.extend(cythonize(numpy_demo))
+except ImportError:
+    pass
 
 setup(
   name = 'Demos',
-  cmdclass = {'build_ext': build_ext},
   ext_modules = ext_modules,
 )

Demos/spam.pyx

@@ -5,7 +5,7 @@
 cdef class Spam:
   cdef public int amount
 
-  def __new__(self):
+  def __cinit__(self):
     self.amount = 0
 
   def __dealloc__(self):

runtests.py

@@ -645,7 +645,11 @@ class CythonUnitTestCase(CythonCompileTestCase):
             pass
 
 
-include_debugger = sys.version_info[:2] > (2, 5)
+try:
+    import gdb
+    include_debugger = sys.version_info[:2] > (2, 5)
+except:
+    include_debugger = False
 
 def collect_unittests(path, module_prefix, suite, selectors):
     def file_matches(filename):
@@ -703,14 +707,14 @@ def collect_doctests(path, module_prefix, suite, selectors):
                 filename in blacklist)
     import doctest, types
     for dirpath, dirnames, filenames in os.walk(path):
-        parentname = os.path.split(dirpath)[-1]
-        if package_matches(parentname):
+        for dir in list(dirnames):
+            if not package_matches(dir):
+                dirnames.remove(dir)
         for f in filenames:
             if file_matches(f):
                 if not f.endswith('.py'): continue
                 filepath = os.path.join(dirpath, f)
                 if os.path.getsize(filepath) == 0: continue
-                    if 'no doctest' in open(filepath).next(): continue
                 filepath = filepath[:-len(".py")]
                 modulename = module_prefix + filepath[len(path)+1:].replace(os.path.sep, '.')
                 if not [ 1 for match in selectors if match(modulename) ]:

setup.py

@@ -264,6 +264,7 @@ packages = [
     'Cython.Distutils',
     'Cython.Plex',
     'Cython.Tests',
+    'Cython.Build.Tests',
     'Cython.Compiler.Tests',
 ]
 

tests/run/complex_numbers_T305.pyx

@@ -162,6 +162,11 @@ def test_conjugate_typedef(cdouble z):
     """
     return z.conjugate()
 
+cdef cdouble test_conjugate_nogil(cdouble z) nogil:
+    # Really just a compile test.
+    return z.conjugate()
+test_conjugate_nogil(0) # use it
+
 ## cdef extern from "complex_numbers_T305.h":
 ##     ctypedef double double_really_float "myfloat"
 ##     ctypedef float float_really_double "mydouble"