Merge branch 'master' into mypy

.gitignore

@@ -11,14 +11,15 @@ Cython/Compiler/*.c
 Cython/Plex/*.c
 Cython/Runtime/refnanny.c
 Cython/Tempita/*.c
+Cython/*.c
 
 Tools/*.elc
 
-TEST_TMP/
-build/
-wheelhouse*/
+/TEST_TMP/
+/build/
+/wheelhouse*/
 !tests/build/
-dist/
+/dist/
 .gitrev
 .coverage
 *.orig

.travis.yml

@@ -4,10 +4,17 @@ sudo: false
 
 addons:
   apt:
+    sources:
+      - ubuntu-toolchain-r-test
     packages:
       - gdb
       - python-dbg
       - python3-dbg
+      - gcc-6
+      - g++-6
+      # GCC-7 currently takes 5-7 *minutes* to download on travis
+      #- gcc-7
+      #- g++-7
 
 cache:
   pip: true
@@ -32,14 +39,16 @@ env:
     - USE_CCACHE=1
     - CCACHE_SLOPPINESS=pch_defines,time_macros
     - CCACHE_COMPRESS=1
-    - CCACHE_MAXSIZE=100M
-    - PATH="/usr/lib/ccache:$PATH"
+    - CCACHE_MAXSIZE=150M
+    - PATH="/usr/lib/ccache:$HOME/gcc-symlinks:$PATH"
   matrix:
     - BACKEND=c
     - BACKEND=cpp
 
 matrix:
   include:
+    #- python: 3.7-dev
+    #  env: BACKEND=c PY=3 CC=gcc-7
     - os: osx
       osx_image: xcode6.4
       env: BACKEND=c PY=2
@@ -71,7 +80,6 @@ matrix:
   allow_failures:
     - python: pypy
     - python: pypy3
-    - python: 3.7-dev
   exclude:
     - python: pypy
       env: BACKEND=cpp
@@ -84,6 +92,15 @@ branches:
     - release
 
 before_install:
+  - |
+    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+      mkdir "$HOME/gcc-symlinks"
+      ln -s /usr/bin/gcc-6 $HOME/gcc-symlinks/gcc 
+      ln -s /usr/bin/g++-6 $HOME/gcc-symlinks/g++ 
+      
+      if [ -n "$CC" ]; then "$CC" --version; else gcc --version; fi
+    fi
+
   - |
     if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then # Install Miniconda
       curl -s -o miniconda.sh https://repo.continuum.io/miniconda/Miniconda$PY-latest-MacOSX-x86_64.sh;
@@ -94,7 +111,7 @@ before_install:
 
 install:
   - python -c 'import sys; print("Python %s" % (sys.version,))'
-  - if [ -n "${TRAVIS_PYTHON_VERSION##*-dev}" -a -n "${TRAVIS_PYTHON_VERSION##2.6*}" ]; then pip install -r test-requirements.txt $( [ -z "${TRAVIS_PYTHON_VERSION##pypy*}" ] || echo " -r test-requirements-cpython.txt" ) ; fi
+  - if [ -n "${TRAVIS_PYTHON_VERSION##*-dev}" -a -n "${TRAVIS_PYTHON_VERSION##2.6*}" ]; then pip install -r test-requirements.txt $( [ -z "${TRAVIS_PYTHON_VERSION##pypy*}" ] || echo " -r test-requirements-cpython.txt" ) $( [ -n "${TRAVIS_PYTHON_VERSION##3.3*}" ] || echo " tornado<5.0" ) ; fi
   - CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build
 
 before_script: ccache -s || true

Cython/Build/IpythonMagic.py

@@ -14,7 +14,7 @@ Magic command interface for interactive work with Cython
 Usage
 =====
 
-To enable the magics below, execute ``%load_ext cythonmagic``.
+To enable the magics below, execute ``%load_ext cython``.
 
 ``%%cython``
 

Cython/Build/Tests/TestCyCache.py

+import difflib
+import glob
+import gzip
+import os
+import tempfile
+
+import Cython.Build.Dependencies
+import Cython.Utils
+from Cython.TestUtils import CythonTest
+
+
+class TestCyCache(CythonTest):
+
+    def setUp(self):
+        CythonTest.setUp(self)
+        self.temp_dir = tempfile.mkdtemp(
+            prefix='cycache-test',
+            dir='TEST_TMP' if os.path.isdir('TEST_TMP') else None)
+        self.src_dir = tempfile.mkdtemp(prefix='src', dir=self.temp_dir)
+        self.cache_dir = tempfile.mkdtemp(prefix='cache', dir=self.temp_dir)
+
+    def cache_files(self, file_glob):
+        return glob.glob(os.path.join(self.cache_dir, file_glob))
+
+    def fresh_cythonize(self, *args, **kwargs):
+        Cython.Utils.clear_function_caches()
+        Cython.Build.Dependencies._dep_tree = None  # discard method caches
+        Cython.Build.Dependencies.cythonize(*args, **kwargs)
+
+    def test_cycache_switch(self):
+        content1 = 'value = 1\n'
+        content2 = 'value = 2\n'
+        a_pyx = os.path.join(self.src_dir, 'a.pyx')
+        a_c = a_pyx[:-4] + '.c'
+
+        open(a_pyx, 'w').write(content1)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        self.assertEqual(1, len(self.cache_files('a.c*')))
+        a_contents1 = open(a_c).read()
+        os.unlink(a_c)
+
+        open(a_pyx, 'w').write(content2)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        a_contents2 = open(a_c).read()
+        os.unlink(a_c)
+
+        self.assertNotEqual(a_contents1, a_contents2, 'C file not changed!')
+        self.assertEqual(2, len(self.cache_files('a.c*')))
+
+        open(a_pyx, 'w').write(content1)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        self.assertEqual(2, len(self.cache_files('a.c*')))
+        a_contents = open(a_c).read()
+        self.assertEqual(
+            a_contents, a_contents1,
+            msg='\n'.join(list(difflib.unified_diff(
+                a_contents.split('\n'), a_contents1.split('\n')))[:10]))
+
+    def test_cycache_uses_cache(self):
+        a_pyx = os.path.join(self.src_dir, 'a.pyx')
+        a_c = a_pyx[:-4] + '.c'
+        open(a_pyx, 'w').write('pass')
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        a_cache = os.path.join(self.cache_dir, os.listdir(self.cache_dir)[0])
+        gzip.GzipFile(a_cache, 'wb').write('fake stuff'.encode('ascii'))
+        os.unlink(a_c)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        a_contents = open(a_c).read()
+        self.assertEqual(a_contents, 'fake stuff',
+                         'Unexpected contents: %s...' % a_contents[:100])
+
+    def test_multi_file_output(self):
+        a_pyx = os.path.join(self.src_dir, 'a.pyx')
+        a_c = a_pyx[:-4] + '.c'
+        a_h = a_pyx[:-4] + '.h'
+        a_api_h = a_pyx[:-4] + '_api.h'
+        open(a_pyx, 'w').write('cdef public api int foo(int x): return x\n')
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        expected = [a_c, a_h, a_api_h]
+        for output in expected:
+            self.assertTrue(os.path.exists(output), output)
+            os.unlink(output)
+        self.fresh_cythonize(a_pyx, cache=self.cache_dir)
+        for output in expected:
+            self.assertTrue(os.path.exists(output), output)
+
+    def test_options_invalidation(self):
+        hash_pyx = os.path.join(self.src_dir, 'options.pyx')
+        hash_c = hash_pyx[:-len('.pyx')] + '.c'
+
+        open(hash_pyx, 'w').write('pass')
+        self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False)
+        self.assertEqual(1, len(self.cache_files('options.c*')))
+
+        os.unlink(hash_c)
+        self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=True)
+        self.assertEqual(2, len(self.cache_files('options.c*')))
+
+        os.unlink(hash_c)
+        self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False, show_version=False)
+        self.assertEqual(2, len(self.cache_files('options.c*')))
+
+        os.unlink(hash_c)
+        self.fresh_cythonize(hash_pyx, cache=self.cache_dir, cplus=False, show_version=True)
+        self.assertEqual(2, len(self.cache_files('options.c*')))

Cython/Compiler/Annotate.py

@@ -79,14 +79,6 @@ class AnnotationCCodeWriter(CCodeWriter):
             css.append(HtmlFormatter().get_style_defs('.cython'))
         return '\n'.join(css)
 
-    _js = """
-    function toggleDiv(id) {
-        theDiv = id.nextElementSibling
-        if (theDiv.style.display != 'block') theDiv.style.display = 'block';
-        else theDiv.style.display = 'none';
-    }
-    """.strip()
-
     _css_template = textwrap.dedent("""
         body.cython { font-family: courier; font-size: 12; }
 
@@ -114,6 +106,14 @@ class AnnotationCCodeWriter(CCodeWriter):
         .cython.code .c_call  { color: #0000FF; }
     """)
 
+    # on-click toggle function to show/hide C source code
+    _onclick_attr = ' onclick="{0}"'.format((
+        "(function(s){"
+        "    s.display =  s.display === 'block' ? 'none' : 'block'"
+        "})(this.nextElementSibling.style)"
+        ).replace(' ', '')  # poor dev's JS minification
+    )
+
     def save_annotation(self, source_filename, target_filename, coverage_xml=None):
         with Utils.open_source_file(source_filename) as f:
             code = f.read()
@@ -141,9 +141,6 @@ class AnnotationCCodeWriter(CCodeWriter):
                 <style type="text/css">
                 {css}
                 </style>
-                <script>
-                {js}
-                </script>
             </head>
             <body class="cython">
             <p><span style="border-bottom: solid 1px grey;">Generated by Cython {watermark}</span>{more_info}</p>
@@ -151,7 +148,7 @@ class AnnotationCCodeWriter(CCodeWriter):
                 <span style="background-color: #FFFF00">Yellow lines</span> hint at Python interaction.<br />
                 Click on a line that starts with a "<code>+</code>" to see the C code that Cython generated for it.
             </p>
-            ''').format(css=self._css(), js=self._js, watermark=Version.watermark,
+            ''').format(css=self._css(), watermark=Version.watermark,
                         filename=os.path.basename(source_filename) if source_filename else '',
                         more_info=coverage_info)
         ]
@@ -253,7 +250,7 @@ class AnnotationCCodeWriter(CCodeWriter):
                      calls['py_macro_api'] + calls['pyx_macro_api'])
 
             if c_code:
-                onclick = " onclick='toggleDiv(this)'"
+                onclick = self._onclick_attr
                 expandsymbol = '+'
             else:
                 onclick = ''

Cython/Compiler/Buffer.py

@@ -370,7 +370,7 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buf_entry,
     pybuffernd_struct = buffer_aux.buflocal_nd_var.cname
     flags = get_flags(buffer_aux, buffer_type)
 
-    code.putln("{")  # Set up necesarry stack for getbuffer
+    code.putln("{")  # Set up necessary stack for getbuffer
     code.putln("__Pyx_BufFmt_StackElem __pyx_stack[%d];" % buffer_type.dtype.struct_nesting_depth())
 
     getbuffer = get_getbuffer_call(code, "%s", buffer_aux, buffer_type) # fill in object below

Cython/Compiler/Builtin.py

@@ -124,7 +124,8 @@ builtin_function_table = [
                                   PyrexTypes.c_double_complex_type,
                                   PyrexTypes.c_longdouble_complex_type)
                         ) + [
-    BuiltinFunction('abs',        "O",    "O",     "PyNumber_Absolute"),
+    BuiltinFunction('abs',        "O",    "O",     "__Pyx_PyNumber_Absolute",
+                    utility_code=UtilityCode.load("py_abs", "Builtins.c")),
     #('all',       "",     "",      ""),
     #('any',       "",     "",      ""),
     #('ascii',     "",     "",      ""),
@@ -328,7 +329,10 @@ builtin_types_table = [
     ("set",       "PySet_Type",    [BuiltinMethod("__contains__",  "TO",   "b", "PySequence_Contains"),
                                     BuiltinMethod("clear",   "T",  "r", "PySet_Clear"),
                                     # discard() and remove() have a special treatment for unhashable values
-#                                    BuiltinMethod("discard", "TO", "r", "PySet_Discard"),
+                                    BuiltinMethod("discard", "TO", "r", "__Pyx_PySet_Discard",
+                                                  utility_code=UtilityCode.load("py_set_discard", "Optimize.c")),
+                                    BuiltinMethod("remove",  "TO", "r", "__Pyx_PySet_Remove",
+                                                  utility_code=UtilityCode.load("py_set_remove", "Optimize.c")),
                                     # update is actually variadic (see Github issue #1645)
 #                                    BuiltinMethod("update",     "TO", "r", "__Pyx_PySet_Update",
 #                                                  utility_code=UtilityCode.load_cached("PySet_Update", "Builtins.c")),
@@ -388,6 +392,8 @@ def init_builtin_types():
         utility = builtin_utility_code.get(name)
         if name == 'frozenset':
             objstruct_cname = 'PySetObject'
+        elif name == 'bytearray':
+            objstruct_cname = 'PyByteArrayObject'
         elif name == 'bool':
             objstruct_cname = None
         elif name == 'Exception':

Cython/Compiler/CmdLine.py

@@ -154,6 +154,8 @@ def parse_command_line(args):
                 options.capi_reexport_cincludes = True
             elif option == "--fast-fail":
                 Options.fast_fail = True
+            elif option == "--cimport-from-pyx":
+                Options.cimport_from_pyx = True
             elif option in ('-Werror', '--warning-errors'):
                 Options.warning_errors = True
             elif option in ('-Wextra', '--warning-extra'):

Cython/Compiler/Code.pxd

@@ -2,6 +2,7 @@
 from __future__ import absolute_import
 
 cimport cython
+from ..StringIOTree cimport StringIOTree
 
 
 cdef class UtilityCodeBase(object):
@@ -95,7 +96,27 @@ cdef class StringConst:
 
 #def funccontext_property(name):
 
-#class CCodeWriter(object):
+cdef class CCodeWriter(object):
+    cdef readonly StringIOTree buffer
+    cdef readonly list pyclass_stack
+    cdef readonly object globalstate
+    cdef readonly object funcstate
+    cdef object code_config
+    cdef object last_pos
+    cdef object last_marked_pos
+    cdef Py_ssize_t level
+    cdef public Py_ssize_t call_level  # debug-only, see Nodes.py
+    cdef bint bol
+
+    cpdef write(self, s)
+    cpdef put(self, code)
+    cpdef put_safe(self, code)
+    cpdef putln(self, code=*, bint safe=*)
+    @cython.final
+    cdef increase_indent(self)
+    @cython.final
+    cdef decrease_indent(self)
+
 
 cdef class PyrexCodeWriter:
     cdef public object f

Cython/Compiler/CythonScope.py

@@ -26,6 +26,10 @@ class CythonScope(ModuleScope):
                                          cname='<error>')
             entry.in_cinclude = True
 
+    def is_cpp(self):
+        # Allow C++ utility code in C++ contexts.
+        return self.context.cpp
+
     def lookup_type(self, name):
         # This function should go away when types are all first-level objects.
         type = parse_basic_type(name)

Cython/Compiler/FusedNode.py

@@ -276,7 +276,7 @@ class FusedCFuncDefNode(StatListNode):
 
     def _fused_instance_checks(self, normal_types, pyx_code, env):
         """
-        Genereate Cython code for instance checks, matching an object to
+        Generate Cython code for instance checks, matching an object to
         specialized types.
         """
         for specialized_type in normal_types:
@@ -390,7 +390,7 @@ class FusedCFuncDefNode(StatListNode):
             coerce_from_py_func=memslice_type.from_py_function,
             dtype=dtype)
         decl_code.putln(
-            "{{memviewslice_cname}} {{coerce_from_py_func}}(object)")
+            "{{memviewslice_cname}} {{coerce_from_py_func}}(object, int)")
 
         pyx_code.context.update(
             specialized_type_name=specialized_type.specialization_string,
@@ -400,7 +400,7 @@ class FusedCFuncDefNode(StatListNode):
             u"""
                 # try {{dtype}}
                 if itemsize == -1 or itemsize == {{sizeof_dtype}}:
-                    memslice = {{coerce_from_py_func}}(arg)
+                    memslice = {{coerce_from_py_func}}(arg, 0)
                     if memslice.memview:
                         __PYX_XDEC_MEMVIEW(&memslice, 1)
                         # print 'found a match for the buffer through format parsing'
@@ -421,10 +421,11 @@ class FusedCFuncDefNode(StatListNode):
         # The first thing to find a match in this loop breaks out of the loop
         pyx_code.put_chunk(
             u"""
+                """ + (u"arg_is_pythran_compatible = False" if pythran_types else u"") + u"""
                 if ndarray is not None:
                     if isinstance(arg, ndarray):
                         dtype = arg.dtype
-                        arg_is_pythran_compatible = True
+                        """ + (u"arg_is_pythran_compatible = True" if pythran_types else u"") + u"""
                     elif __pyx_memoryview_check(arg):
                         arg_base = arg.base
                         if isinstance(arg_base, ndarray):
@@ -438,24 +439,30 @@ class FusedCFuncDefNode(StatListNode):
                     if dtype is not None:
                         itemsize = dtype.itemsize
                         kind = ord(dtype.kind)
-                        # We only support the endianness of the current compiler
+                        dtype_signed = kind == 'i'
+            """)
+        pyx_code.indent(2)
+        if pythran_types:
+            pyx_code.put_chunk(
+                u"""
+                        # Pythran only supports the endianness of the current compiler
                         byteorder = dtype.byteorder
                         if byteorder == "<" and not __Pyx_Is_Little_Endian():
                             arg_is_pythran_compatible = False
-                        if byteorder == ">" and __Pyx_Is_Little_Endian():
+                        elif byteorder == ">" and __Pyx_Is_Little_Endian():
                             arg_is_pythran_compatible = False
-                        dtype_signed = kind == 'i'
                         if arg_is_pythran_compatible:
                             cur_stride = itemsize
-                            for dim,stride in zip(reversed(arg.shape),reversed(arg.strides)):
-                                if stride != cur_stride:
+                            shape = arg.shape
+                            strides = arg.strides
+                            for i in range(arg.ndim-1, -1, -1):
+                                if (<Py_ssize_t>strides[i]) != cur_stride:
                                     arg_is_pythran_compatible = False
                                     break
-                                cur_stride *= dim
+                                cur_stride *= <Py_ssize_t> shape[i]
                             else:
-                                arg_is_pythran_compatible = not (arg.flags.f_contiguous and arg.ndim > 1)
+                                arg_is_pythran_compatible = not (arg.flags.f_contiguous and (<Py_ssize_t>arg.ndim) > 1)
                 """)
-        pyx_code.indent(2)
         pyx_code.named_insertion_point("numpy_dtype_checks")
         self._buffer_check_numpy_dtype(pyx_code, buffer_types, pythran_types)
         pyx_code.dedent(2)
@@ -464,7 +471,7 @@ class FusedCFuncDefNode(StatListNode):
             self._buffer_parse_format_string_check(
                     pyx_code, decl_code, specialized_type, env)
 
-    def _buffer_declarations(self, pyx_code, decl_code, all_buffer_types):
+    def _buffer_declarations(self, pyx_code, decl_code, all_buffer_types, pythran_types):
         """
         If we have any buffer specializations, write out some variable
         declarations and imports.
@@ -484,10 +491,14 @@ class FusedCFuncDefNode(StatListNode):
                 cdef Py_ssize_t itemsize
                 cdef bint dtype_signed
                 cdef char kind
-                cdef bint arg_is_pythran_compatible
 
                 itemsize = -1
-                arg_is_pythran_compatible = False
+            """)
+
+        if pythran_types:
+            pyx_code.local_variable_declarations.put_chunk(u"""
+                cdef bint arg_is_pythran_compatible
+                cdef Py_ssize_t cur_stride
             """)
 
         pyx_code.imports.put_chunk(
@@ -514,7 +525,7 @@ class FusedCFuncDefNode(StatListNode):
                     pyx_code.local_variable_declarations.put_chunk(
                         u"""
                             cdef bint {{dtype_name}}_is_signed
-                            {{dtype_name}}_is_signed = <{{dtype_type}}> -1 < 0
+                            {{dtype_name}}_is_signed = not (<{{dtype_type}}> -1 > 0)
                         """)
 
     def _split_fused_types(self, arg):
@@ -670,7 +681,7 @@ class FusedCFuncDefNode(StatListNode):
                 default_idx += 1
 
         if all_buffer_types:
-            self._buffer_declarations(pyx_code, decl_code, all_buffer_types)
+            self._buffer_declarations(pyx_code, decl_code, all_buffer_types, pythran_types)
             env.use_utility_code(Code.UtilityCode.load_cached("Import", "ImportExport.c"))
             env.use_utility_code(Code.UtilityCode.load_cached("ImportNumPyArray", "ImportExport.c"))
 

Cython/Compiler/MemoryView.py

@@ -28,12 +28,12 @@ def concat_flags(*flags):
 
 format_flag = "PyBUF_FORMAT"
 
-memview_c_contiguous = "(PyBUF_C_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
-memview_f_contiguous = "(PyBUF_F_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
-memview_any_contiguous = "(PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT | PyBUF_WRITABLE)"
-memview_full_access = "PyBUF_FULL"
-#memview_strided_access = "PyBUF_STRIDED"
-memview_strided_access = "PyBUF_RECORDS"
+memview_c_contiguous = "(PyBUF_C_CONTIGUOUS | PyBUF_FORMAT)"
+memview_f_contiguous = "(PyBUF_F_CONTIGUOUS | PyBUF_FORMAT)"
+memview_any_contiguous = "(PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT)"
+memview_full_access = "PyBUF_FULL_RO"
+#memview_strided_access = "PyBUF_STRIDED_RO"
+memview_strided_access = "PyBUF_RECORDS_RO"
 
 MEMVIEW_DIRECT = '__Pyx_MEMVIEW_DIRECT'
 MEMVIEW_PTR    = '__Pyx_MEMVIEW_PTR'
@@ -484,18 +484,23 @@ def copy_c_or_fortran_cname(memview):
     return "__pyx_memoryview_copy_slice_%s_%s" % (
             memview.specialization_suffix(), c_or_f)
 
+
 def get_copy_new_utility(pos, from_memview, to_memview):
-    if from_memview.dtype != to_memview.dtype:
-        return error(pos, "dtypes must be the same!")
+    if (from_memview.dtype != to_memview.dtype and
+            not (from_memview.dtype.is_const and from_memview.dtype.const_base_type == to_memview.dtype)):
+        error(pos, "dtypes must be the same!")
+        return
     if len(from_memview.axes) != len(to_memview.axes):
-        return error(pos, "number of dimensions must be same")
+        error(pos, "number of dimensions must be same")
+        return
     if not (to_memview.is_c_contig or to_memview.is_f_contig):
-        return error(pos, "to_memview must be c or f contiguous.")
+        error(pos, "to_memview must be c or f contiguous.")
+        return
 
     for (access, packing) in from_memview.axes:
         if access != 'direct':
-            return error(
-                    pos, "cannot handle 'full' or 'ptr' access at this time.")
+            error(pos, "cannot handle 'full' or 'ptr' access at this time.")
+            return
 
     if to_memview.is_c_contig:
         mode = 'c'
@@ -516,6 +521,7 @@ def get_copy_new_utility(pos, from_memview, to_memview):
             dtype_is_object=int(to_memview.dtype.is_pyobject)),
         requires=[copy_contents_new_utility])
 
+
 def get_axes_specs(env, axes):
     '''
     get_axes_specs(env, axes) -> list of (access, packing) specs for each axis.

Cython/Compiler/Options.py

@@ -171,6 +171,8 @@ _directive_defaults = {
     'language_level': 2,
     'fast_getattr': False, # Undocumented until we come up with a better way to handle this everywhere.
     'py2_import': False, # For backward compatibility of Cython's source code in Py3 source mode
+    'preliminary_late_includes_cy28': False,  # Temporary directive in 0.28, to be removed in a later version (see GH#2079).
+    'iterable_coroutine': False,  # Make async coroutines backwards compatible with the old asyncio yield-from syntax.
     'c_string_type': 'bytes',
     'c_string_encoding': '',
     'type_version_tag': True,   # enables Py_TPFLAGS_HAVE_VERSION_TAG on extension types
@@ -194,6 +196,7 @@ _directive_defaults = {
 # optimizations
     'optimize.inline_defnode_calls': True,
     'optimize.unpack_method_calls': True,   # increases code size when True
+    'optimize.unpack_method_calls_in_pyinit': False,   # uselessly increases code size when True
     'optimize.use_switch': True,
 
 # remove unreachable code
@@ -319,6 +322,7 @@ directive_scopes = { # defaults to available everywhere
     'old_style_globals': ('module',),
     'np_pythran': ('module',),
     'fast_gil': ('module',),
+    'iterable_coroutine': ('module', 'function'),
 }
 
 

Cython/Compiler/ParseTreeTransforms.pxd

@@ -7,9 +7,6 @@ from .Visitor cimport (
     CythonTransform, VisitorTransform, TreeVisitor,
     ScopeTrackingTransform, EnvTransform)
 
-cdef class NameNodeCollector(TreeVisitor):
-    cdef list name_nodes
-
 cdef class SkipDeclarations: # (object):
     pass
 

Cython/Compiler/ParseTreeTransforms.py

@@ -25,20 +25,6 @@ from .StringEncoding import EncodedString, _unicode
 from .Errors import error, warning, CompileError, InternalError
 from .Code import UtilityCode
 
-class NameNodeCollector(TreeVisitor):
-    """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
-    attribute.
-    """
-    def __init__(self):
-        super(NameNodeCollector, self).__init__()
-        self.name_nodes = []
-
-    def visit_NameNode(self, node):
-        self.name_nodes.append(node)
-
-    def visit_Node(self, node):
-        self._visitchildren(node, None)
-
 
 class SkipDeclarations(object):
     """
@@ -66,6 +52,7 @@ class SkipDeclarations(object):
     def visit_CStructOrUnionDefNode(self, node):
         return node
 
+
 class NormalizeTree(CythonTransform):
     """
     This transform fixes up a few things after parsing
@@ -1033,7 +1020,8 @@ class InterpretCompilerDirectives(CythonTransform):
         directives = []
         realdecs = []
         both = []
-        for dec in node.decorators:
+        # Decorators coming first take precedence.
+        for dec in node.decorators[::-1]:
             new_directives = self.try_to_parse_directives(dec.decorator)
             if new_directives is not None:
                 for directive in new_directives:
@@ -1043,15 +1031,17 @@ class InterpretCompilerDirectives(CythonTransform):
                             directives.append(directive)
                         if directive[0] == 'staticmethod':
                             both.append(dec)
+                    # Adapt scope type based on decorators that change it.
+                    if directive[0] == 'cclass' and scope_name == 'class':
+                        scope_name = 'cclass'
             else:
                 realdecs.append(dec)
-        if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode)):
+        if realdecs and (scope_name == 'cclass' or
+                         isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode))):
             raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
-        else:
-            node.decorators = realdecs + both
+        node.decorators = realdecs[::-1] + both[::-1]
         # merge or override repeated directives
         optdict = {}
-        directives.reverse() # Decorators coming first take precedence
         for directive in directives:
             name, value = directive
             if name in optdict:
@@ -1940,6 +1930,8 @@ if VALUE is not None:
             binding = self.current_directives.get('binding')
             rhs = ExprNodes.PyCFunctionNode.from_defnode(node, binding)
             node.code_object = rhs.code_object
+            if node.is_generator:
+                node.gbody.code_object = node.code_object
 
         if env.is_py_class_scope:
             rhs.binding = True
@@ -2591,10 +2583,13 @@ class MarkClosureVisitor(CythonTransform):
         collector.visitchildren(node)
 
         if node.is_async_def:
-            coroutine_type = Nodes.AsyncGenNode if collector.has_yield else Nodes.AsyncDefNode
+            coroutine_type = Nodes.AsyncDefNode
             if collector.has_yield:
+                coroutine_type = Nodes.AsyncGenNode
                 for yield_expr in collector.yields + collector.returns:
                     yield_expr.in_async_gen = True
+            elif self.current_directives['iterable_coroutine']:
+                coroutine_type = Nodes.IterableAsyncDefNode
         elif collector.has_await:
             found = next(y for y in collector.yields if y.is_await)
             error(found.pos, "'await' not allowed in generators (use 'yield')")

Cython/Compiler/Parsing.pxd

@@ -191,7 +191,7 @@ cdef p_c_class_options(PyrexScanner s)
 cdef p_property_decl(PyrexScanner s)
 cdef p_doc_string(PyrexScanner s)
 cdef p_ignorable_statement(PyrexScanner s)
-cdef p_compiler_directive_comments(PyrexScanner s)
+cdef dict p_compiler_directive_comments(PyrexScanner s)
 cdef p_template_definition(PyrexScanner s)
 cdef p_cpp_class_definition(PyrexScanner s, pos, ctx)
 cdef p_cpp_class_attribute(PyrexScanner s, ctx)

Cython/Compiler/StringEncoding.py

@@ -191,6 +191,14 @@ def bytes_literal(s, encoding):
     return s
 
 
+def encoded_string(s, encoding):
+    assert isinstance(s, (_unicode, bytes))
+    s = EncodedString(s)
+    if encoding is not None:
+        s.encoding = encoding
+    return s
+
+
 char_from_escape_sequence = {
     r'\a' : u'\a',
     r'\b' : u'\b',

Cython/Compiler/Tests/TestTreeFragment.py

@@ -45,7 +45,7 @@ class TestTreeFragments(CythonTest):
         T = F.substitute({"v" : NameNode(pos=None, name="a")})
         v = F.root.stats[1].rhs.operand2.operand1
         a = T.stats[1].rhs.operand2.operand1
-        self.assertEquals(v.pos, a.pos)
+        self.assertEqual(v.pos, a.pos)
 
     def test_temps(self):
         TemplateTransform.temp_name_counter = 0

Cython/Includes/cpython/long.pxd

@@ -146,4 +146,4 @@ cdef extern from "Python.h":
     # pointer. If pylong cannot be converted, an OverflowError will be
     # raised. This is only assured to produce a usable void pointer
     # for values created with PyLong_FromVoidPtr(). For values outside
-    # 0..LONG_MAX, both signed and unsigned integers are acccepted.
+    # 0..LONG_MAX, both signed and unsigned integers are accepted.