Merge remote-tracking branch 'upstream/master'

.gitignore

@@ -4,9 +4,13 @@ __pycache__
 *.so
 *.o
 
+*.egg
+*.egg-info
+
 Cython/Compiler/*.c
 Cython/Plex/*.c
 Cython/Runtime/refnanny.c
+Cython/Tempita/*.c
 
 Tools/*.elc
 
@@ -24,3 +28,5 @@ dist/
 
 tags
 TAGS
+
+.tox

.hgignore

@@ -3,6 +3,8 @@ syntax: glob
 *.pyc
 *.pyo
 __pycache__
+*.egg
+*.egg-info
 
 Cython/Compiler/*.c
 Cython/Plex/*.c

.hgtags

@@ -39,3 +39,13 @@ ef9d2c680684d0df7d81f529cda29e9e1741f575 cython-0.10.1
 8412b39fbc3eb709a543e2f1e95c0c8881ea9ed4 0.14.beta2
 a6b9f0a6d02d23fc3d3a9d0587867faa3afb2fcd 0.14.rc0
 15bf34c9387444e262acb1de594405444dd571a4 0.14
+5320ddd8c3a60d00e0513f9f70d6846cd449409b 0.17.beta1
+275fb550c1d802da3df35ebae41e04eadc60e49e 0.17.2
+b0faba6967e74f652286a0de6d02d736845b0708 0.17.3
+c1a18ab6b0808e87f68d2f9d914c01934510aef5 0.18b1
+9a11631e0edb0394b17554cde8bec5d4784117e4 0.18rc1
+76f33728e8534e698267e097cf603ea59ade6f30 0.18
+4f782ac7b3fdf3b4408bbf9f2ed4c38f31e60920 0.19b1
+52beb5b16df5b8a92bb6c8c47faf42370d73cb0f 0.19b2
+4818f5b68eb4b4ea6ad7e415f6672b491e2461bc 0.19rc1
+48407fa3f3c9da84ab3dc103a6a2b1ca4c1beb2a 0.19

.travis.yml

+language: python
+
+python:
+  - 2.6
+  - 2.7
+  - 3.2
+  - 3.3
+
+#  - pypy
+
+branches:
+  only:
+    - master
+
+install: pip install .
+
+script: CFLAGS=-O0 python runtests.py -vv
+
+matrix:
+  allow_failures:
+    - python: pypy

Cython/Build/BuildExecutable.py

@@ -13,8 +13,8 @@ import sys
 import os
 from distutils import sysconfig
 
-def get_config_var(name):
-    return sysconfig.get_config_var(name) or ''
+def get_config_var(name, default=''):
+    return sysconfig.get_config_var(name) or default
 
 INCDIR = sysconfig.get_python_inc()
 LIBDIR1 = get_config_var('LIBDIR')
@@ -27,9 +27,9 @@ if PYLIB_DYN == PYLIB:
 else:
     PYLIB_DYN = os.path.splitext(PYLIB_DYN[3:])[0] # 'lib(XYZ).so' -> XYZ
 
-CC = get_config_var('CC')
+CC = get_config_var('CC', os.environ.get('CC', ''))
 CFLAGS = get_config_var('CFLAGS') + ' ' + os.environ.get('CFLAGS', '')
-LINKCC = get_config_var('LINKCC')
+LINKCC = get_config_var('LINKCC', os.environ.get('LINKCC', CC))
 LINKFORSHARED = get_config_var('LINKFORSHARED')
 LIBS = get_config_var('LIBS')
 SYSLIBS = get_config_var('SYSLIBS')

Cython/Build/Cythonize.py

+#!/usr/bin/env python
+
+import os
+import shutil
+import tempfile
+
+from distutils.core import setup
+from Cython.Build.Dependencies import cythonize, extended_iglob
+from Cython.Utils import is_package_dir
+from Cython.Compiler import Options
+
+try:
+    import multiprocessing
+    parallel_compiles = int(multiprocessing.cpu_count() * 1.5)
+except ImportError:
+    multiprocessing = None
+    parallel_compiles = 0
+
+
+class _FakePool(object):
+    def map_async(self, func, args):
+        from itertools import imap
+        for _ in imap(func, args):
+            pass
+
+    def close(self): pass
+    def terminate(self): pass
+    def join(self): pass
+
+
+def parse_directives(option, name, value, parser):
+    dest = option.dest
+    old_directives = dict(getattr(parser.values, dest,
+                                  Options.directive_defaults))
+    directives = Options.parse_directive_list(
+        value, relaxed_bool=True, current_settings=old_directives)
+    setattr(parser.values, dest, directives)
+
+
+def parse_options(option, name, value, parser):
+    dest = option.dest
+    options = dict(getattr(parser.values, dest, {}))
+    for opt in value.split(','):
+        if '=' in opt:
+            n, v = opt.split('=', 1)
+            v = v.lower() not in ('false', 'f', '0', 'no')
+        else:
+            n, v = opt, True
+        options[n] = v
+    setattr(parser.values, dest, options)
+
+
+def find_package_base(path):
+    base_dir, package_path = os.path.split(path)
+    while os.path.isfile(os.path.join(base_dir, '__init__.py')):
+        base_dir, parent = os.path.split(base_dir)
+        package_path = '%s/%s' % (parent, package_path)
+    return base_dir, package_path
+
+
+def cython_compile(path_pattern, options):
+    pool = None
+    paths = map(os.path.abspath, extended_iglob(path_pattern))
+    try:
+        for path in paths:
+            if options.build_inplace:
+                base_dir = path
+                while not os.path.isdir(base_dir) or is_package_dir(base_dir):
+                    base_dir = os.path.dirname(base_dir)
+            else:
+                base_dir = None
+
+            if os.path.isdir(path):
+                # recursively compiling a package
+                paths = [os.path.join(path, '**', '*.%s' % ext)
+                         for ext in ('py', 'pyx')]
+            else:
+                # assume it's a file(-like thing)
+                paths = [path]
+
+            ext_modules = cythonize(
+                paths,
+                nthreads=options.parallel,
+                exclude_failures=options.keep_going,
+                exclude=options.excludes,
+                compiler_directives=options.directives,
+                force=options.force,
+                quiet=options.quiet,
+                **options.options)
+
+            if ext_modules and options.build:
+                if len(ext_modules) > 1 and options.parallel > 1:
+                    if pool is None:
+                        try:
+                            pool = multiprocessing.Pool(options.parallel)
+                        except OSError:
+                            pool = _FakePool()
+                    pool.map_async(run_distutils, [
+                        (base_dir, [ext]) for ext in ext_modules])
+                else:
+                    run_distutils((base_dir, ext_modules))
+    except:
+        if pool is not None:
+            pool.terminate()
+        raise
+    else:
+        if pool is not None:
+            pool.close()
+            pool.join()
+
+
+def run_distutils(args):
+    base_dir, ext_modules = args
+    script_args = ['build_ext', '-i']
+    cwd = os.getcwd()
+    temp_dir = None
+    try:
+        if base_dir:
+            os.chdir(base_dir)
+            temp_dir = tempfile.mkdtemp(dir=base_dir)
+            script_args.extend(['--build-temp', temp_dir])
+        setup(
+            script_name='setup.py',
+            script_args=script_args,
+            ext_modules=ext_modules,
+        )
+    finally:
+        if base_dir:
+            os.chdir(cwd)
+            if temp_dir and os.path.isdir(temp_dir):
+                shutil.rmtree(temp_dir)
+
+
+def parse_args(args):
+    from optparse import OptionParser
+    parser = OptionParser(usage='%prog [options] [sources and packages]+')
+
+    parser.add_option('-X', '--directive', metavar='NAME=VALUE,...', dest='directives',
+                      type=str, action='callback', callback=parse_directives, default={},
+                      help='set a compiler directive')
+    parser.add_option('-s', '--option', metavar='NAME=VALUE', dest='options',
+                      type=str, action='callback', callback=parse_options, default={},
+                      help='set a cythonize option')
+    parser.add_option('-3', dest='python3_mode', action='store_true',
+                      help='use Python 3 syntax mode by default')
+
+    parser.add_option('-x', '--exclude', metavar='PATTERN', dest='excludes',
+                      action='append', default=[],
+                      help='exclude certain file patterns from the compilation')
+
+    parser.add_option('-b', '--build', dest='build', action='store_true',
+                      help='build extension modules using distutils')
+    parser.add_option('-i', '--inplace', dest='build_inplace', action='store_true',
+                      help='build extension modules in place using distutils (implies -b)')
+    parser.add_option('-j', '--parallel', dest='parallel', metavar='N',
+                      type=int, default=parallel_compiles,
+                      help=('run builds in N parallel jobs (default: %d)' %
+                            parallel_compiles or 1))
+    parser.add_option('-f', '--force', dest='force', action='store_true',
+                      help='force recompilation')
+    parser.add_option('-q', '--quiet', dest='quiet', action='store_true',
+                      help='be less verbose during compilation')
+
+    parser.add_option('--lenient', dest='lenient', action='store_true',
+                      help='increase Python compatibility by ignoring some compile time errors')
+    parser.add_option('-k', '--keep-going', dest='keep_going', action='store_true',
+                      help='compile as much as possible, ignore compilation failures')
+
+    options, args = parser.parse_args(args)
+    if not args:
+        parser.error("no source files provided")
+    if options.build_inplace:
+        options.build = True
+    if multiprocessing is None:
+        options.parallel = 0
+    if options.python3_mode:
+        options.options['language_level'] = 3
+    return options, args
+
+
+def main(args=None):
+    options, paths = parse_args(args)
+
+    if options.lenient:
+        # increase Python compatibility by ignoring compile time errors
+        Options.error_on_unknown_names = False
+        Options.error_on_uninitialized = False
+
+    for path in paths:
+        cython_compile(path, options)
+
+
+if __name__ == '__main__':
+    main()

Cython/Build/Inline.py

@@ -16,6 +16,7 @@ from Cython.Compiler.ParseTreeTransforms import CythonTransform, SkipDeclaration
 from Cython.Compiler.TreeFragment import parse_from_strings
 from Cython.Build.Dependencies import strip_string_literals, cythonize, cached_function
 from Cython.Compiler import Pipeline
+from Cython.Utils import get_cython_cache_dir
 import cython as cython_module
 
 # A utility function to convert user-supplied ASCII strings to unicode.
@@ -28,8 +29,6 @@ if sys.version_info[0] < 3:
 else:
     to_unicode = lambda x: x
 
-_code_cache = {}
-
 
 class AllSymbols(CythonTransform, SkipDeclarations):
     def __init__(self):
@@ -93,9 +92,23 @@ def safe_type(arg, context=None):
                     return '%s.%s' % (base_type.__module__, base_type.__name__)
         return 'object'
 
+def _get_build_extension():
+    dist = Distribution()
+    # Ensure the build respects distutils configuration by parsing
+    # the configuration files
+    config_files = dist.find_config_files()
+    dist.parse_config_files(config_files)
+    build_extension = build_ext(dist)
+    build_extension.finalize_options()
+    return build_extension
+
+@cached_function
+def _create_context(cython_include_dirs):
+    return Context(list(cython_include_dirs), default_options)
+
 def cython_inline(code,
                   get_type=unsafe_type,
-                  lib_dir=os.path.expanduser('~/.cython/inline'),
+                  lib_dir=os.path.join(get_cython_cache_dir(), 'inline'),
                   cython_include_dirs=['.'],
                   force=False,
                   quiet=False,
@@ -108,7 +121,7 @@ def cython_inline(code,
     orig_code = code
     code, literals = strip_string_literals(code)
     code = strip_common_indent(code)
-    ctx = Context(cython_include_dirs, default_options)
+    ctx = _create_context(tuple(cython_include_dirs))
     if locals is None:
         locals = inspect.currentframe().f_back.f_back.f_locals
     if globals is None:
@@ -137,58 +150,71 @@ def cython_inline(code,
     arg_sigs = tuple([(get_type(kwds[arg], ctx), arg) for arg in arg_names])
     key = orig_code, arg_sigs, sys.version_info, sys.executable, Cython.__version__
     module_name = "_cython_inline_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
+    
+    if module_name in sys.modules:
+        module = sys.modules[module_name]
+    
+    else:
+        build_extension = None
+        if cython_inline.so_ext is None:
+            # Figure out and cache current extension suffix
+            build_extension = _get_build_extension()
+            cython_inline.so_ext = build_extension.get_ext_filename('')
 
-    so_ext = [ ext for ext,_,mod_type in imp.get_suffixes() if mod_type == imp.C_EXTENSION ][0]
-    module_path = os.path.join(lib_dir, module_name+so_ext)
+        module_path = os.path.join(lib_dir, module_name + cython_inline.so_ext)
 
-    if not os.path.exists(lib_dir):
-        os.makedirs(lib_dir)
-    if force or not os.path.isfile(module_path):
-        cflags = []
-        c_include_dirs = []
-        qualified = re.compile(r'([.\w]+)[.]')
-        for type, _ in arg_sigs:
-            m = qualified.match(type)
-            if m:
-                cimports.append('\ncimport %s' % m.groups()[0])
-                # one special case
-                if m.groups()[0] == 'numpy':
-                    import numpy
-                    c_include_dirs.append(numpy.get_include())
-                    # cflags.append('-Wno-unused')
-        module_body, func_body = extract_func_code(code)
-        params = ', '.join(['%s %s' % a for a in arg_sigs])
-        module_code = """
+        if not os.path.exists(lib_dir):
+            os.makedirs(lib_dir)
+        if force or not os.path.isfile(module_path):
+            cflags = []
+            c_include_dirs = []
+            qualified = re.compile(r'([.\w]+)[.]')
+            for type, _ in arg_sigs:
+                m = qualified.match(type)
+                if m:
+                    cimports.append('\ncimport %s' % m.groups()[0])
+                    # one special case
+                    if m.groups()[0] == 'numpy':
+                        import numpy
+                        c_include_dirs.append(numpy.get_include())
+                        # cflags.append('-Wno-unused')
+            module_body, func_body = extract_func_code(code)
+            params = ', '.join(['%s %s' % a for a in arg_sigs])
+            module_code = """
 %(module_body)s
 %(cimports)s
 def __invoke(%(params)s):
 %(func_body)s
-        """ % {'cimports': '\n'.join(cimports), 'module_body': module_body, 'params': params, 'func_body': func_body }
-        for key, value in literals.items():
-            module_code = module_code.replace(key, value)
-        pyx_file = os.path.join(lib_dir, module_name + '.pyx')
-        fh = open(pyx_file, 'w')
-        try: 
-            fh.write(module_code)
-        finally:
-            fh.close()
-        extension = Extension(
-            name = module_name,
-            sources = [pyx_file],
-            include_dirs = c_include_dirs,
-            extra_compile_args = cflags)
-        build_extension = build_ext(Distribution())
-        build_extension.finalize_options()
-        build_extension.extensions = cythonize([extension], ctx=ctx, quiet=quiet)
-        build_extension.build_temp = os.path.dirname(pyx_file)
-        build_extension.build_lib  = lib_dir
-        build_extension.run()
-        _code_cache[key] = module_name
+            """ % {'cimports': '\n'.join(cimports), 'module_body': module_body, 'params': params, 'func_body': func_body }
+            for key, value in literals.items():
+                module_code = module_code.replace(key, value)
+            pyx_file = os.path.join(lib_dir, module_name + '.pyx')
+            fh = open(pyx_file, 'w')
+            try: 
+                fh.write(module_code)
+            finally:
+                fh.close()
+            extension = Extension(
+                name = module_name,
+                sources = [pyx_file],
+                include_dirs = c_include_dirs,
+                extra_compile_args = cflags)
+            if build_extension is None:
+                build_extension = _get_build_extension()
+            build_extension.extensions = cythonize([extension], include_path=cython_include_dirs, quiet=quiet)
+            build_extension.build_temp = os.path.dirname(pyx_file)
+            build_extension.build_lib  = lib_dir
+            build_extension.run()
+
+        module = imp.load_dynamic(module_name, module_path)
 
-    module = imp.load_dynamic(module_name, module_path)
     arg_list = [kwds[arg] for arg in arg_names]
     return module.__invoke(*arg_list)
 
+# Cached suffix used by cython_inline above.  None should get
+# overridden with actual value upon the first cython_inline invocation
+cython_inline.so_ext = None
+
 non_space = re.compile('[^ ]')
 def strip_common_indent(code):
     min_indent = None

Cython/Build/Tests/TestStripLiterals.py

@@ -35,10 +35,10 @@ class TestStripLiterals(CythonTest):
         self.t("u'abc'", "u'_L1_'")
 
     def test_raw(self):
-        self.t(r"r'abc\'", "r'_L1_'")
+        self.t(r"r'abc\\'", "r'_L1_'")
 
     def test_raw_unicode(self):
-        self.t(r"ru'abc\'", "ru'_L1_'")
+        self.t(r"ru'abc\\'", "ru'_L1_'")
 
     def test_comment(self):
         self.t("abc # foo", "abc #_L1_")

Cython/Compiler/Annotate.py

@@ -11,9 +11,11 @@ from Code import CCodeWriter
 from Cython import Utils
 
 # need one-characters subsitutions (for now) so offsets aren't off
-special_chars = [(u'<', u'\xF0', u'&lt;'),
-                 (u'>', u'\xF1', u'&gt;'),
-                 (u'&', u'\xF2', u'&amp;')]
+special_chars = [
+    (u'&', u'\xF2', u'&amp;'),
+    (u'<', u'\xF0', u'&lt;'),
+    (u'>', u'\xF1', u'&gt;'),
+]
 
 line_pos_comment = re.compile(r'/\*.*?<<<<<<<<<<<<<<.*?\*/\n*', re.DOTALL)
 
@@ -57,8 +59,7 @@ class AnnotationCCodeWriter(CCodeWriter):
         self.mark_pos(None)
         f = Utils.open_source_file(source_filename)
         lines = f.readlines()
-        for k in range(len(lines)):
-            line = lines[k]
+        for k, line in enumerate(lines):
             for c, cc, html in special_chars:
                 line = line.replace(c, cc)
             lines[k] = line
@@ -75,8 +76,7 @@ class AnnotationCCodeWriter(CCodeWriter):
                     else:
                         all.append((pos, start+end))
 
-        all.sort()
-        all.reverse()
+        all.sort(reverse=True)
         for pos, item in all:
             _, line_no, col = pos
             line_no -= 1
@@ -86,7 +86,8 @@ class AnnotationCCodeWriter(CCodeWriter):
 
         html_filename = os.path.splitext(target_filename)[0] + ".html"
         f = codecs.open(html_filename, "w", encoding="UTF-8")
-        f.write(u'<!-- Generated by Cython %s on %s -->\n' % (Version.version, time.asctime()))
+        f.write(u'<!DOCTYPE html>\n')
+        f.write(u'<!-- Generated by Cython %s -->\n' % Version.watermark)
         f.write(u'<html>\n')
         f.write(u"""
 <head>
@@ -120,14 +121,14 @@ body { font-family: courier; font-size: 12; }
 <script>
 function toggleDiv(id) {
     theDiv = document.getElementById(id);
-    if (theDiv.style.display == 'none') theDiv.style.display = 'block';
+    if (theDiv.style.display != 'block') theDiv.style.display = 'block';
     else theDiv.style.display = 'none';
 }
 </script>
 </head>
         """)
         f.write(u'<body>\n')
-        f.write(u'<p>Generated by Cython %s on %s\n' % (Version.version, time.asctime()))
+        f.write(u'<p>Generated by Cython %s\n' % Version.watermark)
         c_file = Utils.decode_filename(os.path.basename(target_filename))
         f.write(u'<p>Raw output: <a href="%s">%s</a>\n' % (c_file, c_file))
         k = 0
@@ -147,8 +148,9 @@ function toggleDiv(id) {
                 code = code_source_file[k]
             except KeyError:
                 code = ''
-
-            code = code.replace('<', '<code><</code>')
+            else:
+                for c, cc, html in special_chars:
+                    code = code.replace(c, html)
 
             code, py_c_api_calls = py_c_api.subn(ur"<span class='py_c_api'>\1</span>(", code)
             code, pyx_c_api_calls = pyx_c_api.subn(ur"<span class='pyx_c_api'>\1</span>(", code)

Cython/Compiler/AutoDocTransforms.py

@@ -11,6 +11,40 @@ class EmbedSignature(CythonTransform):
         self.class_name = None
         self.class_node = None
 
+    unop_precedence = 11
+    binop_precedence = {
+        'or': 1,
+        'and': 2,
+        'not': 3,
+        'in': 4, 'not in': 4, 'is': 4, 'is not': 4, '<': 4, '<=': 4, '>': 4, '>=': 4, '!=': 4, '==': 4,
+        '|': 5,
+        '^': 6,
+        '&': 7,
+        '<<': 8, '>>': 8,
+        '+': 9, '-': 9,
+        '*': 10, '/': 10, '//': 10, '%': 10,
+        # unary: '+': 11, '-': 11, '~': 11
+        '**': 12}
+
+    def _fmt_expr_node(self, node, precedence=0):
+        if isinstance(node, ExprNodes.BinopNode) and not node.inplace:
+            new_prec = self.binop_precedence.get(node.operator, 0)
+            result = '%s %s %s' % (self._fmt_expr_node(node.operand1, new_prec),
+                                   node.operator,
+                                   self._fmt_expr_node(node.operand2, new_prec))
+            if precedence > new_prec:
+                result = '(%s)' % result
+        elif isinstance(node, ExprNodes.UnopNode):
+            result = '%s%s' % (node.operator,
+                               self._fmt_expr_node(node.operand, self.unop_precedence))
+            if precedence > self.unop_precedence:
+                result = '(%s)' % result
+        elif isinstance(node, ExprNodes.AttributeNode):
+            result = '%s.%s' % (self._fmt_expr_node(node.obj), node.attribute)
+        else:
+            result = node.name
+        return result
+
     def _fmt_arg_defv(self, arg):
         default_val = arg.default
         if not default_val:
@@ -31,8 +65,8 @@ class EmbedSignature(CythonTransform):
             return repr_val
         except Exception:
             try:
-                return default_val.name # XXX
-            except AttributeError:
+                return self._fmt_expr_node(default_val)
+            except AttributeError, e:
                 return '<???>'
 
     def _fmt_arg(self, arg):
@@ -93,7 +127,6 @@ class EmbedSignature(CythonTransform):
         else:
             return signature
 
-
     def __call__(self, node):
         if not Options.docstrings:
             return node
@@ -172,8 +205,25 @@ class EmbedSignature(CythonTransform):
                 old_doc = node.py_func.entry.doc
             else:
                 old_doc = None
-            new_doc  = self._embed_signature(signature, old_doc)
+            new_doc = self._embed_signature(signature, old_doc)
             node.entry.doc = EncodedString(new_doc)
             if hasattr(node, 'py_func') and node.py_func is not None:
                 node.py_func.entry.doc = EncodedString(new_doc)
         return node
+
+    def visit_PropertyNode(self, node):
+        if not self.current_directives['embedsignature']:
+            return node
+
+        entry = node.entry
+        if entry.visibility == 'public':
+            # property synthesised from a cdef public attribute
+            type_name = entry.type.declaration_code("", for_display=1)
+            if not entry.type.is_pyobject:
+                type_name = "'%s'" % type_name
+            elif entry.type.is_extension_type:
+                type_name = entry.type.module_name + '.' + type_name
+            signature = '%s: %s' % (entry.name, type_name)
+            new_doc = self._embed_signature(signature, entry.doc)
+            entry.doc = EncodedString(new_doc)
+        return node

Cython/Compiler/Buffer.py

-from Visitor import CythonTransform
-from ModuleNode import ModuleNode
-from ExprNodes import *
-from Errors import CompileError
-from UtilityCode import CythonUtilityCode
-from Code import UtilityCode, TempitaUtilityCode
-import Interpreter
-import PyrexTypes
-import Naming
-import Symtab
+from Cython.Compiler.Visitor import CythonTransform
+from Cython.Compiler.ModuleNode import ModuleNode
+from Cython.Compiler.Errors import CompileError
+from Cython.Compiler.UtilityCode import CythonUtilityCode
+from Cython.Compiler.Code import UtilityCode, TempitaUtilityCode
+
+from Cython.Compiler import Options
+from Cython.Compiler import Interpreter
+from Cython.Compiler import PyrexTypes
+from Cython.Compiler import Naming
+from Cython.Compiler import Symtab
 
 
 def dedent(text, reindent=0):
@@ -600,10 +601,14 @@ class GetAndReleaseBufferUtilityCode(object):
 
         find_buffer_types(env)
 
-        proto, impl = TempitaUtilityCode.load_as_string(
+        util_code = TempitaUtilityCode.load(
             "GetAndReleaseBuffer", from_file="Buffer.c",
             context=dict(types=types))
 
+        proto = util_code.format_code(util_code.proto)
+        impl = util_code.format_code(
+            util_code.inject_string_constants(util_code.impl, output)[1])
+
         proto_code.putln(proto)
         code.putln(impl)
 
@@ -680,32 +685,28 @@ def get_type_information_cname(code, dtype, maxdepth=None):
         rep = str(dtype)
 
         flags = "0"
-
-        if dtype.is_int:
-            if dtype.signed == 0:
-                typegroup = 'U'
-            else:
-                typegroup = 'I'
+        is_unsigned = "0"
+        if dtype is PyrexTypes.c_char_type:
+            is_unsigned = "IS_UNSIGNED(%s)" % declcode
+            typegroup = "'H'"
+        elif dtype.is_int:
+            is_unsigned = "IS_UNSIGNED(%s)" % declcode
+            typegroup = "%s ? 'U' : 'I'" % is_unsigned
         elif complex_possible or dtype.is_complex:
-            typegroup = 'C'
+            typegroup = "'C'"
         elif dtype.is_float:
-            typegroup = 'R'
+            typegroup = "'R'"
         elif dtype.is_struct:
-            typegroup = 'S'
+            typegroup = "'S'"
             if dtype.packed:
                 flags = "__PYX_BUF_FLAGS_PACKED_STRUCT"
         elif dtype.is_pyobject:
-            typegroup = 'O'
+            typegroup = "'O'"
         else:
-            assert False
-
-        if dtype.is_int:
-            is_unsigned = "IS_UNSIGNED(%s)" % declcode
-        else:
-            is_unsigned = "0"
+            assert False, dtype
 
         typeinfo = ('static __Pyx_TypeInfo %s = '
-                        '{ "%s", %s, sizeof(%s), { %s }, %s, \'%s\', %s, %s };')
+                        '{ "%s", %s, sizeof(%s), { %s }, %s, %s, %s, %s };')
         tup = (name, rep, structinfo_name, declcode,
                ', '.join([str(x) for x in arraysizes]) or '0', len(arraysizes),
                typegroup, is_unsigned, flags)
@@ -729,12 +730,6 @@ buffer_struct_declare_code = load_buffer_utility("BufferStructDeclare",
 raise_indexerror_code = load_buffer_utility("BufferIndexError")
 raise_indexerror_nogil = load_buffer_utility("BufferIndexErrorNogil")
 
-parse_typestring_repeat_code = UtilityCode(
-proto = """
-""",
-impl = """
-""")
-
 raise_buffer_fallback_code = load_buffer_utility("BufferFallbackError")
 buffer_structs_code = load_buffer_utility(
         "BufferFormatStructs", proto_block='utility_code_proto_before_types')

Cython/Compiler/CmdLine.py

@@ -20,7 +20,6 @@ Options:
   -o, --output-file <filename>   Specify name of generated C file
   -t, --timestamps               Only compile newer source files
   -f, --force                    Compile all source files (overrides implied -t)
-  -q, --quiet                    Don't print module names in recursive mode
   -v, --verbose                  Be verbose, print file names on multiple compilation
   -p, --embed-positions          If specified, the positions in Cython files of each
                                  function definition is embedded in its docstring.
@@ -29,6 +28,7 @@ Options:
   -w, --working <directory>      Sets the working directory for Cython (the directory modules
                                  are searched from)
   --gdb                          Output debug information for cygdb
+  --gdb-outdir <directory>       Specify gdb debug information output directory. Implies --gdb.
 
   -D, --no-docstrings            Strip docstrings from the compiled module.
   -a, --annotate                 Produce a colorized HTML version of the source.
@@ -37,16 +37,15 @@ Options:
   --embed[=<method_name>]        Generate a main() function that embeds the Python interpreter.
   -2                             Compile based on Python-2 syntax and code semantics.
   -3                             Compile based on Python-3 syntax and code semantics.
+  --lenient                      Change some compile time errors to runtime errors to
+                                 improve Python compatibility
+  --capi-reexport-cincludes      Add cincluded headers to any auto-generated header files.
   --fast-fail                    Abort the compilation on the first error
-  --warning-error, -Werror       Make all warnings into errors
+  --warning-errors, -Werror      Make all warnings into errors
   --warning-extra, -Wextra       Enable extra warnings
   -X, --directive <name>=<value>[,<name=value,...] Overrides a compiler directive
 """
 
-# The following is broken http://trac.cython.org/cython_trac/ticket/379
-#  -r, --recursive                Recursively find and compile dependencies (implies -t)
-
-
 #The following experimental options are supported only on MacOSX:
 #  -C, --compile    Compile generated .c file to .o file
 #  --link           Link .o file to produce extension module (implies -C)
@@ -98,8 +97,6 @@ def parse_command_line(args):
                 options.working_path = pop_arg()
             elif option in ("-o", "--output-file"):
                 options.output_file = pop_arg()
-            elif option in ("-r", "--recursive"):
-                options.recursive = 1
             elif option in ("-t", "--timestamps"):
                 options.timestamps = 1
             elif option in ("-f", "--force"):
@@ -125,18 +122,24 @@ def parse_command_line(args):
             elif option == "--gdb":
                 options.gdb_debug = True
                 options.output_dir = os.curdir
+            elif option == "--gdb-outdir":
+                options.gdb_debug = True
+                options.output_dir = pop_arg()
+            elif option == "--lenient":
+                Options.error_on_unknown_names = False
+                Options.error_on_uninitialized = False
             elif option == '-2':
                 options.language_level = 2
             elif option == '-3':
                 options.language_level = 3
+            elif option == "--capi-reexport-cincludes":
+                options.capi_reexport_cincludes = True
             elif option == "--fast-fail":
                 Options.fast_fail = True
             elif option in ('-Werror', '--warning-errors'):
                 Options.warning_errors = True
             elif option in ('-Wextra', '--warning-extra'):
                 options.compiler_directives.update(Options.extra_warnings)
-            elif option == "--disable-function-redefinition":
-                Options.disable_function_redefinition = True
             elif option == "--old-style-globals":
                 Options.old_style_globals = True
             elif option == "--directive" or option.startswith('-X'):

Cython/Compiler/Code.pxd

@@ -30,6 +30,7 @@ cdef class FunctionState:
 
     cdef public bint in_try_finally
     cdef public object exc_vars
+    cdef public bint can_trace
 
     cdef public list temps_allocated
     cdef public dict temps_free
@@ -39,6 +40,7 @@ cdef class FunctionState:
 
     cdef public object closure_temps
     cdef public bint should_declare_error_indicator
+    cdef public bint uses_error_indicator
 
     @cython.locals(n=size_t)
     cpdef new_label(self, name=*)

Cython/Compiler/Errors.py

@@ -48,10 +48,17 @@ class CompileError(PyrexError):
     def __init__(self, position = None, message = u""):
         self.position = position
         self.message_only = message
+        self.formatted_message = format_error(message, position)
         self.reported = False
     # Deprecated and withdrawn in 2.6:
     #   self.message = message
-        Exception.__init__(self, format_error(message, position))
+        Exception.__init__(self, self.formatted_message)
+        # Python Exception subclass pickling is broken,
+        # see http://bugs.python.org/issue1692335
+        self.args = (position, message)
+
+    def __str__(self):
+        return self.formatted_message
 
 class CompileWarning(PyrexWarning):
 
@@ -96,6 +103,9 @@ class CompilerCrash(CompileError):
                 message += u'\n'
             message += u'%s: %s' % (cause.__class__.__name__, cause)
         CompileError.__init__(self, pos, message)
+        # Python Exception subclass pickling is broken,
+        # see http://bugs.python.org/issue1692335
+        self.args = (pos, context, message, cause, stacktrace)
 
 class NoElementTreeInstalledException(PyrexError):
     """raised when the user enabled options.gdb_debug but no ElementTree

Cython/Compiler/FlowControl.pxd

 cimport cython
 
+from Cython.Compiler.Visitor cimport CythonTransform, TreeVisitor
+
 cdef class ControlBlock:
      cdef public set children
      cdef public set parents
@@ -26,18 +28,24 @@ cdef class ExitBlock(ControlBlock):
 
 cdef class NameAssignment:
     cdef public bint is_arg
+    cdef public bint is_deletion
     cdef public object lhs
     cdef public object rhs
     cdef public object entry
     cdef public object pos
     cdef public set refs
     cdef public object bit
+    cdef public object inferred_type
 
 cdef class AssignmentList:
     cdef public object bit
     cdef public object mask
     cdef public list stats
 
+cdef class AssignmentCollector(TreeVisitor):
+    cdef list assignments
+
+@cython.final
 cdef class ControlFlow:
      cdef public set blocks
      cdef public set entries
@@ -50,17 +58,20 @@ cdef class ControlFlow:
 
      cdef public dict assmts
 
-     cpdef newblock(self, parent=*)
-     cpdef nextblock(self, parent=*)
+     cpdef newblock(self, ControlBlock parent=*)
+     cpdef nextblock(self, ControlBlock parent=*)
      cpdef bint is_tracked(self, entry)
+     cpdef bint is_statically_assigned(self, entry)
      cpdef mark_position(self, node)
      cpdef mark_assignment(self, lhs, rhs, entry)
      cpdef mark_argument(self, lhs, rhs, entry)
      cpdef mark_deletion(self, node, entry)
      cpdef mark_reference(self, node, entry)
+
+     @cython.locals(block=ControlBlock, parent=ControlBlock, unreachable=set)
      cpdef normalize(self)
 
-     @cython.locals(offset=object, assmts=AssignmentList,
+     @cython.locals(bit=object, assmts=AssignmentList,
                     block=ControlBlock)
      cpdef initialize(self)
 
@@ -73,5 +84,22 @@ cdef class ControlFlow:
 cdef class Uninitialized:
      pass
 
-@cython.locals(dirty=bint, block=ControlBlock, parent=ControlBlock)
+cdef class Unknown:
+    pass
+
+@cython.locals(dirty=bint, block=ControlBlock, parent=ControlBlock,
+               assmt=NameAssignment)
 cdef check_definitions(ControlFlow flow, dict compiler_directives)
+
+@cython.final
+cdef class ControlFlowAnalysis(CythonTransform):
+    cdef object gv_ctx
+    cdef set reductions
+    cdef list env_stack
+    cdef list stack
+    cdef object env
+    cdef ControlFlow flow
+    cdef bint in_inplace_assignment
+
+    cpdef mark_assignment(self, lhs, rhs=*)
+    cpdef mark_position(self, node)

Cython/Compiler/Future.py

 def _get_feature(name):
     import __future__
-    try:
-        return getattr(__future__, name)
-    except AttributeError:
-        # unique fake object for earlier Python versions or Python 3
-        return object()
+    # fall back to a unique fake object for earlier Python versions or Python 3
+    return getattr(__future__, name, object())
 
 unicode_literals = _get_feature("unicode_literals")
 with_statement = _get_feature("with_statement")
 division = _get_feature("division")
 print_function = _get_feature("print_function")
+absolute_import = _get_feature("absolute_import")
 nested_scopes = _get_feature("nested_scopes")  # dummy
 generators = _get_feature("generators")  # dummy
 

Cython/Compiler/Lexicon.py

@@ -4,7 +4,10 @@
 #
 
 raw_prefixes = "rR"
-string_prefixes = "cCuUbB"
+bytes_prefixes = "bB"
+string_prefixes = "uU" + bytes_prefixes
+char_prefixes = "cC"
+any_string_prefix = raw_prefixes + string_prefixes + char_prefixes
 IDENT = 'IDENT'
 
 def make_lexicon():
@@ -34,46 +37,23 @@ def make_lexicon():
     fltconst = (decimal_fract + Opt(exponent)) | (decimal + exponent)
     imagconst = (intconst | fltconst) + Any("jJ")
 
-    sq_string = (
-        Str("'") +
-        Rep(AnyBut("\\\n'") | (Str("\\") + AnyChar)) +
-        Str("'")
-    )
-
-    dq_string = (
-        Str('"') +
-        Rep(AnyBut('\\\n"') | (Str("\\") + AnyChar)) +
-        Str('"')
-    )
-
-    non_sq = AnyBut("'") | (Str('\\') + AnyChar)
-    tsq_string = (
-        Str("'''")
-        + Rep(non_sq | (Str("'") + non_sq) | (Str("''") + non_sq))
-        + Str("'''")
-    )
-
-    non_dq = AnyBut('"') | (Str('\\') + AnyChar)
-    tdq_string = (
-        Str('"""')
-        + Rep(non_dq | (Str('"') + non_dq) | (Str('""') + non_dq))
-        + Str('"""')
-    )
-
-    beginstring = Opt(Any(string_prefixes)) + Opt(Any(raw_prefixes)) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
+    beginstring = Opt(Any(string_prefixes) + Opt(Any(raw_prefixes)) |
+                      Any(raw_prefixes) + Opt(Any(bytes_prefixes)) |
+                      Any(char_prefixes)
+                      ) + (Str("'") | Str('"') | Str("'''") | Str('"""'))
     two_oct = octdigit + octdigit
     three_oct = octdigit + octdigit + octdigit
     two_hex = hexdigit + hexdigit
     four_hex = two_hex + two_hex
     escapeseq = Str("\\") + (two_oct | three_oct |
+                             Str('N{') + Rep(AnyBut('}')) + Str('}') |
                              Str('u') + four_hex | Str('x') + two_hex |
                              Str('U') + four_hex + four_hex | AnyChar)
 
-
     deco = Str("@")
     bra = Any("([{")
     ket = Any(")]}")
-    punct = Any(":,;+-*/|&<>=.%`~^?")
+    punct = Any(":,;+-*/|&<>=.%`~^?!")
     diphthong = Str("==", "<>", "!=", "<=", ">=", "<<", ">>", "**", "//",
                     "+=", "-=", "*=", "/=", "%=", "|=", "^=", "&=",
                     "<<=", ">>=", "**=", "//=", "->")
@@ -95,7 +75,6 @@ def make_lexicon():
         (ket, Method('close_bracket_action')),
         (lineterm, Method('newline_action')),
 
-        #(stringlit, 'STRING'),
         (beginstring, Method('begin_string_action')),
 
         (comment, IGNORE),

Cython/Compiler/MemoryView.py

@@ -180,6 +180,9 @@ def valid_memslice_dtype(dtype, i=0):
     if dtype.is_complex and dtype.real_type.is_int:
         return False
 
+    if dtype is PyrexTypes.c_bint_type:
+        return False
+
     if dtype.is_struct and dtype.kind == 'struct':
         for member in dtype.scope.var_entries:
             if not valid_memslice_dtype(member.type):
@@ -273,12 +276,12 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
         return bufp
 
     def generate_buffer_slice_code(self, code, indices, dst, have_gil,
-                                   have_slices):
+                                   have_slices, directives):
         """
         Slice a memoryviewslice.
 
-        indices     - list of index nodes. If not a SliceNode, then it must be
-                      coercible to Py_ssize_t
+        indices     - list of index nodes. If not a SliceNode, or NoneNode,
+                      then it must be coercible to Py_ssize_t
 
         Simply call __pyx_memoryview_slice_memviewslice with the right
         arguments.
@@ -307,28 +310,14 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
         code.putln("%(dst)s.memview = %(src)s.memview;" % locals())
         code.put_incref_memoryviewslice(dst)
 
-        for dim, index in enumerate(indices):
+        dim = -1
+        for index in indices:
             error_goto = code.error_goto(index.pos)
-
-            if not isinstance(index, ExprNodes.SliceNode):
-                # normal index
-                idx = index.result()
-
+            if not index.is_none:
+                dim += 1
                 access, packing = self.type.axes[dim]
-                if access == 'direct':
-                    indirect = False
-                else:
-                    indirect = True
-                    generic = (access == 'full')
-                    if new_ndim != 0:
-                        return error(index.pos,
-                                     "All preceding dimensions must be "
-                                     "indexed and not sliced")
-
-                d = locals()
-                code.put(load_slice_util("SliceIndex", d))
-            else:
 
+            if isinstance(index, ExprNodes.SliceNode):
                 # slice, unspecified dimension, or part of ellipsis
                 d = locals()
                 for s in "start stop step".split():
@@ -344,7 +333,6 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
                     not d['have_step']):
                     # full slice (:), simply copy over the extent, stride
                     # and suboffset. Also update suboffset_dim if needed
-                    access, packing = self.type.axes[dim]
                     d['access'] = access
                     code.put(load_slice_util("SimpleSlice", d))
                 else:
@@ -352,6 +340,33 @@ class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
 
                 new_ndim += 1
 
+            elif index.is_none:
+                # newaxis
+                attribs = [('shape', 1), ('strides', 0), ('suboffsets', -1)]
+                for attrib, value in attribs:
+                    code.putln("%s.%s[%d] = %d;" % (dst, attrib, new_ndim, value))
+
+                new_ndim += 1
+
+            else:
+                # normal index
+                idx = index.result()
+
+                if access == 'direct':
+                    indirect = False
+                else:
+                    indirect = True
+                    generic = (access == 'full')
+                    if new_ndim != 0:
+                        return error(index.pos,
+                                     "All preceding dimensions must be "
+                                     "indexed and not sliced")
+
+                wraparound = int(directives['wraparound'])
+                boundscheck = int(directives['boundscheck'])
+                d = locals()
+                code.put(load_slice_util("SliceIndex", d))
+
         if not no_suboffset_dim:
             code.funcstate.release_temp(suboffset_dim)
 
@@ -361,11 +376,13 @@ def empty_slice(pos):
     return ExprNodes.SliceNode(pos, start=none,
                                stop=none, step=none)
 
-def unellipsify(indices, ndim):
+def unellipsify(indices, newaxes, ndim):
     result = []
     seen_ellipsis = False
     have_slices = False
 
+    n_indices = len(indices) - len(newaxes)
+
     for index in indices:
         if isinstance(index, ExprNodes.EllipsisNode):
             have_slices = True
@@ -374,16 +391,19 @@ def unellipsify(indices, ndim):
             if seen_ellipsis:
                 result.append(full_slice)
             else:
-                nslices = ndim - len(indices) + 1
+                nslices = ndim - n_indices + 1
                 result.extend([full_slice] * nslices)
                 seen_ellipsis = True
         else:
-            have_slices = have_slices or isinstance(index, ExprNodes.SliceNode)
+            have_slices = (have_slices or
+                           isinstance(index, ExprNodes.SliceNode) or
+                           index.is_none)
             result.append(index)
 
-    if len(result) < ndim:
+    result_length = len(result) - len(newaxes)
+    if result_length < ndim:
         have_slices = True
-        nslices = ndim - len(result)
+        nslices = ndim - result_length
         result.extend([empty_slice(indices[-1].pos)] * nslices)
 
     return have_slices, result
@@ -708,6 +728,14 @@ def get_axes_specs(env, axes):
 
     return axes_specs
 
+def validate_axes(pos, axes):
+    if len(axes) >= Options.buffer_max_dims:
+        error(pos, "More dimensions than the maximum number"
+                   " of buffer dimensions were used.")
+        return False
+
+    return True
+
 def all(it):
     for item in it:
         if not item:
@@ -891,8 +919,7 @@ memviewslice_init_code = load_memview_c_utility(
     context=dict(context, BUF_MAX_NDIMS=Options.buffer_max_dims),
     requires=[memviewslice_declare_code,
               Buffer.acquire_utility_code,
-              atomic_utility,
-              Buffer.typeinfo_compare_code],
+              atomic_utility],
 )
 
 memviewslice_index_helpers = load_memview_c_utility("MemviewSliceIndex")

Cython/Compiler/Naming.py

@@ -103,6 +103,8 @@ global_code_object_cache_find = pyrex_prefix + 'find_code_object'
 global_code_object_cache_insert = pyrex_prefix + 'insert_code_object'
 
 genexpr_id_ref = 'genexpr'
+freelist_name  = 'freelist'
+freecount_name = 'freecount'
 
 line_c_macro = "__LINE__"
 
@@ -131,7 +133,7 @@ h_guard_prefix   = "__PYX_HAVE__"
 api_guard_prefix = "__PYX_HAVE_API__"
 api_func_guard   = "__PYX_HAVE_API_FUNC_"
 
-PYX_NAN          = "__PYX_NAN"
+PYX_NAN          = "__PYX_NAN()"
 
 def py_version_hex(major, minor=0, micro=0, release_level=0, release_serial=0):
     return (major << 24) | (minor << 16) | (micro << 8) | (release_level << 4) | (release_serial)

Cython/Compiler/ParseTreeTransforms.pxd

@@ -34,10 +34,10 @@ cdef map_starred_assignment(list lhs_targets, list starred_assignments, list lhs
 #class WithTransform(CythonTransform, SkipDeclarations):
 #class DecoratorTransform(CythonTransform, SkipDeclarations):
 
-#class AnalyseDeclarationsTransform(CythonTransform):
+#class AnalyseDeclarationsTransform(EnvTransform):
 
 cdef class AnalyseExpressionsTransform(CythonTransform):
-    cdef list env_stack
+    pass
 
 cdef class ExpandInplaceOperators(EnvTransform):
     pass
@@ -63,6 +63,7 @@ cdef class CreateClosureClasses(CythonTransform):
 cdef class GilCheck(VisitorTransform):
     cdef list env_stack
     cdef bint nogil
+    cdef bint nogil_declarator_only
 
 cdef class TransformBuiltinMethods(EnvTransform):
     cdef visit_cython_attribute(self, node)

Cython/Compiler/TreeFragment.py

@@ -26,7 +26,7 @@ class StringParseContext(Main.Context):
         self.module_name = name
 
     def find_module(self, module_name, relative_to = None, pos = None, need_pxd = 1):
-        if module_name != self.module_name:
+        if module_name not in (self.module_name, 'cython'):
             raise AssertionError("Not yet supporting any cimports/includes from string code snippets")
         return ModuleScope(module_name, parent_module = None, context = self)
 
@@ -231,6 +231,12 @@ class TreeFragment(object):
                                    substitutions = nodes,
                                    temps = self.temps + temps, pos = pos)
 
+class SetPosTransform(VisitorTransform):
+    def __init__(self, pos):
+        super(SetPosTransform, self).__init__()
+        self.pos = pos
 
-
-
+    def visit_Node(self, node):
+        node.pos = self.pos
+        self.visitchildren(node)
+        return node