merge (including cwitty's change to get_exception_utility_code)

Cython/Compiler/Builtin.py

+#
+#   Pyrex - Builtin Definitions
+#
+
+from Symtab import BuiltinScope
+from TypeSlots import Signature
+
+builtin_function_table = [
+    # name,        args,   return,  C API func,           py equiv = "*"
+    ('abs',        "O",    "O",     "PyNumber_Absolute"),
+    #('chr',       "",     "",      ""),
+    #('cmp', "",   "",     "",      ""), # int PyObject_Cmp(PyObject *o1, PyObject *o2, int *result)
+    #('compile',   "",     "",      ""), # PyObject* Py_CompileString(	char *str, char *filename, int start)
+    ('delattr',    "OO",   "r",     "PyObject_DelAttr"),
+    ('dir',        "O",    "O",     "PyObject_Dir"),
+    ('divmod',     "OO",   "O",     "PyNumber_Divmod"),
+    #('eval',      "",     "",      ""),
+    #('execfile',  "",     "",      ""),
+    #('filter',    "",     "",      ""),
+    ('getattr',    "OO",   "O",     "PyObject_GetAttr"),
+    ('getattr3',   "OOO",  "O",     "__Pyx_GetAttr3",       "getattr"),
+    ('hasattr',    "OO",   "i",     "PyObject_HasAttr"),
+    ('hash',       "O",    "i",     "PyObject_Hash"),
+    #('hex',       "",     "",      ""),
+    #('id',        "",     "",      ""),
+    #('input',     "",     "",      ""),
+    ('intern',     "s",    "O",     "PyString_InternFromString"),
+    ('isinstance', "OO",   "i",     "PyObject_IsInstance"),
+    ('issubclass', "OO",   "i",     "PyObject_IsSubclass"),
+    ('iter',       "O",    "O",     "PyObject_GetIter"),
+    ('len',        "O",    "Z",     "PyObject_Length"),
+    #('map',       "",     "",      ""),
+    #('max',       "",     "",      ""),
+    #('min',       "",     "",      ""),
+    #('oct',       "",     "",      ""),
+    # Not worth doing open, when second argument would become mandatory
+    #('open',       "ss",   "O",     "PyFile_FromString"),
+    #('ord',       "",     "",      ""),
+    ('pow',        "OOO",  "O",     "PyNumber_Power"),
+    #('range',     "",     "",      ""),
+    #('raw_input', "",     "",      ""),
+    #('reduce',    "",     "",      ""),
+    ('reload',     "O",    "O",     "PyImport_ReloadModule"),
+    ('repr',       "O",    "O",     "PyObject_Repr"),
+    #('round',     "",     "",      ""),
+    ('setattr',    "OOO",  "r",     "PyObject_SetAttr"),
+    #('sum',       "",     "",      ""),
+    #('unichr',    "",     "",      ""),
+    #('unicode',   "",     "",      ""),
+    #('vars',      "",     "",      ""),
+    #('zip',       "",     "",      ""),
+    #  Can't do these easily until we have builtin type entries.
+    #('typecheck',  "OO",   "i",     "PyObject_TypeCheck", False),
+    #('issubtype',  "OO",   "i",     "PyType_IsSubtype",   False),
+]
+
+# Builtin types
+#  bool
+#  buffer
+#  classmethod
+#  dict
+#  enumerate
+#  file
+#  float
+#  int
+#  list
+#  long
+#  object
+#  property
+#  slice
+#  staticmethod
+#  super
+#  str
+#  tuple
+#  type
+#  xrange
+
+getattr3_utility_code = ["""
+static PyObject *__Pyx_GetAttr3(PyObject *, PyObject *, PyObject *); /*proto*/
+""","""
+static PyObject *__Pyx_GetAttr3(PyObject *o, PyObject *n, PyObject *d) {
+    PyObject *r = PyObject_GetAttr(o, n);
+    if (!r) {
+        if (!PyErr_ExceptionMatches(PyExc_AttributeError))
+            goto bad;
+        PyErr_Clear();
+        r = d;
+        Py_INCREF(d);
+    }
+    return r;
+bad:
+    return 0;
+}
+"""]
+
+builtin_utility_code = {
+    'getattr3': getattr3_utility_code,
+}
+
+builtin_scope = BuiltinScope()
+
+def declare_builtin_func(name, args, ret, cname, py_equiv = "*"):
+    sig = Signature(args, ret)
+    type = sig.function_type()
+    utility = builtin_utility_code.get(name)
+    builtin_scope.declare_builtin_cfunction(name, type, cname, py_equiv, utility)
+
+def init_builtin_funcs():
+    for desc in builtin_function_table:
+        declare_builtin_func(*desc)
+
+def init_builtins():
+    init_builtin_funcs()
+
+init_builtins()

Cython/Compiler/Code.py

@@ -21,10 +21,11 @@ class CCodeWriter:
     # in_try_finally   boolean         inside try of try...finally
     # filename_table   {string : int}  for finding filename table indexes
     # filename_list    [string]        filenames in filename table order
+    # exc_vars         (string * 3)    exception variables for reraise, or None
     # input_file_contents dict         contents (=list of lines) of any file that was used as input
     #                                  to create this output C code.  This is
     #                                  used to annotate the comments. 
-    
+   
     in_try_finally = 0
     
     def __init__(self, f):
@@ -37,8 +38,9 @@ class CCodeWriter:
         self.error_label = None
         self.filename_table = {}
         self.filename_list = []
+        self.exc_vars = None
         self.input_file_contents = {}
-    
+
     def putln(self, code = ""):
         if self.marker and self.bol:
             self.emit_marker()
@@ -186,8 +188,10 @@ class CCodeWriter:
         #print "Code.put_var_declaration:", entry.name, "definition =", definition ###
         visibility = entry.visibility
         if visibility == 'private' and not definition:
+            #print "...private and not definition, skipping" ###
             return
         if not entry.used and visibility == "private":
+            #print "not used and private, skipping" ###
             return
         storage_class = ""
         if visibility == 'extern':
@@ -288,13 +292,6 @@ class CCodeWriter:
         #	code = "((PyObject*)%s)" % code
         self.put_init_to_py_none(code, entry.type)
 
-    def put_py_gil_state_ensure(self, cname):
-        self.putln("PyGILState_STATE %s;" % cname)
-        self.putln("%s = PyGILState_Ensure();" % cname)
-
-    def put_py_gil_state_release(self, cname):
-        self.putln("PyGILState_Release(%s);" % cname)
-
     def put_pymethoddef(self, entry, term):
         if entry.doc:
             doc_code = entry.doc_cname
@@ -316,6 +313,10 @@ class CCodeWriter:
 #        return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos)))  # TODO this path is almost _never_ taken, yet this macro makes is slower!
         return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))
 
+    def put_h_guard(self, guard):
+        self.putln("#ifndef %s" % guard)
+        self.putln("#define %s" % guard)
+    
     def error_goto(self, pos):
         lbl = self.error_label
         self.use_label(lbl)

Cython/Compiler/Main.py

@@ -17,12 +17,14 @@ import Parsing
 from Symtab import BuiltinScope, ModuleScope
 import Code
 from Cython.Utils import replace_suffix
+import Builtin
+from Cython import Utils
 
 verbose = 0
 
 class Context:
     #  This class encapsulates the context needed for compiling
-    #  one or more Pyrex implementation files along with their
+    #  one or more Cython implementation files along with their
     #  associated and imported declaration files. It includes
     #  the root of the module import namespace and the list
     #  of directories to search for include files.
@@ -31,7 +33,8 @@ class Context:
     #  include_directories   [string]
     
     def __init__(self, include_directories):
-        self.modules = {"__builtin__" : BuiltinScope()}
+        #self.modules = {"__builtin__" : BuiltinScope()}
+        self.modules = {"__builtin__" : Builtin.builtin_scope}
         self.include_directories = include_directories
         
     def find_module(self, module_name, 
@@ -180,22 +183,29 @@ class Context:
             else:
                 c_suffix = ".c"
             result.c_file = replace_suffix(source, c_suffix)
+        c_stat = None
+        if result.c_file:
+            try:
+                c_stat = os.stat(result.c_file)
+            except EnvironmentError:
+                pass
         module_name = self.extract_module_name(source, options)
         initial_pos = (source, 1, 0)
         scope = self.find_module(module_name, pos = initial_pos, need_pxd = 0)
         errors_occurred = False
         try:
             tree = self.parse(source, scope.type_names, pxd = 0, full_module_name = full_module_name)
-            tree.process_implementation(scope, result)
+            tree.process_implementation(scope, options, result)
         except CompileError:
             errors_occurred = True
         Errors.close_listing_file()
         result.num_errors = Errors.num_errors
         if result.num_errors > 0:
             errors_occurred = True
-        if errors_occurred:
+        if errors_occurred and result.c_file:
             try:
-                os.unlink(result.c_file)
+                #os.unlink(result.c_file)
+                Utils.castrate_file(result.c_file, c_stat)
             except EnvironmentError:
                 pass
             result.c_file = None
@@ -225,6 +235,7 @@ class CompilationOptions:
     errors_to_stderr  boolean   Echo errors to stderr when using .lis
     include_path      [string]  Directories to search for include files
     output_file       string    Name of generated .c file
+    generate_pxi      boolean   Generate .pxi file for public declarations
     
     Following options are experimental and only used on MacOSX:
     
@@ -238,7 +249,11 @@ class CompilationOptions:
         self.include_path = []
         self.objects = []
         if defaults:
-            self.__dict__.update(defaults.__dict__)
+            if isinstance(defaults, CompilationOptions):
+                defaults = defaults.__dict__
+        else:
+            defaults = default_options
+        self.__dict__.update(defaults)
         self.__dict__.update(kw)
 
 
@@ -249,6 +264,7 @@ class CompilationResult:
     c_file           string or None   The generated C source file
     h_file           string or None   The generated C header file
     i_file           string or None   The generated .pxi file
+    api_file         string or None   The generated C API .h file
     listing_file     string or None   File of error messages
     object_file      string or None   Result of compiling the C file
     extension_file   string or None   Result of linking the object file
@@ -259,6 +275,7 @@ class CompilationResult:
         self.c_file = None
         self.h_file = None
         self.i_file = None
+        self.api_file = None
         self.listing_file = None
         self.object_file = None
         self.extension_file = None
@@ -317,18 +334,19 @@ def main(command_line = 0):
 #
 #------------------------------------------------------------------------
 
-default_options = CompilationOptions(
+default_options = dict(
     show_version = 0,
     use_listing_file = 0,
     errors_to_stderr = 1,
     c_only = 1,
     obj_only = 1,
     cplus = 0,
-    output_file = None)
+    output_file = None,
+    generate_pxi = 0)
     
 if sys.platform == "mac":
     from Cython.Mac.MacSystem import c_compile, c_link, CCompilerError
-    default_options.use_listing_file = 1
+    default_options['use_listing_file'] = 1
 elif sys.platform == "darwin":
     from Cython.Mac.DarwinSystem import c_compile, c_link, CCompilerError
 else:

Cython/Compiler/Naming.py

@@ -20,6 +20,7 @@ label_prefix      = pyrex_prefix + "L"
 pymethdef_prefix  = pyrex_prefix + "mdef_"
 methtab_prefix    = pyrex_prefix + "methods_"
 memtab_prefix     = pyrex_prefix + "members_"
+modules_prefix    = pyrex_prefix + "module_"
 interned_prefix   = pyrex_prefix + "n_"
 interned_num_prefix = pyrex_prefix + "num_"
 objstruct_prefix  = pyrex_prefix + "obj_"
@@ -50,6 +51,7 @@ module_cname     = pyrex_prefix + "m"
 moddoc_cname     = pyrex_prefix + "mdoc"
 methtable_cname  = pyrex_prefix + "methods"
 retval_cname     = pyrex_prefix + "r"
+reqd_kwds_cname  = pyrex_prefix + "reqd_kwds"
 self_cname       = pyrex_prefix + "self"
 stringtab_cname  = pyrex_prefix + "string_tab"
 vtabslot_cname   = pyrex_prefix + "vtab"
@@ -58,6 +60,18 @@ gilstate_cname   = pyrex_prefix + "state"
 
 extern_c_macro  = pyrex_prefix.upper() + "EXTERN_C"
 
+exc_type_name   = pyrex_prefix + "exc_type"
+exc_value_name  = pyrex_prefix + "exc_value"
+exc_tb_name     = pyrex_prefix + "exc_tb"
+exc_lineno_name = pyrex_prefix + "exc_lineno"
+
+exc_vars = (exc_type_name, exc_value_name, exc_tb_name)
+
+api_name        = pyrex_prefix + "capi__"
+
+h_guard_prefix   = "__PYX_HAVE__"
+api_guard_prefix = "__PYX_HAVE_API__"
+api_func_guard   = "__PYX_HAVE_API_FUNC_"
 
 def py_version_hex(major, minor=0, micro=0, release_level=0, release_serial=0):
     return (major << 24) | (minor << 16) | (micro << 8) | (release_level << 4) | (release_serial)

Cython/Compiler/Options.py

@@ -3,7 +3,7 @@
 #
 
 intern_names = 1    #  Intern global variable and attribute names
-cache_builtins = 1  #  Perform lookups on builtin names only once
+cache_builtins = 0  #  Perform lookups on builtin names only once
 
 embed_pos_in_docstring = 0
 gcc_branch_hints = 1

Cython/Compiler/Scanning.py

@@ -6,6 +6,7 @@
 import cPickle as pickle
 
 import os
+import platform
 import stat
 import sys
 from time import time
@@ -138,11 +139,7 @@ reserved_words = [
     "raise", "import", "exec", "try", "except", "finally",
     "while", "if", "elif", "else", "for", "in", "assert",
     "and", "or", "not", "is", "in", "lambda", "from",
-    "NULL", "cimport", "by", "with", "rdef"
-]
-
-function_contexts = [ # allowed arguments to the "with" option
-    "GIL"
+    "NULL", "cimport", "by", "with", "rdef", "DEF", "IF", "ELIF", "ELSE"
 ]
 
 class Method:
@@ -164,7 +161,53 @@ def build_resword_dict():
 
 #------------------------------------------------------------------
 
+class CompileTimeScope(object):
+
+    def __init__(self, outer = None):
+        self.entries = {}
+        self.outer = outer
+    
+    def declare(self, name, value):
+        self.entries[name] = value
+    
+    def lookup_here(self, name):
+        return self.entries[name]
+    
+    def lookup(self, name):
+        try:
+            return self.lookup_here(name)
+        except KeyError:
+            outer = self.outer
+            if outer:
+                return outer.lookup(name)
+            else:
+                raise
+
+def initial_compile_time_env():
+    benv = CompileTimeScope()
+    names = ('UNAME_SYSNAME', 'UNAME_NODENAME', 'UNAME_RELEASE',
+        'UNAME_VERSION', 'UNAME_MACHINE')
+    for name, value in zip(names, platform.uname()):
+        benv.declare(name, value)
+    import __builtin__
+    names = ('False', 'True',
+        'abs', 'bool', 'chr', 'cmp', 'complex', 'dict', 'divmod', 'enumerate',
+        'float', 'hash', 'hex', 'int', 'len', 'list', 'long', 'map', 'max', 'min',
+        'oct', 'ord', 'pow', 'range', 'reduce', 'repr', 'round', 'slice', 'str',
+        'sum', 'tuple', 'xrange', 'zip')
+    for name in names:
+        benv.declare(name, getattr(__builtin__, name))
+    denv = CompileTimeScope(benv)
+    return denv
+
+#------------------------------------------------------------------
+
 class PyrexScanner(Scanner):
+    #  context            Context  Compilation context
+    #  type_names         set      Identifiers to be treated as type names
+    #  compile_time_env   dict     Environment for conditional compilation
+    #  compile_time_eval  boolean  In a true conditional compilation context
+    #  compile_time_expr  boolean  In a compile-time expression context
     
     resword_dict = build_resword_dict()
 
@@ -174,9 +217,15 @@ class PyrexScanner(Scanner):
         if parent_scanner:
             self.context = parent_scanner.context
             self.type_names = parent_scanner.type_names
+            self.compile_time_env = parent_scanner.compile_time_env
+            self.compile_time_eval = parent_scanner.compile_time_eval
+            self.compile_time_expr = parent_scanner.compile_time_expr
         else:
             self.context = context
             self.type_names = type_names
+            self.compile_time_env = initial_compile_time_env()
+            self.compile_time_eval = 1
+            self.compile_time_expr = 0
         self.trace = trace_scanner
         self.indentation_stack = [0]
         self.indentation_char = None
@@ -307,10 +356,19 @@ class PyrexScanner(Scanner):
         if self.sy == what:
             self.next()
         else:
-            if message:
-                self.error(message)
-            else:
-                self.error("Expected '%s'" % what)
+            self.expected(what, message)
+    
+    def expect_keyword(self, what, message = None):
+        if self.sy == 'IDENT' and self.systring == what:
+            self.next()
+        else:
+            self.expected(what, message)
+    
+    def expected(self, what, message):
+        if message:
+            self.error(message)
+        else:
+            self.error("Expected '%s'" % what)
         
     def expect_indent(self):
         self.expect('INDENT',
@@ -320,7 +378,7 @@ class PyrexScanner(Scanner):
         self.expect('DEDENT',
             "Expected a decrease in indentation level")
 
-    def expect_newline(self, message):
+    def expect_newline(self, message = "Expected a newline"):
         # Expect either a newline or end of file
         if self.sy <> 'EOF':
             self.expect('NEWLINE', message)

Cython/Compiler/TypeSlots.py

@@ -83,7 +83,20 @@ class Signature:
     
     def return_type(self):
         return self.format_map[self.ret_format]
-        
+
+    def exception_value(self):
+        return self.error_value_map.get(self.ret_format)
+    
+    def function_type(self):
+        #  Construct a C function type descriptor for this signature
+        args = []
+        for i in xrange(self.num_fixed_args()):
+            arg_type = self.fixed_arg_type(i)
+            args.append(PyrexTypes.CFuncTypeArg("", arg_type, None))
+        ret_type = self.return_type()
+        exc_value = self.exception_value()
+        return PyrexTypes.CFuncType(ret_type, args, exception_value = exc_value)
+
     def method_flags(self):
         if self.ret_format == "O":
             full_args = self.fixed_arg_format
@@ -104,22 +117,23 @@ class SlotDescriptor:
     #
     #  slot_name    string           Member name of the slot in the type object
     #  is_initialised_dynamically    Is initialised by code in the module init function
-
-    def __init__(self, slot_name, dynamic = 0, min_python_version = None):
+    #  flag                          Py_TPFLAGS_XXX value indicating presence of slot
+    
+    def __init__(self, slot_name, dynamic = 0, flag = None):
         self.slot_name = slot_name
         self.is_initialised_dynamically = dynamic
-        self.min_python_version = min_python_version
+        self.flag = flag
 
     def generate(self, scope, code):
         if self.is_initialised_dynamically:
             value = 0
         else:
             value = self.slot_code(scope)
-        if self.min_python_version is not None:
-            code.putln("#if PY_VERSION_HEX >= " +
-                       code.get_py_version_hex(self.min_python_version))
+        flag = self.flag
+        if flag:
+            code.putln("#if Py_TPFLAGS_DEFAULT & %s" % flag)
         code.putln("%s, /*%s*/" % (value, self.slot_name))
-        if self.min_python_version is not None:
+        if flag:
             code.putln("#endif")
 
     # Some C implementations have trouble statically 
@@ -182,10 +196,8 @@ class MethodSlot(SlotDescriptor):
     #  method_name  string           The __xxx__ name of the method
     #  default      string or None   Default value of the slot
     
-    def __init__(self, signature, slot_name, method_name,
-                 default = None, min_python_version = None):
-        SlotDescriptor.__init__(self, slot_name,
-                                min_python_version = min_python_version)
+    def __init__(self, signature, slot_name, method_name, default = None, flag = None):
+        SlotDescriptor.__init__(self, slot_name, flag = flag)
         self.signature = signature
         self.slot_name = slot_name
         self.method_name = method_name
@@ -411,7 +423,7 @@ getcharbufferproc = Signature("TiS", 'i')  # typedef int (*getcharbufferproc)(Py
 readbufferproc = Signature("TZP", "Z")     # typedef Py_ssize_t (*readbufferproc)(PyObject *, Py_ssize_t, void **);
 writebufferproc = Signature("TZP", "Z")    # typedef Py_ssize_t (*writebufferproc)(PyObject *, Py_ssize_t, void **);
 segcountproc = Signature("TZ", "Z")        # typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
-writebufferproc = Signature("TZS", "Z")    # typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
+charbufferproc = Signature("TZS", "Z")     # typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
 objargproc = Signature("TO", 'r')          # typedef int (*objobjproc)(PyObject *, PyObject *);
                                            # typedef int (*visitproc)(PyObject *, void *);
                                            # typedef int (*traverseproc)(PyObject *, visitproc, void *);
@@ -502,11 +514,11 @@ PyNumberMethods = (
     MethodSlot(ibinaryfunc, "nb_inplace_true_divide", "__itruediv__"),
 
     # Added in release 2.5
-    MethodSlot(unaryfunc, "nb_index", "__index__", min_python_version=(2,5)),
+    MethodSlot(unaryfunc, "nb_index", "__index__", flag = "Py_TPFLAGS_HAVE_INDEX")
 )
 
 PySequenceMethods = (
-    MethodSlot(lenfunc, "sq_length", "__len__"),    # EmptySlot("sq_length"), # mp_length used instead
+    MethodSlot(lenfunc, "sq_length", "__len__"),
     EmptySlot("sq_concat"), # nb_add used instead
     EmptySlot("sq_repeat"), # nb_multiply used instead
     SyntheticSlot("sq_item", ["__getitem__"], "0"),    #EmptySlot("sq_item"),   # mp_subscript used instead
@@ -610,7 +622,7 @@ slot_table = (
 #
 #------------------------------------------------------------------------------------------
 
-MethodSlot(initproc, "", "__new__")
+MethodSlot(initproc, "", "__cinit__")
 MethodSlot(destructor, "", "__dealloc__")
 MethodSlot(objobjargproc, "", "__setitem__")
 MethodSlot(objargproc, "", "__delitem__")

Cython/Debugging.py

@@ -7,7 +7,7 @@
 def print_call_chain(*args):
     import sys
     print " ".join(map(str, args))
-    f = sys._getframe(2)
+    f = sys._getframe(1)
     while f:
         name = f.f_code.co_name
         s = f.f_locals.get('self', None)

Cython/Distutils/__init__.py

 # July 2002, Graham Fawcett
-
 #
-
 # this hack was inspired by the way Thomas Heller got py2exe
-
 # to appear as a distutil command
-
 #
-
 # we replace distutils.command.build_ext with our own version
-
 # and keep the old one under the module name _build_ext,
-
 # so that *our* build_ext can make use of it.
 
-
-
 from build_ext import build_ext
-
-
-
+from extension import Extension

Cython/Mac/DarwinSystem.py

@@ -6,15 +6,20 @@ verbose = 0
 gcc_pendantic = True
 gcc_warnings_are_errors = True
 gcc_all_warnings = True
+gcc_optimize = False
 
-import os
+import os, sys
 from Cython.Utils import replace_suffix
 from Cython.Compiler.Errors import PyrexError
 
+version_string = "%s.%s" % sys.version_info[:2]
+
 py_include_dirs = [
-    "/Library/Frameworks/Python.framework/Headers"
+    "/Library/Frameworks/Python.framework/Versions/%s/Headers" % version_string
 ]
 
+os.environ["MACOSX_DEPLOYMENT_TARGET"] = "10.3"
+
 compilers = ["gcc", "g++"]
 compiler_options = \
     "-g -c -fno-strict-aliasing -Wno-long-double -no-cpp-precomp " \
@@ -27,11 +32,16 @@ if gcc_warnings_are_errors:
 if gcc_all_warnings:
     compiler_options.append("-Wall")
     compiler_options.append("-Wno-unused-function")
+if gcc_optimize:
+    compiler_options.append("-O")
 
 linkers = ["gcc", "g++"]
 linker_options = \
-    "-Wl,-F.,-w -bundle -framework Python" \
+    "-Wl,-F.,-w -bundle -undefined dynamic_lookup" \
     .split()
+#linker_options = \
+#	"-Wl,-F.,-w -bundle -framework Python" \
+#	.split()
 
 class CCompilerError(PyrexError):
     pass