preliminary merge of Pyrex 0.9.6.2 -> crashes

Cython/Compiler/Code.py

@@ -21,10 +21,11 @@ class CCodeWriter:
     # in_try_finally   boolean         inside try of try...finally
     # filename_table   {string : int}  for finding filename table indexes
     # filename_list    [string]        filenames in filename table order
+    # exc_vars         (string * 3)    exception variables for reraise, or None
     # input_file_contents dict         contents (=list of lines) of any file that was used as input
     #                                  to create this output C code.  This is
     #                                  used to annotate the comments. 
-    
+   
     in_try_finally = 0
     
     def __init__(self, f):
@@ -37,8 +38,9 @@ class CCodeWriter:
         self.error_label = None
         self.filename_table = {}
         self.filename_list = []
+        self.exc_vars = None
         self.input_file_contents = {}
-    
+
     def putln(self, code = ""):
         if self.marker and self.bol:
             self.emit_marker()
@@ -186,8 +188,10 @@ class CCodeWriter:
         #print "Code.put_var_declaration:", entry.name, "definition =", definition ###
         visibility = entry.visibility
         if visibility == 'private' and not definition:
+            #print "...private and not definition, skipping" ###
             return
         if not entry.used and visibility == "private":
+            #print "not used and private, skipping" ###
             return
         storage_class = ""
         if visibility == 'extern':
@@ -288,13 +292,6 @@ class CCodeWriter:
         #	code = "((PyObject*)%s)" % code
         self.put_init_to_py_none(code, entry.type)
 
-    def put_py_gil_state_ensure(self, cname):
-        self.putln("PyGILState_STATE %s;" % cname)
-        self.putln("%s = PyGILState_Ensure();" % cname)
-
-    def put_py_gil_state_release(self, cname):
-        self.putln("PyGILState_Release(%s);" % cname)
-
     def put_pymethoddef(self, entry, term):
         if entry.doc:
             doc_code = entry.doc_cname
@@ -316,6 +313,10 @@ class CCodeWriter:
 #        return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos)))  # TODO this path is almost _never_ taken, yet this macro makes is slower!
         return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))
 
+    def put_h_guard(self, guard):
+        self.putln("#ifndef %s" % guard)
+        self.putln("#define %s" % guard)
+    
     def error_goto(self, pos):
         lbl = self.error_label
         self.use_label(lbl)

Cython/Compiler/Main.py

@@ -17,12 +17,14 @@ import Parsing
 from Symtab import BuiltinScope, ModuleScope
 import Code
 from Cython.Utils import replace_suffix
+import Builtin
+from Cython import Utils
 
 verbose = 0
 
 class Context:
     #  This class encapsulates the context needed for compiling
-    #  one or more Pyrex implementation files along with their
+    #  one or more Cython implementation files along with their
     #  associated and imported declaration files. It includes
     #  the root of the module import namespace and the list
     #  of directories to search for include files.
@@ -31,7 +33,8 @@ class Context:
     #  include_directories   [string]
     
     def __init__(self, include_directories):
-        self.modules = {"__builtin__" : BuiltinScope()}
+        #self.modules = {"__builtin__" : BuiltinScope()}
+        self.modules = {"__builtin__" : Builtin.builtin_scope}
         self.include_directories = include_directories
         
     def find_module(self, module_name, 
@@ -180,22 +183,29 @@ class Context:
             else:
                 c_suffix = ".c"
             result.c_file = replace_suffix(source, c_suffix)
+        c_stat = None
+        if result.c_file:
+            try:
+                c_stat = os.stat(result.c_file)
+            except EnvironmentError:
+                pass
         module_name = self.extract_module_name(source, options)
         initial_pos = (source, 1, 0)
         scope = self.find_module(module_name, pos = initial_pos, need_pxd = 0)
         errors_occurred = False
         try:
             tree = self.parse(source, scope.type_names, pxd = 0, full_module_name = full_module_name)
-            tree.process_implementation(scope, result)
+            tree.process_implementation(scope, options, result)
         except CompileError:
             errors_occurred = True
         Errors.close_listing_file()
         result.num_errors = Errors.num_errors
         if result.num_errors > 0:
             errors_occurred = True
-        if errors_occurred:
+        if errors_occurred and result.c_file:
             try:
-                os.unlink(result.c_file)
+                #os.unlink(result.c_file)
+                Utils.castrate_file(result.c_file, c_stat)
             except EnvironmentError:
                 pass
             result.c_file = None
@@ -225,6 +235,7 @@ class CompilationOptions:
     errors_to_stderr  boolean   Echo errors to stderr when using .lis
     include_path      [string]  Directories to search for include files
     output_file       string    Name of generated .c file
+    generate_pxi      boolean   Generate .pxi file for public declarations
     
     Following options are experimental and only used on MacOSX:
     
@@ -238,7 +249,11 @@ class CompilationOptions:
         self.include_path = []
         self.objects = []
         if defaults:
-            self.__dict__.update(defaults.__dict__)
+            if isinstance(defaults, CompilationOptions):
+                defaults = defaults.__dict__
+        else:
+            defaults = default_options
+        self.__dict__.update(defaults)
         self.__dict__.update(kw)
 
 
@@ -249,6 +264,7 @@ class CompilationResult:
     c_file           string or None   The generated C source file
     h_file           string or None   The generated C header file
     i_file           string or None   The generated .pxi file
+    api_file         string or None   The generated C API .h file
     listing_file     string or None   File of error messages
     object_file      string or None   Result of compiling the C file
     extension_file   string or None   Result of linking the object file
@@ -259,6 +275,7 @@ class CompilationResult:
         self.c_file = None
         self.h_file = None
         self.i_file = None
+        self.api_file = None
         self.listing_file = None
         self.object_file = None
         self.extension_file = None
@@ -317,18 +334,19 @@ def main(command_line = 0):
 #
 #------------------------------------------------------------------------
 
-default_options = CompilationOptions(
+default_options = dict(
     show_version = 0,
     use_listing_file = 0,
     errors_to_stderr = 1,
     c_only = 1,
     obj_only = 1,
     cplus = 0,
-    output_file = None)
+    output_file = None,
+    generate_pxi = 0)
     
 if sys.platform == "mac":
     from Cython.Mac.MacSystem import c_compile, c_link, CCompilerError
-    default_options.use_listing_file = 1
+    default_options['use_listing_file'] = 1
 elif sys.platform == "darwin":
     from Cython.Mac.DarwinSystem import c_compile, c_link, CCompilerError
 else:

Cython/Compiler/Naming.py

@@ -50,6 +50,7 @@ module_cname     = pyrex_prefix + "m"
 moddoc_cname     = pyrex_prefix + "mdoc"
 methtable_cname  = pyrex_prefix + "methods"
 retval_cname     = pyrex_prefix + "r"
+reqd_kwds_cname  = pyrex_prefix + "reqd_kwds"
 self_cname       = pyrex_prefix + "self"
 stringtab_cname  = pyrex_prefix + "string_tab"
 vtabslot_cname   = pyrex_prefix + "vtab"
@@ -58,6 +59,16 @@ gilstate_cname   = pyrex_prefix + "state"
 
 extern_c_macro  = pyrex_prefix.upper() + "EXTERN_C"
 
+exc_type_name   = pyrex_prefix + "exc_type"
+exc_value_name  = pyrex_prefix + "exc_value"
+exc_tb_name     = pyrex_prefix + "exc_tb"
+exc_lineno_name = pyrex_prefix + "exc_lineno"
+
+exc_vars = (exc_type_name, exc_value_name, exc_tb_name)
+
+h_guard_prefix   = "__PYX_HAVE__"
+api_guard_prefix = "__PYX_HAVE_API__"
+api_func_guard   = "__PYX_HAVE_API_FUNC_"
 
 def py_version_hex(major, minor=0, micro=0, release_level=0, release_serial=0):
     return (major << 24) | (minor << 16) | (micro << 8) | (release_level << 4) | (release_serial)

Cython/Compiler/Options.py

@@ -3,7 +3,7 @@
 #
 
 intern_names = 1    #  Intern global variable and attribute names
-cache_builtins = 1  #  Perform lookups on builtin names only once
+cache_builtins = 0  #  Perform lookups on builtin names only once
 
 embed_pos_in_docstring = 0
 gcc_branch_hints = 1

Cython/Compiler/Scanning.py

@@ -6,6 +6,7 @@
 import cPickle as pickle
 
 import os
+import platform
 import stat
 import sys
 from time import time
@@ -138,11 +139,7 @@ reserved_words = [
     "raise", "import", "exec", "try", "except", "finally",
     "while", "if", "elif", "else", "for", "in", "assert",
     "and", "or", "not", "is", "in", "lambda", "from",
-    "NULL", "cimport", "by", "with", "rdef"
-]
-
-function_contexts = [ # allowed arguments to the "with" option
-    "GIL"
+    "NULL", "cimport", "by", "with", "rdef", "DEF", "IF", "ELIF", "ELSE"
 ]
 
 class Method:
@@ -164,7 +161,53 @@ def build_resword_dict():
 
 #------------------------------------------------------------------
 
+class CompileTimeScope(object):
+
+    def __init__(self, outer = None):
+        self.entries = {}
+        self.outer = outer
+    
+    def declare(self, name, value):
+        self.entries[name] = value
+    
+    def lookup_here(self, name):
+        return self.entries[name]
+    
+    def lookup(self, name):
+        try:
+            return self.lookup_here(name)
+        except KeyError:
+            outer = self.outer
+            if outer:
+                return outer.lookup(name)
+            else:
+                raise
+
+def initial_compile_time_env():
+    benv = CompileTimeScope()
+    names = ('UNAME_SYSNAME', 'UNAME_NODENAME', 'UNAME_RELEASE',
+        'UNAME_VERSION', 'UNAME_MACHINE')
+    for name, value in zip(names, platform.uname()):
+        benv.declare(name, value)
+    import __builtin__
+    names = ('False', 'True',
+        'abs', 'bool', 'chr', 'cmp', 'complex', 'dict', 'divmod', 'enumerate',
+        'float', 'hash', 'hex', 'int', 'len', 'list', 'long', 'map', 'max', 'min',
+        'oct', 'ord', 'pow', 'range', 'reduce', 'repr', 'round', 'slice', 'str',
+        'sum', 'tuple', 'xrange', 'zip')
+    for name in names:
+        benv.declare(name, getattr(__builtin__, name))
+    denv = CompileTimeScope(benv)
+    return denv
+
+#------------------------------------------------------------------
+
 class PyrexScanner(Scanner):
+    #  context            Context  Compilation context
+    #  type_names         set      Identifiers to be treated as type names
+    #  compile_time_env   dict     Environment for conditional compilation
+    #  compile_time_eval  boolean  In a true conditional compilation context
+    #  compile_time_expr  boolean  In a compile-time expression context
     
     resword_dict = build_resword_dict()
 
@@ -174,9 +217,15 @@ class PyrexScanner(Scanner):
         if parent_scanner:
             self.context = parent_scanner.context
             self.type_names = parent_scanner.type_names
+            self.compile_time_env = parent_scanner.compile_time_env
+            self.compile_time_eval = parent_scanner.compile_time_eval
+            self.compile_time_expr = parent_scanner.compile_time_expr
         else:
             self.context = context
             self.type_names = type_names
+            self.compile_time_env = initial_compile_time_env()
+            self.compile_time_eval = 1
+            self.compile_time_expr = 0
         self.trace = trace_scanner
         self.indentation_stack = [0]
         self.indentation_char = None
@@ -307,10 +356,19 @@ class PyrexScanner(Scanner):
         if self.sy == what:
             self.next()
         else:
-            if message:
-                self.error(message)
-            else:
-                self.error("Expected '%s'" % what)
+            self.expected(what, message)
+    
+    def expect_keyword(self, what, message = None):
+        if self.sy == 'IDENT' and self.systring == what:
+            self.next()
+        else:
+            self.expected(what, message)
+    
+    def expected(self, what, message):
+        if message:
+            self.error(message)
+        else:
+            self.error("Expected '%s'" % what)
         
     def expect_indent(self):
         self.expect('INDENT',
@@ -320,7 +378,7 @@ class PyrexScanner(Scanner):
         self.expect('DEDENT',
             "Expected a decrease in indentation level")
 
-    def expect_newline(self, message):
+    def expect_newline(self, message = "Expected a newline"):
         # Expect either a newline or end of file
         if self.sy <> 'EOF':
             self.expect('NEWLINE', message)

Cython/Debugging.py

@@ -7,7 +7,7 @@
 def print_call_chain(*args):
     import sys
     print " ".join(map(str, args))
-    f = sys._getframe(2)
+    f = sys._getframe(1)
     while f:
         name = f.f_code.co_name
         s = f.f_locals.get('self', None)

Cython/Distutils/__init__.py

 # July 2002, Graham Fawcett
-
 #
-
 # this hack was inspired by the way Thomas Heller got py2exe
-
 # to appear as a distutil command
-
 #
-
 # we replace distutils.command.build_ext with our own version
-
 # and keep the old one under the module name _build_ext,
-
 # so that *our* build_ext can make use of it.
 
-
-
 from build_ext import build_ext
-
-
-
+from extension import Extension

Cython/Unix/LinuxSystem.py

@@ -7,7 +7,7 @@ gcc_pendantic = True
 gcc_warnings_are_errors = True
 gcc_all_warnings = True
 
-import os
+import os, sys
 from Cython.Utils import replace_suffix
 from Cython.Compiler.Errors import PyrexError