diff --git a/Cython/Compiler/Builtin.py b/Cython/Compiler/Builtin.py index f9c6605f37cbee2e16db093ee91be8115cb21fbe..94106760107c2197615a6b11fc56b9c8f8a83572 100644 --- a/Cython/Compiler/Builtin.py +++ b/Cython/Compiler/Builtin.py @@ -82,7 +82,7 @@ builtin_function_table = [ builtin_types_table = [ ("type", "PyType_Type", []), -# ("str", "PyString_Type", []), +# ("str", "PyBytes_Type", []), ("unicode", "PyUnicode_Type", []), ("file", "PyFile_Type", []), # ("slice", "PySlice_Type", []), diff --git a/Cython/Compiler/CmdLine.py b/Cython/Compiler/CmdLine.py index ce182ceab5519c99f2518fcdef3ac15df64323b4..3275a224aeb83a58bb78b7264bac5e2155ac8304 100644 --- a/Cython/Compiler/CmdLine.py +++ b/Cython/Compiler/CmdLine.py @@ -17,6 +17,10 @@ Options: -I, --include-dir <directory> Search for include files in named directory (multiply include directories are allowed). -o, --output-file <filename> Specify name of generated C file + -r, --recursive Recursively find and compile dependencies + -t, --timestamps Only compile newer source files (implied with -r) + -f, --force Compile all source files (overrides implied -t) + -q, --quiet Don't print module names in recursive mode -p, --embed-positions If specified, the positions in Cython files of each function definition is embedded in its docstring. -z, --pre-import <module> If specified, assume undeclared names in this @@ -111,6 +115,12 @@ def parse_command_line(args): options.working_path = pop_arg() elif option in ("-o", "--output-file"): options.output_file = pop_arg() + elif option in ("-r", "--recursive"): + options.recursive = 1 + elif option in ("-t", "--timestamps"): + options.timestamps = 1 + elif option in ("-f", "--force"): + options.timestamps = 0 elif option in ("-p", "--embed-positions"): Options.embed_pos_in_docstring = 1 elif option in ("-z", "--pre-import"): diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index a410311f962e02c715311a763699a7d646fad5e1..2a95b4f2c75ea777cec82ca83f7baadfaffc4950 100644 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -300,7 +300,11 @@ class ExprNode(Node): def analyse_target_types(self, env): self.analyse_types(env) - + + def gil_assignment_check(self, env): + if env.nogil and self.type.is_pyobject: + error(self.pos, "Assignment of Python object not allowed without gil") + def check_const(self): self.not_const() @@ -312,7 +316,11 @@ class ExprNode(Node): def addr_not_const(self): error(self.pos, "Address is not constant") - + + def gil_check(self, env): + if env.nogil and self.type.is_pyobject: + self.gil_error() + # ----------------- Result Allocation ----------------- def result_in_temp(self): @@ -758,11 +766,16 @@ class LongNode(AtomicExprNode): def compile_time_value(self, denv): return long(self.value) + + gil_message = "Constructing Python long int" def analyse_types(self, env): self.type = py_object_type + self.gil_check(env) self.is_temp = 1 - + + gil_message = "Constructing Python long int" + def generate_evaluation_code(self, code): code.putln( '%s = PyLong_FromString("%s", 0, 0); %s' % ( @@ -781,8 +794,11 @@ class ImagNode(AtomicExprNode): def analyse_types(self, env): self.type = py_object_type + self.gil_check(env) self.is_temp = 1 - + + gil_message = "Constructing complex number" + def generate_evaluation_code(self, code): code.putln( "%s = PyComplex_FromDoubles(0.0, %s); %s" % ( @@ -883,7 +899,10 @@ class NameNode(AtomicExprNode): else: self.is_temp = 1 env.use_utility_code(get_name_interned_utility_code) - + self.gil_check(env) + + gil_message = "Accessing Python global or builtin" + def analyse_entry(self, env): #print "NameNode.analyse_entry:", self.name ### self.check_identifier_kind() @@ -1060,8 +1079,11 @@ class BackquoteNode(ExprNode): self.arg.analyse_types(env) self.arg = self.arg.coerce_to_pyobject(env) self.type = py_object_type + self.gil_check(env) self.is_temp = 1 - + + gil_message = "Backquote expression" + def generate_result_code(self, code): code.putln( "%s = PyObject_Repr(%s); %s" % ( @@ -1086,9 +1108,12 @@ class ImportNode(ExprNode): if self.name_list: self.name_list.analyse_types(env) self.type = py_object_type + self.gil_check(env) self.is_temp = 1 env.use_utility_code(import_utility_code) - + + gil_message = "Python import" + def generate_result_code(self, code): if self.name_list: name_list_code = self.name_list.py_result() @@ -1114,11 +1139,14 @@ class IteratorNode(ExprNode): self.sequence.analyse_types(env) self.sequence = self.sequence.coerce_to_pyobject(env) self.type = py_object_type + self.gil_check(env) self.is_temp = 1 self.counter = TempNode(self.pos, PyrexTypes.c_py_ssize_t_type, env) self.counter.allocate_temp(env) - + + gil_message = "Iterating over Python object" + def release_temp(self, env): env.release_temp(self.result_code) self.counter.release_temp(env) @@ -1265,6 +1293,7 @@ class IndexNode(ExprNode): else: self.index = self.index.coerce_to_pyobject(env) self.type = py_object_type + self.gil_check(env) self.is_temp = 1 else: if self.base.type.is_ptr or self.base.type.is_array: @@ -1281,7 +1310,9 @@ class IndexNode(ExprNode): error(self.pos, "Invalid index type '%s'" % self.index.type) - + + gil_message = "Indexing Python object" + def check_const_addr(self): self.base.check_const_addr() self.index.check_const() @@ -1408,8 +1439,11 @@ class SliceIndexNode(ExprNode): if self.stop: self.stop = self.stop.coerce_to(c_int, env) self.type = py_object_type + self.gil_check(env) self.is_temp = 1 - + + gil_message = "Slicing Python object" + def generate_result_code(self, code): code.putln( "%s = PySequence_GetSlice(%s, %s, %s); %s" % ( @@ -1482,8 +1516,11 @@ class SliceNode(ExprNode): self.stop = self.stop.coerce_to_pyobject(env) self.step = self.step.coerce_to_pyobject(env) self.type = py_object_type + self.gil_check(env) self.is_temp = 1 - + + gil_message = "Constructing Python slice object" + def generate_result_code(self, code): code.putln( "%s = PySlice_New(%s, %s, %s); %s" % ( @@ -1493,7 +1530,15 @@ class SliceNode(ExprNode): self.step.py_result(), code.error_goto_if_null(self.result_code, self.pos))) -class SimpleCallNode(ExprNode): + +class CallNode(ExprNode): + def gil_check(self, env): + # Make sure we're not in a nogil environment + if env.nogil: + error(self.pos, "Calling gil-requiring function without gil") + + +class SimpleCallNode(CallNode): # Function call without keyword, * or ** args. # # function ExprNode @@ -1542,6 +1587,7 @@ class SimpleCallNode(ExprNode): self.arg_tuple.analyse_types(env) self.args = None self.type = py_object_type + self.gil_check(env) self.is_temp = 1 else: for arg in self.args: @@ -1614,6 +1660,9 @@ class SimpleCallNode(ExprNode): if func_type.exception_check == '+': if func_type.exception_value is None: env.use_utility_code(cpp_exception_utility_code) + # Check gil + if not func_type.nogil: + self.gil_check(env) def calculate_result_code(self): return self.c_call_code() @@ -1708,7 +1757,7 @@ class SimpleCallNode(ExprNode): rhs, code.error_goto_if(" && ".join(exc_checks), self.pos))) -class GeneralCallNode(ExprNode): +class GeneralCallNode(CallNode): # General Python function call, including keyword, # * and ** arguments. # @@ -1744,6 +1793,7 @@ class GeneralCallNode(ExprNode): self.starstar_arg = \ self.starstar_arg.coerce_to_pyobject(env) self.type = py_object_type + self.gil_check(env) self.is_temp = 1 def generate_result_code(self, code): @@ -1794,8 +1844,11 @@ class AsTupleNode(ExprNode): self.arg.analyse_types(env) self.arg = self.arg.coerce_to_pyobject(env) self.type = py_object_type + self.gil_check(env) self.is_temp = 1 - + + gil_message = "Constructing Python tuple" + def generate_result_code(self, code): code.putln( "%s = PySequence_Tuple(%s); %s" % ( @@ -2000,12 +2053,15 @@ class AttributeNode(ExprNode): self.type = py_object_type self.is_py_attr = 1 self.interned_attr_cname = env.intern_identifier(self.attribute) + self.gil_check(env) else: if not obj_type.is_error: error(self.pos, "Object of type '%s' has no attribute '%s'" % (obj_type, self.attribute)) - + + gil_message = "Accessing Python attribute" + def is_simple(self): if self.obj: return self.result_in_temp() or self.obj.is_simple() @@ -2122,8 +2178,9 @@ class SequenceNode(ExprNode): arg.analyse_types(env) self.args[i] = arg.coerce_to_pyobject(env) self.type = py_object_type + self.gil_check(env) self.is_temp = 1 - + def analyse_target_types(self, env): self.iterator = PyTempNode(self.pos, env) self.unpacked_items = [] @@ -2220,7 +2277,9 @@ class SequenceNode(ExprNode): class TupleNode(SequenceNode): # Tuple constructor. - + + gil_message = "Constructing Python tuple" + def analyse_types(self, env): if len(self.args) == 0: self.is_temp = 0 @@ -2271,7 +2330,9 @@ class TupleNode(SequenceNode): class ListNode(SequenceNode): # List constructor. - + + gil_message = "Constructing Python list" + def analyse_types(self, env): SequenceNode.analyse_types(self, env) self.type = list_type @@ -2368,8 +2429,11 @@ class DictNode(ExprNode): for item in self.key_value_pairs: item.analyse_types(env) self.type = dict_type + self.gil_check(env) self.is_temp = 1 - + + gil_message = "Constructing Python dict" + def allocate_temps(self, env, result = None): # Custom method used here because key-value # pairs are evaluated and used one at a time. @@ -2444,9 +2508,12 @@ class ClassNode(ExprNode): self.doc = self.doc.coerce_to_pyobject(env) self.module_name = env.global_scope().qualified_name self.type = py_object_type + self.gil_check(env) self.is_temp = 1 env.use_utility_code(create_class_utility_code); + gil_message = "Constructing Python class" + def generate_result_code(self, code): if self.doc: code.put_error_if_neg(self.pos, @@ -2476,8 +2543,11 @@ class UnboundMethodNode(ExprNode): def analyse_types(self, env): self.function.analyse_types(env) self.type = py_object_type + self.gil_check(env) self.is_temp = 1 - + + gil_message = "Constructing an unbound method" + def generate_result_code(self, code): code.putln( "%s = PyMethod_New(%s, 0, %s); %s" % ( @@ -2496,8 +2566,11 @@ class PyCFunctionNode(AtomicExprNode): def analyse_types(self, env): self.type = py_object_type + self.gil_check(env) self.is_temp = 1 - + + gil_message = "Constructing Python function" + def generate_result_code(self, code): code.putln( "%s = PyCFunction_New(&%s, 0); %s" % ( @@ -2549,6 +2622,7 @@ class UnopNode(ExprNode): if self.is_py_operation(): self.coerce_operand_to_pyobject(env) self.type = py_object_type + self.gil_check(env) self.is_temp = 1 else: self.analyse_c_operation(env) @@ -2898,6 +2972,7 @@ class BinopNode(ExprNode): if self.is_py_operation(): self.coerce_operands_to_pyobjects(env) self.type = py_object_type + self.gil_check(env) self.is_temp = 1 if Options.incref_local_binop and self.operand1.type.is_pyobject: self.operand1 = self.operand1.coerce_to_temp(env) @@ -3129,6 +3204,7 @@ class BoolBinopNode(ExprNode): self.operand2 = self.operand2.coerce_to_pyobject(env) self.temp_bool = TempNode(self.pos, PyrexTypes.c_bint_type, env) self.type = py_object_type + self.gil_check(env) else: self.operand1 = self.operand1.coerce_to_boolean(env) self.operand2 = self.operand2.coerce_to_boolean(env) @@ -3137,11 +3213,10 @@ class BoolBinopNode(ExprNode): # both operands be temp nodes. self.operand1 = self.operand1.coerce_to_temp(env) #CTT self.operand2 = self.operand2.coerce_to_temp(env) - # coerce_to_simple does not seem to be sufficient - #self.operand1 = self.operand1.coerce_to_simple(env) - #self.operand2 = self.operand2.coerce_to_simple(env) self.is_temp = 1 - + + gil_message = "Truth-testing Python object" + def allocate_temps(self, env, result_code = None): # We don't need both operands at the same time, and # one of the operands will also be our result. So we @@ -3438,7 +3513,6 @@ class PrimaryCmpNode(ExprNode, CmpNode): if self.has_int_operands(): self.coerce_chars_to_ints(env) if self.cascade: - #self.operand2 = self.operand2.coerce_to_temp(env) #CTT self.operand2 = self.operand2.coerce_to_simple(env) self.cascade.coerce_cascaded_operands_to_temp(env) self.check_operand_types(env) @@ -3688,8 +3762,11 @@ class PyTypeTestNode(CoercionNode): assert dst_type.is_extension_type or dst_type.is_builtin_type, "PyTypeTest on non extension type" CoercionNode.__init__(self, arg) self.type = dst_type + self.gil_check(env) self.result_ctype = arg.ctype() env.use_utility_code(type_test_utility_code) + + gil_message = "Python type test" def analyse_types(self, env): pass @@ -3724,11 +3801,14 @@ class CoerceToPyTypeNode(CoercionNode): def __init__(self, arg, env): CoercionNode.__init__(self, arg) self.type = py_object_type + self.gil_check(env) self.is_temp = 1 if not arg.type.to_py_function: error(arg.pos, "Cannot convert '%s' to Python object" % arg.type) - + + gil_message = "Converting to Python object" + def generate_result_code(self, code): function = self.arg.type.to_py_function code.putln('%s = %s(%s); %s' % ( @@ -3773,7 +3853,11 @@ class CoerceToBooleanNode(CoercionNode): CoercionNode.__init__(self, arg) self.type = PyrexTypes.c_bint_type if arg.type.is_pyobject: + if env.nogil: + self.gil_error() self.is_temp = 1 + + gil_message = "Truth-testing Python object" def check_const(self): if self.is_temp: @@ -3802,8 +3886,11 @@ class CoerceToTempNode(CoercionNode): self.type = self.arg.type self.is_temp = 1 if self.type.is_pyobject: + self.gil_check(env) self.result_ctype = py_object_type - + + gil_message = "Creating temporary Python reference" + def generate_result_code(self, code): #self.arg.generate_evaluation_code(code) # Already done # by generic generate_subexpr_evaluation_code! diff --git a/Cython/Compiler/Main.py b/Cython/Compiler/Main.py index 5d45408741604ae2d7fe3720cbcd7cbd10f77454..d308457571b86e32501c744faaf6d9805d491652 100644 --- a/Cython/Compiler/Main.py +++ b/Cython/Compiler/Main.py @@ -3,8 +3,8 @@ # import os, sys, re, codecs -if sys.version_info[:2] < (2, 2): - sys.stderr.write("Sorry, Cython requires Python 2.2 or later\n") +if sys.version_info[:2] < (2, 3): + sys.stderr.write("Sorry, Cython requires Python 2.3 or later\n") sys.exit(1) try: @@ -14,14 +14,13 @@ except NameError: from sets import Set as set from time import time +import Code +import Errors +import Parsing import Version from Scanning import PyrexScanner, FileSourceDescriptor -import Errors from Errors import PyrexError, CompileError, error -import Parsing from Symtab import BuiltinScope, ModuleScope -import Code -from Cython.Utils import replace_suffix from Cython import Utils # Note: PHASES and TransformSet should be removed soon; but that's for @@ -116,24 +115,27 @@ class Context: pass return scope - def find_pxd_file(self, module_name, pos): - # Search include directories for the .pxd file - # corresponding to the given (full) module name. - if "." in module_name: - pxd_filename = "%s.pxd" % os.path.join(*module_name.split('.')) - else: - pxd_filename = "%s.pxd" % module_name - return self.search_include_directories(pxd_filename, pos) + def find_pxd_file(self, qualified_name, pos): + # Search include path for the .pxd file corresponding to the + # given fully-qualified module name. + return self.search_include_directories(qualified_name, ".pxd", pos) + + def find_pyx_file(self, qualified_name, pos): + # Search include path for the .pyx file corresponding to the + # given fully-qualified module name, as for find_pxd_file(). + return self.search_include_directories(qualified_name, ".pyx", pos) def find_include_file(self, filename, pos): # Search list of include directories for filename. # Reports an error and returns None if not found. - path = self.search_include_directories(filename, pos) + path = self.search_include_directories(filename, "", pos, + split_package=False) if not path: error(pos, "'%s' not found" % filename) return path - def search_include_directories(self, filename, pos): + def search_include_directories(self, qualified_name, suffix, pos, + split_package=True): # Search the list of include directories for the given # file name. If a source file position is given, first # searches the directory containing that file. Returns @@ -145,12 +147,88 @@ class Context: raise RuntimeError("Only file sources for code supported") here_dir = os.path.dirname(file_desc.filename) dirs = [here_dir] + dirs + + dotted_filename = qualified_name + suffix + if split_package: + names = qualified_name.split('.') + package_names = names[:-1] + module_name = names[-1] + module_filename = module_name + suffix + package_filename = "__init__" + suffix + for dir in dirs: - path = os.path.join(dir, filename) + path = os.path.join(dir, dotted_filename) if os.path.exists(path): return path + if split_package: + package_dir = self.check_package_dir(dir, package_names) + if package_dir is not None: + path = os.path.join(package_dir, module_filename) + if os.path.exists(path): + return path + path = os.path.join(dir, package_dir, module_name, + package_filename) + if os.path.exists(path): + return path return None + def check_package_dir(self, dir, package_names): + package_dir = os.path.join(dir, *package_names) + if not os.path.exists(package_dir): + return None + for dirname in package_names: + dir = os.path.join(dir, dirname) + package_init = os.path.join(dir, "__init__.py") + if not os.path.exists(package_init) and \ + not os.path.exists(package_init + "x"): # same with .pyx ? + return None + return package_dir + + def c_file_out_of_date(self, source_path): + c_path = Utils.replace_suffix(source_path, ".c") + if not os.path.exists(c_path): + return 1 + c_time = Utils.modification_time(c_path) + if Utils.file_newer_than(source_path, c_time): + return 1 + pos = [source_path] + pxd_path = Utils.replace_suffix(source_path, ".pxd") + if os.path.exists(pxd_path) and Utils.file_newer_than(pxd_path, c_time): + return 1 + for kind, name in self.read_dependency_file(source_path): + if kind == "cimport": + dep_path = self.find_pxd_file(name, pos) + elif kind == "include": + dep_path = self.search_include_directories(name, pos) + else: + continue + if dep_path and Utils.file_newer_than(dep_path, c_time): + return 1 + return 0 + + def find_cimported_module_names(self, source_path): + return [ name for kind, name in self.read_dependency_file(source_path) + if kind == "cimport" ] + + def is_package_dir(self, dir_path): + # Return true if the given directory is a package directory. + for filename in ("__init__.py", "__init__.pyx"): + path = os.path.join(dir_path, filename) + if os.path.exists(path): + return 1 + + def read_dependency_file(self, source_path): + dep_path = replace_suffix(source_path, ".dep") + if os.path.exists(dep_path): + f = open(dep_path, "rU") + chunks = [ line.strip().split(" ", 1) + for line in f.readlines() + if " " in line.strip() ] + f.close() + return chunks + else: + return () + def lookup_submodule(self, name): # Look up a top-level module. Returns None if not found. return self.modules.get(name, None) @@ -184,10 +262,23 @@ class Context: return tree def extract_module_name(self, path, options): - # Get the module name out of a source file pathname. - _, tail = os.path.split(path) - name, _ = os.path.splitext(tail) - return name + # Find fully_qualified module name from the full pathname + # of a source file. + dir, filename = os.path.split(path) + module_name, _ = os.path.splitext(filename) + if "." in module_name: + return module_name + if module_name == "__init__": + dir, module_name = os.path.split(dir) + names = [module_name] + while self.is_package_dir(dir): + parent, package_name = os.path.split(dir) + if parent == dir: + break + names.append(package_name) + dir = parent + names.reverse() + return ".".join(names) def compile(self, source, options = None, full_module_name = None): # Compile a Pyrex implementation file in this context @@ -197,16 +288,11 @@ class Context: result = CompilationResult() cwd = os.getcwd() - if full_module_name is None: - full_module_name, _ = os.path.splitext(source) - full_module_name = re.sub(r'[\\/]', '.', full_module_name) - full_module_name = re.sub(r'[^\w.]', '_', full_module_name) - source = os.path.join(cwd, source) result.main_source_file = source if options.use_listing_file: - result.listing_file = replace_suffix(source, ".lis") + result.listing_file = Utils.replace_suffix(source, ".lis") Errors.open_listing_file(result.listing_file, echo_to_stderr = options.errors_to_stderr) else: @@ -218,20 +304,21 @@ class Context: c_suffix = ".cpp" else: c_suffix = ".c" - result.c_file = replace_suffix(source, c_suffix) + result.c_file = Utils .replace_suffix(source, c_suffix) c_stat = None if result.c_file: try: c_stat = os.stat(result.c_file) except EnvironmentError: pass - module_name = full_module_name # self.extract_module_name(source, options) + module_name = full_module_name or self.extract_module_name(source, options) source = FileSourceDescriptor(source) initial_pos = (source, 1, 0) scope = self.find_module(module_name, pos = initial_pos, need_pxd = 0) errors_occurred = False try: - tree = self.parse(source, scope.type_names, pxd = 0, full_module_name = full_module_name) + tree = self.parse(source, scope.type_names, pxd = 0, + full_module_name = full_module_name) tree.process_implementation(scope, options, result) except CompileError: errors_occurred = True @@ -241,8 +328,7 @@ class Context: errors_occurred = True if errors_occurred and result.c_file: try: - #os.unlink(result.c_file) - Utils.castrate_file(result.c_file, c_stat) + Utils.castrate_file(result.c_file, os.stat(source)) except EnvironmentError: pass result.c_file = None @@ -259,7 +345,7 @@ class Context: #------------------------------------------------------------------------ # -# Main Python entry point +# Main Python entry points # #------------------------------------------------------------------------ @@ -273,6 +359,11 @@ class CompilationOptions: include_path [string] Directories to search for include files output_file string Name of generated .c file generate_pxi boolean Generate .pxi file for public declarations + recursive boolean Recursively find and compile dependencies + timestamps boolean Only compile changed source files. If None, + defaults to true when recursive is true. + verbose boolean Always print source names being compiled + quiet boolean Don't print source names in recursive mode transforms Transform.TransformSet Transforms to use on the parse tree Following options are experimental and only used on MacOSX: @@ -283,7 +374,7 @@ class CompilationOptions: cplus boolean Compile as c++ code """ - def __init__(self, defaults = None, **kw): + def __init__(self, defaults = None, c_compile = 0, c_link = 0, **kw): self.include_path = [] self.objects = [] if defaults: @@ -293,6 +384,10 @@ class CompilationOptions: defaults = default_options self.__dict__.update(defaults) self.__dict__.update(kw) + if c_compile: + self.c_only = 0 + if c_link: + self.obj_only = 0 class CompilationResult: @@ -320,24 +415,90 @@ class CompilationResult: self.main_source_file = None -def compile(source, options = None, c_compile = 0, c_link = 0, - full_module_name = None): +class CompilationResultSet(dict): + """ + Results from compiling multiple Pyrex source files. A mapping + from source file paths to CompilationResult instances. Also + has the following attributes: + + num_errors integer Total number of compilation errors + """ + + num_errors = 0 + + def add(self, source, result): + self[source] = result + self.num_errors += result.num_errors + + +def compile_single(source, options, full_module_name = None): """ - compile(source, options = default_options) + compile_single(source, options, full_module_name) - Compile the given Cython implementation file and return - a CompilationResult object describing what was produced. + Compile the given Pyrex implementation file and return a CompilationResult. + Always compiles a single file; does not perform timestamp checking or + recursion. """ - if not options: - options = default_options - options = CompilationOptions(defaults = options) - if c_compile: - options.c_only = 0 - if c_link: - options.obj_only = 0 context = Context(options.include_path) return context.compile(source, options, full_module_name) +def compile_multiple(sources, options): + """ + compile_multiple(sources, options) + + Compiles the given sequence of Pyrex implementation files and returns + a CompilationResultSet. Performs timestamp checking and/or recursion + if these are specified in the options. + """ + sources = [os.path.abspath(source) for source in sources] + processed = set() + results = CompilationResultSet() + context = Context(options.include_path) + recursive = options.recursive + timestamps = options.timestamps + if timestamps is None: + timestamps = recursive + verbose = options.verbose or ((recursive or timestamps) and not options.quiet) + for source in sources: + if source not in processed: + if not timestamps or context.c_file_out_of_date(source): + if verbose: + sys.stderr.write("Compiling %s\n" % source) + result = context.compile(source, options) + # Compiling multiple sources in one context doesn't quite + # work properly yet. + context = Context(options.include_path) # to be removed later + results.add(source, result) + processed.add(source) + if recursive: + for module_name in context.find_cimported_module_names(source): + path = context.find_pyx_file(module_name, [source]) + if path: + sources.append(path) + else: + sys.stderr.write( + "Cannot find .pyx file for cimported module '%s'\n" % module_name) + return results + +def compile(source, options = None, c_compile = 0, c_link = 0, + full_module_name = None, **kwds): + """ + compile(source [, options], [, <option> = <value>]...) + + Compile one or more Pyrex implementation files, with optional timestamp + checking and recursing on dependecies. The source argument may be a string + or a sequence of strings If it is a string and no recursion or timestamp + checking is requested, a CompilationResult is returned, otherwise a + CompilationResultSet is returned. + """ + options = CompilationOptions(defaults = options, c_compile = c_compile, + c_link = c_link, **kwds) + if isinstance(source, basestring) and not options.timestamps \ + and not options.recursive: + return compile_single(source, options, full_module_name) + else: + return compile_multiple(source, options) + #------------------------------------------------------------------------ # # Main command-line entry point @@ -351,21 +512,19 @@ def main(command_line = 0): from CmdLine import parse_command_line options, sources = parse_command_line(args) else: - options = default_options + options = CompilationOptions(default_options) sources = args if options.show_version: sys.stderr.write("Cython version %s\n" % Version.version) if options.working_path!="": os.chdir(options.working_path) - context = Context(options.include_path) - for source in sources: - try: - result = context.compile(source, options) - if result.num_errors > 0: - any_failures = 1 - except PyrexError, e: - sys.stderr.write(str(e) + '\n') + try: + result = compile(sources, options) + if result.num_errors > 0: any_failures = 1 + except (EnvironmentError, PyrexError), e: + sys.stderr.write(str(e) + '\n') + any_failures = 1 if any_failures: sys.exit(1) @@ -388,8 +547,11 @@ default_options = dict( annotate = False, generate_pxi = 0, transforms = TransformSet(), - working_path = "") - + working_path = "", + recursive = 0, + timestamps = None, + verbose = 0, + quiet = 0) if sys.platform == "mac": from Cython.Mac.MacSystem import c_compile, c_link, CCompilerError default_options['use_listing_file'] = 1 diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py index 61df6912cecea355c0381a4dede8101cb8b97e1c..5f8fcd97e974ba5af4bf1dae02f10fbc14ae8731 100644 --- a/Cython/Compiler/ModuleNode.py +++ b/Cython/Compiler/ModuleNode.py @@ -54,6 +54,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if self.has_imported_c_functions(): self.module_temp_cname = env.allocate_temp_pyobject() env.release_temp(self.module_temp_cname) + self.generate_dep_file(env, result) self.generate_c_code(env, options, result) self.generate_h_code(env, options, result) self.generate_api_code(env, result) @@ -65,6 +66,20 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): return 1 return 0 + def generate_dep_file(self, env, result): + modules = self.referenced_modules + if len(modules) > 1 or env.included_files: + dep_file = replace_suffix(result.c_file, ".dep") + f = open(dep_file, "w") + try: + for module in modules: + if module is not env: + f.write("cimport %s\n" % module.qualified_name) + for path in module.included_files: + f.write("include %s\n" % path) + finally: + f.close() + def generate_h_code(self, env, options, result): def h_entries(entries, pxd = 0): return [entry for entry in entries @@ -348,12 +363,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def generate_declarations_for_modules(self, env, modules, code): code.putln("") - code.putln("/* Declarations */") + code.putln("/* Type declarations */") vtab_list, vtabslot_list = self.sort_type_hierarchy(modules, env) self.generate_type_definitions( env, modules, vtab_list, vtabslot_list, code) for module in modules: defined_here = module is env + code.putln("/* Module declarations from %s */" % + module.qualified_name.encode("ASCII", "ignore")) self.generate_global_declarations(module, code, defined_here) self.generate_cfunction_predeclarations(module, code, defined_here) @@ -440,6 +457,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(" #define PyInt_AsUnsignedLongMask PyLong_AsUnsignedLongMask") code.putln(" #define PyInt_AsUnsignedLongLongMask PyLong_AsUnsignedLongLongMask") code.putln(" #define PyNumber_Divide(x,y) PyNumber_TrueDivide(x,y)") + code.putln("#else") + code.putln(" #define PyBytes_Type PyString_Type") code.putln("#endif") code.putln("#if PY_MAJOR_VERSION >= 3") @@ -686,8 +705,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): entry.type.typeptr_cname) code.put_var_declarations(env.var_entries, static = 1, dll_linkage = "DL_EXPORT", definition = definition) - code.put_var_declarations(env.default_entries, static = 1, - definition = definition) + if definition: + code.put_var_declarations(env.default_entries, static = 1, + definition = definition) def generate_cfunction_predeclarations(self, env, code, definition): for entry in env.cfunc_entries: @@ -2124,8 +2144,13 @@ __Pyx_import_all_from(PyObject *locals, PyObject *v) break; } if (skip_leading_underscores && +#if PY_MAJOR_VERSION < 3 PyString_Check(name) && PyString_AS_STRING(name)[0] == '_') +#else + PyUnicode_Check(name) && + PyUnicode_AS_UNICODE(name)[0] == '_') +#endif { Py_DECREF(name); continue; @@ -2147,10 +2172,11 @@ __Pyx_import_all_from(PyObject *locals, PyObject *v) } -static int %s(PyObject* m) { +static int %(IMPORT_STAR)s(PyObject* m) { int i; int ret = -1; + char* s; PyObject *locals = 0; PyObject *list = 0; PyObject *name; @@ -2163,7 +2189,13 @@ static int %s(PyObject* m) { for(i=0; i<PyList_GET_SIZE(list); i++) { name = PyTuple_GET_ITEM(PyList_GET_ITEM(list, i), 0); item = PyTuple_GET_ITEM(PyList_GET_ITEM(list, i), 1); - if (%s(item, name, PyString_AsString(name)) < 0) goto bad; +#if PY_MAJOR_VERSION < 3 + s = PyString_AsString(name); +#else + s = PyUnicode_AsString(name); +#endif + if (!s) goto bad; + if (%(IMPORT_STAR_SET)s(item, name, s) < 0) goto bad; } ret = 0; @@ -2172,4 +2204,5 @@ bad: Py_XDECREF(list); return ret; } -""" % ( Naming.import_star, Naming.import_star_set ) +""" % {'IMPORT_STAR' : Naming.import_star, + 'IMPORT_STAR_SET' : Naming.import_star_set } diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py index c280caa533e3a0b80dee644c99341af184a48b26..829c4ffb841d3d7cd35b4ee14e00d7bdc369ebc6 100644 --- a/Cython/Compiler/Nodes.py +++ b/Cython/Compiler/Nodes.py @@ -83,6 +83,15 @@ class Node(object): self.pos = pos self.__dict__.update(kw) + gil_message = "Operation" + + def gil_check(self, env): + if env.nogil: + self.gil_error() + + def gil_error(self): + error(self.pos, "%s not allowed without gil" % self.gil_message) + def clone_node(self): """Clone the node. This is defined as a shallow copy, except for member lists amongst the child attributes (from get_child_accessors) which are also @@ -431,9 +440,9 @@ class CFuncDeclaratorNode(CDeclaratorNode): # Catch attempted C-style func(void) decl if type.is_void: error(arg_node.pos, "Use spam() rather than spam(void) to declare a function with no arguments.") - if type.is_pyobject and self.nogil: - error(self.pos, - "Function with Python argument cannot be declared nogil") +# if type.is_pyobject and self.nogil: +# error(self.pos, +# "Function with Python argument cannot be declared nogil") func_type_args.append( PyrexTypes.CFuncTypeArg(name, type, arg_node.pos)) if arg_node.default: @@ -480,9 +489,6 @@ class CFuncDeclaratorNode(CDeclaratorNode): error(self.exception_value.pos, "Exception value incompatible with function return type") exc_check = self.exception_check - if return_type.is_pyobject and self.nogil: - error(self.pos, - "Function with Python return type cannot be declared nogil") if return_type.is_array: error(self.pos, "Function cannot return an array") @@ -810,6 +816,9 @@ class FuncDefNode(StatNode, BlockNode): genv = env.global_scope() lenv = LocalScope(name = self.entry.name, outer_scope = genv) lenv.return_type = self.return_type + type = self.entry.type + if type.is_cfunction: + lenv.nogil = type.nogil and not type.with_gil code.init_labels() self.declare_arguments(lenv) transforms.run('before_analyse_function', self, env=env, lenv=lenv, genv=genv) @@ -890,24 +899,19 @@ class FuncDefNode(StatNode, BlockNode): exc_check = self.caller_will_check_exceptions() if err_val is not None or exc_check: code.putln('__Pyx_AddTraceback("%s");' % self.entry.qualified_name) - if err_val is not None: - code.putln( - "%s = %s;" % ( - Naming.retval_cname, - err_val)) else: code.putln( '__Pyx_WriteUnraisable("%s");' % self.entry.qualified_name) env.use_utility_code(unraisable_exception_utility_code) - #if not self.return_type.is_void: - default_retval = self.return_type.default_value - if default_retval: - code.putln( - "%s = %s;" % ( - Naming.retval_cname, - default_retval)) - #self.return_type.default_value)) + default_retval = self.return_type.default_value + if err_val is None and default_retval: + err_val = default_retval + if err_val is not None: + code.putln( + "%s = %s;" % ( + Naming.retval_cname, + err_val)) # ----- Return cleanup code.put_label(code.return_label) if not Options.init_local_none: @@ -1056,8 +1060,12 @@ class CFuncDefNode(FuncDefNode): self.declare_argument(env, arg) def need_gil_acquisition(self, lenv): + type = self.type with_gil = self.type.with_gil - if self.type.nogil and not with_gil: + if type.nogil and not with_gil: + if type.return_type.is_pyobject: + error(self.pos, + "Function with Python return type cannot be declared nogil") for entry in lenv.var_entries + lenv.temp_entries: if entry.type.is_pyobject: error(self.pos, "Function declared nogil has Python locals or temporaries") @@ -2159,6 +2167,7 @@ class SingleAssignmentNode(AssignmentNode): def analyse_types(self, env, use_temp = 0): self.rhs.analyse_types(env) self.lhs.analyse_target_types(env) + self.lhs.gil_assignment_check(env) self.rhs = self.rhs.coerce_to(self.lhs.type, env) if use_temp: self.rhs = self.rhs.coerce_to_temp(env) @@ -2223,6 +2232,7 @@ class CascadedAssignmentNode(AssignmentNode): self.coerced_rhs_list = [] for lhs in self.lhs_list: lhs.analyse_target_types(env) + lhs.gil_assignment_check(env) rhs = CloneNode(self.rhs) rhs = rhs.coerce_to(lhs.type, env) self.coerced_rhs_list.append(rhs) @@ -2465,15 +2475,9 @@ class PrintStatNode(StatNode): self.arg_tuple = self.arg_tuple.coerce_to_pyobject(env) self.arg_tuple.release_temp(env) env.use_utility_code(printing_utility_code) - return - for i in range(len(self.args)): - arg = self.args[i] - arg.analyse_types(env) - arg = arg.coerce_to_pyobject(env) - arg.allocate_temps(env) - arg.release_temp(env) - self.args[i] = arg - #env.recycle_pending_temps() # TEMPORARY + self.gil_check(env) + + gil_message = "Python print statement" def generate_execution_code(self, code): self.arg_tuple.generate_evaluation_code(code) @@ -2502,10 +2506,14 @@ class DelStatNode(StatNode): def analyse_expressions(self, env): for arg in self.args: arg.analyse_target_expression(env, None) - if not arg.type.is_pyobject: + if arg.type.is_pyobject: + self.gil_check(env) + else: error(arg.pos, "Deletion of non-Python object") #arg.release_target_temp(env) - + + gil_message = "Deleting Python object" + def generate_execution_code(self, code): for arg in self.args: if arg.type.is_pyobject: @@ -2592,7 +2600,11 @@ class ReturnStatNode(StatNode): and not return_type.is_pyobject and not return_type.is_returncode): error(self.pos, "Return value required") - + if return_type.is_pyobject: + self.gil_check(env) + + gil_message = "Returning Python object" + def generate_execution_code(self, code): code.mark_pos(self.pos) if not self.return_type: @@ -2654,11 +2666,11 @@ class RaiseStatNode(StatNode): self.exc_value.release_temp(env) if self.exc_tb: self.exc_tb.release_temp(env) -# if not (self.exc_type or self.exc_value or self.exc_tb): -# env.use_utility_code(reraise_utility_code) -# else: env.use_utility_code(raise_utility_code) - + self.gil_check(env) + + gil_message = "Raising exception" + def generate_execution_code(self, code): if self.exc_type: self.exc_type.generate_evaluation_code(code) @@ -2707,8 +2719,11 @@ class ReraiseStatNode(StatNode): child_attrs = [] def analyse_expressions(self, env): + self.gil_check(env) env.use_utility_code(raise_utility_code) + gil_message = "Raising exception" + def generate_execution_code(self, code): vars = code.exc_vars if vars: @@ -2735,7 +2750,10 @@ class AssertStatNode(StatNode): self.cond.release_temp(env) if self.value: self.value.release_temp(env) + self.gil_check(env) #env.recycle_pending_temps() # TEMPORARY + + gil_message = "Raising exception" def generate_execution_code(self, code): code.putln("#ifndef PYREX_WITHOUT_ASSERTIONS") @@ -2831,7 +2849,6 @@ class IfClauseNode(Node): self.condition = \ self.condition.analyse_temp_boolean_expression(env) self.condition.release_temp(env) - #env.recycle_pending_temps() # TEMPORARY self.body.analyse_expressions(env) def generate_execution_code(self, code, end_label): @@ -3236,6 +3253,7 @@ class TryExceptStatNode(StatNode): except_clause.analyse_declarations(env) if self.else_clause: self.else_clause.analyse_declarations(env) + self.gil_check(env) def analyse_expressions(self, env): self.body.analyse_expressions(env) @@ -3244,7 +3262,10 @@ class TryExceptStatNode(StatNode): except_clause.analyse_expressions(env) if self.else_clause: self.else_clause.analyse_expressions(env) - + self.gil_check(env) + + gil_message = "Try-except statement" + def generate_execution_code(self, code): old_error_label = code.new_error_label() our_error_label = code.error_label @@ -3404,16 +3425,11 @@ class TryFinallyStatNode(StatNode): def analyse_expressions(self, env): self.body.analyse_expressions(env) self.cleanup_list = env.free_temp_entries[:] - #self.exc_vars = ( - # env.allocate_temp(PyrexTypes.py_object_type), - # env.allocate_temp(PyrexTypes.py_object_type), - # env.allocate_temp(PyrexTypes.py_object_type)) - #self.lineno_var = \ - # env.allocate_temp(PyrexTypes.c_int_type) self.finally_clause.analyse_expressions(env) - #for var in self.exc_vars: - # env.release_temp(var) - + self.gil_check(env) + + gil_message = "Try-finally statement" + def generate_execution_code(self, code): old_error_label = code.error_label old_labels = code.all_new_labels() @@ -3558,6 +3574,15 @@ class GILStatNode(TryFinallyStatNode): body = body, finally_clause = GILExitNode(pos, state = state)) + def analyse_expressions(self, env): + was_nogil = env.nogil + env.nogil = 1 + TryFinallyStatNode.analyse_expressions(self, env) + env.nogil = was_nogil + + def gil_check(self, env): + pass + def generate_execution_code(self, code): code.putln("/*with %s:*/ {" % self.state) if self.state == 'gil': @@ -3568,19 +3593,6 @@ class GILStatNode(TryFinallyStatNode): TryFinallyStatNode.generate_execution_code(self, code) code.putln("}") -#class GILEntryNode(StatNode): -# # state string 'gil' or 'nogil' -# -# def analyse_expressions(self, env): -# pass -# -# def generate_execution_code(self, code): -# if self.state == 'gil': -# code.putln("PyGILState_STATE _save = PyGILState_Ensure();") -# else: -# code.putln("PyThreadState *_save;") -# code.putln("Py_UNBLOCK_THREADS") - class GILExitNode(StatNode): # Used as the 'finally' block in a GILStatNode diff --git a/Cython/Compiler/Parsing.py b/Cython/Compiler/Parsing.py index 7bdfa4d2d690371f258a36558184bbddc289de1a..32ecde7d67eae88b441a4d8c3936a0b7b4be658b 100644 --- a/Cython/Compiler/Parsing.py +++ b/Cython/Compiler/Parsing.py @@ -3,7 +3,6 @@ # import os, re -from string import join, replace from types import ListType, TupleType from Scanning import PyrexScanner, FileSourceDescriptor import Nodes @@ -13,6 +12,26 @@ from Errors import error, InternalError from Cython import Utils import Future +class Ctx(object): + # Parsing context + level = 'other' + visibility = 'private' + cdef_flag = 0 + typedef_flag = 0 + api = 0 + overridable = 0 + nogil = 0 + + def __init__(self, **kwds): + self.__dict__.update(kwds) + + def __call__(self, **kwds): + ctx = Ctx() + d = ctx.__dict__ + d.update(self.__dict__) + d.update(kwds) + return ctx + def p_ident(s, message = "Expected an identifier"): if s.sy == 'IDENT': name = s.systring @@ -38,9 +57,7 @@ def p_ident_list(s): #------------------------------------------ def p_binop_expr(s, ops, p_sub_expr): - #print "p_binop_expr:", ops, p_sub_expr ### n1 = p_sub_expr(s) - #print "p_binop_expr(%s):" % p_sub_expr, s.sy ### while s.sy in ops: op = s.sy pos = s.position() @@ -74,7 +91,6 @@ def p_test(s): #or_test: and_test ('or' and_test)* def p_or_test(s): - #return p_binop_expr(s, ('or',), p_and_test) return p_rassoc_binop_expr(s, ('or',), p_and_test) def p_rassoc_binop_expr(s, ops, p_subexpr): @@ -982,7 +998,7 @@ def p_from_import_statement(s, first_statement = 0): items.append( (name, ExprNodes.NameNode(name_pos, - name = as_name or name))) + name = as_name or name))) import_list = ExprNodes.ListNode( imported_names[0][0], args = imported_name_strings) dotted_name = Utils.EncodedString(dotted_name) @@ -1008,7 +1024,7 @@ def p_dotted_name(s, as_allowed): names.append(p_ident(s)) if as_allowed: as_name = p_as_name(s) - return (pos, target_name, join(names, "."), as_name) + return (pos, target_name, u'.'.join(names), as_name) def p_as_name(s): if s.sy == 'IDENT' and s.systring == 'as': @@ -1196,7 +1212,7 @@ def p_except_clause(s): return Nodes.ExceptClauseNode(pos, pattern = exc_type, target = exc_value, body = body) -def p_include_statement(s, level): +def p_include_statement(s, ctx): pos = s.position() s.next() # 'include' _, include_file_name = p_string_literal(s) @@ -1204,11 +1220,12 @@ def p_include_statement(s, level): if s.compile_time_eval: include_file_path = s.context.find_include_file(include_file_name, pos) if include_file_path: + s.included_files.append(include_file_name) f = Utils.open_source_file(include_file_path, mode="rU") source_desc = FileSourceDescriptor(include_file_path) s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding) try: - tree = p_statement_list(s2, level) + tree = p_statement_list(s2, ctx) finally: f.close() return tree @@ -1258,7 +1275,7 @@ def p_simple_statement(s, first_statement = 0): node = p_expression_or_assignment(s) return node -def p_simple_statement_list(s, first_statement = 0): +def p_simple_statement_list(s, ctx, first_statement = 0): # Parse a series of simple statements on one line # separated by semicolons. stat = p_simple_statement(s, first_statement = first_statement) @@ -1294,7 +1311,7 @@ def p_DEF_statement(s): s.expect_newline() return Nodes.PassStatNode(pos) -def p_IF_statement(s, level, cdef_flag, visibility, api): +def p_IF_statement(s, ctx): pos = s.position() saved_eval = s.compile_time_eval current_eval = saved_eval @@ -1304,7 +1321,7 @@ def p_IF_statement(s, level, cdef_flag, visibility, api): s.next() # 'IF' or 'ELIF' expr = p_compile_time_expr(s) s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv)) - body = p_suite(s, level, cdef_flag, visibility, api = api) + body = p_suite(s, ctx) if s.compile_time_eval: result = body current_eval = 0 @@ -1313,7 +1330,7 @@ def p_IF_statement(s, level, cdef_flag, visibility, api): if s.sy == 'ELSE': s.next() s.compile_time_eval = current_eval - body = p_suite(s, level, cdef_flag, visibility, api = api) + body = p_suite(s, ctx) if current_eval: result = body if not result: @@ -1321,18 +1338,18 @@ def p_IF_statement(s, level, cdef_flag, visibility, api): s.compile_time_eval = saved_eval return result -def p_statement(s, level, cdef_flag = 0, visibility = 'private', api = 0, - first_statement = 0): +def p_statement(s, ctx, first_statement = 0): + cdef_flag = ctx.cdef_flag if s.sy == 'ctypedef': - if level not in ('module', 'module_pxd'): + if ctx.level not in ('module', 'module_pxd'): s.error("ctypedef statement not allowed here") - if api: + if ctx.api: error(s.position(), "'api' not allowed with 'ctypedef'") - return p_ctypedef_statement(s, level, visibility, api) + return p_ctypedef_statement(s, ctx) elif s.sy == 'DEF': return p_DEF_statement(s) elif s.sy == 'IF': - return p_IF_statement(s, level, cdef_flag, visibility, api) + return p_IF_statement(s, ctx) else: overridable = 0 if s.sy == 'cdef': @@ -1343,36 +1360,32 @@ def p_statement(s, level, cdef_flag = 0, visibility = 'private', api = 0, overridable = 1 s.next() if cdef_flag: - if level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'): + if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'): s.error('cdef statement not allowed here') - s.level = level - return p_cdef_statement(s, level, visibility = visibility, - api = api, overridable = overridable) - # elif s.sy == 'cpdef': - # s.next() - # return p_c_func_or_var_declaration(s, level, s.position(), visibility = visibility, api = api, overridable = True) + s.level = ctx.level + return p_cdef_statement(s, ctx(overridable = overridable)) else: - if api: + if ctx.api: error(s.pos, "'api' not allowed with this statement") elif s.sy == 'def': - if level not in ('module', 'class', 'c_class', 'property'): + if ctx.level not in ('module', 'class', 'c_class', 'property'): s.error('def statement not allowed here') - s.level = level + s.level = ctx.level return p_def_statement(s) elif s.sy == 'class': - if level != 'module': + if ctx.level != 'module': s.error("class definition not allowed here") return p_class_statement(s) elif s.sy == 'include': - if level not in ('module', 'module_pxd'): + if ctx.level not in ('module', 'module_pxd'): s.error("include statement not allowed here") - return p_include_statement(s, level) - elif level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property': + return p_include_statement(s, ctx) + elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property': return p_property_decl(s) - elif s.sy == 'pass' and level != 'property': + elif s.sy == 'pass' and ctx.level != 'property': return p_pass_statement(s, with_newline = 1) else: - if level in ('c_class_pxd', 'property'): + if ctx.level in ('c_class_pxd', 'property'): s.error("Executable statement not allowed here") if s.sy == 'if': return p_if_statement(s) @@ -1385,25 +1398,22 @@ def p_statement(s, level, cdef_flag = 0, visibility = 'private', api = 0, elif s.sy == 'with': return p_with_statement(s) else: - return p_simple_statement_list(s, first_statement = first_statement) + return p_simple_statement_list( + s, ctx, first_statement = first_statement) -def p_statement_list(s, level, - cdef_flag = 0, visibility = 'private', api = 0, first_statement = 0): +def p_statement_list(s, ctx, first_statement = 0): # Parse a series of statements separated by newlines. pos = s.position() stats = [] while s.sy not in ('DEDENT', 'EOF'): - stats.append(p_statement(s, level, - cdef_flag = cdef_flag, visibility = visibility, api = api, - first_statement = first_statement)) + stats.append(p_statement(s, ctx, first_statement = first_statement)) first_statement = 0 if len(stats) == 1: return stats[0] else: return Nodes.StatListNode(pos, stats = stats) -def p_suite(s, level = 'other', cdef_flag = 0, - visibility = 'private', with_doc = 0, with_pseudo_doc = 0, api = 0): +def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0): pos = s.position() s.expect(':') doc = None @@ -1413,17 +1423,13 @@ def p_suite(s, level = 'other', cdef_flag = 0, s.expect_indent() if with_doc or with_pseudo_doc: doc = p_doc_string(s) - body = p_statement_list(s, - level = level, - cdef_flag = cdef_flag, - visibility = visibility, - api = api) + body = p_statement_list(s, ctx) s.expect_dedent() else: - if api: + if ctx.api: error(s.pos, "'api' not allowed with this statement") - if level in ('module', 'class', 'function', 'other'): - body = p_simple_statement_list(s) + if ctx.level in ('module', 'class', 'function', 'other'): + body = p_simple_statement_list(s, ctx) else: body = p_pass_statement(s) s.expect_newline("Syntax error in declarations") @@ -1556,8 +1562,9 @@ def p_opt_cname(s): cname = None return cname -def p_c_declarator(s, empty = 0, is_type = 0, cmethod_flag = 0, assignable = 0, - nonempty = 0, calling_convention_allowed = 0): +def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0, + assignable = 0, nonempty = 0, + calling_convention_allowed = 0): # If empty is true, the declarator must be empty. If nonempty is true, # the declarator must be nonempty. Otherwise we don't care. # If cmethod_flag is true, then if this declarator declares @@ -1567,13 +1574,16 @@ def p_c_declarator(s, empty = 0, is_type = 0, cmethod_flag = 0, assignable = 0, s.next() if s.sy == ')' or looking_at_type(s): base = Nodes.CNameDeclaratorNode(pos, name = "", cname = None) - result = p_c_func_declarator(s, pos, base, cmethod_flag) + result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag) else: - result = p_c_declarator(s, empty, is_type, cmethod_flag, nonempty = nonempty, - calling_convention_allowed = 1) + result = p_c_declarator(s, ctx, empty = empty, is_type = is_type, + cmethod_flag = cmethod_flag, + nonempty = nonempty, + calling_convention_allowed = 1) s.expect(')') else: - result = p_c_simple_declarator(s, empty, is_type, cmethod_flag, assignable, nonempty) + result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, + assignable, nonempty) if not calling_convention_allowed and result.calling_convention and s.sy != '(': error(s.position(), "%s on something that is not a function" % result.calling_convention) @@ -1583,7 +1593,7 @@ def p_c_declarator(s, empty = 0, is_type = 0, cmethod_flag = 0, assignable = 0, result = p_c_array_declarator(s, result) else: # sy == '(' s.next() - result = p_c_func_declarator(s, pos, result, cmethod_flag) + result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag) cmethod_flag = 0 return result @@ -1597,10 +1607,10 @@ def p_c_array_declarator(s, base): s.expect(']') return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim) -def p_c_func_declarator(s, pos, base, cmethod_flag): +def p_c_func_declarator(s, pos, ctx, base, cmethod_flag): # Opening paren has already been skipped - args = p_c_arg_list(s, in_pyfunc = 0, cmethod_flag = cmethod_flag, - nonempty_declarators = 0) + args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag, + nonempty_declarators = 0) ellipsis = p_optional_ellipsis(s) s.expect(')') nogil = p_nogil(s) @@ -1609,19 +1619,24 @@ def p_c_func_declarator(s, pos, base, cmethod_flag): return Nodes.CFuncDeclaratorNode(pos, base = base, args = args, has_varargs = ellipsis, exception_value = exc_val, exception_check = exc_check, - nogil = nogil or with_gil, with_gil = with_gil) + nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil) -def p_c_simple_declarator(s, empty, is_type, cmethod_flag, assignable, nonempty): +def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag, + assignable, nonempty): pos = s.position() calling_convention = p_calling_convention(s) if s.sy == '*': s.next() - base = p_c_declarator(s, empty, is_type, cmethod_flag, assignable, nonempty) + base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, + cmethod_flag = cmethod_flag, + assignable = assignable, nonempty = nonempty) result = Nodes.CPtrDeclaratorNode(pos, base = base) elif s.sy == '**': # scanner returns this as a single token s.next() - base = p_c_declarator(s, empty, is_type, cmethod_flag, assignable, nonempty) + base = p_c_declarator(s, ctx, empty = empty, is_type = is_type, + cmethod_flag = cmethod_flag, + assignable = assignable, nonempty = nonempty) result = Nodes.CPtrDeclaratorNode(pos, base = Nodes.CPtrDeclaratorNode(pos, base = base)) @@ -1687,28 +1702,14 @@ def p_exception_value_clause(s): c_arg_list_terminators = ('*', '**', '.', ')') -#def p_c_arg_list(s, in_pyfunc, cmethod_flag = 0, nonempty_declarators = 0, -# kw_only = 0): -# args = [] -# if s.sy not in c_arg_list_terminators: -# args.append(p_c_arg_decl(s, in_pyfunc, cmethod_flag, -# nonempty = nonempty_declarators, kw_only = kw_only)) -# while s.sy == ',': -# s.next() -# if s.sy in c_arg_list_terminators: -# break -# args.append(p_c_arg_decl(s, in_pyfunc), nonempty = nonempty_declarators, -# kw_only = kw_only) -# return args - -def p_c_arg_list(s, in_pyfunc, cmethod_flag = 0, nonempty_declarators = 0, - kw_only = 0): +def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0, + nonempty_declarators = 0, kw_only = 0): # Comma-separated list of C argument declarations, possibly empty. # May have a trailing comma. args = [] is_self_arg = cmethod_flag while s.sy not in c_arg_list_terminators: - args.append(p_c_arg_decl(s, in_pyfunc, is_self_arg, + args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg, nonempty = nonempty_declarators, kw_only = kw_only)) if s.sy != ',': break @@ -1723,12 +1724,12 @@ def p_optional_ellipsis(s): else: return 0 -def p_c_arg_decl(s, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0): +def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0): pos = s.position() not_none = 0 default = None base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty) - declarator = p_c_declarator(s, nonempty = nonempty) + declarator = p_c_declarator(s, ctx, nonempty = nonempty) if s.sy == 'not': s.next() if s.sy == 'IDENT' and s.systring == 'None': @@ -1761,57 +1762,60 @@ def p_api(s): else: return 0 -def p_cdef_statement(s, level, visibility = 'private', api = 0, - overridable = False): +def p_cdef_statement(s, ctx): pos = s.position() - visibility = p_visibility(s, visibility) - api = api or p_api(s) - if api: - if visibility not in ('private', 'public'): - error(pos, "Cannot combine 'api' with '%s'" % visibility) - if (visibility == 'extern') and s.sy == 'from': - return p_cdef_extern_block(s, level, pos) + ctx.visibility = p_visibility(s, ctx.visibility) + ctx.api = ctx.api or p_api(s) + if ctx.api: + if ctx.visibility not in ('private', 'public'): + error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility) + if (ctx.visibility == 'extern') and s.sy == 'from': + return p_cdef_extern_block(s, pos, ctx) elif s.sy == 'import': s.next() - return p_cdef_extern_block(s, level, pos) - elif s.sy == ':': - return p_cdef_block(s, level, visibility, api) + return p_cdef_extern_block(s, pos, ctx) + if p_nogil(s): + ctx.nogil = 1 + if s.sy == ':': + return p_cdef_block(s, ctx) elif s.sy == 'class': - if level not in ('module', 'module_pxd'): + if ctx.level not in ('module', 'module_pxd'): error(pos, "Extension type definition not allowed here") - #if api: + #if ctx.api: # error(pos, "'api' not allowed with extension class") - return p_c_class_definition(s, level, pos, visibility = visibility, api = api) + return p_c_class_definition(s, pos, ctx) elif s.sy == 'IDENT' and s.systring in struct_union_or_enum: - if level not in ('module', 'module_pxd'): + if ctx.level not in ('module', 'module_pxd'): error(pos, "C struct/union/enum definition not allowed here") - #if visibility == 'public': + #if ctx.visibility == 'public': # error(pos, "Public struct/union/enum definition not implemented") - #if api: + #if ctx.api: # error(pos, "'api' not allowed with '%s'" % s.systring) if s.systring == "enum": - return p_c_enum_definition(s, pos, level, visibility) + return p_c_enum_definition(s, pos, ctx) else: - return p_c_struct_or_union_definition(s, pos, level, visibility) + return p_c_struct_or_union_definition(s, pos, ctx) elif s.sy == 'pass': node = p_pass_statement(s) s.expect_newline('Expected a newline') return node else: - return p_c_func_or_var_declaration(s, level, pos, visibility, api, - overridable) + return p_c_func_or_var_declaration(s, pos, ctx) -def p_cdef_block(s, level, visibility, api): - return p_suite(s, level, cdef_flag = 1, visibility = visibility, api = api) +def p_cdef_block(s, ctx): + return p_suite(s, ctx(cdef_flag = 1)) -def p_cdef_extern_block(s, level, pos): +def p_cdef_extern_block(s, pos, ctx): include_file = None s.expect('from') if s.sy == '*': s.next() else: _, include_file = p_string_literal(s) - body = p_suite(s, level, cdef_flag = 1, visibility = 'extern') + ctx = ctx(cdef_flag = 1, visibility = 'extern') + if p_nogil(s): + ctx.nogil = 1 + body = p_suite(s, ctx) return Nodes.CDefExternNode(pos, include_file = include_file, body = body) @@ -1820,7 +1824,7 @@ struct_union_or_enum = ( "struct", "union", "enum" ) -def p_c_enum_definition(s, pos, level, visibility, typedef_flag = 0): +def p_c_enum_definition(s, pos, ctx): # s.sy == ident 'enum' s.next() if s.sy == 'IDENT': @@ -1842,9 +1846,10 @@ def p_c_enum_definition(s, pos, level, visibility, typedef_flag = 0): while s.sy not in ('DEDENT', 'EOF'): p_c_enum_line(s, items) s.expect_dedent() - return Nodes.CEnumDefNode(pos, name = name, cname = cname, - items = items, typedef_flag = typedef_flag, visibility = visibility, - in_pxd = level == 'module_pxd') + return Nodes.CEnumDefNode( + pos, name = name, cname = cname, items = items, + typedef_flag = ctx.typedef_flag, visibility = ctx.visibility, + in_pxd = ctx.level == 'module_pxd') def p_c_enum_line(s, items): if s.sy != 'pass': @@ -1869,7 +1874,7 @@ def p_c_enum_item(s, items): items.append(Nodes.CEnumDefItemNode(pos, name = name, cname = cname, value = value)) -def p_c_struct_or_union_definition(s, pos, level, visibility, typedef_flag = 0): +def p_c_struct_or_union_definition(s, pos, ctx): # s.sy == ident 'struct' or 'union' kind = s.systring s.next() @@ -1882,10 +1887,11 @@ def p_c_struct_or_union_definition(s, pos, level, visibility, typedef_flag = 0): s.expect('NEWLINE') s.expect_indent() attributes = [] + body_ctx = Ctx() while s.sy != 'DEDENT': if s.sy != 'pass': attributes.append( - p_c_func_or_var_declaration(s, level = 'other', pos = s.position())) + p_c_func_or_var_declaration(s, s.position(), body_ctx)) else: s.next() s.expect_newline("Expected a newline") @@ -1894,8 +1900,8 @@ def p_c_struct_or_union_definition(s, pos, level, visibility, typedef_flag = 0): s.expect_newline("Syntax error in struct or union definition") return Nodes.CStructOrUnionDefNode(pos, name = name, cname = cname, kind = kind, attributes = attributes, - typedef_flag = typedef_flag, visibility = visibility, - in_pxd = level == 'module_pxd') + typedef_flag = ctx.typedef_flag, visibility = ctx.visibility, + in_pxd = ctx.level == 'module_pxd') def p_visibility(s, prev_visibility): pos = s.position() @@ -1915,26 +1921,26 @@ def p_c_modifiers(s): return [modifier] + p_c_modifiers(s) return [] -def p_c_func_or_var_declaration(s, level, pos, visibility = 'private', api = 0, - overridable = False): - cmethod_flag = level in ('c_class', 'c_class_pxd') +def p_c_func_or_var_declaration(s, pos, ctx): + cmethod_flag = ctx.level in ('c_class', 'c_class_pxd') modifiers = p_c_modifiers(s) base_type = p_c_base_type(s, nonempty = 1) - declarator = p_c_declarator(s, cmethod_flag = cmethod_flag, assignable = 1, nonempty = 1) - declarator.overridable = overridable + declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag, + assignable = 1, nonempty = 1) + declarator.overridable = ctx.overridable if s.sy == ':': - if level not in ('module', 'c_class'): + if ctx.level not in ('module', 'c_class'): s.error("C function definition not allowed here") - doc, suite = p_suite(s, 'function', with_doc = 1) + doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1) result = Nodes.CFuncDefNode(pos, - visibility = visibility, + visibility = ctx.visibility, base_type = base_type, declarator = declarator, body = suite, doc = doc, modifiers = modifiers, - api = api, - overridable = overridable) + api = ctx.api, + overridable = ctx.overridable) else: #if api: # error(s.pos, "'api' not allowed with variable declaration") @@ -1943,39 +1949,40 @@ def p_c_func_or_var_declaration(s, level, pos, visibility = 'private', api = 0, s.next() if s.sy == 'NEWLINE': break - declarator = p_c_declarator(s, cmethod_flag = cmethod_flag, assignable = 1, nonempty = 1) + declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag, + assignable = 1, nonempty = 1) declarators.append(declarator) s.expect_newline("Syntax error in C variable declaration") result = Nodes.CVarDefNode(pos, - visibility = visibility, - base_type = base_type, + visibility = ctx.visibility, + base_type = base_type, declarators = declarators, - in_pxd = level == 'module_pxd', - api = api, - overridable = overridable) + in_pxd = ctx.level == 'module_pxd', + api = ctx.api, + overridable = ctx.overridable) return result -def p_ctypedef_statement(s, level, visibility = 'private', api = 0): +def p_ctypedef_statement(s, ctx): # s.sy == 'ctypedef' pos = s.position() s.next() - visibility = p_visibility(s, visibility) + visibility = p_visibility(s, ctx.visibility) + ctx = ctx(typedef_flag = 1, visibility = visibility) if s.sy == 'class': - return p_c_class_definition(s, level, pos, - visibility = visibility, typedef_flag = 1, api = api) + return p_c_class_definition(s, pos, ctx) elif s.sy == 'IDENT' and s.systring in ('struct', 'union', 'enum'): if s.systring == 'enum': - return p_c_enum_definition(s, pos, level, visibility, typedef_flag = 1) + return p_c_enum_definition(s, pos, ctx) else: - return p_c_struct_or_union_definition(s, pos, level, visibility, - typedef_flag = 1) + return p_c_struct_or_union_definition(s, pos, ctx) else: base_type = p_c_base_type(s, nonempty = 1) - declarator = p_c_declarator(s, is_type = 1, nonempty = 1) + declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1) s.expect_newline("Syntax error in ctypedef statement") - return Nodes.CTypeDefNode(pos, - base_type = base_type, declarator = declarator, visibility = visibility, - in_pxd = level == 'module_pxd') + return Nodes.CTypeDefNode( + pos, base_type = base_type, + declarator = declarator, visibility = visibility, + in_pxd = ctx.level == 'module_pxd') def p_def_statement(s): # s.sy == 'def' @@ -2003,7 +2010,7 @@ def p_def_statement(s): s.expect(')') if p_nogil(s): error(s.pos, "Python function cannot be declared nogil") - doc, body = p_suite(s, 'function', with_doc = 1) + doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1) return Nodes.DefNode(pos, name = name, args = args, star_arg = star_arg, starstar_arg = starstar_arg, doc = doc, body = body) @@ -2025,14 +2032,13 @@ def p_class_statement(s): s.expect(')') else: base_list = [] - doc, body = p_suite(s, 'class', with_doc = 1) + doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1) return Nodes.PyClassDefNode(pos, name = class_name, bases = ExprNodes.TupleNode(pos, args = base_list), doc = doc, body = body) -def p_c_class_definition(s, level, pos, - visibility = 'private', typedef_flag = 0, api = 0): +def p_c_class_definition(s, pos, ctx): # s.sy == 'class' s.next() module_path = [] @@ -2041,7 +2047,7 @@ def p_c_class_definition(s, level, pos, s.next() module_path.append(class_name) class_name = p_ident(s) - if module_path and visibility != 'extern': + if module_path and ctx.visibility != 'extern': error(pos, "Qualified class name only allowed for 'extern' C class") if module_path and s.sy == 'IDENT' and s.systring == 'as': s.next() @@ -2065,38 +2071,38 @@ def p_c_class_definition(s, level, pos, base_class_module = ".".join(base_class_path[:-1]) base_class_name = base_class_path[-1] if s.sy == '[': - if visibility not in ('public', 'extern'): + if ctx.visibility not in ('public', 'extern'): error(s.position(), "Name options only allowed for 'public' or 'extern' C class") objstruct_name, typeobj_name = p_c_class_options(s) if s.sy == ':': - if level == 'module_pxd': + if ctx.level == 'module_pxd': body_level = 'c_class_pxd' else: body_level = 'c_class' - doc, body = p_suite(s, body_level, with_doc = 1) + doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1) else: s.expect_newline("Syntax error in C class definition") doc = None body = None - if visibility == 'extern': + if ctx.visibility == 'extern': if not module_path: error(pos, "Module name required for 'extern' C class") if typeobj_name: error(pos, "Type object name specification not allowed for 'extern' C class") - elif visibility == 'public': + elif ctx.visibility == 'public': if not objstruct_name: error(pos, "Object struct name specification required for 'public' C class") if not typeobj_name: error(pos, "Type object name specification required for 'public' C class") - elif visibility == 'private': - if api: + elif ctx.visibility == 'private': + if ctx.api: error(pos, "Only 'public' C class can be declared 'api'") else: - error(pos, "Invalid class visibility '%s'" % visibility) + error(pos, "Invalid class visibility '%s'" % ctx.visibility) return Nodes.CClassDefNode(pos, - visibility = visibility, - typedef_flag = typedef_flag, - api = api, + visibility = ctx.visibility, + typedef_flag = ctx.typedef_flag, + api = ctx.api, module_name = ".".join(module_path), class_name = class_name, as_name = as_name, @@ -2104,7 +2110,7 @@ def p_c_class_definition(s, level, pos, base_class_name = base_class_name, objstruct_name = objstruct_name, typeobj_name = typeobj_name, - in_pxd = level == 'module_pxd', + in_pxd = ctx.level == 'module_pxd', doc = doc, body = body) @@ -2131,7 +2137,7 @@ def p_property_decl(s): pos = s.position() s.next() # 'property' name = p_ident(s) - doc, body = p_suite(s, 'property', with_doc = 1) + doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1) return Nodes.PropertyNode(pos, name = name, doc = doc, body = body) def p_doc_string(s): @@ -2152,7 +2158,7 @@ def p_module(s, pxd, full_module_name): level = 'module_pxd' else: level = 'module' - body = p_statement_list(s, level, first_statement = 1) + body = p_statement_list(s, Ctx(level = level), first_statement = 1) if s.sy != 'EOF': s.error("Syntax error in statement [%s,%s]" % ( repr(s.sy), repr(s.systring))) @@ -2164,7 +2170,7 @@ def p_module(s, pxd, full_module_name): # #---------------------------------------------- -def print_parse_tree(f, node, level, key = None): +def print_parse_tree(f, node, level, key = None): from Nodes import Node ind = " " * level if node: diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py index cb71d828fc8dec508220010e0c261023e5450d3d..45346675092a215179fd23c50e151b90dd0b31af 100644 --- a/Cython/Compiler/PyrexTypes.py +++ b/Cython/Compiler/PyrexTypes.py @@ -695,9 +695,10 @@ class CFuncType(CType): return 0 if not self.same_calling_convention_as(other_type): return 0 + if self.nogil and not other_type.nogil: + return 0 return 1 - - + def compatible_signature_with(self, other_type, as_cmethod = 0): return self.compatible_signature_with_resolved_type(other_type.resolve(), as_cmethod) @@ -789,22 +790,24 @@ class CFuncType(CType): arg_decl_code = ", ".join(arg_decl_list) if not arg_decl_code and not pyrex: arg_decl_code = "void" - exc_clause = "" + trailer = "" if (pyrex or for_display) and not self.return_type.is_pyobject: if self.exception_value and self.exception_check: - exc_clause = " except? %s" % self.exception_value + trailer = " except? %s" % self.exception_value elif self.exception_value: - exc_clause = " except %s" % self.exception_value + trailer = " except %s" % self.exception_value elif self.exception_check == '+': - exc_clause = " except +" + trailer = " except +" else: - " except *" + " except *" # ignored + if self.nogil: + trailer += " nogil" cc = self.calling_convention_prefix() if (not entity_code and cc) or entity_code.startswith("*"): entity_code = "(%s%s)" % (cc, entity_code) cc = "" return self.return_type.declaration_code( - "%s%s(%s)%s" % (cc, entity_code, arg_decl_code, exc_clause), + "%s%s(%s)%s" % (cc, entity_code, arg_decl_code, trailer), for_display, dll_linkage, pyrex) def function_header_code(self, func_name, arg_code): diff --git a/Cython/Compiler/Scanning.py b/Cython/Compiler/Scanning.py index 4ba1718c7d4ca87903acf273021ff9a73d8f711d..2e20d18add461ca9780a99d780f4fc99fa6e854f 100644 --- a/Cython/Compiler/Scanning.py +++ b/Cython/Compiler/Scanning.py @@ -284,11 +284,10 @@ class PyrexScanner(Scanner): # compile_time_env dict Environment for conditional compilation # compile_time_eval boolean In a true conditional compilation context # compile_time_expr boolean In a compile-time expression context - resword_dict = build_resword_dict() def __init__(self, file, filename, parent_scanner = None, - type_names = None, context = None, source_encoding=None): + type_names = None, context = None, source_encoding=None): Scanner.__init__(self, get_lexicon(), file, filename) if parent_scanner: self.context = parent_scanner.context diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.py index 1fa1ede66ca6c74d953af350c783fab3e6cffa5f..11d0a73d4139571faafc153203d53f75b63a315f 100644 --- a/Cython/Compiler/Symtab.py +++ b/Cython/Compiler/Symtab.py @@ -150,12 +150,14 @@ class Scope: # pystring_entries [Entry] String const entries newly used as # Python strings in this scope # control_flow ControlFlow Used for keeping track of environment state + # nogil boolean In a nogil section is_py_class_scope = 0 is_c_class_scope = 0 is_module_scope = 0 scope_prefix = "" in_cinclude = 0 + nogil = 0 def __init__(self, name, outer_scope, parent_scope): # The outer_scope is the next scope in the lookup chain. @@ -554,10 +556,6 @@ class Scope: return [entry for entry in self.temp_entries if entry not in self.free_temp_entries] - #def recycle_pending_temps(self): - # # Obsolete - # pass - def use_utility_code(self, new_code): self.global_scope().use_utility_code(new_code) @@ -652,7 +650,7 @@ class BuiltinScope(Scope): "long": ["((PyObject*)&PyLong_Type)", py_object_type], "float": ["((PyObject*)&PyFloat_Type)", py_object_type], - "str": ["((PyObject*)&PyString_Type)", py_object_type], + "str": ["((PyObject*)&PyBytes_Type)", py_object_type], "unicode":["((PyObject*)&PyUnicode_Type)", py_object_type], "tuple": ["((PyObject*)&PyTuple_Type)", py_object_type], "list": ["((PyObject*)&PyList_Type)", py_object_type], @@ -687,6 +685,7 @@ class ModuleScope(Scope): # parent_module Scope Parent in the import namespace # module_entries {string : Entry} For cimport statements # type_names {string : 1} Set of type names (used during parsing) + # included_files [string] Cython sources included with 'include' # pxd_file_loaded boolean Corresponding .pxd file has been processed # cimported_modules [ModuleScope] Modules imported with cimport # new_interned_string_entries [Entry] New interned strings waiting to be declared @@ -723,6 +722,7 @@ class ModuleScope(Scope): self.interned_objs = [] self.all_pystring_entries = [] self.types_imported = {} + self.included_files = [] self.pynum_entries = [] self.has_extern_class = 0 self.cached_builtins = [] @@ -1326,6 +1326,7 @@ class CClassScope(ClassScope): # entry.type = type else: error(pos, "Signature not compatible with previous declaration") + error(entry.pos, "Previous declaration is here") else: if self.defined: error(pos, @@ -1365,8 +1366,8 @@ class CClassScope(ClassScope): entry.is_variable = 1 self.inherited_var_entries.append(entry) for base_entry in base_scope.cfunc_entries: - entry = self.add_cfunction(base_entry.name, base_entry.type, None, - adapt(base_entry.cname), base_entry.visibility) + entry = self.add_cfunction(base_entry.name, base_entry.type, + base_entry.pos, adapt(base_entry.cname), base_entry.visibility) entry.is_inherited = 1 def allocate_temp(self, type): diff --git a/Cython/Compiler/TypeSlots.py b/Cython/Compiler/TypeSlots.py index 78c59e2fc6896875efc1db792014ac7cbb54b475..c1890730a00e851ed43d90e63247176e38521a7f 100644 --- a/Cython/Compiler/TypeSlots.py +++ b/Cython/Compiler/TypeSlots.py @@ -177,8 +177,8 @@ class FixedSlot(SlotDescriptor): # # value string - def __init__(self, slot_name, value): - SlotDescriptor.__init__(self, slot_name) + def __init__(self, slot_name, value, py3k=True): + SlotDescriptor.__init__(self, slot_name, py3k=py3k) self.value = value def slot_code(self, scope): @@ -188,8 +188,8 @@ class FixedSlot(SlotDescriptor): class EmptySlot(FixedSlot): # Descriptor for a type slot whose value is always 0. - def __init__(self, slot_name): - FixedSlot.__init__(self, slot_name, "0") + def __init__(self, slot_name, py3k=True): + FixedSlot.__init__(self, slot_name, "0", py3k=py3k) class MethodSlot(SlotDescriptor): @@ -553,12 +553,12 @@ PyNumberMethods = ( MethodSlot(binaryfunc, "nb_and", "__and__"), MethodSlot(binaryfunc, "nb_xor", "__xor__"), MethodSlot(binaryfunc, "nb_or", "__or__"), - EmptySlot("nb_coerce"), + EmptySlot("nb_coerce", py3k = False), MethodSlot(unaryfunc, "nb_int", "__int__"), MethodSlot(unaryfunc, "nb_long", "__long__"), MethodSlot(unaryfunc, "nb_float", "__float__"), - MethodSlot(unaryfunc, "nb_oct", "__oct__"), - MethodSlot(unaryfunc, "nb_hex", "__hex__"), + MethodSlot(unaryfunc, "nb_oct", "__oct__", py3k = False), + MethodSlot(unaryfunc, "nb_hex", "__hex__", py3k = False), # Added in release 2.0 MethodSlot(ibinaryfunc, "nb_inplace_add", "__iadd__"), diff --git a/Cython/Utils.py b/Cython/Utils.py index de22a6a9da14e3d12ea2e5221c064cd155896898..c7ecf2c469a608ee99306ff23fa539529bc09e33 100644 --- a/Cython/Utils.py +++ b/Cython/Utils.py @@ -24,7 +24,6 @@ def castrate_file(path, st): except EnvironmentError: pass else: - #st = os.stat(path) f.seek(0, 0) f.truncate() f.write( @@ -33,6 +32,14 @@ def castrate_file(path, st): if st: os.utime(path, (st.st_atime, st.st_mtime)) +def modification_time(path): + st = os.stat(path) + return st.st_mtime + +def file_newer_than(path, time): + ftime = modification_time(path) + return ftime > time + # support for source file encoding detection and unicode decoding def encode_filename(filename): diff --git a/runtests.py b/runtests.py index 1d86bd0e0b7afa96b12d1559a481b0325826ee68..75ab875b912a12fc3b9a2b3095ea70d4a70b88d4 100644 --- a/runtests.py +++ b/runtests.py @@ -17,7 +17,7 @@ CFLAGS = os.getenv('CFLAGS', '').split() class ErrorWriter(object): - match_error = re.compile('(warning:)?(?:.*:)?([-0-9]+):([-0-9]+):(.*)').match + match_error = re.compile('(warning:)?(?:.*:)?\s*([-0-9]+)\s*:\s*([-0-9]+)\s*:\s*(.*)').match def __init__(self): self.output = [] self.write = self.output.append @@ -31,8 +31,9 @@ class ErrorWriter(object): is_warning, line, column, message = match.groups() if (is_warning and collect_warnings) or \ (not is_warning and collect_errors): - result.append( "%d:%d:%s" % (int(line), int(column), message.strip()) ) - return result + result.append( (int(line), int(column), message.strip()) ) + result.sort() + return [ "%d:%d: %s" % values for values in result ] def geterrors(self): return self._collect(True, False) diff --git a/tests/compile/dotted_cimport_submodule/__init__.pyx b/tests/compile/dotted_cimport_submodule/__init__.pyx new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/compile/extcoerce.pyx b/tests/compile/extcoerce.pyx index 613ead8a752e54ca41f729134b0d2ef169c1507a..374238e08f584b55047aede6f01a186a54317dd1 100644 --- a/tests/compile/extcoerce.pyx +++ b/tests/compile/extcoerce.pyx @@ -7,7 +7,7 @@ cdef class Swallow: pass def f(Grail g): - cdef int i + cdef int i = 0 cdef Swallow s g = x x = g diff --git a/tests/compile/nogil.h b/tests/compile/nogil.h new file mode 100644 index 0000000000000000000000000000000000000000..fb2e4d4e2c446355da6f6fca1194b9963884f280 --- /dev/null +++ b/tests/compile/nogil.h @@ -0,0 +1,2 @@ +void e1(void); +void *e2(void); diff --git a/tests/compile/nogil.pyx b/tests/compile/nogil.pyx index 8f747ad0df0d3ef4516f2c569a36c8d349a93273..ba6f856a6f72fe472543e545d9f478d813cb8a15 100644 --- a/tests/compile/nogil.pyx +++ b/tests/compile/nogil.pyx @@ -1,5 +1,18 @@ -cdef extern void g(int x) nogil +cdef extern object g(object x) nogil +cdef extern void g2(object x) nogil + +cdef extern from "nogil.h": + void e1() nogil + int *e2() nogil cdef void f(int x) nogil: - cdef int y - y = 42 + cdef int y + y = 42 + +cdef void h(object x) nogil: + cdef void *p + g2(x) + g2(<object>p) + p = <void *>x + e1() + e2() diff --git a/tests/compile/varargdecl.pyx b/tests/compile/varargdecl.pyx deleted file mode 100644 index 2f627265fcca63babd7eee489979ea191015f003..0000000000000000000000000000000000000000 --- a/tests/compile/varargdecl.pyx +++ /dev/null @@ -1,2 +0,0 @@ -cdef grail(char *blarg, ...): - pass diff --git a/tests/compile/while.pyx b/tests/compile/while.pyx index 31e36bc15b822bad6edb69946180156fdd5da5ea..8d6990d3af81dbdbc800b0d80c7c1c1ac5cd3dfc 100644 --- a/tests/compile/while.pyx +++ b/tests/compile/while.pyx @@ -1,5 +1,5 @@ def f(a, b): - cdef int i + cdef int i = 5 while a: x = 1 @@ -23,4 +23,4 @@ def f(a, b): x = 1 continue x = 2 - \ No newline at end of file + diff --git a/tests/compile/withgil.pyx b/tests/compile/withgil.pyx index 7d5d54846d126577a4f1a9dc59a64a72784eccfe..fa77d6ed1b6c4aa7acdbb429e033f997d0d4c2c6 100644 --- a/tests/compile/withgil.pyx +++ b/tests/compile/withgil.pyx @@ -3,3 +3,6 @@ cdef void f() with gil: cdef int g(void* x) with gil: pass + +f() +g("test") diff --git a/tests/errors/cmethbasematch.pyx b/tests/errors/cmethbasematch.pyx new file mode 100644 index 0000000000000000000000000000000000000000..b0b1222e3bb6d65e6ef2f5451c027c63def8ea19 --- /dev/null +++ b/tests/errors/cmethbasematch.pyx @@ -0,0 +1,12 @@ +cdef class C: + cdef void f(self): + pass + +cdef class D(C): + cdef void f(self, int x): + pass + +_ERRORS = u""" +6: 9: Signature not compatible with previous declaration +2: 9: Previous declaration is here +""" diff --git a/tests/errors/nogil.pyx b/tests/errors/nogil.pyx new file mode 100644 index 0000000000000000000000000000000000000000..17e2e4343c905ed642930231af54c4278a6a994d --- /dev/null +++ b/tests/errors/nogil.pyx @@ -0,0 +1,136 @@ +cdef object f(object x) nogil: + pass + +cdef void g(int x) nogil: + cdef object z + z = None + +cdef void h(int x) nogil: + p() + +cdef object p() nogil: + pass + +cdef void r() nogil: + q() + +cdef object m(): + cdef object x, y, obj + cdef int i, j, k + global fred + q() + with nogil: + r() + q() + i = 42 + obj = None + 17L + 7j + help + `"Hello"` + import fred + from fred import obj + for x in obj: + pass + obj[i] + obj[i:j] + obj[i:j:k] + obj.fred + (x, y) + [x, y] + {x: y} + obj and x + t(obj) +# f(42) # Cython handles this internally + x + obj + -obj + x = y = obj + x, y = y, x + obj[i] = x + obj.fred = x + print obj + del fred + return obj + raise obj + if obj: + pass + while obj: + pass + for x <= obj <= y: + pass + try: + pass + except: + pass + try: + pass + finally: + pass + +cdef void q(): + pass + +cdef class C: + pass + +cdef void t(C c) nogil: + pass + + +_ERRORS = u""" + 1: 5: Function with Python return type cannot be declared nogil + 6: 6: Assignment of Python object not allowed without gil + 4: 5: Function declared nogil has Python locals or temporaries + 8: 5: Function declared nogil has Python locals or temporaries +11: 5: Function with Python return type cannot be declared nogil +15: 5: Calling gil-requiring function without gil +24: 9: Calling gil-requiring function without gil +26:12: Assignment of Python object not allowed without gil +27: 8: Constructing Python long int not allowed without gil +28: 8: Constructing complex number not allowed without gil +29:12: Accessing Python global or builtin not allowed without gil +30: 8: Backquote expression not allowed without gil +31:15: Python import not allowed without gil +31:15: Assignment of Python object not allowed without gil +32:13: Python import not allowed without gil +32:25: Constructing Python list not allowed without gil +33:17: Iterating over Python object not allowed without gil +35:11: Indexing Python object not allowed without gil +36:11: Slicing Python object not allowed without gil +37:11: Constructing Python slice object not allowed without gil +37:11: Indexing Python object not allowed without gil +37:13: Converting to Python object not allowed without gil +37:15: Converting to Python object not allowed without gil +37:17: Converting to Python object not allowed without gil +38:11: Accessing Python attribute not allowed without gil +39: 9: Constructing Python tuple not allowed without gil +40: 8: Constructing Python list not allowed without gil +41: 8: Constructing Python dict not allowed without gil +42:12: Creating temporary Python reference not allowed without gil +42:12: Truth-testing Python object not allowed without gil +42:17: Creating temporary Python reference not allowed without gil +43:13: Python type test not allowed without gil +#44: 4: Converting to Python object not allowed without gil +45:10: Operation not allowed without gil +46: 8: Operation not allowed without gil +47:10: Assignment of Python object not allowed without gil +47:14: Assignment of Python object not allowed without gil +48: 9: Assignment of Python object not allowed without gil +48:13: Assignment of Python object not allowed without gil +48:16: Creating temporary Python reference not allowed without gil +48:19: Creating temporary Python reference not allowed without gil +49:11: Indexing Python object not allowed without gil +49:11: Assignment of Python object not allowed without gil +50:11: Accessing Python attribute not allowed without gil +50:11: Assignment of Python object not allowed without gil +51: 8: Constructing Python tuple not allowed without gil +51: 8: Python print statement not allowed without gil +52: 8: Deleting Python object not allowed without gil +53: 8: Returning Python object not allowed without gil +54: 8: Raising exception not allowed without gil +55:14: Truth-testing Python object not allowed without gil +57:17: Truth-testing Python object not allowed without gil +59: 8: Converting to Python object not allowed without gil +61: 8: Try-except statement not allowed without gil +65: 8: Try-finally statement not allowed without gil +""" diff --git a/tests/errors/nogilfunctype.pyx b/tests/errors/nogilfunctype.pyx new file mode 100644 index 0000000000000000000000000000000000000000..9b5580f09f70e458386af2306d8d6888b6be5fa4 --- /dev/null +++ b/tests/errors/nogilfunctype.pyx @@ -0,0 +1,14 @@ +cdef extern from *: + cdef void f() + cdef void (*fp)() nogil + + cdef void g() nogil + cdef void (*gp)() + +gp = g + +fp = f + +_ERRORS = u""" +10:6: Cannot assign type 'void (void)' to 'void (*)(void) nogil' +""" diff --git a/tests/run/big_indices.pyx b/tests/run/big_indices.pyx index d8f3a765b6cef03f1da849255d2b9c327d2260a3..7a787912f987e97273d5ff39d5aab2aa4b934b1a 100644 --- a/tests/run/big_indices.pyx +++ b/tests/run/big_indices.pyx @@ -1,7 +1,7 @@ __doc__ = u""" >>> test() - neg -1 - pos 4294967294 + neg False + pos True neg pos neg @@ -12,8 +12,8 @@ def test(): cdef long neg = -1 cdef unsigned long pos = -2 # will be a large positive number - print "neg", neg - print "pos", pos + print "neg", neg > 0 + print "pos", pos > - D = { neg: 'neg', pos: 'pos' } diff --git a/tests/run/exceptionpropagation.pyx b/tests/run/exceptionpropagation.pyx new file mode 100644 index 0000000000000000000000000000000000000000..e324a70a40269f65a3ed23fd664526442a99a2f5 --- /dev/null +++ b/tests/run/exceptionpropagation.pyx @@ -0,0 +1,17 @@ +__doc__ = u""" +>>> foo(0) +>>> foo(1) +Traceback (most recent call last): +RuntimeError +""" + +cdef int CHKERR(int ierr) except -1: + if ierr==0: return 0 + raise RuntimeError + +cdef int obj2int(object ob) except *: + return ob + +def foo(a): + cdef int i = obj2int(a) + CHKERR(i) diff --git a/tests/run/inplace.pyx b/tests/run/inplace.pyx index 22f8ac839c32c97e9db9548bc41a42c612c6cf4a..54859fb3f3a914048454eb3adc1355e28a3526b7 100644 --- a/tests/run/inplace.pyx +++ b/tests/run/inplace.pyx @@ -1,6 +1,6 @@ __doc__ = u""" >>> f(5, 7) - 29509034655744 + 29509034655744L >>> g(13, 4) 32 @@ -9,6 +9,10 @@ __doc__ = u""" 105.0 """ +import sys +if sys.version_info[0] >= 3: + __doc__ = __doc__.replace(u"L", u"") + def f(a,b): a += b a *= b diff --git a/tests/run/modop.pyx b/tests/run/modop.pyx index 60d493534421b7af4217d8f010ad58fff989b94f..be55648dd5a4cba89f93b53eb84558e06589931f 100644 --- a/tests/run/modop.pyx +++ b/tests/run/modop.pyx @@ -6,6 +6,9 @@ __doc__ = u""" >>> modint(9,2) 1 + + >>> print modptr() + spameggs """ def modobj(obj2, obj3): @@ -17,11 +20,10 @@ def modint(int int2, int int3): int1 = int2 % int3 return int1 -cdef modptr(): - # FIXME!!! +def modptr(): cdef char *str2, *str3 - str2 = "spam" + str2 = "spam%s" str3 = "eggs" obj1 = str2 % str3 - return obj1 + return obj1.decode(u"ASCII") diff --git a/tests/run/varargdecl.pyx b/tests/run/varargdecl.pyx new file mode 100644 index 0000000000000000000000000000000000000000..c59a0d16a40ea6e135a26fcb5280dcb1ba1f3caf --- /dev/null +++ b/tests/run/varargdecl.pyx @@ -0,0 +1,10 @@ +__doc__ = u""" +>>> test() +""" + +cdef grail(char *blarg, ...): + pass + +def test(): + grail("test") + grail("test", "toast") diff --git a/tests/run/withnogil.pyx b/tests/run/withnogil.pyx index 02a42b742a77dcd88a00e6aea747dadd422ac9e3..8d1f8d366ca07e7337aa23c063ea1442b44c2816 100644 --- a/tests/run/withnogil.pyx +++ b/tests/run/withnogil.pyx @@ -18,5 +18,5 @@ def g(): h() return 1 -cdef int h() except -1: +cdef int h() nogil except -1: pass