Skip to content

C
cython
Commit f6a3f4c6 authored by Stefan Behnel's avatar Stefan Behnel
Browse files
Options

merged (and partially rewrote) dependency tracking and package resolution changes from Pyrex 0.9.8

parent 23814b66
Hide whitespace changes
Inline Side-by-side
Showing with 235 additions and 64 deletions
Cython/Compiler/CmdLine.py
View file @ f6a3f4c6
......@@ -18,6 +18,10 @@ Options:
-I, --include-dir <directory> Search for include files in named directory
(multiply include directories are allowed).
-o, --output-file <filename> Specify name of generated C file
-r, --recursive Recursively find and compile dependencies
-t, --timestamps Only compile newer source files (implied with -r)
-f, --force Compile all source files (overrides implied -t)
-q, --quiet Don't print module names in recursive mode
-p, --embed-positions If specified, the positions in Cython files of each
function definition is embedded in its docstring.
-z, --pre-import <module> If specified, assume undeclared names in this
......@@ -111,6 +115,12 @@ def parse_command_line(args):
options.working_path = pop_arg()
elif option in ("-o", "--output-file"):
options.output_file = pop_arg()
elif option in ("-r", "--recursive"):
options.recursive = 1
elif option in ("-t", "--timestamps"):
options.timestamps = 1
elif option in ("-f", "--force"):
options.timestamps = 0
elif option in ("-p", "--embed-positions"):
Options.embed_pos_in_docstring = 1
elif option in ("-z", "--pre-import"):
......
Cython/Compiler/Main.py
View file @ f6a3f4c6
......@@ -3,8 +3,8 @@
#
import os, sys, re, codecs
if sys.version_info[:2] < (2, 2):
sys.stderr.write("Sorry, Cython requires Python 2.2 or later\n")
if sys.version_info[:2] < (2, 3):
sys.stderr.write("Sorry, Cython requires Python 2.3 or later\n")
sys.exit(1)
try:
......@@ -14,14 +14,13 @@ except NameError:
from sets import Set as set
from time import time
import Version
from Scanning import PyrexScanner
import Code
import Errors
from Errors import PyrexError, CompileError, error
import Parsing
import Version
from Errors import PyrexError, CompileError, error
from Scanning import PyrexScanner
from Symtab import BuiltinScope, ModuleScope
import Code
from Cython.Utils import replace_suffix
from Cython import Utils
import Transform
......@@ -93,31 +92,34 @@ class Context:
try:
if debug_find_module:
print("Context.find_module: Parsing %s" % pxd_pathname)
pxd_tree = self.parse(pxd_pathname, scope.type_names, pxd = 1,
pxd_tree = self.parse(pxd_pathname, scope, pxd = 1,
full_module_name = module_name)
pxd_tree.analyse_declarations(scope)
except CompileError:
pass
return scope
def find_pxd_file(self, module_name, pos):
# Search include directories for the .pxd file
# corresponding to the given (full) module name.
if "." in module_name:
pxd_filename = "%s.pxd" % os.path.join(*module_name.split('.'))
else:
pxd_filename = "%s.pxd" % module_name
return self.search_include_directories(pxd_filename, pos)
def find_pxd_file(self, qualified_name, pos):
# Search include path for the .pxd file corresponding to the
# given fully-qualified module name.
return self.search_include_directories(qualified_name, ".pxd", pos)
def find_pyx_file(self, qualified_name, pos):
# Search include path for the .pyx file corresponding to the
# given fully-qualified module name, as for find_pxd_file().
return self.search_include_directories(qualified_name, ".pyx", pos)
def find_include_file(self, filename, pos):
# Search list of include directories for filename.
# Reports an error and returns None if not found.
path = self.search_include_directories(filename, pos)
path = self.search_include_directories(filename, "", pos,
split_package=False)
if not path:
error(pos, "'%s' not found" % filename)
return path
def search_include_directories(self, filename, pos):
def search_include_directories(self, qualified_name, suffix, pos,
split_package=True):
# Search the list of include directories for the given
# file name. If a source file position is given, first
# searches the directory containing that file. Returns
......@@ -126,12 +128,81 @@ class Context:
if pos:
here_dir = os.path.dirname(pos[0])
dirs = [here_dir] + dirs
dotted_filename = qualified_name + suffix
if split_package:
names = qualified_name.split('.')
package_names = names[:-1]
module_name = names[-1]
module_filename = module_name + suffix
package_filename = "__init__" + suffix
for dir in dirs:
path = os.path.join(dir, filename)
path = os.path.join(dir, dotted_filename)
if os.path.exists(path):
return path
if split_package:
package_dir = self.check_package_dir(dir, package_names)
if package_dir is not None:
path = os.path.join(package_dir, module_filename)
if os.path.exists(path):
return path
path = os.path.join(dir, package_dir, module_name,
package_filename)
if os.path.exists(path):
return path
return None
def check_package_dir(self, dir, package_names):
package_dir = os.path.join(dir, *package_names)
if not os.path.exists(package_dir):
return None
for dirname in package_names:
dir = os.path.join(dir, dirname)
package_init = os.path.join(dir, "__init__.py")
if not os.path.exists(package_init) and \
not os.path.exists(package_init + "x"): # same with .pyx ?
return None
return package_dir
def c_file_out_of_date(self, source_path):
c_path = Utils.replace_suffix(source_path, ".c")
if not os.path.exists(c_path):
return 1
c_time = Utils.modification_time(c_path)
if Utils.file_newer_than(source_path, c_time):
return 1
pos = [source_path]
pxd_path = Utils.replace_suffix(source_path, ".pxd")
if os.path.exists(pxd_path) and Utils.file_newer_than(pxd_path, c_time):
return 1
for kind, name in self.read_dependency_file(source_path):
if kind == "cimport":
dep_path = self.find_pxd_file(name, pos)
elif kind == "include":
dep_path = self.search_include_directories(name, pos)
else:
continue
if dep_path and Utils.file_newer_than(dep_path, c_time):
return 1
return 0
def find_cimported_module_names(self, source_path):
return [ name for kind, name in self.read_dependency_file(source_path)
if kind == "cimport" ]
def read_dependency_file(self, source_path):
dep_path = replace_suffix(source_path, ".dep")
if os.path.exists(dep_path):
f = open(dep_path, "rU")
chunks = [ line.strip().split(" ", 1)
for line in f.readlines()
if " " in line.strip() ]
f.close()
return chunks
else:
return ()
def lookup_submodule(self, name):
# Look up a top-level module. Returns None if not found.
return self.modules.get(name, None)
......@@ -145,14 +216,14 @@ class Context:
self.modules[name] = scope
return scope
def parse(self, source_filename, type_names, pxd, full_module_name):
def parse(self, source_filename, scope, pxd, full_module_name):
name = Utils.encode_filename(source_filename)
# Parse the given source file and return a parse tree.
try:
f = Utils.open_source_file(source_filename, "rU")
try:
s = PyrexScanner(f, name, source_encoding = f.encoding,
type_names = type_names, context = self)
scope = scope, context = self)
tree = Parsing.p_module(s, pxd, full_module_name)
finally:
f.close()
......@@ -185,7 +256,7 @@ class Context:
result.main_source_file = source
if options.use_listing_file:
result.listing_file = replace_suffix(source, ".lis")
result.listing_file = Utils.replace_suffix(source, ".lis")
Errors.open_listing_file(result.listing_file,
echo_to_stderr = options.errors_to_stderr)
else:
......@@ -197,19 +268,14 @@ class Context:
c_suffix = ".cpp"
else:
c_suffix = ".c"
result.c_file = replace_suffix(source, c_suffix)
c_stat = None
if result.c_file:
try:
c_stat = os.stat(result.c_file)
except EnvironmentError:
pass
result.c_file = Utils.replace_suffix(source, c_suffix)
module_name = full_module_name # self.extract_module_name(source, options)
initial_pos = (source, 1, 0)
scope = self.find_module(module_name, pos = initial_pos, need_pxd = 0)
errors_occurred = False
try:
tree = self.parse(source, scope.type_names, pxd = 0, full_module_name = full_module_name)
tree = self.parse(source, scope, pxd = 0,
full_module_name = full_module_name)
tree.process_implementation(scope, options, result)
except CompileError:
errors_occurred = True
......@@ -219,8 +285,7 @@ class Context:
errors_occurred = True
if errors_occurred and result.c_file:
try:
#os.unlink(result.c_file)
Utils.castrate_file(result.c_file, c_stat)
Utils.castrate_file(result.c_file, os.stat(source))
except EnvironmentError:
pass
result.c_file = None
......@@ -237,7 +302,7 @@ class Context:
#------------------------------------------------------------------------
#
# Main Python entry point
# Main Python entry points
#
#------------------------------------------------------------------------
......@@ -251,6 +316,10 @@ class CompilationOptions:
include_path [string] Directories to search for include files
output_file string Name of generated .c file
generate_pxi boolean Generate .pxi file for public declarations
recursive boolean Recursively find and compile dependencies
timestamps boolean Only compile changed source files. If None,
defaults to true when recursive is true.
quiet boolean Don't print source names in recursive mode
transforms Transform.TransformSet Transforms to use on the parse tree
Following options are experimental and only used on MacOSX:
......@@ -261,7 +330,7 @@ class CompilationOptions:
cplus boolean Compile as c++ code
"""
def __init__(self, defaults = None, **kw):
def __init__(self, defaults = None, c_compile = 0, c_link = 0, **kw):
self.include_path = []
self.objects = []
if defaults:
......@@ -271,6 +340,10 @@ class CompilationOptions:
defaults = default_options
self.__dict__.update(defaults)
self.__dict__.update(kw)
if c_compile:
self.c_only = 0
if c_link:
self.obj_only = 0
class CompilationResult:
......@@ -298,24 +371,87 @@ class CompilationResult:
self.main_source_file = None
def compile(source, options = None, c_compile = 0, c_link = 0,
full_module_name = None):
class CompilationResultSet(dict):
"""
compile(source, options = default_options)
Results from compiling multiple Pyrex source files. A mapping
from source file paths to CompilationResult instances. Also
has the following attributes:
Compile the given Cython implementation file and return
a CompilationResult object describing what was produced.
num_errors integer Total number of compilation errors
"""
num_errors = 0
def add(self, source, result):
self[source] = result
self.num_errors += result.num_errors
def compile_single(source, options, full_module_name = None):
"""
compile_single(source, options, full_module_name)
Compile the given Pyrex implementation file and return a CompilationResult.
Always compiles a single file; does not perform timestamp checking or
recursion.
"""
if not options:
options = default_options
options = CompilationOptions(defaults = options)
if c_compile:
options.c_only = 0
if c_link:
options.obj_only = 0
context = Context(options.include_path)
return context.compile(source, options, full_module_name)
def compile_multiple(sources, options):
"""
compile_multiple(sources, options)
Compiles the given sequence of Pyrex implementation files and returns
a CompilationResultSet. Performs timestamp checking and/or recursion
if these are specified in the options.
"""
sources = [os.path.abspath(source) for source in sources]
processed = set()
results = CompilationResultSet()
context = Context(options.include_path)
recursive = options.recursive
timestamps = options.timestamps
if timestamps is None:
timestamps = recursive
verbose = recursive and not options.quiet
for source in sources:
if source not in processed:
if not timestamps or context.c_file_out_of_date(source):
if verbose:
sys.stderr.write("Compiling %s\n" % source)
result = context.compile(source, options)
results.add(source, result)
processed.add(source)
if recursive:
for module_name in context.find_cimported_module_names(source):
path = context.find_pyx_file(module_name, [source])
if path:
sources.append(path)
else:
sys.stderr.write(
"Cannot find .pyx file for cimported module '%s'\n" % module_name)
return results
def compile(source, options = None, c_compile = 0, c_link = 0,
full_module_name = None, **kwds):
"""
compile(source [, options], [, <option> = <value>]...)
Compile one or more Pyrex implementation files, with optional timestamp
checking and recursing on dependecies. The source argument may be a string
or a sequence of strings If it is a string and no recursion or timestamp
checking is requested, a CompilationResult is returned, otherwise a
CompilationResultSet is returned.
"""
options = CompilationOptions(defaults = options, c_compile = c_compile,
c_link = c_link, **kwds)
if isinstance(source, basestring) and not options.timestamps \
and not options.recursive:
return compile_single(source, options, full_module_name)
else:
return compile_multiple(source, options)
#------------------------------------------------------------------------
#
# Main command-line entry point
......@@ -329,21 +465,19 @@ def main(command_line = 0):
from CmdLine import parse_command_line
options, sources = parse_command_line(args)
else:
options = default_options
options = CompilationOptions(default_options)
sources = args
if options.show_version:
sys.stderr.write("Cython version %s\n" % Version.version)
if options.working_path!="":
os.chdir(options.working_path)
context = Context(options.include_path)
for source in sources:
try:
result = context.compile(source, options)
if result.num_errors > 0:
any_failures = 1
except PyrexError, e:
sys.stderr.write(str(e) + '\n')
try:
result = compile(sources, options)
if result.num_errors > 0:
any_failures = 1
except (EnvironmentError, PyrexError), e:
sys.stderr.write(str(e) + '\n')
any_failures = 1
if any_failures:
sys.exit(1)
......@@ -363,6 +497,9 @@ default_options = dict(
output_file = None,
annotate = False,
generate_pxi = 0,
recursive = 0,
timestamps = None,
quiet = 0,
transforms = Transform.TransformSet(),
working_path = "")
......
Cython/Compiler/ModuleNode.py
View file @ f6a3f4c6
......@@ -54,6 +54,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
if self.has_imported_c_functions():
self.module_temp_cname = env.allocate_temp_pyobject()
env.release_temp(self.module_temp_cname)
self.generate_dep_file(env, result)
self.generate_c_code(env, options, result)
self.generate_h_code(env, options, result)
self.generate_api_code(env, result)
......@@ -65,6 +66,20 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
return 1
return 0
def generate_dep_file(self, env, result):
modules = self.referenced_modules
if len(modules) > 1 or env.included_files:
dep_file = replace_suffix(result.c_file, ".dep")
f = open(dep_file, "w")
try:
for module in modules:
if module is not env:
f.write("cimport %s\n" % module.qualified_name)
for path in module.included_files:
f.write("include %s\n" % path)
finally:
f.close()
def generate_h_code(self, env, options, result):
def h_entries(entries, pxd = 0):
return [entry for entry in entries
......
Cython/Compiler/Parsing.py
View file @ f6a3f4c6
......@@ -1203,8 +1203,10 @@ def p_include_statement(s, level):
if s.compile_time_eval:
include_file_path = s.context.find_include_file(include_file_name, pos)
if include_file_path:
s.included_files.append(include_file_name)
f = Utils.open_source_file(include_file_path, mode="rU")
s2 = PyrexScanner(f, include_file_path, s, source_encoding=f.encoding)
s2 = PyrexScanner(f, include_file_path, parent_scanner = s,
source_encoding=f.encoding)
try:
tree = p_statement_list(s2, level)
finally:
......
Cython/Compiler/Scanning.py
View file @ f6a3f4c6
......@@ -206,24 +206,26 @@ def initial_compile_time_env():
class PyrexScanner(Scanner):
# context Context Compilation context
# type_names set Identifiers to be treated as type names
# included_files [string] Files included with 'include' statement
# compile_time_env dict Environment for conditional compilation
# compile_time_eval boolean In a true conditional compilation context
# compile_time_expr boolean In a compile-time expression context
resword_dict = build_resword_dict()
def __init__(self, file, filename, parent_scanner = None,
type_names = None, context = None, source_encoding=None):
scope = None, context = None, source_encoding=None):
Scanner.__init__(self, get_lexicon(), file, filename)
if parent_scanner:
self.context = parent_scanner.context
self.type_names = parent_scanner.type_names
self.included_files = parent_scanner.included_files
self.compile_time_env = parent_scanner.compile_time_env
self.compile_time_eval = parent_scanner.compile_time_eval
self.compile_time_expr = parent_scanner.compile_time_expr
else:
self.context = context
self.type_names = type_names
self.type_names = scope.type_names
self.included_files = scope.included_files
self.compile_time_env = initial_compile_time_env()
self.compile_time_eval = 1
self.compile_time_expr = 0
......
Cython/Compiler/Symtab.py
View file @ f6a3f4c6
......@@ -554,10 +554,6 @@ class Scope:
return [entry for entry in self.temp_entries
if entry not in self.free_temp_entries]
#def recycle_pending_temps(self):
# # Obsolete
# pass
def use_utility_code(self, new_code):
self.global_scope().use_utility_code(new_code)
......@@ -687,6 +683,7 @@ class ModuleScope(Scope):
# parent_module Scope Parent in the import namespace
# module_entries {string : Entry} For cimport statements
# type_names {string : 1} Set of type names (used during parsing)
# included_files [string] Cython sources included with 'include'
# pxd_file_loaded boolean Corresponding .pxd file has been processed
# cimported_modules [ModuleScope] Modules imported with cimport
# new_interned_string_entries [Entry] New interned strings waiting to be declared
......@@ -723,6 +720,7 @@ class ModuleScope(Scope):
self.interned_objs = []
self.all_pystring_entries = []
self.types_imported = {}
self.included_files = []
self.pynum_entries = []
self.has_extern_class = 0
self.cached_builtins = []
......
Cython/Utils.py
View file @ f6a3f4c6
......@@ -24,7 +24,6 @@ def castrate_file(path, st):
except EnvironmentError:
pass
else:
#st = os.stat(path)
f.seek(0, 0)
f.truncate()
f.write(
......@@ -33,6 +32,14 @@ def castrate_file(path, st):
if st:
os.utime(path, (st.st_atime, st.st_mtime))
def modification_time(path):
st = os.stat(path)
return st.st_mtime
def file_newer_than(path, time):
ftime = modification_time(path)
return ftime > time
# support for source file encoding detection and unicode decoding
def encode_filename(filename):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7