Commit 01a5a332 authored by Stefan Behnel's avatar Stefan Behnel

large merge of cython-dagss as of revision 764

parents df65309f 72d54fb4
from Cython.Compiler.Visitor import TreeVisitor from Cython.Compiler.Visitor import TreeVisitor, get_temp_name_handle_desc
from Cython.Compiler.Nodes import * from Cython.Compiler.Nodes import *
from Cython.Compiler.ExprNodes import *
""" """
Serializes a Cython code tree to Cython code. This is primarily useful for Serializes a Cython code tree to Cython code. This is primarily useful for
...@@ -35,6 +36,7 @@ class CodeWriter(TreeVisitor): ...@@ -35,6 +36,7 @@ class CodeWriter(TreeVisitor):
result = LinesResult() result = LinesResult()
self.result = result self.result = result
self.numindents = 0 self.numindents = 0
self.tempnames = {}
def write(self, tree): def write(self, tree):
self.visit(tree) self.visit(tree)
...@@ -58,6 +60,12 @@ class CodeWriter(TreeVisitor): ...@@ -58,6 +60,12 @@ class CodeWriter(TreeVisitor):
self.startline(s) self.startline(s)
self.endline() self.endline()
def putname(self, name):
tmpdesc = get_temp_name_handle_desc(name)
if tmpdesc is not None:
name = self.tempnames.setdefault(tmpdesc, u"$" +tmpdesc)
self.put(name)
def comma_seperated_list(self, items, output_rhs=False): def comma_seperated_list(self, items, output_rhs=False):
if len(items) > 0: if len(items) > 0:
for item in items[:-1]: for item in items[:-1]:
...@@ -116,7 +124,7 @@ class CodeWriter(TreeVisitor): ...@@ -116,7 +124,7 @@ class CodeWriter(TreeVisitor):
self.endline() self.endline()
def visit_NameNode(self, node): def visit_NameNode(self, node):
self.put(node.name) self.putname(node.name)
def visit_IntNode(self, node): def visit_IntNode(self, node):
self.put(node.value) self.put(node.value)
...@@ -185,10 +193,23 @@ class CodeWriter(TreeVisitor): ...@@ -185,10 +193,23 @@ class CodeWriter(TreeVisitor):
self.comma_seperated_list(node.args) # Might need to discover whether we need () around tuples...hmm... self.comma_seperated_list(node.args) # Might need to discover whether we need () around tuples...hmm...
def visit_SimpleCallNode(self, node): def visit_SimpleCallNode(self, node):
self.put(node.function.name + u"(") self.visit(node.function)
self.put(u"(")
self.comma_seperated_list(node.args) self.comma_seperated_list(node.args)
self.put(")") self.put(")")
def visit_GeneralCallNode(self, node):
self.visit(node.function)
self.put(u"(")
posarg = node.positional_args
if isinstance(posarg, AsTupleNode):
self.visit(posarg.arg)
else:
self.comma_seperated_list(posarg)
if node.keyword_args is not None or node.starstar_arg is not None:
raise Exception("Not implemented yet")
self.put(u")")
def visit_ExprStatNode(self, node): def visit_ExprStatNode(self, node):
self.startline() self.startline()
self.visit(node.expr) self.visit(node.expr)
...@@ -197,9 +218,73 @@ class CodeWriter(TreeVisitor): ...@@ -197,9 +218,73 @@ class CodeWriter(TreeVisitor):
def visit_InPlaceAssignmentNode(self, node): def visit_InPlaceAssignmentNode(self, node):
self.startline() self.startline()
self.visit(node.lhs) self.visit(node.lhs)
self.put(" %s= " % node.operator) self.put(u" %s= " % node.operator)
self.visit(node.rhs) self.visit(node.rhs)
self.endline() self.endline()
def visit_WithStatNode(self, node):
self.startline()
self.put(u"with ")
self.visit(node.manager)
if node.target is not None:
self.put(u" as ")
self.visit(node.target)
self.endline(u":")
self.indent()
self.visit(node.body)
self.dedent()
def visit_AttributeNode(self, node):
self.visit(node.obj)
self.put(u".%s" % node.attribute)
def visit_BoolNode(self, node):
self.put(str(node.value))
def visit_TryFinallyStatNode(self, node):
self.line(u"try:")
self.indent()
self.visit(node.body)
self.dedent()
self.line(u"finally:")
self.indent()
self.visit(node.finally_clause)
self.dedent()
def visit_TryExceptStatNode(self, node):
self.line(u"try:")
self.indent()
self.visit(node.body)
self.dedent()
for x in node.except_clauses:
self.visit(x)
if node.else_clause is not None:
self.visit(node.else_clause)
def visit_ExceptClauseNode(self, node):
self.startline(u"except")
if node.pattern is not None:
self.put(u" ")
self.visit(node.pattern)
if node.target is not None:
self.put(u", ")
self.visit(node.target)
self.endline(":")
self.indent()
self.visit(node.body)
self.dedent()
def visit_ReraiseStatNode(self, node):
self.line("raise")
def visit_NoneNode(self, node):
self.put(u"None")
def visit_ImportNode(self, node):
self.put(u"(import %s)" % node.module_name.value)
def visit_NotNode(self, node):
self.put(u"(not ")
self.visit(node.operand)
self.put(u")")
...@@ -200,6 +200,8 @@ class CCodeWriter: ...@@ -200,6 +200,8 @@ class CCodeWriter:
def put_var_declaration(self, entry, static = 0, dll_linkage = None, def put_var_declaration(self, entry, static = 0, dll_linkage = None,
definition = True): definition = True):
#print "Code.put_var_declaration:", entry.name, "definition =", definition ### #print "Code.put_var_declaration:", entry.name, "definition =", definition ###
if entry.in_closure:
return
visibility = entry.visibility visibility = entry.visibility
if visibility == 'private' and not definition: if visibility == 'private' and not definition:
#print "...private and not definition, skipping" ### #print "...private and not definition, skipping" ###
......
...@@ -32,6 +32,7 @@ class CompileError(PyrexError): ...@@ -32,6 +32,7 @@ class CompileError(PyrexError):
def __init__(self, position = None, message = ""): def __init__(self, position = None, message = ""):
self.position = position self.position = position
self.message_only = message
# Deprecated and withdrawn in 2.6: # Deprecated and withdrawn in 2.6:
# self.message = message # self.message = message
if position: if position:
...@@ -91,6 +92,7 @@ def error(position, message): ...@@ -91,6 +92,7 @@ def error(position, message):
#print "Errors.error:", repr(position), repr(message) ### #print "Errors.error:", repr(position), repr(message) ###
global num_errors global num_errors
err = CompileError(position, message) err = CompileError(position, message)
# if position is not None: raise Exception(err) # debug
line = "%s\n" % err line = "%s\n" % err
if listing_file: if listing_file:
listing_file.write(line) listing_file.write(line)
......
...@@ -972,7 +972,7 @@ class NameNode(AtomicExprNode): ...@@ -972,7 +972,7 @@ class NameNode(AtomicExprNode):
if entry.is_builtin: if entry.is_builtin:
namespace = Naming.builtins_cname namespace = Naming.builtins_cname
else: # entry.is_pyglobal else: # entry.is_pyglobal
namespace = entry.namespace_cname namespace = entry.scope.namespace_cname
code.putln( code.putln(
'%s = __Pyx_GetName(%s, %s); %s' % ( '%s = __Pyx_GetName(%s, %s); %s' % (
self.result_code, self.result_code,
...@@ -997,7 +997,7 @@ class NameNode(AtomicExprNode): ...@@ -997,7 +997,7 @@ class NameNode(AtomicExprNode):
# is_pyglobal seems to be True for module level-globals only. # is_pyglobal seems to be True for module level-globals only.
# We use this to access class->tp_dict if necessary. # We use this to access class->tp_dict if necessary.
if entry.is_pyglobal: if entry.is_pyglobal:
namespace = self.entry.namespace_cname namespace = self.entry.scope.namespace_cname
if entry.is_member: if entry.is_member:
# if the entry is a member we have to cheat: SetAttr does not work # if the entry is a member we have to cheat: SetAttr does not work
# on types, so we create a descriptor which is then added to tp_dict # on types, so we create a descriptor which is then added to tp_dict
...@@ -1060,7 +1060,6 @@ class NameNode(AtomicExprNode): ...@@ -1060,7 +1060,6 @@ class NameNode(AtomicExprNode):
else: else:
code.annotate(pos, AnnotationItem('c_call', 'c function', size=len(self.name))) code.annotate(pos, AnnotationItem('c_call', 'c function', size=len(self.name)))
class BackquoteNode(ExprNode): class BackquoteNode(ExprNode):
# `expr` # `expr`
# #
...@@ -1212,6 +1211,9 @@ class ExcValueNode(AtomicExprNode): ...@@ -1212,6 +1211,9 @@ class ExcValueNode(AtomicExprNode):
def generate_result_code(self, code): def generate_result_code(self, code):
pass pass
def analyse_types(self, env):
pass
class TempNode(AtomicExprNode): class TempNode(AtomicExprNode):
# Node created during analyse_types phase # Node created during analyse_types phase
...@@ -1273,8 +1275,31 @@ class IndexNode(ExprNode): ...@@ -1273,8 +1275,31 @@ class IndexNode(ExprNode):
self.analyse_base_and_index_types(env, setting = 1) self.analyse_base_and_index_types(env, setting = 1)
def analyse_base_and_index_types(self, env, getting = 0, setting = 0): def analyse_base_and_index_types(self, env, getting = 0, setting = 0):
self.is_buffer_access = False
self.base.analyse_types(env) self.base.analyse_types(env)
self.index.analyse_types(env)
if self.base.type.buffer_options is not None:
if isinstance(self.index, TupleNode):
indices = self.index.args
# is_int_indices = 0 == sum([1 for i in self.index.args if not i.type.is_int])
else:
# is_int_indices = self.index.type.is_int
indices = [self.index]
all_ints = True
for index in indices:
index.analyse_types(env)
if not index.type.is_int:
all_ints = False
if all_ints:
self.indices = indices
self.index = None
self.type = self.base.type.buffer_options.dtype
self.is_temp = 1
self.is_buffer_access = True
if not self.is_buffer_access:
self.index.analyse_types(env) # ok to analyse as tuple
if self.base.type.is_pyobject: if self.base.type.is_pyobject:
if self.index.type.is_int: if self.index.type.is_int:
self.original_index_type = self.index.type self.original_index_type = self.index.type
...@@ -1328,11 +1353,17 @@ class IndexNode(ExprNode): ...@@ -1328,11 +1353,17 @@ class IndexNode(ExprNode):
def generate_subexpr_evaluation_code(self, code): def generate_subexpr_evaluation_code(self, code):
self.base.generate_evaluation_code(code) self.base.generate_evaluation_code(code)
if self.index is not None:
self.index.generate_evaluation_code(code) self.index.generate_evaluation_code(code)
else:
for i in self.indices: i.generate_evaluation_code(code)
def generate_subexpr_disposal_code(self, code): def generate_subexpr_disposal_code(self, code):
self.base.generate_disposal_code(code) self.base.generate_disposal_code(code)
if self.index is not None:
self.index.generate_disposal_code(code) self.index.generate_disposal_code(code)
else:
for i in self.indices: i.generate_disposal_code(code)
def generate_result_code(self, code): def generate_result_code(self, code):
if self.type.is_pyobject: if self.type.is_pyobject:
......
...@@ -7,5 +7,6 @@ def _get_feature(name): ...@@ -7,5 +7,6 @@ def _get_feature(name):
return object() return object()
unicode_literals = _get_feature("unicode_literals") unicode_literals = _get_feature("unicode_literals")
with_statement = _get_feature("with_statement")
del _get_feature del _get_feature
...@@ -25,22 +25,6 @@ from Cython import Utils ...@@ -25,22 +25,6 @@ from Cython import Utils
module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$") module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$")
# Note: PHASES and TransformSet should be removed soon; but that's for
# another day and another commit.
PHASES = [
'before_analyse_function', # run in FuncDefNode.generate_function_definitions
'after_analyse_function' # run in FuncDefNode.generate_function_definitions
]
class TransformSet(dict):
def __init__(self):
for name in PHASES:
self[name] = []
def run(self, name, node, **options):
assert name in self, "Transform phase %s not defined" % name
for transform in self[name]:
transform(node, phase=name, **options)
verbose = 0 verbose = 0
class Context: class Context:
...@@ -58,6 +42,8 @@ class Context: ...@@ -58,6 +42,8 @@ class Context:
#self.modules = {"__builtin__" : BuiltinScope()} #self.modules = {"__builtin__" : BuiltinScope()}
import Builtin import Builtin
self.modules = {"__builtin__" : Builtin.builtin_scope} self.modules = {"__builtin__" : Builtin.builtin_scope}
self.pxds = {}
self.pyxs = {}
self.include_directories = include_directories self.include_directories = include_directories
self.future_directives = set() self.future_directives = set()
...@@ -305,55 +291,25 @@ class Context: ...@@ -305,55 +291,25 @@ class Context:
names.reverse() names.reverse()
return ".".join(names) return ".".join(names)
def compile(self, source, options = None, full_module_name = None): def setup_errors(self, options):
# Compile a Pyrex implementation file in this context
# and return a CompilationResult.
if not options:
options = default_options
result = CompilationResult()
cwd = os.getcwd()
source = os.path.join(cwd, source)
result.main_source_file = source
if options.use_listing_file: if options.use_listing_file:
result.listing_file = Utils.replace_suffix(source, ".lis") result.listing_file = Utils.replace_suffix(source, ".lis")
Errors.open_listing_file(result.listing_file, Errors.open_listing_file(result.listing_file,
echo_to_stderr = options.errors_to_stderr) echo_to_stderr = options.errors_to_stderr)
else: else:
Errors.open_listing_file(None) Errors.open_listing_file(None)
if options.output_file:
result.c_file = os.path.join(cwd, options.output_file) def teardown_errors(self, errors_occurred, options, result):
else: source_desc = result.compilation_source.source_desc
if options.cplus: if not isinstance(source_desc, FileSourceDescriptor):
c_suffix = ".cpp" raise RuntimeError("Only file sources for code supported")
else:
c_suffix = ".c"
result.c_file = Utils.replace_suffix(source, c_suffix)
c_stat = None
if result.c_file:
try:
c_stat = os.stat(result.c_file)
except EnvironmentError:
pass
full_module_name = full_module_name or self.extract_module_name(source, options)
source = FileSourceDescriptor(source)
initial_pos = (source, 1, 0)
scope = self.find_module(full_module_name, pos = initial_pos, need_pxd = 0)
errors_occurred = False
try:
tree = self.parse(source, scope, pxd = 0,
full_module_name = full_module_name)
tree.process_implementation(scope, options, result)
except CompileError:
errors_occurred = True
Errors.close_listing_file() Errors.close_listing_file()
result.num_errors = Errors.num_errors result.num_errors = Errors.num_errors
if result.num_errors > 0: if result.num_errors > 0:
errors_occurred = True errors_occurred = True
if errors_occurred and result.c_file: if errors_occurred and result.c_file:
try: try:
Utils.castrate_file(result.c_file, os.stat(source.filename)) Utils.castrate_file(result.c_file, os.stat(source_desc.filename))
except EnvironmentError: except EnvironmentError:
pass pass
result.c_file = None result.c_file = None
...@@ -366,6 +322,97 @@ class Context: ...@@ -366,6 +322,97 @@ class Context:
extra_objects = options.objects, extra_objects = options.objects,
verbose_flag = options.show_version, verbose_flag = options.show_version,
cplus = options.cplus) cplus = options.cplus)
def run_pipeline(self, pipeline, source):
errors_occurred = False
data = source
try:
for phase in pipeline:
data = phase(data)
except CompileError:
errors_occurred = True
return (errors_occurred, data)
def create_parse(context):
def parse(compsrc):
source_desc = compsrc.source_desc
full_module_name = compsrc.full_module_name
initial_pos = (source_desc, 1, 0)
scope = context.find_module(full_module_name, pos = initial_pos, need_pxd = 0)
tree = context.parse(source_desc, scope, pxd = 0, full_module_name = full_module_name)
tree.compilation_source = compsrc
tree.scope = scope
return tree
return parse
def create_generate_code(context, options, result):
def generate_code(module_node):
scope = module_node.scope
module_node.process_implementation(options, result)
result.compilation_source = module_node.compilation_source
return result
return generate_code
def create_default_pipeline(context, options, result):
from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, BufferTransform
from ParseTreeTransforms import AnalyseDeclarationsTransform, AnalyseExpressionsTransform
from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor
from ModuleNode import check_c_classes
return [
create_parse(context),
NormalizeTree(context),
PostParse(context),
WithTransform(context),
AnalyseDeclarationsTransform(context),
check_c_classes,
AnalyseExpressionsTransform(context),
BufferTransform(context),
# CreateClosureClasses(context),
create_generate_code(context, options, result)
]
def create_default_resultobj(compilation_source, options):
result = CompilationResult()
result.main_source_file = compilation_source.source_desc.filename
result.compilation_source = compilation_source
source_desc = compilation_source.source_desc
if options.output_file:
result.c_file = os.path.join(compilation_source.cwd, options.output_file)
else:
if options.cplus:
c_suffix = ".cpp"
else:
c_suffix = ".c"
result.c_file = Utils.replace_suffix(source_desc.filename, c_suffix)
# The below doesn't make any sense? Why is it there?
c_stat = None
if result.c_file:
try:
c_stat = os.stat(result.c_file)
except EnvironmentError:
pass
return result
def run_pipeline(source, options, full_module_name = None):
# Set up context
context = Context(options.include_path)
# Set up source object
cwd = os.getcwd()
source_desc = FileSourceDescriptor(os.path.join(cwd, source))
full_module_name = full_module_name or context.extract_module_name(source, options)
source = CompilationSource(source_desc, full_module_name, cwd)
# Set up result object
result = create_default_resultobj(source, options)
# Get pipeline
pipeline = create_default_pipeline(context, options, result)
context.setup_errors(options)
errors_occurred, enddata = context.run_pipeline(pipeline, source)
context.teardown_errors(errors_occurred, options, result)
return result return result
#------------------------------------------------------------------------ #------------------------------------------------------------------------
...@@ -374,6 +421,16 @@ class Context: ...@@ -374,6 +421,16 @@ class Context:
# #
#------------------------------------------------------------------------ #------------------------------------------------------------------------
class CompilationSource(object):
"""
Contains the data necesarry to start up a compilation pipeline for
a single compilation unit.
"""
def __init__(self, source_desc, full_module_name, cwd):
self.source_desc = source_desc
self.full_module_name = full_module_name
self.cwd = cwd
class CompilationOptions: class CompilationOptions:
""" """
Options to the Cython compiler: Options to the Cython compiler:
...@@ -389,7 +446,6 @@ class CompilationOptions: ...@@ -389,7 +446,6 @@ class CompilationOptions:
defaults to true when recursive is true. defaults to true when recursive is true.
verbose boolean Always print source names being compiled verbose boolean Always print source names being compiled
quiet boolean Don't print source names in recursive mode quiet boolean Don't print source names in recursive mode
transforms Transform.TransformSet Transforms to use on the parse tree
Following options are experimental and only used on MacOSX: Following options are experimental and only used on MacOSX:
...@@ -427,6 +483,7 @@ class CompilationResult: ...@@ -427,6 +483,7 @@ class CompilationResult:
object_file string or None Result of compiling the C file object_file string or None Result of compiling the C file
extension_file string or None Result of linking the object file extension_file string or None Result of linking the object file
num_errors integer Number of compilation errors num_errors integer Number of compilation errors
compilation_source CompilationSource
""" """
def __init__(self): def __init__(self):
...@@ -464,8 +521,10 @@ def compile_single(source, options, full_module_name = None): ...@@ -464,8 +521,10 @@ def compile_single(source, options, full_module_name = None):
Always compiles a single file; does not perform timestamp checking or Always compiles a single file; does not perform timestamp checking or
recursion. recursion.
""" """
context = Context(options.include_path) return run_pipeline(source, options, full_module_name)
return context.compile(source, options, full_module_name) # context = Context(options.include_path)
# return context.compile(source, options, full_module_name)
def compile_multiple(sources, options): def compile_multiple(sources, options):
""" """
...@@ -478,21 +537,21 @@ def compile_multiple(sources, options): ...@@ -478,21 +537,21 @@ def compile_multiple(sources, options):
sources = [os.path.abspath(source) for source in sources] sources = [os.path.abspath(source) for source in sources]
processed = set() processed = set()
results = CompilationResultSet() results = CompilationResultSet()
context = Context(options.include_path)
recursive = options.recursive recursive = options.recursive
timestamps = options.timestamps timestamps = options.timestamps
if timestamps is None: if timestamps is None:
timestamps = recursive timestamps = recursive
verbose = options.verbose or ((recursive or timestamps) and not options.quiet) verbose = options.verbose or ((recursive or timestamps) and not options.quiet)
for source in sources: for source in sources:
context = Context(options.include_path) # to be removed later
if source not in processed: if source not in processed:
if not timestamps or context.c_file_out_of_date(source): if not timestamps or context.c_file_out_of_date(source):
if verbose: if verbose:
sys.stderr.write("Compiling %s\n" % source) sys.stderr.write("Compiling %s\n" % source)
result = context.compile(source, options) result = context.compile(source, options)
# Compiling multiple sources in one context doesn't quite # Compiling multiple sources in one context doesn't quite
# work properly yet. # work properly yet.
context = Context(options.include_path) # to be removed later
results.add(source, result) results.add(source, result)
processed.add(source) processed.add(source)
if recursive: if recursive:
...@@ -522,7 +581,10 @@ def compile(source, options = None, c_compile = 0, c_link = 0, ...@@ -522,7 +581,10 @@ def compile(source, options = None, c_compile = 0, c_link = 0,
and not options.recursive: and not options.recursive:
return compile_single(source, options, full_module_name) return compile_single(source, options, full_module_name)
else: else:
return compile_multiple(source, options) # Hack it for wednesday dev1
assert len(source) == 1
return compile_single(source[0], options)
# return compile_multiple(source, options)
#------------------------------------------------------------------------ #------------------------------------------------------------------------
# #
...@@ -571,9 +633,9 @@ default_options = dict( ...@@ -571,9 +633,9 @@ default_options = dict(
output_file = None, output_file = None,
annotate = False, annotate = False,
generate_pxi = 0, generate_pxi = 0,
transforms = TransformSet(),
working_path = "", working_path = "",
recursive = 0, recursive = 0,
transforms = None, # deprecated
timestamps = None, timestamps = None,
verbose = 0, verbose = 0,
quiet = 0) quiet = 0)
......
...@@ -25,6 +25,10 @@ from PyrexTypes import py_object_type ...@@ -25,6 +25,10 @@ from PyrexTypes import py_object_type
from Cython.Utils import open_new_file, replace_suffix from Cython.Utils import open_new_file, replace_suffix
def check_c_classes(module_node):
module_node.scope.check_c_classes()
return module_node
class ModuleNode(Nodes.Node, Nodes.BlockNode): class ModuleNode(Nodes.Node, Nodes.BlockNode):
# doc string or None # doc string or None
# body StatListNode # body StatListNode
...@@ -32,6 +36,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -32,6 +36,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
# referenced_modules [ModuleScope] # referenced_modules [ModuleScope]
# module_temp_cname string # module_temp_cname string
# full_module_name string # full_module_name string
#
# scope The module scope.
# compilation_source A CompilationSource (see Main)
child_attrs = ["body"] child_attrs = ["body"]
...@@ -44,10 +51,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -44,10 +51,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
env.doc = self.doc env.doc = self.doc
self.body.analyse_declarations(env) self.body.analyse_declarations(env)
def process_implementation(self, env, options, result): def process_implementation(self, options, result):
self.analyse_declarations(env) env = self.scope
env.check_c_classes()
self.body.analyse_expressions(env)
env.return_type = PyrexTypes.c_void_type env.return_type = PyrexTypes.c_void_type
self.referenced_modules = [] self.referenced_modules = []
self.find_referenced_modules(env, self.referenced_modules, {}) self.find_referenced_modules(env, self.referenced_modules, {})
...@@ -254,6 +259,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -254,6 +259,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
self.generate_module_cleanup_func(env, code) self.generate_module_cleanup_func(env, code)
self.generate_filename_table(code) self.generate_filename_table(code)
self.generate_utility_functions(env, code) self.generate_utility_functions(env, code)
self.generate_buffer_compatability_functions(env, code)
self.generate_declarations_for_modules(env, modules, code.h) self.generate_declarations_for_modules(env, modules, code.h)
...@@ -433,6 +439,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -433,6 +439,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
code.putln(" #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES)") code.putln(" #define PyBUF_F_CONTIGUOUS (0x0040 | PyBUF_STRIDES)")
code.putln(" #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES)") code.putln(" #define PyBUF_ANY_CONTIGUOUS (0x0080 | PyBUF_STRIDES)")
code.putln(" #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES)") code.putln(" #define PyBUF_INDIRECT (0x0100 | PyBUF_STRIDES)")
code.putln("")
code.putln(" static int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags);")
code.putln(" static void PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view);")
code.putln("#endif") code.putln("#endif")
code.put(builtin_module_name_utility_code[0]) code.put(builtin_module_name_utility_code[0])
...@@ -1940,6 +1949,60 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): ...@@ -1940,6 +1949,60 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
code.h.put(utility_code[0]) code.h.put(utility_code[0])
code.put(utility_code[1]) code.put(utility_code[1])
code.put(PyrexTypes.type_conversion_functions) code.put(PyrexTypes.type_conversion_functions)
code.putln("")
def generate_buffer_compatability_functions(self, env, code):
# will be refactored
code.put("""
static int numpy_getbuffer(PyObject *obj, Py_buffer *view, int flags) {
/* This function is always called after a type-check */
PyArrayObject *arr = (PyArrayObject*)obj;
PyArray_Descr *type = (PyArray_Descr*)arr->descr;
view->buf = arr->data;
view->readonly = 0; /*fixme*/
view->format = "B"; /*fixme*/
view->ndim = arr->nd;
view->strides = arr->strides;
view->shape = arr->dimensions;
view->suboffsets = 0;
view->itemsize = type->elsize;
view->internal = 0;
return 0;
}
static void numpy_releasebuffer(PyObject *obj, Py_buffer *view) {
}
""")
# For now, hard-code numpy imported as "numpy"
ndarrtype = env.entries[u'numpy'].as_module.entries['ndarray'].type
types = [
(ndarrtype.typeptr_cname, "numpy_getbuffer", "numpy_releasebuffer")
]
# typeptr_cname = ndarrtype.typeptr_cname
code.putln("static int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags) {")
clause = "if"
for t, get, release in types:
code.putln("%s (__Pyx_TypeTest(obj, %s)) return %s(obj, view, flags);" % (clause, t, get))
clause = "else if"
code.putln("else {")
code.putln("PyErr_Format(PyExc_TypeError, \"'%100s' does not have the buffer interface\", Py_TYPE(obj)->tp_name);")
code.putln("return -1;")
code.putln("}")
code.putln("}")
code.putln("")
code.putln("static void PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view) {")
clause = "if"
for t, get, release in types:
code.putln("%s (__Pyx_TypeTest(obj, %s)) %s(obj, view);" % (clause, t, release))
clause = "else if"
code.putln("}")
code.putln("")
#------------------------------------------------------------------------------------ #------------------------------------------------------------------------------------
# #
......
...@@ -8,6 +8,8 @@ ...@@ -8,6 +8,8 @@
pyrex_prefix = "__pyx_" pyrex_prefix = "__pyx_"
temp_prefix = u"__cyt_"
builtin_prefix = pyrex_prefix + "builtin_" builtin_prefix = pyrex_prefix + "builtin_"
arg_prefix = pyrex_prefix + "arg_" arg_prefix = pyrex_prefix + "arg_"
funcdoc_prefix = pyrex_prefix + "doc_" funcdoc_prefix = pyrex_prefix + "doc_"
...@@ -70,6 +72,8 @@ optional_args_cname = pyrex_prefix + "optional_args" ...@@ -70,6 +72,8 @@ optional_args_cname = pyrex_prefix + "optional_args"
no_opt_args = pyrex_prefix + "no_opt_args" no_opt_args = pyrex_prefix + "no_opt_args"
import_star = pyrex_prefix + "import_star" import_star = pyrex_prefix + "import_star"
import_star_set = pyrex_prefix + "import_star_set" import_star_set = pyrex_prefix + "import_star_set"
cur_scope_cname = pyrex_prefix + "cur_scope"
enc_scope_cname = pyrex_prefix + "enc_scope"
line_c_macro = "__LINE__" line_c_macro = "__LINE__"
......
...@@ -10,7 +10,7 @@ import Naming ...@@ -10,7 +10,7 @@ import Naming
import PyrexTypes import PyrexTypes
import TypeSlots import TypeSlots
from PyrexTypes import py_object_type, error_type, CTypedefType, CFuncType from PyrexTypes import py_object_type, error_type, CTypedefType, CFuncType
from Symtab import ModuleScope, LocalScope, \ from Symtab import ModuleScope, LocalScope, GeneratorLocalScope, \
StructOrUnionScope, PyClassScope, CClassScope StructOrUnionScope, PyClassScope, CClassScope
from Cython.Utils import open_new_file, replace_suffix, EncodedString from Cython.Utils import open_new_file, replace_suffix, EncodedString
import Options import Options
...@@ -172,6 +172,26 @@ class Node(object): ...@@ -172,6 +172,26 @@ class Node(object):
self._end_pos = pos self._end_pos = pos
return pos return pos
def dump(self, level=0, filter_out=("pos",)):
def dump_child(x, level):
if isinstance(x, Node):
return x.dump(level)
elif isinstance(x, list):
return "[%s]" % ", ".join(dump_child(item, level) for item in x)
else:
return repr(x)
attrs = [(key, value) for key, value in self.__dict__.iteritems() if key not in filter_out]
if len(attrs) == 0:
return "<%s>" % self.__class__.__name__
else:
indent = " " * level
res = "<%s\n" % (self.__class__.__name__)
for key, value in attrs:
res += "%s %s: %s\n" % (indent, key, dump_child(value, level + 1))
res += "%s>" % indent
return res
class BlockNode: class BlockNode:
# Mixin class for nodes representing a declaration block. # Mixin class for nodes representing a declaration block.
...@@ -545,7 +565,6 @@ class CBaseTypeNode(Node): ...@@ -545,7 +565,6 @@ class CBaseTypeNode(Node):
pass pass
class CSimpleBaseTypeNode(CBaseTypeNode): class CSimpleBaseTypeNode(CBaseTypeNode):
# name string # name string
# module_path [string] Qualifying name components # module_path [string] Qualifying name components
...@@ -587,6 +606,30 @@ class CSimpleBaseTypeNode(CBaseTypeNode): ...@@ -587,6 +606,30 @@ class CSimpleBaseTypeNode(CBaseTypeNode):
else: else:
return PyrexTypes.error_type return PyrexTypes.error_type
class CBufferAccessTypeNode(Node):
# After parsing:
# positional_args [ExprNode] List of positional arguments
# keyword_args DictNode Keyword arguments
# base_type_node CBaseTypeNode
# After PostParse:
# dtype_node CBaseTypeNode
# ndim int
# After analysis:
# type PyrexType.PyrexType
child_attrs = ["base_type_node", "positional_args", "keyword_args",
"dtype_node"]
dtype_node = None
def analyse(self, env):
base_type = self.base_type_node.analyse(env)
dtype = self.dtype_node.analyse(env)
options = PyrexTypes.BufferOptions(dtype=dtype, ndim=self.ndim)
self.type = PyrexTypes.create_buffer_type(base_type, options)
return self.type
class CComplexBaseTypeNode(CBaseTypeNode): class CComplexBaseTypeNode(CBaseTypeNode):
# base_type CBaseTypeNode # base_type CBaseTypeNode
...@@ -773,9 +816,11 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -773,9 +816,11 @@ class FuncDefNode(StatNode, BlockNode):
# return_type PyrexType # return_type PyrexType
# #filename string C name of filename string const # #filename string C name of filename string const
# entry Symtab.Entry # entry Symtab.Entry
# needs_closure boolean Whether or not this function has inner functions/classes/yield
py_func = None py_func = None
assmt = None assmt = None
needs_closure = False
def analyse_default_values(self, env): def analyse_default_values(self, env):
genv = env.global_scope() genv = env.global_scope()
...@@ -803,25 +848,27 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -803,25 +848,27 @@ class FuncDefNode(StatNode, BlockNode):
def need_gil_acquisition(self, lenv): def need_gil_acquisition(self, lenv):
return 0 return 0
def generate_function_definitions(self, env, code, transforms): def create_local_scope(self, env):
code.mark_pos(self.pos) genv = env
# Generate C code for header and body of function while env.is_py_class_scope or env.is_c_class_scope:
genv = env.global_scope() env = env.outer_scope
if self.needs_closure:
lenv = GeneratorLocalScope(name = self.entry.name, outer_scope = genv)
else:
lenv = LocalScope(name = self.entry.name, outer_scope = genv) lenv = LocalScope(name = self.entry.name, outer_scope = genv)
lenv.return_type = self.return_type lenv.return_type = self.return_type
type = self.entry.type type = self.entry.type
if type.is_cfunction: if type.is_cfunction:
lenv.nogil = type.nogil and not type.with_gil lenv.nogil = type.nogil and not type.with_gil
self.local_scope = lenv
return lenv
def generate_function_definitions(self, env, code, transforms):
# Generate C code for header and body of function
code.init_labels() code.init_labels()
self.declare_arguments(lenv) lenv = self.local_scope
transforms.run('before_analyse_function', self, env=env, lenv=lenv, genv=genv)
self.body.analyse_control_flow(lenv)
self.body.analyse_declarations(lenv)
self.body.analyse_expressions(lenv)
transforms.run('after_analyse_function', self, env=env, lenv=lenv, genv=genv)
# Code for nested function definitions would go here
# if we supported them, which we probably won't.
# ----- Top-level constants used by this function # ----- Top-level constants used by this function
code.mark_pos(self.pos)
self.generate_interned_num_decls(lenv, code) self.generate_interned_num_decls(lenv, code)
self.generate_interned_string_decls(lenv, code) self.generate_interned_string_decls(lenv, code)
self.generate_py_string_decls(lenv, code) self.generate_py_string_decls(lenv, code)
...@@ -838,7 +885,10 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -838,7 +885,10 @@ class FuncDefNode(StatNode, BlockNode):
self.generate_function_header(code, self.generate_function_header(code,
with_pymethdef = env.is_py_class_scope) with_pymethdef = env.is_py_class_scope)
# ----- Local variable declarations # ----- Local variable declarations
lenv.mangle_closure_cnames(Naming.cur_scope_cname)
self.generate_argument_declarations(lenv, code) self.generate_argument_declarations(lenv, code)
if self.needs_closure:
code.putln("/* TODO: declare and create scope object */")
code.put_var_declarations(lenv.var_entries) code.put_var_declarations(lenv.var_entries)
init = "" init = ""
if not self.return_type.is_void: if not self.return_type.is_void:
...@@ -919,6 +969,7 @@ class FuncDefNode(StatNode, BlockNode): ...@@ -919,6 +969,7 @@ class FuncDefNode(StatNode, BlockNode):
self.put_stararg_decrefs(code) self.put_stararg_decrefs(code)
if acquire_gil: if acquire_gil:
code.putln("PyGILState_Release(_save);") code.putln("PyGILState_Release(_save);")
code.putln("/* TODO: decref scope object */")
# ----- Return # ----- Return
if not self.return_type.is_void: if not self.return_type.is_void:
code.putln("return %s;" % Naming.retval_cname) code.putln("return %s;" % Naming.retval_cname)
...@@ -975,8 +1026,9 @@ class CFuncDefNode(FuncDefNode): ...@@ -975,8 +1026,9 @@ class CFuncDefNode(FuncDefNode):
# #
# with_gil boolean Acquire GIL around body # with_gil boolean Acquire GIL around body
# type CFuncType # type CFuncType
# py_func wrapper for calling from Python
child_attrs = ["base_type", "declarator", "body"] child_attrs = ["base_type", "declarator", "body", "py_func"]
def unqualified_name(self): def unqualified_name(self):
return self.entry.name return self.entry.name
...@@ -1839,6 +1891,8 @@ class OverrideCheckNode(StatNode): ...@@ -1839,6 +1891,8 @@ class OverrideCheckNode(StatNode):
child_attrs = ['body'] child_attrs = ['body']
body = None
def analyse_expressions(self, env): def analyse_expressions(self, env):
self.args = env.arg_entries self.args = env.arg_entries
if self.py_func.is_module_scope: if self.py_func.is_module_scope:
...@@ -1881,9 +1935,10 @@ class OverrideCheckNode(StatNode): ...@@ -1881,9 +1935,10 @@ class OverrideCheckNode(StatNode):
# code.put_decref(self.func_temp, PyrexTypes.py_object_type) # code.put_decref(self.func_temp, PyrexTypes.py_object_type)
code.putln("}") code.putln("}")
class ClassDefNode(StatNode, BlockNode):
pass
class PyClassDefNode(ClassDefNode):
class PyClassDefNode(StatNode, BlockNode):
# A Python class definition. # A Python class definition.
# #
# name EncodedString Name of the class # name EncodedString Name of the class
...@@ -1916,18 +1971,26 @@ class PyClassDefNode(StatNode, BlockNode): ...@@ -1916,18 +1971,26 @@ class PyClassDefNode(StatNode, BlockNode):
bases = bases, dict = self.dict, doc = doc_node) bases = bases, dict = self.dict, doc = doc_node)
self.target = ExprNodes.NameNode(pos, name = name) self.target = ExprNodes.NameNode(pos, name = name)
def create_scope(self, env):
genv = env
while env.is_py_class_scope or env.is_c_class_scope:
env = env.outer_scope
cenv = self.scope = PyClassScope(name = self.name, outer_scope = genv)
return cenv
def analyse_declarations(self, env): def analyse_declarations(self, env):
self.target.analyse_target_declaration(env) self.target.analyse_target_declaration(env)
cenv = self.create_scope(env)
cenv.class_obj_cname = self.target.entry.cname
self.body.analyse_declarations(cenv)
def analyse_expressions(self, env): def analyse_expressions(self, env):
self.dict.analyse_expressions(env) self.dict.analyse_expressions(env)
self.classobj.analyse_expressions(env) self.classobj.analyse_expressions(env)
genv = env.global_scope() genv = env.global_scope()
cenv = PyClassScope(name = self.name, outer_scope = genv) cenv = self.scope
cenv.class_dict_cname = self.dict.result_code cenv.class_dict_cname = self.dict.result_code
cenv.class_obj_cname = self.classobj.result_code cenv.namespace_cname = cenv.class_obj_cname = self.classobj.result_code
self.scope = cenv
self.body.analyse_declarations(cenv)
self.body.analyse_expressions(cenv) self.body.analyse_expressions(cenv)
self.target.analyse_target_expression(env, self.classobj) self.target.analyse_target_expression(env, self.classobj)
self.dict.release_temp(env) self.dict.release_temp(env)
...@@ -1947,7 +2010,7 @@ class PyClassDefNode(StatNode, BlockNode): ...@@ -1947,7 +2010,7 @@ class PyClassDefNode(StatNode, BlockNode):
self.dict.generate_disposal_code(code) self.dict.generate_disposal_code(code)
class CClassDefNode(StatNode, BlockNode): class CClassDefNode(ClassDefNode):
# An extension type definition. # An extension type definition.
# #
# visibility 'private' or 'public' or 'extern' # visibility 'private' or 'public' or 'extern'
...@@ -2356,7 +2419,7 @@ class InPlaceAssignmentNode(AssignmentNode): ...@@ -2356,7 +2419,7 @@ class InPlaceAssignmentNode(AssignmentNode):
# Fortunately, the type of the lhs node is fairly constrained # Fortunately, the type of the lhs node is fairly constrained
# (it must be a NameNode, AttributeNode, or IndexNode). # (it must be a NameNode, AttributeNode, or IndexNode).
child_attrs = ["lhs", "rhs", "dup"] child_attrs = ["lhs", "rhs"]
dup = None dup = None
def analyse_declarations(self, env): def analyse_declarations(self, env):
...@@ -2991,7 +3054,7 @@ class ForInStatNode(LoopNode, StatNode): ...@@ -2991,7 +3054,7 @@ class ForInStatNode(LoopNode, StatNode):
# else_clause StatNode # else_clause StatNode
# item NextNode used internally # item NextNode used internally
child_attrs = ["target", "iterator", "body", "else_clause", "item"] child_attrs = ["target", "iterator", "body", "else_clause"]
item = None item = None
def analyse_declarations(self, env): def analyse_declarations(self, env):
...@@ -3225,6 +3288,18 @@ class ForFromStatNode(LoopNode, StatNode): ...@@ -3225,6 +3288,18 @@ class ForFromStatNode(LoopNode, StatNode):
self.else_clause.annotate(code) self.else_clause.annotate(code)
class WithStatNode(StatNode):
"""
Represents a Python with statement.
This is only used at parse tree level; and is not present in
analysis or generation phases.
"""
# manager The with statement manager object
# target Node (lhs expression)
# body StatNode
child_attrs = ["manager", "target", "body"]
class TryExceptStatNode(StatNode): class TryExceptStatNode(StatNode):
# try .. except statement # try .. except statement
# #
...@@ -3317,17 +3392,26 @@ class ExceptClauseNode(Node): ...@@ -3317,17 +3392,26 @@ class ExceptClauseNode(Node):
# pattern ExprNode # pattern ExprNode
# target ExprNode or None # target ExprNode or None
# body StatNode # body StatNode
# excinfo_target NameNode or None optional target for exception info
# match_flag string result of exception match # match_flag string result of exception match
# exc_value ExcValueNode used internally # exc_value ExcValueNode used internally
# function_name string qualified name of enclosing function # function_name string qualified name of enclosing function
# exc_vars (string * 3) local exception variables # exc_vars (string * 3) local exception variables
child_attrs = ["pattern", "target", "body", "exc_value"] # excinfo_target is never set by the parser, but can be set by a transform
# in order to extract more extensive information about the exception as a
# sys.exc_info()-style tuple into a target variable
child_attrs = ["pattern", "target", "body", "exc_value", "excinfo_target"]
exc_value = None exc_value = None
excinfo_target = None
def analyse_declarations(self, env): def analyse_declarations(self, env):
if self.target: if self.target:
self.target.analyse_target_declaration(env) self.target.analyse_target_declaration(env)
if self.excinfo_target is not None:
self.excinfo_target.analyse_target_declaration(env)
self.body.analyse_declarations(env) self.body.analyse_declarations(env)
def analyse_expressions(self, env): def analyse_expressions(self, env):
...@@ -3345,6 +3429,17 @@ class ExceptClauseNode(Node): ...@@ -3345,6 +3429,17 @@ class ExceptClauseNode(Node):
self.exc_value = ExprNodes.ExcValueNode(self.pos, env, self.exc_vars[1]) self.exc_value = ExprNodes.ExcValueNode(self.pos, env, self.exc_vars[1])
self.exc_value.allocate_temps(env) self.exc_value.allocate_temps(env)
self.target.analyse_target_expression(env, self.exc_value) self.target.analyse_target_expression(env, self.exc_value)
if self.excinfo_target is not None:
import ExprNodes
self.excinfo_tuple = ExprNodes.TupleNode(pos=self.pos, args=[
ExprNodes.ExcValueNode(pos=self.pos, env=env, var=self.exc_vars[0]),
ExprNodes.ExcValueNode(pos=self.pos, env=env, var=self.exc_vars[1]),
ExprNodes.ExcValueNode(pos=self.pos, env=env, var=self.exc_vars[2])
])
self.excinfo_tuple.analyse_expressions(env)
self.excinfo_tuple.allocate_temps(env)
self.excinfo_target.analyse_target_expression(env, self.excinfo_tuple)
self.body.analyse_expressions(env) self.body.analyse_expressions(env)
for var in self.exc_vars: for var in self.exc_vars:
env.release_temp(var) env.release_temp(var)
...@@ -3374,6 +3469,10 @@ class ExceptClauseNode(Node): ...@@ -3374,6 +3469,10 @@ class ExceptClauseNode(Node):
if self.target: if self.target:
self.exc_value.generate_evaluation_code(code) self.exc_value.generate_evaluation_code(code)
self.target.generate_assignment_code(self.exc_value, code) self.target.generate_assignment_code(self.exc_value, code)
if self.excinfo_target is not None:
self.excinfo_tuple.generate_evaluation_code(code)
self.excinfo_target.generate_assignment_code(self.excinfo_tuple, code)
old_exc_vars = code.exc_vars old_exc_vars = code.exc_vars
code.exc_vars = self.exc_vars code.exc_vars = self.exc_vars
self.body.generate_execution_code(code) self.body.generate_execution_code(code)
...@@ -4484,6 +4583,7 @@ bad: ...@@ -4484,6 +4583,7 @@ bad:
Py_XDECREF(*tb); Py_XDECREF(*tb);
return -1; return -1;
} }
"""] """]
#------------------------------------------------------------------------------------ #------------------------------------------------------------------------------------
from Cython.Compiler.Visitor import VisitorTransform, temp_name_handle, CythonTransform
from Cython.Compiler.ModuleNode import ModuleNode
from Cython.Compiler.Nodes import *
from Cython.Compiler.ExprNodes import *
from Cython.Compiler.TreeFragment import TreeFragment
from Cython.Utils import EncodedString
from Cython.Compiler.Errors import CompileError
from sets import Set as set
class NormalizeTree(CythonTransform):
"""
This transform fixes up a few things after parsing
in order to make the parse tree more suitable for
transforms.
a) After parsing, blocks with only one statement will
be represented by that statement, not by a StatListNode.
When doing transforms this is annoying and inconsistent,
as one cannot in general remove a statement in a consistent
way and so on. This transform wraps any single statements
in a StatListNode containing a single statement.
b) The PassStatNode is a noop and serves no purpose beyond
plugging such one-statement blocks; i.e., once parsed a
` "pass" can just as well be represented using an empty
StatListNode. This means less special cases to worry about
in subsequent transforms (one always checks to see if a
StatListNode has no children to see if the block is empty).
"""
def __init__(self, context):
super(NormalizeTree, self).__init__(context)
self.is_in_statlist = False
self.is_in_expr = False
def visit_ExprNode(self, node):
stacktmp = self.is_in_expr
self.is_in_expr = True
self.visitchildren(node)
self.is_in_expr = stacktmp
return node
def visit_StatNode(self, node, is_listcontainer=False):
stacktmp = self.is_in_statlist
self.is_in_statlist = is_listcontainer
self.visitchildren(node)
self.is_in_statlist = stacktmp
if not self.is_in_statlist and not self.is_in_expr:
return StatListNode(pos=node.pos, stats=[node])
else:
return node
def visit_PassStatNode(self, node):
if not self.is_in_statlist:
return StatListNode(pos=node.pos, stats=[])
else:
return []
def visit_StatListNode(self, node):
self.is_in_statlist = True
self.visitchildren(node)
self.is_in_statlist = False
return node
def visit_ParallelAssignmentNode(self, node):
return self.visit_StatNode(node, True)
def visit_CEnumDefNode(self, node):
return self.visit_StatNode(node, True)
def visit_CStructOrUnionDefNode(self, node):
return self.visit_StatNode(node, True)
class PostParseError(CompileError): pass
# error strings checked by unit tests, so define them
ERR_BUF_OPTION_UNKNOWN = '"%s" is not a buffer option'
ERR_BUF_TOO_MANY = 'Too many buffer options'
ERR_BUF_DUP = '"%s" buffer option already supplied'
ERR_BUF_MISSING = '"%s" missing'
ERR_BUF_INT = '"%s" must be an integer'
ERR_BUF_NONNEG = '"%s" must be non-negative'
class PostParse(CythonTransform):
"""
Basic interpretation of the parse tree, as well as validity
checking that can be done on a very basic level on the parse
tree (while still not being a problem with the basic syntax,
as such).
Specifically:
- CBufferAccessTypeNode has its options interpreted:
Any first positional argument goes into the "dtype" attribute,
any "ndim" keyword argument goes into the "ndim" attribute and
so on. Also it is checked that the option combination is valid.
Note: Currently Parsing.py does a lot of interpretation and
reorganization that can be refactored into this transform
if a more pure Abstract Syntax Tree is wanted.
"""
buffer_options = ("dtype", "ndim") # ordered!
def visit_CBufferAccessTypeNode(self, node):
options = {}
# Fetch positional arguments
if len(node.positional_args) > len(self.buffer_options):
self.context.error(ERR_BUF_TOO_MANY)
for arg, unicode_name in zip(node.positional_args, self.buffer_options):
name = str(unicode_name)
options[name] = arg
# Fetch named arguments
for item in node.keyword_args.key_value_pairs:
name = str(item.key.value)
if not name in self.buffer_options:
raise PostParseError(item.key.pos,
ERR_BUF_UNKNOWN % name)
if name in options.keys():
raise PostParseError(item.key.pos,
ERR_BUF_DUP % key)
options[name] = item.value
provided = options.keys()
# get dtype
dtype = options.get("dtype")
if dtype is None: raise PostParseError(node.pos, ERR_BUF_MISSING % 'dtype')
node.dtype_node = dtype
# get ndim
if "ndim" in provided:
ndimnode = options["ndim"]
if not isinstance(ndimnode, IntNode):
# Compile-time values (DEF) are currently resolved by the parser,
# so nothing more to do here
raise PostParseError(ndimnode.pos, ERR_BUF_INT % 'ndim')
ndim_value = int(ndimnode.value)
if ndim_value < 0:
raise PostParseError(ndimnode.pos, ERR_BUF_NONNEG % 'ndim')
node.ndim = int(ndimnode.value)
else:
node.ndim = 1
# We're done with the parse tree args
node.positional_args = None
node.keyword_args = None
return node
class BufferTransform(CythonTransform):
"""
Run after type analysis. Takes care of the buffer functionality.
"""
scope = None
def __call__(self, node):
cymod = self.context.modules[u'__cython__']
self.buffer_type = cymod.entries[u'Py_buffer'].type
return super(BufferTransform, self).__call__(node)
def handle_scope(self, node, scope):
# For all buffers, insert extra variables in the scope.
# The variables are also accessible from the buffer_info
# on the buffer entry
bufvars = [(name, entry) for name, entry
in scope.entries.iteritems()
if entry.type.buffer_options is not None]
for name, entry in bufvars:
# Variable has buffer opts, declare auxiliary vars
bufopts = entry.type.buffer_options
bufinfo = scope.declare_var(temp_name_handle(u"%s_bufinfo" % name),
self.buffer_type, node.pos)
temp_var = scope.declare_var(temp_name_handle(u"%s_tmp" % name),
entry.type, node.pos)
stridevars = []
shapevars = []
for idx in range(bufopts.ndim):
# stride
varname = temp_name_handle(u"%s_%s%d" % (name, "stride", idx))
var = scope.declare_var(varname, PyrexTypes.c_int_type, node.pos, is_cdef=True)
stridevars.append(var)
# shape
varname = temp_name_handle(u"%s_%s%d" % (name, "shape", idx))
var = scope.declare_var(varname, PyrexTypes.c_uint_type, node.pos, is_cdef=True)
shapevars.append(var)
entry.buffer_aux = Symtab.BufferAux(bufinfo, stridevars,
shapevars)
entry.buffer_aux.temp_var = temp_var
self.scope = scope
def visit_ModuleNode(self, node):
self.handle_scope(node, node.scope)
self.visitchildren(node)
return node
def visit_FuncDefNode(self, node):
self.handle_scope(node, node.local_scope)
self.visitchildren(node)
return node
acquire_buffer_fragment = TreeFragment(u"""
TMP = LHS
if TMP is not None:
__cython__.PyObject_ReleaseBuffer(<__cython__.PyObject*>TMP, &BUFINFO)
TMP = RHS
__cython__.PyObject_GetBuffer(<__cython__.PyObject*>TMP, &BUFINFO, 0)
ASSIGN_AUX
LHS = TMP
""")
fetch_strides = TreeFragment(u"""
TARGET = BUFINFO.strides[IDX]
""")
fetch_shape = TreeFragment(u"""
TARGET = BUFINFO.shape[IDX]
""")
# ass = SingleAssignmentNode(pos=node.pos,
# lhs=NameNode(node.pos, name=entry.name),
# rhs=IndexNode(node.pos,
# base=AttributeNode(node.pos,
# obj=NameNode(node.pos, name=bufaux.buffer_info_var.name),
# attribute=EncodedString("strides")),
# index=IntNode(node.pos, value=EncodedString(idx))))
# print ass.dump()
def visit_SingleAssignmentNode(self, node):
self.visitchildren(node)
bufaux = node.lhs.entry.buffer_aux
if bufaux is not None:
auxass = []
for idx, entry in enumerate(bufaux.stridevars):
entry.used = True
ass = self.fetch_strides.substitute({
u"TARGET": NameNode(node.pos, name=entry.name),
u"BUFINFO": NameNode(node.pos, name=bufaux.buffer_info_var.name),
u"IDX": IntNode(node.pos, value=EncodedString(idx))
})
auxass.append(ass)
for idx, entry in enumerate(bufaux.shapevars):
entry.used = True
ass = self.fetch_shape.substitute({
u"TARGET": NameNode(node.pos, name=entry.name),
u"BUFINFO": NameNode(node.pos, name=bufaux.buffer_info_var.name),
u"IDX": IntNode(node.pos, value=EncodedString(idx))
})
auxass.append(ass)
bufaux.buffer_info_var.used = True
acq = self.acquire_buffer_fragment.substitute({
u"TMP" : NameNode(pos=node.pos, name=bufaux.temp_var.name),
u"LHS" : node.lhs,
u"RHS": node.rhs,
u"ASSIGN_AUX": StatListNode(node.pos, stats=auxass),
u"BUFINFO": NameNode(pos=node.pos, name=bufaux.buffer_info_var.name)
}, pos=node.pos)
# Note: The below should probably be refactored into something
# like fragment.substitute(..., context=self.context), with
# TreeFragment getting context.pipeline_until_now() and
# applying it on the fragment.
acq.analyse_declarations(self.scope)
acq.analyse_expressions(self.scope)
stats = acq.stats
# stats += [node] # Do assignment after successful buffer acquisition
# print acq.dump()
return stats
else:
return node
buffer_access = TreeFragment(u"""
(<unsigned char*>(BUF.buf + OFFSET))[0]
""")
def visit_IndexNode(self, node):
if node.is_buffer_access:
assert node.index is None
assert node.indices is not None
bufaux = node.base.entry.buffer_aux
assert bufaux is not None
to_sum = [ IntBinopNode(node.pos, operator='*', operand1=index,
operand2=NameNode(node.pos, name=stride.name))
for index, stride in zip(node.indices, bufaux.stridevars)]
print to_sum
indices = node.indices
# reduce * on indices
expr = to_sum[0]
for next in to_sum[1:]:
expr = IntBinopNode(node.pos, operator='+', operand1=expr, operand2=next)
tmp= self.buffer_access.substitute({
'BUF': NameNode(node.pos, name=bufaux.buffer_info_var.name),
'OFFSET': expr
})
tmp.analyse_expressions(self.scope)
return tmp.stats[0].expr
else:
return node
def visit_CallNode(self, node):
### print node.dump()
return node
# def visit_FuncDefNode(self, node):
# print node.dump()
class WithTransform(CythonTransform):
# EXCINFO is manually set to a variable that contains
# the exc_info() tuple that can be generated by the enclosing except
# statement.
template_without_target = TreeFragment(u"""
MGR = EXPR
EXIT = MGR.__exit__
MGR.__enter__()
EXC = True
try:
try:
BODY
except:
EXC = False
if not EXIT(*EXCINFO):
raise
finally:
if EXC:
EXIT(None, None, None)
""", temps=[u'MGR', u'EXC', u"EXIT", u"SYS)"],
pipeline=[NormalizeTree(None)])
template_with_target = TreeFragment(u"""
MGR = EXPR
EXIT = MGR.__exit__
VALUE = MGR.__enter__()
EXC = True
try:
try:
TARGET = VALUE
BODY
except:
EXC = False
if not EXIT(*EXCINFO):
raise
finally:
if EXC:
EXIT(None, None, None)
""", temps=[u'MGR', u'EXC', u"EXIT", u"VALUE", u"SYS"],
pipeline=[NormalizeTree(None)])
def visit_WithStatNode(self, node):
excinfo_name = temp_name_handle('EXCINFO')
excinfo_namenode = NameNode(pos=node.pos, name=excinfo_name)
excinfo_target = NameNode(pos=node.pos, name=excinfo_name)
if node.target is not None:
result = self.template_with_target.substitute({
u'EXPR' : node.manager,
u'BODY' : node.body,
u'TARGET' : node.target,
u'EXCINFO' : excinfo_namenode
}, pos = node.pos)
# Set except excinfo target to EXCINFO
result.stats[4].body.stats[0].except_clauses[0].excinfo_target = excinfo_target
else:
result = self.template_without_target.substitute({
u'EXPR' : node.manager,
u'BODY' : node.body,
u'EXCINFO' : excinfo_namenode
}, pos = node.pos)
# Set except excinfo target to EXCINFO
result.stats[4].body.stats[0].except_clauses[0].excinfo_target = excinfo_target
return result.stats
class AnalyseDeclarationsTransform(CythonTransform):
def __call__(self, root):
self.env_stack = [root.scope]
return super(AnalyseDeclarationsTransform, self).__call__(root)
def visit_ModuleNode(self, node):
node.analyse_declarations(self.env_stack[-1])
self.visitchildren(node)
return node
def visit_FuncDefNode(self, node):
lenv = node.create_local_scope(self.env_stack[-1])
node.body.analyse_control_flow(lenv) # this will be totally refactored
node.declare_arguments(lenv)
node.body.analyse_declarations(lenv)
self.env_stack.append(lenv)
self.visitchildren(node)
self.env_stack.pop()
return node
class AnalyseExpressionsTransform(CythonTransform):
def visit_ModuleNode(self, node):
node.body.analyse_expressions(node.scope)
self.visitchildren(node)
return node
def visit_FuncDefNode(self, node):
node.body.analyse_expressions(node.local_scope)
self.visitchildren(node)
return node
class MarkClosureVisitor(CythonTransform):
needs_closure = False
def visit_FuncDefNode(self, node):
self.needs_closure = False
self.visitchildren(node)
node.needs_closure = self.needs_closure
self.needs_closure = True
return node
def visit_ClassDefNode(self, node):
self.visitchildren(node)
self.needs_closure = True
return node
def visit_YieldNode(self, node):
self.needs_closure = True
class CreateClosureClasses(CythonTransform):
# Output closure classes in module scope for all functions
# that need it.
def visit_ModuleNode(self, node):
self.module_scope = node.scope
self.visitchildren(node)
return node
def create_class_from_scope(self, node, target_module_scope):
as_name = temp_name_handle("closure")
func_scope = node.local_scope
entry = target_module_scope.declare_c_class(name = as_name,
pos = node.pos, defining = True, implementing = True)
class_scope = entry.type.scope
for entry in func_scope.entries.values():
class_scope.declare_var(pos=node.pos,
name=entry.name,
cname=entry.cname,
type=entry.type,
is_cdef=True)
def visit_FuncDefNode(self, node):
self.create_class_from_scope(node, self.module_scope)
return node
...@@ -312,6 +312,7 @@ def p_call(s, function): ...@@ -312,6 +312,7 @@ def p_call(s, function):
if s.sy != ',': if s.sy != ',':
break break
s.next() s.next()
if s.sy == '*': if s.sy == '*':
s.next() s.next()
star_arg = p_simple_expr(s) star_arg = p_simple_expr(s)
...@@ -1159,13 +1160,13 @@ def p_for_from_step(s): ...@@ -1159,13 +1160,13 @@ def p_for_from_step(s):
inequality_relations = ('<', '<=', '>', '>=') inequality_relations = ('<', '<=', '>', '>=')
def p_for_target(s): def p_target(s, terminator):
pos = s.position() pos = s.position()
expr = p_bit_expr(s) expr = p_bit_expr(s)
if s.sy == ',': if s.sy == ',':
s.next() s.next()
exprs = [expr] exprs = [expr]
while s.sy != 'in': while s.sy != terminator:
exprs.append(p_bit_expr(s)) exprs.append(p_bit_expr(s))
if s.sy != ',': if s.sy != ',':
break break
...@@ -1174,6 +1175,9 @@ def p_for_target(s): ...@@ -1174,6 +1175,9 @@ def p_for_target(s):
else: else:
return expr return expr
def p_for_target(s):
return p_target(s, 'in')
def p_for_iterator(s): def p_for_iterator(s):
pos = s.position() pos = s.position()
expr = p_testlist(s) expr = p_testlist(s)
...@@ -1253,8 +1257,17 @@ def p_with_statement(s): ...@@ -1253,8 +1257,17 @@ def p_with_statement(s):
body = p_suite(s) body = p_suite(s)
return Nodes.GILStatNode(pos, state = state, body = body) return Nodes.GILStatNode(pos, state = state, body = body)
else: else:
s.error("Only 'with gil' and 'with nogil' implemented", manager = p_expr(s)
pos = pos) target = None
if s.sy == 'IDENT' and s.systring == 'as':
s.next()
allow_multi = (s.sy == '(')
target = p_target(s, ':')
if not allow_multi and isinstance(target, ExprNodes.TupleNode):
s.error("Multiple with statement target values not allowed without paranthesis")
body = p_suite(s)
return Nodes.WithStatNode(pos, manager = manager,
target = target, body = body)
def p_simple_statement(s, first_statement = 0): def p_simple_statement(s, first_statement = 0):
#print "p_simple_statement:", s.sy, s.systring ### #print "p_simple_statement:", s.sy, s.systring ###
...@@ -1447,6 +1460,71 @@ def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0): ...@@ -1447,6 +1460,71 @@ def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
else: else:
return body return body
def p_positional_and_keyword_args(s, end_sy_set, type_positions=(), type_keywords=()):
"""
Parses positional and keyword arguments. end_sy_set
should contain any s.sy that terminate the argument list.
Argument expansion (* and **) are not allowed.
type_positions and type_keywords specifies which argument
positions and/or names which should be interpreted as
types. Other arguments will be treated as expressions.
Returns: (positional_args, keyword_args)
"""
positional_args = []
keyword_args = []
pos_idx = 0
while s.sy not in end_sy_set:
if s.sy == '*' or s.sy == '**':
s.error('Argument expansion not allowed here.')
was_keyword = False
parsed_type = False
if s.sy == 'IDENT':
# Since we can have either types or expressions as positional args,
# we use a strategy of looking an extra step forward for a '=' and
# if it is a positional arg we backtrack.
ident = s.systring
s.next()
if s.sy == '=':
s.next()
# Is keyword arg
if ident in type_keywords:
arg = p_c_base_type(s)
parsed_type = True
else:
arg = p_simple_expr(s)
keyword_node = ExprNodes.IdentifierStringNode(arg.pos,
value = Utils.EncodedString(ident))
keyword_args.append((keyword_node, arg))
was_keyword = True
else:
s.put_back('IDENT', ident)
if not was_keyword:
if pos_idx in type_positions:
arg = p_c_base_type(s)
parsed_type = True
else:
arg = p_simple_expr(s)
positional_args.append(arg)
pos_idx += 1
if len(keyword_args) > 0:
s.error("Non-keyword arg following keyword arg",
pos = arg.pos)
if s.sy != ',':
if s.sy not in end_sy_set:
if parsed_type:
s.error("Expected: type")
else:
s.error("Expected: expression")
break
s.next()
return positional_args, keyword_args
def p_c_base_type(s, self_flag = 0, nonempty = 0): def p_c_base_type(s, self_flag = 0, nonempty = 0):
# If self_flag is true, this is the base type for the # If self_flag is true, this is the base type for the
# self argument of a C method of an extension type. # self argument of a C method of an extension type.
...@@ -1519,11 +1597,43 @@ def p_c_simple_base_type(s, self_flag, nonempty): ...@@ -1519,11 +1597,43 @@ def p_c_simple_base_type(s, self_flag, nonempty):
else: else:
#print "p_c_simple_base_type: not looking at type at", s.position() #print "p_c_simple_base_type: not looking at type at", s.position()
name = None name = None
return Nodes.CSimpleBaseTypeNode(pos,
type_node = Nodes.CSimpleBaseTypeNode(pos,
name = name, module_path = module_path, name = name, module_path = module_path,
is_basic_c_type = is_basic, signed = signed, is_basic_c_type = is_basic, signed = signed,
longness = longness, is_self_arg = self_flag) longness = longness, is_self_arg = self_flag)
# Treat trailing [] on type as buffer access
if s.sy == '[':
if is_basic:
p.error("Basic C types do not support buffer access")
return p_buffer_access(s, type_node)
else:
return type_node
def p_buffer_access(s, type_node):
# s.sy == '['
pos = s.position()
s.next()
positional_args, keyword_args = (
p_positional_and_keyword_args(s, (']',), (0,), ('dtype',))
)
s.expect(']')
keyword_dict = ExprNodes.DictNode(pos,
key_value_pairs = [
ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
for key, value in keyword_args
])
result = Nodes.CBufferAccessTypeNode(pos,
positional_args = positional_args,
keyword_args = keyword_dict,
base_type_node = type_node)
return result
def looking_at_type(s): def looking_at_type(s):
return looking_at_base_type(s) or s.looking_at_type_name() return looking_at_base_type(s) or s.looking_at_type_name()
......
...@@ -4,6 +4,23 @@ ...@@ -4,6 +4,23 @@
from Cython import Utils from Cython import Utils
import Naming import Naming
import copy
class BufferOptions:
# dtype PyrexType
# ndim int
def __init__(self, dtype, ndim):
self.dtype = dtype
self.ndim = ndim
def create_buffer_type(base_type, buffer_options):
# Make a shallow copy of base_type and then annotate it
# with the buffer information
result = copy.copy(base_type)
result.buffer_options = buffer_options
return result
class BaseType: class BaseType:
# #
...@@ -92,6 +109,7 @@ class PyrexType(BaseType): ...@@ -92,6 +109,7 @@ class PyrexType(BaseType):
default_value = "" default_value = ""
parsetuple_format = "" parsetuple_format = ""
pymemberdef_typecode = None pymemberdef_typecode = None
buffer_options = None # can contain a BufferOptions instance
def resolve(self): def resolve(self):
# If a typedef, returns the base type. # If a typedef, returns the base type.
...@@ -183,7 +201,6 @@ class CTypedefType(BaseType): ...@@ -183,7 +201,6 @@ class CTypedefType(BaseType):
def __getattr__(self, name): def __getattr__(self, name):
return getattr(self.typedef_base_type, name) return getattr(self.typedef_base_type, name)
class PyObjectType(PyrexType): class PyObjectType(PyrexType):
# #
# Base class for all Python object types (reference-counted). # Base class for all Python object types (reference-counted).
......
# #
# Pyrex - Symbol Table # Symbol Table
# #
import re import re
...@@ -19,6 +19,14 @@ import __builtin__ ...@@ -19,6 +19,14 @@ import __builtin__
possible_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match possible_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match
nice_identifier = re.compile('^[a-zA-Z0-0_]+$').match nice_identifier = re.compile('^[a-zA-Z0-0_]+$').match
class BufferAux:
def __init__(self, buffer_info_var, stridevars, shapevars):
self.buffer_info_var = buffer_info_var
self.stridevars = stridevars
self.shapevars = shapevars
def __repr__(self):
return "<BufferAux %r>" % self.__dict__
class Entry: class Entry:
# A symbol table entry in a Scope or ModuleNamespace. # A symbol table entry in a Scope or ModuleNamespace.
# #
...@@ -47,6 +55,7 @@ class Entry: ...@@ -47,6 +55,7 @@ class Entry:
# is_self_arg boolean Is the "self" arg of an exttype method # is_self_arg boolean Is the "self" arg of an exttype method
# is_arg boolean Is the arg of a method # is_arg boolean Is the arg of a method
# is_local boolean Is a local variable # is_local boolean Is a local variable
# in_closure boolean Is referenced in an inner scope
# is_readonly boolean Can't be assigned to # is_readonly boolean Can't be assigned to
# func_cname string C func implementing Python func # func_cname string C func implementing Python func
# pos position Source position where declared # pos position Source position where declared
...@@ -75,6 +84,8 @@ class Entry: ...@@ -75,6 +84,8 @@ class Entry:
# defined_in_pxd boolean Is defined in a .pxd file (not just declared) # defined_in_pxd boolean Is defined in a .pxd file (not just declared)
# api boolean Generate C API for C class or function # api boolean Generate C API for C class or function
# utility_code string Utility code needed when this entry is used # utility_code string Utility code needed when this entry is used
#
# buffer_aux BufferAux or None Extra information needed for buffer variables
borrowed = 0 borrowed = 0
init = "" init = ""
...@@ -96,6 +107,7 @@ class Entry: ...@@ -96,6 +107,7 @@ class Entry:
is_self_arg = 0 is_self_arg = 0
is_arg = 0 is_arg = 0
is_local = 0 is_local = 0
in_closure = 0
is_declared_generic = 0 is_declared_generic = 0
is_readonly = 0 is_readonly = 0
func_cname = None func_cname = None
...@@ -115,6 +127,7 @@ class Entry: ...@@ -115,6 +127,7 @@ class Entry:
api = 0 api = 0
utility_code = None utility_code = None
is_overridable = 0 is_overridable = 0
buffer_aux = None
def __init__(self, name, cname, type, pos = None, init = None): def __init__(self, name, cname, type, pos = None, init = None):
self.name = name self.name = name
...@@ -163,6 +176,8 @@ class Scope: ...@@ -163,6 +176,8 @@ class Scope:
in_cinclude = 0 in_cinclude = 0
nogil = 0 nogil = 0
temp_prefix = Naming.pyrex_prefix
def __init__(self, name, outer_scope, parent_scope): def __init__(self, name, outer_scope, parent_scope):
# The outer_scope is the next scope in the lookup chain. # The outer_scope is the next scope in the lookup chain.
# The parent_scope is used to derive the qualified name of this scope. # The parent_scope is used to derive the qualified name of this scope.
...@@ -447,7 +462,14 @@ class Scope: ...@@ -447,7 +462,14 @@ class Scope:
# Look up name in this scope or an enclosing one. # Look up name in this scope or an enclosing one.
# Return None if not found. # Return None if not found.
return (self.lookup_here(name) return (self.lookup_here(name)
or (self.outer_scope and self.outer_scope.lookup(name)) or (self.outer_scope and self.outer_scope.lookup_from_inner(name))
or None)
def lookup_from_inner(self, name):
# Look up name in this scope or an enclosing one.
# This is only called from enclosing scopes.
return (self.lookup_here(name)
or (self.outer_scope and self.outer_scope.lookup_from_inner(name))
or None) or None)
def lookup_here(self, name): def lookup_here(self, name):
...@@ -562,7 +584,7 @@ class Scope: ...@@ -562,7 +584,7 @@ class Scope:
return entry.cname return entry.cname
n = self.temp_counter n = self.temp_counter
self.temp_counter = n + 1 self.temp_counter = n + 1
cname = "%s%d" % (Naming.pyrex_prefix, n) cname = "%s%d" % (self.temp_prefix, n)
entry = Entry("", cname, type) entry = Entry("", cname, type)
entry.used = 1 entry.used = 1
if type.is_pyobject or type == PyrexTypes.c_py_ssize_t_type: if type.is_pyobject or type == PyrexTypes.c_py_ssize_t_type:
...@@ -608,6 +630,9 @@ class Scope: ...@@ -608,6 +630,9 @@ class Scope:
return 0 return 0
class PreImportScope(Scope): class PreImportScope(Scope):
namespace_cname = Naming.preimport_cname
def __init__(self): def __init__(self):
Scope.__init__(self, Options.pre_import, None, None) Scope.__init__(self, Options.pre_import, None, None)
...@@ -615,7 +640,6 @@ class PreImportScope(Scope): ...@@ -615,7 +640,6 @@ class PreImportScope(Scope):
entry = self.declare(name, name, py_object_type, pos) entry = self.declare(name, name, py_object_type, pos)
entry.is_variable = True entry.is_variable = True
entry.is_pyglobal = True entry.is_pyglobal = True
entry.namespace_cname = Naming.preimport_cname
return entry return entry
...@@ -761,6 +785,7 @@ class ModuleScope(Scope): ...@@ -761,6 +785,7 @@ class ModuleScope(Scope):
self.has_extern_class = 0 self.has_extern_class = 0
self.cached_builtins = [] self.cached_builtins = []
self.undeclared_cached_builtins = [] self.undeclared_cached_builtins = []
self.namespace_cname = self.module_cname
def qualifying_scope(self): def qualifying_scope(self):
return self.parent_module return self.parent_module
...@@ -876,7 +901,6 @@ class ModuleScope(Scope): ...@@ -876,7 +901,6 @@ class ModuleScope(Scope):
raise InternalError( raise InternalError(
"Non-cdef global variable is not a generic Python object") "Non-cdef global variable is not a generic Python object")
entry.is_pyglobal = 1 entry.is_pyglobal = 1
entry.namespace_cname = self.module_cname
else: else:
entry.is_cglobal = 1 entry.is_cglobal = 1
self.var_entries.append(entry) self.var_entries.append(entry)
...@@ -1075,7 +1099,6 @@ class ModuleScope(Scope): ...@@ -1075,7 +1099,6 @@ class ModuleScope(Scope):
var_entry.is_readonly = 1 var_entry.is_readonly = 1
entry.as_variable = var_entry entry.as_variable = var_entry
class LocalScope(Scope): class LocalScope(Scope):
def __init__(self, name, outer_scope): def __init__(self, name, outer_scope):
...@@ -1119,6 +1142,33 @@ class LocalScope(Scope): ...@@ -1119,6 +1142,33 @@ class LocalScope(Scope):
entry = self.global_scope().lookup_target(name) entry = self.global_scope().lookup_target(name)
self.entries[name] = entry self.entries[name] = entry
def lookup_from_inner(self, name):
entry = self.lookup_here(name)
if entry:
entry.in_closure = 1
return entry
else:
return (self.outer_scope and self.outer_scope.lookup_from_inner(name)) or None
def mangle_closure_cnames(self, scope_var):
for entry in self.entries.values():
if entry.in_closure:
if not hasattr(entry, 'orig_cname'):
entry.orig_cname = entry.cname
entry.cname = scope_var + "->" + entry.cname
class GeneratorLocalScope(LocalScope):
temp_prefix = Naming.cur_scope_cname + "->" + LocalScope.temp_prefix
def mangle_closure_cnames(self, scope_var):
for entry in self.entries.values() + self.temp_entries:
entry.in_closure = 1
LocalScope.mangle_closure_cnames(self, scope_var)
# def mangle(self, prefix, name):
# return "%s->%s" % (Naming.scope_obj_cname, name)
class StructOrUnionScope(Scope): class StructOrUnionScope(Scope):
# Namespace of a C struct or union. # Namespace of a C struct or union.
...@@ -1198,7 +1248,6 @@ class PyClassScope(ClassScope): ...@@ -1198,7 +1248,6 @@ class PyClassScope(ClassScope):
entry = Scope.declare_var(self, name, type, pos, entry = Scope.declare_var(self, name, type, pos,
cname, visibility, is_cdef) cname, visibility, is_cdef)
entry.is_pyglobal = 1 entry.is_pyglobal = 1
entry.namespace_cname = self.class_obj_cname
return entry return entry
def allocate_temp(self, type): def allocate_temp(self, type):
...@@ -1295,7 +1344,7 @@ class CClassScope(ClassScope): ...@@ -1295,7 +1344,7 @@ class CClassScope(ClassScope):
entry.is_pyglobal = 1 # xxx: is_pyglobal changes behaviour in so many places that entry.is_pyglobal = 1 # xxx: is_pyglobal changes behaviour in so many places that
# I keep it in for now. is_member should be enough # I keep it in for now. is_member should be enough
# later on # later on
entry.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname self.namespace_cname = "(PyObject *)%s" % self.parent_type.typeptr_cname
entry.interned_cname = self.intern_identifier(name) entry.interned_cname = self.intern_identifier(name)
return entry return entry
......
from Cython.TestUtils import CythonTest
import Cython.Compiler.Errors as Errors
from Cython.Compiler.Nodes import *
from Cython.Compiler.ParseTreeTransforms import *
class TestBufferParsing(CythonTest):
# First, we only test the raw parser, i.e.
# the number and contents of arguments are NOT checked.
# However "dtype"/the first positional argument is special-cased
# to parse a type argument rather than an expression
def parse(self, s):
return self.should_not_fail(lambda: self.fragment(s)).root
def not_parseable(self, expected_error, s):
e = self.should_fail(lambda: self.fragment(s), Errors.CompileError)
self.assertEqual(expected_error, e.message_only)
def test_basic(self):
t = self.parse(u"cdef object[float, 4, ndim=2, foo=foo] x")
bufnode = t.stats[0].base_type
self.assert_(isinstance(bufnode, CBufferAccessTypeNode))
self.assertEqual(2, len(bufnode.positional_args))
# print bufnode.dump()
# should put more here...
def test_type_fail(self):
self.not_parseable("Expected: type",
u"cdef object[2] x")
def test_type_pos(self):
self.parse(u"cdef object[short unsigned int, 3] x")
def test_type_keyword(self):
self.parse(u"cdef object[foo=foo, dtype=short unsigned int] x")
def test_notype_as_expr1(self):
self.not_parseable("Expected: expression",
u"cdef object[foo2=short unsigned int] x")
def test_notype_as_expr2(self):
self.not_parseable("Expected: expression",
u"cdef object[int, short unsigned int] x")
def test_pos_after_key(self):
self.not_parseable("Non-keyword arg following keyword arg",
u"cdef object[foo=1, 2] x")
class TestBufferOptions(CythonTest):
# Tests the full parsing of the options within the brackets
def parse_opts(self, opts):
s = u"cdef object[%s] x" % opts
root = self.fragment(s, pipeline=[PostParse(self)]).root
buftype = root.stats[0].base_type
self.assert_(isinstance(buftype, CBufferAccessTypeNode))
self.assert_(isinstance(buftype.base_type_node, CSimpleBaseTypeNode))
self.assertEqual(u"object", buftype.base_type_node.name)
return buftype
def non_parse(self, expected_err, opts):
e = self.should_fail(lambda: self.parse_opts(opts))
self.assertEqual(expected_err, e.message_only)
def test_basic(self):
buf = self.parse_opts(u"unsigned short int, 3")
self.assert_(isinstance(buf.dtype_node, CSimpleBaseTypeNode))
self.assert_(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1)
self.assertEqual(3, buf.ndim)
def test_dict(self):
buf = self.parse_opts(u"ndim=3, dtype=unsigned short int")
self.assert_(isinstance(buf.dtype_node, CSimpleBaseTypeNode))
self.assert_(buf.dtype_node.signed == 0 and buf.dtype_node.longness == -1)
self.assertEqual(3, buf.ndim)
def test_dtype(self):
self.non_parse(ERR_BUF_MISSING % 'dtype', u"")
def test_ndim(self):
self.parse_opts(u"int, 2")
self.non_parse(ERR_BUF_INT % 'ndim', u"int, 'a'")
self.non_parse(ERR_BUF_NONNEG % 'ndim', u"int, -34")
def test_use_DEF(self):
t = self.fragment(u"""
DEF ndim = 3
cdef object[int, ndim] x
cdef object[ndim=ndim, dtype=int] y
""", pipeline=[PostParse(self)]).root
self.assert_(t.stats[1].base_type.ndim == 3)
self.assert_(t.stats[2].base_type.ndim == 3)
# add exotic and impossible combinations as they come along
from Cython.TestUtils import TransformTest
from Cython.Compiler.ParseTreeTransforms import *
from Cython.Compiler.Nodes import *
class TestNormalizeTree(TransformTest):
def test_parserbehaviour_is_what_we_coded_for(self):
t = self.fragment(u"if x: y").root
self.assertLines(u"""
(root): StatListNode
stats[0]: IfStatNode
if_clauses[0]: IfClauseNode
condition: NameNode
body: ExprStatNode
expr: NameNode
""", self.treetypes(t))
def test_wrap_singlestat(self):
t = self.run_pipeline([NormalizeTree(None)], u"if x: y")
self.assertLines(u"""
(root): StatListNode
stats[0]: IfStatNode
if_clauses[0]: IfClauseNode
condition: NameNode
body: StatListNode
stats[0]: ExprStatNode
expr: NameNode
""", self.treetypes(t))
def test_wrap_multistat(self):
t = self.run_pipeline([NormalizeTree(None)], u"""
if z:
x
y
""")
self.assertLines(u"""
(root): StatListNode
stats[0]: IfStatNode
if_clauses[0]: IfClauseNode
condition: NameNode
body: StatListNode
stats[0]: ExprStatNode
expr: NameNode
stats[1]: ExprStatNode
expr: NameNode
""", self.treetypes(t))
def test_statinexpr(self):
t = self.run_pipeline([NormalizeTree(None)], u"""
a, b = x, y
""")
self.assertLines(u"""
(root): StatListNode
stats[0]: ParallelAssignmentNode
stats[0]: SingleAssignmentNode
lhs: NameNode
rhs: NameNode
stats[1]: SingleAssignmentNode
lhs: NameNode
rhs: NameNode
""", self.treetypes(t))
def test_wrap_offagain(self):
t = self.run_pipeline([NormalizeTree(None)], u"""
x
y
if z:
x
""")
self.assertLines(u"""
(root): StatListNode
stats[0]: ExprStatNode
expr: NameNode
stats[1]: ExprStatNode
expr: NameNode
stats[2]: IfStatNode
if_clauses[0]: IfClauseNode
condition: NameNode
body: StatListNode
stats[0]: ExprStatNode
expr: NameNode
""", self.treetypes(t))
def test_pass_eliminated(self):
t = self.run_pipeline([NormalizeTree(None)], u"pass")
self.assert_(len(t.stats) == 0)
class TestWithTransform(TransformTest):
def test_simplified(self):
t = self.run_pipeline([WithTransform(None)], u"""
with x:
y = z ** 3
""")
self.assertCode(u"""
$MGR = x
$EXIT = $MGR.__exit__
$MGR.__enter__()
$EXC = True
try:
try:
y = z ** 3
except:
$EXC = False
if (not $EXIT($EXCINFO)):
raise
finally:
if $EXC:
$EXIT(None, None, None)
""", t)
def test_basic(self):
t = self.run_pipeline([WithTransform(None)], u"""
with x as y:
y = z ** 3
""")
self.assertCode(u"""
$MGR = x
$EXIT = $MGR.__exit__
$VALUE = $MGR.__enter__()
$EXC = True
try:
try:
y = $VALUE
y = z ** 3
except:
$EXC = False
if (not $EXIT($EXCINFO)):
raise
finally:
if $EXC:
$EXIT(None, None, None)
""", t)
if __name__ == "__main__":
import unittest
unittest.main()
from Cython.TestUtils import CythonTest from Cython.TestUtils import CythonTest
from Cython.Compiler.TreeFragment import * from Cython.Compiler.TreeFragment import *
from Cython.Compiler.Nodes import * from Cython.Compiler.Nodes import *
import Cython.Compiler.Naming as Naming
class TestTreeFragments(CythonTest): class TestTreeFragments(CythonTest):
def test_basic(self): def test_basic(self):
F = self.fragment(u"x = 4") F = self.fragment(u"x = 4")
T = F.copy() T = F.copy()
...@@ -12,15 +14,15 @@ class TestTreeFragments(CythonTest): ...@@ -12,15 +14,15 @@ class TestTreeFragments(CythonTest):
F = self.fragment(u"if True: x = 4") F = self.fragment(u"if True: x = 4")
T1 = F.root T1 = F.root
T2 = F.copy() T2 = F.copy()
self.assertEqual("x", T2.body.if_clauses[0].body.lhs.name) self.assertEqual("x", T2.stats[0].if_clauses[0].body.lhs.name)
T2.body.if_clauses[0].body.lhs.name = "other" T2.stats[0].if_clauses[0].body.lhs.name = "other"
self.assertEqual("x", T1.body.if_clauses[0].body.lhs.name) self.assertEqual("x", T1.stats[0].if_clauses[0].body.lhs.name)
def test_substitutions_are_copied(self): def test_substitutions_are_copied(self):
T = self.fragment(u"y + y").substitute({"y": NameNode(pos=None, name="x")}) T = self.fragment(u"y + y").substitute({"y": NameNode(pos=None, name="x")})
self.assertEqual("x", T.body.expr.operand1.name) self.assertEqual("x", T.stats[0].expr.operand1.name)
self.assertEqual("x", T.body.expr.operand2.name) self.assertEqual("x", T.stats[0].expr.operand2.name)
self.assert_(T.body.expr.operand1 is not T.body.expr.operand2) self.assert_(T.stats[0].expr.operand1 is not T.stats[0].expr.operand2)
def test_substitution(self): def test_substitution(self):
F = self.fragment(u"x = 4") F = self.fragment(u"x = 4")
...@@ -32,7 +34,7 @@ class TestTreeFragments(CythonTest): ...@@ -32,7 +34,7 @@ class TestTreeFragments(CythonTest):
F = self.fragment(u"PASS") F = self.fragment(u"PASS")
pass_stat = PassStatNode(pos=None) pass_stat = PassStatNode(pos=None)
T = F.substitute({"PASS" : pass_stat}) T = F.substitute({"PASS" : pass_stat})
self.assert_(isinstance(T.body, PassStatNode), T.body) self.assert_(isinstance(T.stats[0], PassStatNode), T)
def test_pos_is_transferred(self): def test_pos_is_transferred(self):
F = self.fragment(u""" F = self.fragment(u"""
...@@ -40,11 +42,22 @@ class TestTreeFragments(CythonTest): ...@@ -40,11 +42,22 @@ class TestTreeFragments(CythonTest):
x = u * v ** w x = u * v ** w
""") """)
T = F.substitute({"v" : NameNode(pos=None, name="a")}) T = F.substitute({"v" : NameNode(pos=None, name="a")})
v = F.root.body.stats[1].rhs.operand2.operand1 v = F.root.stats[1].rhs.operand2.operand1
a = T.body.stats[1].rhs.operand2.operand1 a = T.stats[1].rhs.operand2.operand1
self.assertEquals(v.pos, a.pos) self.assertEquals(v.pos, a.pos)
def test_temps(self):
import Cython.Compiler.Visitor as v
v.tmpnamectr = 0
F = self.fragment(u"""
TMP
x = TMP
""")
T = F.substitute(temps=[u"TMP"])
s = T.stats
self.assert_(s[0].expr.name == Naming.temp_prefix + u"1_TMP", s[0].expr.name)
self.assert_(s[1].rhs.name == Naming.temp_prefix + u"1_TMP")
self.assert_(s[0].expr.name != u"TMP")
if __name__ == "__main__": if __name__ == "__main__":
import unittest import unittest
......
...@@ -6,8 +6,8 @@ import re ...@@ -6,8 +6,8 @@ import re
from cStringIO import StringIO from cStringIO import StringIO
from Scanning import PyrexScanner, StringSourceDescriptor from Scanning import PyrexScanner, StringSourceDescriptor
from Symtab import BuiltinScope, ModuleScope from Symtab import BuiltinScope, ModuleScope
from Visitor import VisitorTransform from Visitor import VisitorTransform, temp_name_handle
from Nodes import Node from Nodes import Node, StatListNode
from ExprNodes import NameNode from ExprNodes import NameNode
import Parsing import Parsing
import Main import Main
...@@ -53,7 +53,7 @@ def parse_from_strings(name, code, pxds={}): ...@@ -53,7 +53,7 @@ def parse_from_strings(name, code, pxds={}):
buf = StringIO(code.encode(encoding)) buf = StringIO(code.encode(encoding))
scanner = PyrexScanner(buf, code_source, source_encoding = encoding, scanner = PyrexScanner(buf, code_source, source_encoding = encoding,
type_names = scope.type_names, context = context) scope = scope, context = context)
tree = Parsing.p_module(scanner, 0, module_name) tree = Parsing.p_module(scanner, 0, module_name)
return tree return tree
...@@ -93,25 +93,54 @@ class TemplateTransform(VisitorTransform): ...@@ -93,25 +93,54 @@ class TemplateTransform(VisitorTransform):
same way. It is the responsibility of the caller to make sure same way. It is the responsibility of the caller to make sure
that the replacement nodes is a valid expression. that the replacement nodes is a valid expression.
Also a list "temps" should be passed. Any names listed will
be transformed into anonymous, temporary names.
Currently supported for tempnames is:
NameNode
(various function and class definition nodes etc. should be added to this)
Each replacement node gets the position of the substituted node Each replacement node gets the position of the substituted node
recursively applied to every member node. recursively applied to every member node.
""" """
def __call__(self, node, substitutions, temps, pos):
self.substitutions = substitutions
tempdict = {}
for key in temps:
tempdict[key] = temp_name_handle(key)
self.temps = tempdict
self.pos = pos
return super(TemplateTransform, self).__call__(node)
def visit_Node(self, node): def visit_Node(self, node):
if node is None: if node is None:
return node return None
else: else:
c = node.clone_node() c = node.clone_node()
if self.pos is not None:
c.pos = self.pos
self.visitchildren(c) self.visitchildren(c)
return c return c
def try_substitution(self, node, key): def try_substitution(self, node, key):
sub = self.substitutions.get(key) sub = self.substitutions.get(key)
if sub is None: if sub is not None:
return self.visit_Node(node) # make copy as usual pos = self.pos
if pos is None: pos = node.pos
return ApplyPositionAndCopy(pos)(sub)
else: else:
return ApplyPositionAndCopy(node.pos)(sub) return self.visit_Node(node) # make copy as usual
def visit_NameNode(self, node): def visit_NameNode(self, node):
tempname = self.temps.get(node.name)
if tempname is not None:
# Replace name with temporary
node.name = tempname
return self.visit_Node(node)
else:
return self.try_substitution(node, node.name) return self.try_substitution(node, node.name)
def visit_ExprStatNode(self, node): def visit_ExprStatNode(self, node):
...@@ -122,10 +151,6 @@ class TemplateTransform(VisitorTransform): ...@@ -122,10 +151,6 @@ class TemplateTransform(VisitorTransform):
else: else:
return self.visit_Node(node) return self.visit_Node(node)
def __call__(self, node, substitutions):
self.substitutions = substitutions
return super(TemplateTransform, self).__call__(node)
def copy_code_tree(node): def copy_code_tree(node):
return TreeCopier()(node) return TreeCopier()(node)
...@@ -133,12 +158,12 @@ INDENT_RE = re.compile(ur"^ *") ...@@ -133,12 +158,12 @@ INDENT_RE = re.compile(ur"^ *")
def strip_common_indent(lines): def strip_common_indent(lines):
"Strips empty lines and common indentation from the list of strings given in lines" "Strips empty lines and common indentation from the list of strings given in lines"
lines = [x for x in lines if x.strip() != u""] lines = [x for x in lines if x.strip() != u""]
minindent = min(len(INDENT_RE.match(x).group(0)) for x in lines) minindent = min([len(INDENT_RE.match(x).group(0)) for x in lines])
lines = [x[minindent:] for x in lines] lines = [x[minindent:] for x in lines]
return lines return lines
class TreeFragment(object): class TreeFragment(object):
def __init__(self, code, name, pxds={}): def __init__(self, code, name="(tree fragment)", pxds={}, temps=[], pipeline=[]):
if isinstance(code, unicode): if isinstance(code, unicode):
def fmt(x): return u"\n".join(strip_common_indent(x.split(u"\n"))) def fmt(x): return u"\n".join(strip_common_indent(x.split(u"\n")))
...@@ -147,18 +172,28 @@ class TreeFragment(object): ...@@ -147,18 +172,28 @@ class TreeFragment(object):
for key, value in pxds.iteritems(): for key, value in pxds.iteritems():
fmt_pxds[key] = fmt(value) fmt_pxds[key] = fmt(value)
self.root = parse_from_strings(name, fmt_code, fmt_pxds) t = parse_from_strings(name, fmt_code, fmt_pxds)
mod = t
t = t.body # Make sure a StatListNode is at the top
if not isinstance(t, StatListNode):
t = StatListNode(pos=mod.pos, stats=[t])
for transform in pipeline:
t = transform(t)
self.root = t
elif isinstance(code, Node): elif isinstance(code, Node):
if pxds != {}: raise NotImplementedError() if pxds != {}: raise NotImplementedError()
self.root = code self.root = code
else: else:
raise ValueError("Unrecognized code format (accepts unicode and Node)") raise ValueError("Unrecognized code format (accepts unicode and Node)")
self.temps = temps
def copy(self): def copy(self):
return copy_code_tree(self.root) return copy_code_tree(self.root)
def substitute(self, nodes={}): def substitute(self, nodes={}, temps=[], pos = None):
return TemplateTransform()(self.root, substitutions = nodes) return TemplateTransform()(self.root,
substitutions = nodes,
temps = self.temps + temps, pos = pos)
......
# #
# Tree visitor and transform framework # Tree visitor and transform framework
# #
import inspect
import Nodes import Nodes
import ExprNodes import ExprNodes
import inspect import Naming
from Cython.Utils import EncodedString
class BasicVisitor(object): class BasicVisitor(object):
"""A generic visitor base class which can be used for visiting any kind of object.""" """A generic visitor base class which can be used for visiting any kind of object."""
...@@ -129,7 +131,6 @@ class VisitorTransform(TreeVisitor): ...@@ -129,7 +131,6 @@ class VisitorTransform(TreeVisitor):
was not, an exception will be raised. (Typically you want to ensure that you was not, an exception will be raised. (Typically you want to ensure that you
are within a StatListNode or similar before doing this.) are within a StatListNode or similar before doing this.)
""" """
def visitchildren(self, parent, attrs=None): def visitchildren(self, parent, attrs=None):
result = super(VisitorTransform, self).visitchildren(parent, attrs) result = super(VisitorTransform, self).visitchildren(parent, attrs)
for attr, newnode in result.iteritems(): for attr, newnode in result.iteritems():
...@@ -150,6 +151,19 @@ class VisitorTransform(TreeVisitor): ...@@ -150,6 +151,19 @@ class VisitorTransform(TreeVisitor):
def __call__(self, root): def __call__(self, root):
return self.visit(root) return self.visit(root)
class CythonTransform(VisitorTransform):
"""
Certain common conventions and utilitues for Cython transforms.
"""
def __init__(self, context):
super(CythonTransform, self).__init__()
self.context = context
def visit_Node(self, node):
self.visitchildren(node)
return node
# Utils # Utils
def ensure_statlist(node): def ensure_statlist(node):
if not isinstance(node, Nodes.StatListNode): if not isinstance(node, Nodes.StatListNode):
...@@ -166,6 +180,19 @@ def replace_node(ptr, value): ...@@ -166,6 +180,19 @@ def replace_node(ptr, value):
else: else:
getattr(parent, attrname)[listidx] = value getattr(parent, attrname)[listidx] = value
tmpnamectr = 0
def temp_name_handle(description):
global tmpnamectr
tmpnamectr += 1
return EncodedString(Naming.temp_prefix + u"%d_%s" % (tmpnamectr, description))
def get_temp_name_handle_desc(handle):
if not handle.startswith(u"__cyt_"):
return None
else:
idx = handle.find(u"_", 6)
return handle[idx+1:]
class PrintTree(TreeVisitor): class PrintTree(TreeVisitor):
"""Prints a representation of the tree to standard output. """Prints a representation of the tree to standard output.
Subclass and override repr_of to provide more information Subclass and override repr_of to provide more information
......
...@@ -4,8 +4,49 @@ import unittest ...@@ -4,8 +4,49 @@ import unittest
from Cython.Compiler.ModuleNode import ModuleNode from Cython.Compiler.ModuleNode import ModuleNode
import Cython.Compiler.Main as Main import Cython.Compiler.Main as Main
from Cython.Compiler.TreeFragment import TreeFragment, strip_common_indent from Cython.Compiler.TreeFragment import TreeFragment, strip_common_indent
from Cython.Compiler.Visitor import TreeVisitor
class NodeTypeWriter(TreeVisitor):
def __init__(self):
super(NodeTypeWriter, self).__init__()
self._indents = 0
self.result = []
def visit_Node(self, node):
if len(self.access_path) == 0:
name = u"(root)"
else:
tip = self.access_path[-1]
if tip[2] is not None:
name = u"%s[%d]" % tip[1:3]
else:
name = tip[1]
self.result.append(u" " * self._indents +
u"%s: %s" % (name, node.__class__.__name__))
self._indents += 1
self.visitchildren(node)
self._indents -= 1
def treetypes(root):
"""Returns a string representing the tree by class names.
There's a leading and trailing whitespace so that it can be
compared by simple string comparison while still making test
cases look ok."""
w = NodeTypeWriter()
w.visit(root)
return u"\n".join([u""] + w.result + [u""])
class CythonTest(unittest.TestCase): class CythonTest(unittest.TestCase):
def assertLines(self, expected, result):
"Checks that the given strings or lists of strings are equal line by line"
if not isinstance(expected, list): expected = expected.split(u"\n")
if not isinstance(result, list): result = result.split(u"\n")
for idx, (expected_line, result_line) in enumerate(zip(expected, result)):
self.assertEqual(expected_line, result_line, "Line %d:\nExp: %s\nGot: %s" % (idx, expected_line, result_line))
self.assertEqual(len(expected), len(result),
"Unmatched lines. Got:\n%s\nExpected:\n%s" % ("\n".join(expected), u"\n".join(result)))
def assertCode(self, expected, result_tree): def assertCode(self, expected, result_tree):
writer = CodeWriter() writer = CodeWriter()
writer.write(result_tree) writer.write(result_tree)
...@@ -18,13 +59,35 @@ class CythonTest(unittest.TestCase): ...@@ -18,13 +59,35 @@ class CythonTest(unittest.TestCase):
self.assertEqual(len(result_lines), len(expected_lines), self.assertEqual(len(result_lines), len(expected_lines),
"Unmatched lines. Got:\n%s\nExpected:\n%s" % ("\n".join(result_lines), expected)) "Unmatched lines. Got:\n%s\nExpected:\n%s" % ("\n".join(result_lines), expected))
def fragment(self, code, pxds={}): def fragment(self, code, pxds={}, pipeline=[]):
"Simply create a tree fragment using the name of the test-case in parse errors." "Simply create a tree fragment using the name of the test-case in parse errors."
name = self.id() name = self.id()
if name.startswith("__main__."): name = name[len("__main__."):] if name.startswith("__main__."): name = name[len("__main__."):]
name = name.replace(".", "_") name = name.replace(".", "_")
return TreeFragment(code, name, pxds) return TreeFragment(code, name, pxds, pipeline=pipeline)
def treetypes(self, root):
return treetypes(root)
def should_fail(self, func, exc_type=Exception):
"""Calls "func" and fails if it doesn't raise the right exception
(any exception by default). Also returns the exception in question.
"""
try:
func()
self.fail("Expected an exception of type %r" % exc_type)
except exc_type, e:
self.assert_(isinstance(e, exc_type))
return e
def should_not_fail(self, func):
"""Calls func and succeeds if and only if no exception is raised
(i.e. converts exception raising into a failed testcase). Returns
the return value of func."""
try:
return func()
except:
self.fail()
class TransformTest(CythonTest): class TransformTest(CythonTest):
""" """
...@@ -37,8 +100,8 @@ class TransformTest(CythonTest): ...@@ -37,8 +100,8 @@ class TransformTest(CythonTest):
To create a test case: To create a test case:
- Call run_pipeline. The pipeline should at least contain the transform you - Call run_pipeline. The pipeline should at least contain the transform you
are testing; pyx should be either a string (passed to the parser to are testing; pyx should be either a string (passed to the parser to
create a post-parse tree) or a ModuleNode representing input to pipeline. create a post-parse tree) or a node representing input to pipeline.
The result will be a transformed result (usually a ModuleNode). The result will be a transformed result.
- Check that the tree is correct. If wanted, assertCode can be used, which - Check that the tree is correct. If wanted, assertCode can be used, which
takes a code string as expected, and a ModuleNode in result_tree takes a code string as expected, and a ModuleNode in result_tree
...@@ -53,7 +116,6 @@ class TransformTest(CythonTest): ...@@ -53,7 +116,6 @@ class TransformTest(CythonTest):
def run_pipeline(self, pipeline, pyx, pxds={}): def run_pipeline(self, pipeline, pyx, pxds={}):
tree = self.fragment(pyx, pxds).root tree = self.fragment(pyx, pxds).root
assert isinstance(tree, ModuleNode)
# Run pipeline # Run pipeline
for T in pipeline: for T in pipeline:
tree = T(tree) tree = T(tree)
......
...@@ -73,6 +73,9 @@ class TestCodeWriter(CythonTest): ...@@ -73,6 +73,9 @@ class TestCodeWriter(CythonTest):
def test_inplace_assignment(self): def test_inplace_assignment(self):
self.t(u"x += 43") self.t(u"x += 43")
def test_attribute(self):
self.t(u"a.x")
if __name__ == "__main__": if __name__ == "__main__":
import unittest import unittest
unittest.main() unittest.main()
......
cdef extern from "Python.h":
ctypedef struct PyObject
ctypedef struct Py_buffer:
void *buf
Py_ssize_t len
int readonly
char *format
int ndim
Py_ssize_t *shape
Py_ssize_t *strides
Py_ssize_t *suboffsets
Py_ssize_t itemsize
void *internal
int PyObject_GetBuffer(PyObject* obj, Py_buffer* view, int flags) except -1
void PyObject_ReleaseBuffer(PyObject* obj, Py_buffer* view)
# int PyObject_GetBuffer(PyObject *obj, Py_buffer *view,
# int flags)
...@@ -238,6 +238,29 @@ class CythonRunTestCase(CythonCompileTestCase): ...@@ -238,6 +238,29 @@ class CythonRunTestCase(CythonCompileTestCase):
except Exception: except Exception:
pass pass
def collect_unittests(path, suite, selectors):
def file_matches(filename):
return filename.startswith("Test") and filename.endswith(".py")
def package_matches(dirname):
return dirname == "Tests"
loader = unittest.TestLoader()
for dirpath, dirnames, filenames in os.walk(path):
parentname = os.path.split(dirpath)[-1]
if package_matches(parentname):
for f in filenames:
if file_matches(f):
filepath = os.path.join(dirpath, f)[:-len(".py")]
modulename = filepath[len(path)+1:].replace(os.path.sep, '.')
if not [ 1 for match in selectors if match(modulename) ]:
continue
module = __import__(modulename)
for x in modulename.split('.')[1:]:
module = getattr(module, x)
suite.addTests(loader.loadTestsFromModule(module))
if __name__ == '__main__': if __name__ == '__main__':
from optparse import OptionParser from optparse import OptionParser
parser = OptionParser() parser = OptionParser()
...@@ -247,6 +270,12 @@ if __name__ == '__main__': ...@@ -247,6 +270,12 @@ if __name__ == '__main__':
parser.add_option("--no-cython", dest="with_cython", parser.add_option("--no-cython", dest="with_cython",
action="store_false", default=True, action="store_false", default=True,
help="do not run the Cython compiler, only the C compiler") help="do not run the Cython compiler, only the C compiler")
parser.add_option("--no-unit", dest="unittests",
action="store_false", default=True,
help="do not run the unit tests")
parser.add_option("--no-file", dest="filetests",
action="store_false", default=True,
help="do not run the file based tests")
parser.add_option("-C", "--coverage", dest="coverage", parser.add_option("-C", "--coverage", dest="coverage",
action="store_true", default=False, action="store_true", default=False,
help="collect source coverage data for the Compiler") help="collect source coverage data for the Compiler")
...@@ -296,9 +325,15 @@ if __name__ == '__main__': ...@@ -296,9 +325,15 @@ if __name__ == '__main__':
if not selectors: if not selectors:
selectors = [ lambda x:True ] selectors = [ lambda x:True ]
tests = TestBuilder(ROOTDIR, WORKDIR, selectors, test_suite = unittest.TestSuite()
if options.unittests:
collect_unittests(os.getcwd(), test_suite, selectors)
if options.filetests:
filetests = TestBuilder(ROOTDIR, WORKDIR, selectors,
options.annotate_source, options.cleanup_workdir) options.annotate_source, options.cleanup_workdir)
test_suite = tests.build_suite() test_suite.addTests(filetests.build_suite())
unittest.TextTestRunner(verbosity=options.verbosity).run(test_suite) unittest.TextTestRunner(verbosity=options.verbosity).run(test_suite)
......
from __future__ import with_statement
__doc__ = u"""
>>> no_as()
enter
hello
exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
>>> basic()
enter
value
exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
>>> with_exception(None)
enter
value
exit <type 'type'> <class 'withstat.MyException'> <type 'traceback'>
outer except
>>> with_exception(True)
enter
value
exit <type 'type'> <class 'withstat.MyException'> <type 'traceback'>
>>> multitarget()
enter
1 2 3 4 5
exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
>>> tupletarget()
enter
(1, 2, (3, (4, 5)))
exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
>>> typed()
enter
10
exit <type 'NoneType'> <type 'NoneType'> <type 'NoneType'>
"""
class MyException(Exception):
pass
class ContextManager:
def __init__(self, value, exit_ret = None):
self.value = value
self.exit_ret = exit_ret
def __exit__(self, a, b, tb):
print "exit", type(a), type(b), type(tb)
return self.exit_ret
def __enter__(self):
print "enter"
return self.value
def no_as():
with ContextManager("value"):
print "hello"
def basic():
with ContextManager("value") as x:
print x
def with_exception(exit_ret):
try:
with ContextManager("value", exit_ret=exit_ret) as value:
print value
raise MyException()
except:
print "outer except"
def multitarget():
with ContextManager((1, 2, (3, (4, 5)))) as (a, b, (c, (d, e))):
print a, b, c, d, e
def tupletarget():
with ContextManager((1, 2, (3, (4, 5)))) as t:
print t
def typed():
cdef unsigned char i
c = ContextManager(255)
with c as i:
i += 11
print i
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment