Skip to content

C
cython
Commit df78cdc7 authored by Dag Sverre Seljebotn's avatar Dag Sverre Seljebotn
Browse files
Options

Introduced code.globalstate.use_utility_code and used it in Buffer.py

parent 218d5959
Hide whitespace changes
Inline Side-by-side
Showing with 206 additions and 164 deletions
Cython/Compiler/Annotate.py
View file @ df78cdc7
......@@ -30,8 +30,8 @@ class AnnotationCCodeWriter(CCodeWriter):
self.annotations = create_from.annotations
self.code = create_from.code
def create_new(self, create_from, buffer):
return AnnotationCCodeWriter(create_from, buffer)
def create_new(self, create_from, buffer, copy_formatting):
return AnnotationCCodeWriter(create_from, buffer, copy_formatting)
def write(self, s):
CCodeWriter.write(self, s)
......
Cython/Compiler/Buffer.py
View file @ df78cdc7
......@@ -33,7 +33,6 @@ class IntroduceBufferAuxiliaryVars(CythonTransform):
node.scope.include_files.append("endian.h")
use_py2_buffer_functions(node.scope)
use_empty_bufstruct_code(node.scope, self.max_ndim)
node.scope.use_utility_code(access_utility_code)
return result
......@@ -60,9 +59,6 @@ class IntroduceBufferAuxiliaryVars(CythonTransform):
if buftype.ndim > self.max_ndim:
self.max_ndim = buftype.ndim
# Get or make a type string checker
tschecker = buffer_type_checker(buftype.dtype, scope)
# Declare auxiliary vars
cname = scope.mangle(Naming.bufstruct_prefix, name)
bufinfo = scope.declare_var(name="$%s" % cname, cname=cname,
......@@ -83,17 +79,13 @@ class IntroduceBufferAuxiliaryVars(CythonTransform):
stridevars = [var(Naming.bufstride_prefix, i, "0") for i in range(entry.type.ndim)]
shapevars = [var(Naming.bufshape_prefix, i, "0") for i in range(entry.type.ndim)]
entry.buffer_aux = Symtab.BufferAux(bufinfo, stridevars, shapevars, tschecker)
mode = entry.type.mode
if mode == 'full':
suboffsetvars = [var(Naming.bufsuboffset_prefix, i, "-1") for i in range(entry.type.ndim)]
entry.buffer_aux.lookup = get_buf_lookup_full(scope, entry.type.ndim)
elif mode == 'strided':
suboffsetvars = None
entry.buffer_aux.lookup = get_buf_lookup_strided(scope, entry.type.ndim)
entry.buffer_aux.suboffsetvars = suboffsetvars
entry.buffer_aux.get_buffer_cname = tschecker
entry.buffer_aux = Symtab.BufferAux(bufinfo, stridevars, shapevars, suboffsetvars)
scope.buffer_entries = bufvars
self.scope = scope
......@@ -144,24 +136,19 @@ def put_unpack_buffer_aux_into_scope(buffer_aux, mode, code):
(s.cname, bufstruct, field, idx)
for idx, s in enumerate(vars)]))
def getbuffer_cond_code(obj_cname, buffer_aux, flags, ndim):
bufstruct = buffer_aux.buffer_info_var.cname
return "%s(%s, &%s, %s, %d) == -1" % (
buffer_aux.get_buffer_cname, obj_cname, bufstruct, flags, ndim)
def put_acquire_arg_buffer(entry, code, pos):
code.globalstate.use_utility_code(acquire_utility_code)
buffer_aux = entry.buffer_aux
cname = entry.cname
bufstruct = buffer_aux.buffer_info_var.cname
flags = get_flags(buffer_aux, entry.type)
getbuffer_cname = get_getbuffer_code(entry.type.dtype, code)
# Acquire any new buffer
code.putln(code.error_goto_if(getbuffer_cond_code(cname,
buffer_aux,
flags,
entry.type.ndim),
pos))
code.putln(code.error_goto_if("%s(%s, &%s, %s, %d) == -1" % (
getbuffer_cname,
entry.cname,
entry.buffer_aux.buffer_info_var.cname,
get_flags(buffer_aux, entry.type),
entry.type.ndim), pos))
# An exception raised in arg parsing cannot be catched, so no
# need to do care about the buffer then.
# need to care about the buffer then.
put_unpack_buffer_aux_into_scope(buffer_aux, entry.type.mode, code)
#def put_release_buffer_normal(entry, code):
......@@ -192,11 +179,12 @@ def put_assign_to_buffer(lhs_cname, rhs_cname, buffer_aux, buffer_type,
(which may or may not succeed).
"""
code.globalstate.use_utility_code(acquire_utility_code)
bufstruct = buffer_aux.buffer_info_var.cname
flags = get_flags(buffer_aux, buffer_type)
getbuffer = "%s(%%s, &%s, %s, %d)" % (buffer_aux.get_buffer_cname,
# note: object is filled in later
getbuffer = "%s(%%s, &%s, %s, %d)" % (get_getbuffer_code(buffer_type.dtype, code),
# note: object is filled in later (%%s)
bufstruct,
flags,
buffer_type.ndim)
......@@ -256,12 +244,14 @@ def put_access(entry, index_signeds, index_cnames, pos, code):
body. The lookup however is delegated to a inline function that is instantiated
once per ndim (lookup with suboffsets tend to get quite complicated).
"""
code.globalstate.use_utility_code(access_utility_code)
bufaux = entry.buffer_aux
bufstruct = bufaux.buffer_info_var.cname
# Check bounds and fix negative indices
boundscheck = True
nonegs = True
tmp_cname = code.funcstate.allocate_temp(PyrexTypes.c_int_type)
if boundscheck:
code.putln("%s = -1;" % tmp_cname)
for idx, (signed, cname, shape) in enumerate(zip(index_signeds, index_cnames,
......@@ -290,19 +280,31 @@ def put_access(entry, index_signeds, index_cnames, pos, code):
code.end_block()
code.funcstate.release_temp(tmp_cname)
# Create buffer lookup and return it
params = []
nd = entry.type.ndim
if entry.type.mode == 'full':
for i, s, o in zip(index_cnames, bufaux.stridevars, bufaux.suboffsetvars):
params.append(i)
params.append(s.cname)
params.append(o.cname)
funcname = "__Pyx_BufPtrFull%dd" % nd
funcgen = buf_lookup_full_code
else:
for i, s in zip(index_cnames, bufaux.stridevars):
params.append(i)
params.append(s.cname)
ptrcode = "%s(%s.buf, %s)" % (bufaux.lookup, bufstruct,
", ".join(params))
funcname = "__Pyx_BufPtrStrided%dd" % nd
funcgen = buf_lookup_strided_code
code.globalstate.use_generated_code(funcgen, name=funcname, nd=nd)
ptrcode = "%s(%s.buf, %s)" % (funcname, bufstruct, ", ".join(params))
valuecode = "*%s" % entry.type.buffer_ptr_type.cast_code(ptrcode)
return valuecode
......@@ -313,54 +315,35 @@ def use_empty_bufstruct_code(env, max_ndim):
Py_ssize_t __Pyx_zeros[] = {%s};
Py_ssize_t __Pyx_minusones[] = {%s};
""") % (", ".join(["0"] * max_ndim), ", ".join(["-1"] * max_ndim))
env.use_utility_code([code, ""])
env.use_utility_code([code, ""], "empty_bufstruct_code")
def get_buf_lookup_strided(env, nd):
def buf_lookup_strided_code(proto, defin, name, nd):
"""
Generates and registers as utility a buffer lookup function for the right number
Generates a buffer lookup function for the right number
of dimensions. The function gives back a void* at the right location.
"""
name = "__Pyx_BufPtrStrided_%dd" % nd
if not env.has_utility_code(name):
# _i_ndex, _s_tride
args = ", ".join(["i%d, s%d" % (i, i) for i in range(nd)])
offset = " + ".join(["i%d * s%d" % (i, i) for i in range(nd)])
proto = dedent("""\
#define %s(buf, %s) ((char*)buf + %s)
""") % (name, args, offset)
env.use_utility_code([proto, ""], name=name)
return name
# _i_ndex, _s_tride
args = ", ".join(["i%d, s%d" % (i, i) for i in range(nd)])
offset = " + ".join(["i%d * s%d" % (i, i) for i in range(nd)])
proto.putln("#define %s(buf, %s) ((char*)buf + %s)" % (name, args, offset))
def get_buf_lookup_full(env, nd):
def buf_lookup_full_code(proto, defin, name, nd):
"""
Generates and registers as utility a buffer lookup function for the right number
Generates a buffer lookup function for the right number
of dimensions. The function gives back a void* at the right location.
"""
name = "__Pyx_BufPtrFull_%dd" % nd
if not env.has_utility_code(name):
# _i_ndex, _s_tride, sub_o_ffset
args = ", ".join(["Py_ssize_t i%d, Py_ssize_t s%d, Py_ssize_t o%d" % (i, i, i) for i in range(nd)])
proto = dedent("""\
static INLINE void* %s(void* buf, %s);
""") % (name, args)
func = dedent("""
# _i_ndex, _s_tride, sub_o_ffset
args = ", ".join(["Py_ssize_t i%d, Py_ssize_t s%d, Py_ssize_t o%d" % (i, i, i) for i in range(nd)])
proto.putln("static INLINE void* %s(void* buf, %s);" % (name, args))
defin.putln(dedent("""
static INLINE void* %s(void* buf, %s) {
char* ptr = (char*)buf;
""") % (name, args) + "".join([dedent("""\
ptr += s%d * i%d;
if (o%d >= 0) ptr = *((char**)ptr) + o%d;
""") % (i, i, i, i) for i in range(nd)]
) + "\nreturn ptr;\n}"
env.use_utility_code([proto, func], name=name)
return name
) + "\nreturn ptr;\n}")
#
......@@ -375,11 +358,11 @@ def mangle_dtype_name(dtype):
prefix = ""
return prefix + dtype.declaration_code("").replace(" ", "_")
def get_ts_check_item(dtype, env):
def get_ts_check_item(dtype, writer):
# See if we can consume one (unnamed) dtype as next item
# Put native types and structs in seperate namespaces (as one could create a struct named unsigned_int...)
name = "__Pyx_BufferTypestringCheck_item_%s" % mangle_dtype_name(dtype)
if not env.has_utility_code(name):
if not writer.globalstate.has_utility_code(name):
char = dtype.typestring
if char is not None:
# Can use direct comparison
......@@ -415,7 +398,7 @@ def get_ts_check_item(dtype, env):
return NULL;
} else return ts + 1;
""", 2)
env.use_utility_code([dedent("""\
writer.globalstate.use_utility_code([dedent("""\
static const char* %s(const char* ts); /*proto*/
""") % name, dedent("""
static const char* %s(const char* ts) {
......@@ -425,7 +408,7 @@ def get_ts_check_item(dtype, env):
return name
def get_getbuffer_code(dtype, env):
def get_getbuffer_code(dtype, code):
"""
Generate a utility function for getting a buffer for the given dtype.
The function will:
......@@ -436,9 +419,9 @@ def get_getbuffer_code(dtype, env):
"""
name = "__Pyx_GetBuffer_%s" % mangle_dtype_name(dtype)
if not env.has_utility_code(name):
env.use_utility_code(acquire_utility_code)
itemchecker = get_ts_check_item(dtype, env)
if not code.globalstate.has_utility_code(name):
code.globalstate.use_utility_code(acquire_utility_code)
itemchecker = get_ts_check_item(dtype, code)
utilcode = [dedent("""
static int %s(PyObject* obj, Py_buffer* buf, int flags, int nd); /*proto*/
""") % name, dedent("""
......@@ -473,72 +456,21 @@ def get_getbuffer_code(dtype, env):
__Pyx_ZeroBuffer(buf);
return -1;
}""") % locals()]
env.use_utility_code(utilcode, name)
code.globalstate.use_utility_code(utilcode, name)
return name
def buffer_type_checker(dtype, env):
def buffer_type_checker(dtype, code):
# Creates a type checker function for the given type.
if dtype.is_struct_or_union:
assert False
elif dtype.is_int or dtype.is_float:
# This includes simple typedef-ed types
funcname = get_getbuffer_code(dtype, env)
funcname = get_getbuffer_code(dtype, code)
else:
assert False
return funcname
def use_py2_buffer_functions(env):
# will be refactored
try:
env.entries[u'numpy']
env.use_utility_code(["","""
static int numpy_getbuffer(PyObject *obj, Py_buffer *view, int flags) {
/* This function is always called after a type-check; safe to cast */
PyArrayObject *arr = (PyArrayObject*)obj;
PyArray_Descr *type = (PyArray_Descr*)arr->descr;
int typenum = PyArray_TYPE(obj);
if (!PyTypeNum_ISNUMBER(typenum)) {
PyErr_Format(PyExc_TypeError, "Only numeric NumPy types currently supported.");
return -1;
}
/*
NumPy format codes doesn't completely match buffer codes;
seems safest to retranslate.
01234567890123456789012345*/
const char* base_codes = "?bBhHiIlLqQfdgfdgO";
char* format = (char*)malloc(4);
char* fp = format;
*fp++ = type->byteorder;
if (PyTypeNum_ISCOMPLEX(typenum)) *fp++ = 'Z';
*fp++ = base_codes[typenum];
*fp = 0;
view->buf = arr->data;
view->readonly = !PyArray_ISWRITEABLE(obj);
view->ndim = PyArray_NDIM(arr);
view->strides = PyArray_STRIDES(arr);
view->shape = PyArray_DIMS(arr);
view->suboffsets = NULL;
view->format = format;
view->itemsize = type->elsize;
view->internal = 0;
return 0;
}
static void numpy_releasebuffer(PyObject *obj, Py_buffer *view) {
free((char*)view->format);
view->format = NULL;
}
"""])
except KeyError:
pass
codename = "PyObject_GetBuffer" # just a representative unique key
# Search all types for __getbuffer__ overloads
......@@ -562,6 +494,7 @@ static void numpy_releasebuffer(PyObject *obj, Py_buffer *view) {
try:
ndarrtype = env.entries[u'numpy'].as_module.entries['ndarray'].type
types.append((ndarrtype.typeptr_cname, "numpy_getbuffer", "numpy_releasebuffer"))
env.use_utility_code(numpy_code)
except KeyError:
pass
......@@ -602,7 +535,7 @@ static void numpy_releasebuffer(PyObject *obj, Py_buffer *view) {
static int PyObject_GetBuffer(PyObject *obj, Py_buffer *view, int flags);
static void PyObject_ReleaseBuffer(PyObject *obj, Py_buffer *view);
#endif
""") ,code], codename)
"""), code], codename)
#
# Static utility code
......@@ -690,3 +623,53 @@ static void __Pyx_RaiseBufferFallbackError(void) {
}
"""]
numpy_code = ["""
static int numpy_getbuffer(PyObject *obj, Py_buffer *view, int flags);
static void numpy_releasebuffer(PyObject *obj, Py_buffer *view);
""","""
static int numpy_getbuffer(PyObject *obj, Py_buffer *view, int flags) {
/* This function is always called after a type-check; safe to cast */
PyArrayObject *arr = (PyArrayObject*)obj;
PyArray_Descr *type = (PyArray_Descr*)arr->descr;
int typenum = PyArray_TYPE(obj);
if (!PyTypeNum_ISNUMBER(typenum)) {
PyErr_Format(PyExc_TypeError, "Only numeric NumPy types currently supported.");
return -1;
}
/*
NumPy format codes doesn't completely match buffer codes;
seems safest to retranslate.
01234567890123456789012345*/
const char* base_codes = "?bBhHiIlLqQfdgfdgO";
char* format = (char*)malloc(4);
char* fp = format;
*fp++ = type->byteorder;
if (PyTypeNum_ISCOMPLEX(typenum)) *fp++ = 'Z';
*fp++ = base_codes[typenum];
*fp = 0;
view->buf = arr->data;
view->readonly = !PyArray_ISWRITEABLE(obj);
view->ndim = PyArray_NDIM(arr);
view->strides = PyArray_STRIDES(arr);
view->shape = PyArray_DIMS(arr);
view->suboffsets = NULL;
view->format = format;
view->itemsize = type->elsize;
view->internal = 0;
return 0;
}
static void numpy_releasebuffer(PyObject *obj, Py_buffer *view) {
free((char*)view->format);
view->format = NULL;
}
"""]
Cython/Compiler/Code.py
View file @ df78cdc7
......@@ -127,6 +127,7 @@ class FunctionState(object):
freelist = self.temps_free.get(type)
if freelist is None:
freelist = []
self.temps_free[type] = freelist
freelist.append(name)
......@@ -136,11 +137,15 @@ class GlobalState(object):
# input_file_contents dict contents (=list of lines) of any file that was used as input
# to create this output C code. This is
# used to annotate the comments.
# used_utility_code set(string|int) Ids of used utility code (to avoid reinsertion)
# utilprotowriter CCodeWriter
# utildefwriter CCodeWriter
def __init__(self):
self.filename_table = {}
self.filename_list = []
self.input_file_contents = {}
self.used_utility_code = set()
def lookup_filename(self, filename):
try:
......@@ -162,6 +167,58 @@ class GlobalState(object):
self.input_file_contents[source_desc] = F
return F
def use_utility_code(self, codetup, name=None):
"""
Adds the given utility code to the C file if needed.
codetup should unpack into one prototype code part and one
definition code part, both strings inserted directly in C.
If name is provided, it is used as an identifier to avoid inserting
code twice. Otherwise, id(codetup) is used as such an identifier.
"""
if name is None: name = id(codetup)
if self.check_utility_code_needed_and_register(name):
proto, _def = codetup
self.utilprotowriter.put(proto)
self.utildefwriter.put(_def)
def has_utility_code(self, name):
return name in self.used_utility_code
def use_generated_code(self, func, name, *args, **kw):
"""
Requests that the utility code that func can generate is used in the C
file. func is called like this:
func(proto, definition, name, *args, **kw)
where proto and definition are two CCodeWriter instances; the
former should have the prototype written to it and the other the definition.
The call might happen at some later point (if compiling multiple modules
into a cache for instance), and will only happen once per utility code.
name is used to identify the utility code, so that it isn't regenerated
when the same code is requested again.
"""
if self.check_utility_code_needed_and_register(name):
func(self.utilprotowriter, self.utildefwriter,
name, *args, **kw)
def check_utility_code_needed_and_register(self, name):
if name in self.used_utility_code:
return False
else:
self.used_utility_code.add(name)
return True
def put_utility_code_protos(self, writer):
writer.insert(self.utilprotowriter)
def put_utility_code_defs(self, writer):
writer.insert(self.utildefwriter)
def funccontext_property(name):
def get(self):
......@@ -198,29 +255,34 @@ class CCodeWriter(object):
# globalstate GlobalState contains state global for a C file (input file info,
# utility code, declared constants etc.)
def __init__(self, create_from=None, buffer=None):
def __init__(self, create_from=None, buffer=None, copy_formatting=False):
if buffer is None: buffer = StringIOTree()
self.buffer = buffer
self.marker = None
self.last_marker_line = 0
self.funcstate = None # always start with no function state
self.funcstate = None
self.level = 0
self.bol = 1
if create_from is None:
# Root CCodeWriter
self.level = 0
self.bol = 1
self.globalstate = GlobalState()
# These needs to be constructed after all state is set, as
# the construction copies over the state
self.globalstate.utilprotowriter = self.new_writer()
self.globalstate.utildefwriter = self.new_writer()
else:
# Use same global state
self.globalstate = create_from.globalstate
# Clone formatting state
self.level = create_from.level
self.bol = create_from.bol
if copy_formatting:
self.level = create_from.level
self.bol = create_from.bol
def create_new(self, create_from, buffer):
def create_new(self, create_from, buffer, copy_formatting):
# polymorphic constructor -- very slightly more versatile
# than using __class__
return CCodeWriter(create_from, buffer)
return CCodeWriter(create_from, buffer, copy_formatting)
def copyto(self, f):
self.buffer.copyto(f)
......@@ -232,9 +294,25 @@ class CCodeWriter(object):
self.buffer.write(s)
def insertion_point(self):
other = self.create_new(create_from=self, buffer=self.buffer.insertion_point())
other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True)
return other
def new_writer(self):
"""
Creates a new CCodeWriter connected to the same global state, which
can later be inserted using insert.
"""
return CCodeWriter(create_from=self)
def insert(self, writer):
"""
Inserts the contents of another code writer (created with
the same global state) in the current location.
It is ok to write to the inserted writer also after insertion.
"""
assert writer.globalstate is self.globalstate
self.buffer.insert(writer.buffer)
# Properties delegated to function scope
label_counter = funccontext_property("label_counter")
......
Cython/Compiler/ModuleNode.py
View file @ df78cdc7
......@@ -1950,6 +1950,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
type.typeptr_cname, type.typeobj_cname))
def generate_utility_functions(self, env, code, h_code):
for codetup, name in env.utility_code_list:
code.globalstate.use_utility_code(codetup, name)
code.globalstate.put_utility_code_protos(h_code)
code.putln("")
code.putln("/* Runtime support code */")
code.putln("")
......@@ -1957,9 +1961,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
code.putln("%s = %s;" %
(Naming.filetable_cname, Naming.filenames_cname))
code.putln("}")
for utility_code in env.utility_code_used:
h_code.put(utility_code[0])
code.put(utility_code[1])
code.globalstate.put_utility_code_defs(code)
code.put(PyrexTypes.type_conversion_functions)
code.putln("")
......
Cython/Compiler/Symtab.py
View file @ df78cdc7
......@@ -23,11 +23,12 @@ nice_identifier = re.compile('^[a-zA-Z0-0_]+$').match
class BufferAux:
writable_needed = False
def __init__(self, buffer_info_var, stridevars, shapevars, tschecker):
def __init__(self, buffer_info_var, stridevars, shapevars,
suboffsetvars):
self.buffer_info_var = buffer_info_var
self.stridevars = stridevars
self.shapevars = shapevars
self.tschecker = tschecker
self.suboffsetvars = suboffsetvars
def __repr__(self):
return "<BufferAux %r>" % self.__dict__
......@@ -620,9 +621,6 @@ class Scope:
def use_utility_code(self, new_code, name=None):
self.global_scope().use_utility_code(new_code, name)
def has_utility_code(self, name):
return self.global_scope().has_utility_code(name)
def generate_library_function_declarations(self, code):
# Generate extern decls for C library funcs used.
......@@ -742,8 +740,7 @@ class ModuleScope(Scope):
# doc string Module doc string
# doc_cname string C name of module doc string
# const_counter integer Counter for naming constants
# utility_code_used [string] Utility code to be included
# utility_code_names set(string) (Optional) names for named (often generated) utility code
# utility_code_list [((string, string), string)] Queuing utility codes for forwarding to Code.py
# default_entries [Entry] Function argument default entries
# python_include_files [string] Standard Python headers to be included
# include_files [string] Other C headers to be included
......@@ -777,8 +774,7 @@ class ModuleScope(Scope):
self.doc = ""
self.doc_cname = Naming.moddoc_cname
self.const_counter = 1
self.utility_code_used = []
self.utility_code_names = set()
self.utility_code_list = []
self.default_entries = []
self.module_entries = {}
self.python_include_files = ["Python.h", "structmember.h"]
......@@ -938,25 +934,8 @@ class ModuleScope(Scope):
return "%s%s%d" % (Naming.const_prefix, prefix, n)
def use_utility_code(self, new_code, name=None):
# Add string to list of utility code to be included,
# if not already there (tested using the provided name,
# or 'is' if name=None -- if the utility code is dynamically
# generated, use the name, otherwise it is not needed).
if name is not None:
if name in self.utility_code_names:
return
for old_code in self.utility_code_used:
if old_code is new_code:
return
self.utility_code_used.append(new_code)
self.utility_code_names.add(name)
def has_utility_code(self, name):
# Checks if utility code (that is registered by name) has
# previously been registered. This is useful if the utility code
# is dynamically generated to avoid re-generation.
return name in self.utility_code_names
self.utility_code_list.append((new_code, name))
def declare_c_class(self, name, pos, defining = 0, implementing = 0,
module_name = None, base_type = None, objstruct_cname = None,
typeobj_cname = None, visibility = 'private', typedef_flag = 0, api = 0):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7