Skip to content

C
cpython
Commit 0d722f3c authored by Jeroen Demeyer's avatar Jeroen Demeyer Committed by Petr Viktorin
Browse files
Options

bpo-36974: separate vectorcall functions for each calling convention (GH-13781)

parent 6e43d073
Hide whitespace changes
Inline Side-by-side
Showing with 368 additions and 99 deletions
Include/descrobject.h
View file @ 0d722f3c
......@@ -91,9 +91,6 @@ PyAPI_FUNC(PyObject *) PyDescr_NewMember(PyTypeObject *,
PyAPI_FUNC(PyObject *) PyDescr_NewGetSet(PyTypeObject *,
struct PyGetSetDef *);
#ifndef Py_LIMITED_API
PyAPI_FUNC(PyObject *) _PyMethodDescr_Vectorcall(
PyObject *descrobj, PyObject *const *args, size_t nargsf, PyObject *kwnames);
PyAPI_FUNC(PyObject *) PyDescr_NewWrapper(PyTypeObject *,
struct wrapperbase *, void *);
#define PyDescr_IsData(d) (Py_TYPE(d)->tp_descr_set != NULL)
......
Include/methodobject.h
View file @ 0d722f3c
......@@ -41,13 +41,6 @@ PyAPI_FUNC(int) PyCFunction_GetFlags(PyObject *);
#endif
PyAPI_FUNC(PyObject *) PyCFunction_Call(PyObject *, PyObject *, PyObject *);
#ifndef Py_LIMITED_API
PyAPI_FUNC(PyObject *) _PyCFunction_Vectorcall(PyObject *func,
PyObject *const *stack,
size_t nargsf,
PyObject *kwnames);
#endif
struct PyMethodDef {
const char *ml_name; /* The name of the built-in function/method */
PyCFunction ml_meth; /* The C function that implements it */
......
Lib/test/test_call.py
View file @ 0d722f3c
......@@ -586,6 +586,8 @@ class TestPEP590(unittest.TestCase):
return super().__call__(*args)
calls += [
(dict.update, ({},), {"key":True}, None),
({}.update, ({},), {"key":True}, None),
(MethodDescriptorHeap(), (0,), {}, True),
(MethodDescriptorOverridden(), (0,), {}, 'new'),
(MethodDescriptorSuper(), (0,), {}, True),
......
Lib/test/test_gdb.py
View file @ 0d722f3c
......@@ -850,10 +850,10 @@ id(42)
# called, so test a variety of calling conventions.
for py_name, py_args, c_name, expected_frame_number in (
('gmtime', '', 'time_gmtime', 1), # METH_VARARGS
('len', '[]', 'builtin_len', 2), # METH_O
('locals', '', 'builtin_locals', 2), # METH_NOARGS
('iter', '[]', 'builtin_iter', 2), # METH_FASTCALL
('sorted', '[]', 'builtin_sorted', 2), # METH_FASTCALL|METH_KEYWORDS
('len', '[]', 'builtin_len', 1), # METH_O
('locals', '', 'builtin_locals', 1), # METH_NOARGS
('iter', '[]', 'builtin_iter', 1), # METH_FASTCALL
('sorted', '[]', 'builtin_sorted', 1), # METH_FASTCALL|METH_KEYWORDS
):
with self.subTest(c_name):
cmd = ('from time import gmtime\n' # (not always needed)
......
Misc/NEWS.d/next/Core and Builtins/2019-06-11-12-59-38.bpo-36974.bVYmSA.rst 0 → 100644
View file @ 0d722f3c
Implemented separate vectorcall functions for every calling convention of
builtin functions and methods. This improves performance for calls.
Objects/call.c
View file @ 0d722f3c
......@@ -216,7 +216,7 @@ PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *kwargs)
PyObject *result = func(callable, args,
nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames);
_PyStack_UnpackDict_Free(args, nargs, kwnames);
return result;
return _Py_CheckFunctionResult(callable, result, NULL);
}
......@@ -625,26 +625,6 @@ exit:
return result;
}
PyObject *
_PyCFunction_Vectorcall(PyObject *func,
PyObject *const *args, size_t nargsf,
PyObject *kwnames)
{
PyObject *result;
assert(func != NULL);
assert(PyCFunction_Check(func));
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
result = _PyMethodDef_RawFastCallKeywords(((PyCFunctionObject*)func)->m_ml,
PyCFunction_GET_SELF(func),
args, nargs, kwnames);
result = _Py_CheckFunctionResult(func, result, NULL);
return result;
}
static PyObject *
cfunction_call_varargs(PyObject *func, PyObject *args, PyObject *kwargs)
{
......
Objects/descrobject.c
View file @ 0d722f3c
......@@ -226,80 +226,199 @@ getset_set(PyGetSetDescrObject *descr, PyObject *obj, PyObject *value)
return -1;
}
static PyObject *
methoddescr_call(PyMethodDescrObject *descr, PyObject *args, PyObject *kwargs)
{
Py_ssize_t nargs;
PyObject *self, *result;
/* Make sure that the first argument is acceptable as 'self' */
assert(PyTuple_Check(args));
nargs = PyTuple_GET_SIZE(args);
/* Vectorcall functions for each of the PyMethodDescr calling conventions.
*
* First, common helpers
*/
static const char *
get_name(PyObject *func) {
assert(PyObject_TypeCheck(func, &PyMethodDescr_Type));
return ((PyMethodDescrObject *)func)->d_method->ml_name;
}
typedef void (*funcptr)(void);
static inline int
method_check_args(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
assert(!PyErr_Occurred());
assert(PyObject_TypeCheck(func, &PyMethodDescr_Type));
if (nargs < 1) {
PyErr_Format(PyExc_TypeError,
"descriptor '%V' of '%.100s' "
"descriptor '%.200s' of '%.100s' "
"object needs an argument",
descr_name((PyDescrObject *)descr), "?",
PyDescr_TYPE(descr)->tp_name);
return NULL;
get_name(func), PyDescr_TYPE(func)->tp_name);
return -1;
}
self = PyTuple_GET_ITEM(args, 0);
PyObject *self = args[0];
if (!_PyObject_RealIsSubclass((PyObject *)Py_TYPE(self),
(PyObject *)PyDescr_TYPE(descr))) {
(PyObject *)PyDescr_TYPE(func)))
{
PyErr_Format(PyExc_TypeError,
"descriptor '%V' for '%.100s' objects "
"descriptor '%.200s' for '%.100s' objects "
"doesn't apply to a '%.100s' object",
descr_name((PyDescrObject *)descr), "?",
PyDescr_TYPE(descr)->tp_name,
self->ob_type->tp_name);
get_name(func), PyDescr_TYPE(func)->tp_name,
Py_TYPE(self)->tp_name);
return -1;
}
if (kwnames && PyTuple_GET_SIZE(kwnames)) {
PyErr_Format(PyExc_TypeError,
"%.200s() takes no keyword arguments", get_name(func));
return -1;
}
return 0;
}
static inline funcptr
method_enter_call(PyObject *func)
{
if (Py_EnterRecursiveCall(" while calling a Python object")) {
return NULL;
}
return (funcptr)((PyMethodDescrObject *)func)->d_method->ml_meth;
}
result = _PyMethodDef_RawFastCallDict(descr->d_method, self,
&_PyTuple_ITEMS(args)[1], nargs - 1,
kwargs);
result = _Py_CheckFunctionResult((PyObject *)descr, result, NULL);
/* Now the actual vectorcall functions */
static PyObject *
method_vectorcall_VARARGS(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
{
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
if (method_check_args(func, args, nargs, kwnames)) {
return NULL;
}
PyObject *argstuple = _PyTuple_FromArray(args+1, nargs-1);
if (argstuple == NULL) {
return NULL;
}
PyCFunction meth = (PyCFunction)method_enter_call(func);
if (meth == NULL) {
Py_DECREF(argstuple);
return NULL;
}
PyObject *result = meth(args[0], argstuple);
Py_DECREF(argstuple);
Py_LeaveRecursiveCall();
return result;
}
// same to methoddescr_call(), but use FASTCALL convention.
PyObject *
_PyMethodDescr_Vectorcall(PyObject *descrobj,
PyObject *const *args, size_t nargsf,
PyObject *kwnames)
static PyObject *
method_vectorcall_VARARGS_KEYWORDS(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
{
assert(Py_TYPE(descrobj) == &PyMethodDescr_Type);
PyMethodDescrObject *descr = (PyMethodDescrObject *)descrobj;
PyObject *self, *result;
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
if (method_check_args(func, args, nargs, NULL)) {
return NULL;
}
PyObject *argstuple = _PyTuple_FromArray(args+1, nargs-1);
if (argstuple == NULL) {
return NULL;
}
PyObject *result = NULL;
/* Create a temporary dict for keyword arguments */
PyObject *kwdict = NULL;
if (kwnames != NULL && PyTuple_GET_SIZE(kwnames) > 0) {
kwdict = _PyStack_AsDict(args + nargs, kwnames);
if (kwdict == NULL) {
goto exit;
}
}
PyCFunctionWithKeywords meth = (PyCFunctionWithKeywords)
method_enter_call(func);
if (meth == NULL) {
goto exit;
}
result = meth(args[0], argstuple, kwdict);
Py_LeaveRecursiveCall();
exit:
Py_DECREF(argstuple);
Py_XDECREF(kwdict);
return result;
}
static PyObject *
method_vectorcall_FASTCALL(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
{
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
if (method_check_args(func, args, nargs, kwnames)) {
return NULL;
}
_PyCFunctionFast meth = (_PyCFunctionFast)
method_enter_call(func);
if (meth == NULL) {
return NULL;
}
PyObject *result = meth(args[0], args+1, nargs-1);
Py_LeaveRecursiveCall();
return result;
}
static PyObject *
method_vectorcall_FASTCALL_KEYWORDS(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
{
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
/* Make sure that the first argument is acceptable as 'self' */
if (nargs < 1) {
PyErr_Format(PyExc_TypeError,
"descriptor '%V' of '%.100s' "
"object needs an argument",
descr_name((PyDescrObject *)descr), "?",
PyDescr_TYPE(descr)->tp_name);
if (method_check_args(func, args, nargs, NULL)) {
return NULL;
}
self = args[0];
if (!_PyObject_RealIsSubclass((PyObject *)Py_TYPE(self),
(PyObject *)PyDescr_TYPE(descr))) {
_PyCFunctionFastWithKeywords meth = (_PyCFunctionFastWithKeywords)
method_enter_call(func);
if (meth == NULL) {
return NULL;
}
PyObject *result = meth(args[0], args+1, nargs-1, kwnames);
Py_LeaveRecursiveCall();
return result;
}
static PyObject *
method_vectorcall_NOARGS(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
{
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
if (method_check_args(func, args, nargs, kwnames)) {
return NULL;
}
if (nargs != 1) {
PyErr_Format(PyExc_TypeError,
"descriptor '%V' for '%.100s' objects "
"doesn't apply to a '%.100s' object",
descr_name((PyDescrObject *)descr), "?",
PyDescr_TYPE(descr)->tp_name,
self->ob_type->tp_name);
"%.200s() takes no arguments (%zd given)", get_name(func), nargs-1);
return NULL;
}
PyCFunction meth = (PyCFunction)method_enter_call(func);
if (meth == NULL) {
return NULL;
}
PyObject *result = meth(args[0], NULL);
Py_LeaveRecursiveCall();
return result;
}
result = _PyMethodDef_RawFastCallKeywords(descr->d_method, self,
args+1, nargs-1, kwnames);
result = _Py_CheckFunctionResult((PyObject *)descr, result, NULL);
static PyObject *
method_vectorcall_O(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
{
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
if (method_check_args(func, args, nargs, kwnames)) {
return NULL;
}
if (nargs != 2) {
PyErr_Format(PyExc_TypeError,
"%.200s() takes exactly one argument (%zd given)",
get_name(func), nargs-1);
return NULL;
}
PyCFunction meth = (PyCFunction)method_enter_call(func);
if (meth == NULL) {
return NULL;
}
PyObject *result = meth(args[0], args[1]);
Py_LeaveRecursiveCall();
return result;
}
/* Instances of classmethod_descriptor are unlikely to be called directly.
For one, the analogous class "classmethod" (for Python classes) is not
callable. Second, users are not likely to access a classmethod_descriptor
......@@ -540,7 +659,7 @@ PyTypeObject PyMethodDescr_Type = {
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
(ternaryfunc)methoddescr_call, /* tp_call */
PyVectorcall_Call, /* tp_call */
0, /* tp_str */
PyObject_GenericGetAttr, /* tp_getattro */
0, /* tp_setattro */
......@@ -738,13 +857,40 @@ descr_new(PyTypeObject *descrtype, PyTypeObject *type, const char *name)
PyObject *
PyDescr_NewMethod(PyTypeObject *type, PyMethodDef *method)
{
/* Figure out correct vectorcall function to use */
vectorcallfunc vectorcall;
switch (method->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS))
{
case METH_VARARGS:
vectorcall = method_vectorcall_VARARGS;
break;
case METH_VARARGS | METH_KEYWORDS:
vectorcall = method_vectorcall_VARARGS_KEYWORDS;
break;
case METH_FASTCALL:
vectorcall = method_vectorcall_FASTCALL;
break;
case METH_FASTCALL | METH_KEYWORDS:
vectorcall = method_vectorcall_FASTCALL_KEYWORDS;
break;
case METH_NOARGS:
vectorcall = method_vectorcall_NOARGS;
break;
case METH_O:
vectorcall = method_vectorcall_O;
break;
default:
PyErr_SetString(PyExc_SystemError, "bad call flags");
return NULL;
}
PyMethodDescrObject *descr;
descr = (PyMethodDescrObject *)descr_new(&PyMethodDescr_Type,
type, method->ml_name);
if (descr != NULL) {
descr->d_method = method;
descr->vectorcall = _PyMethodDescr_Vectorcall;
descr->vectorcall = vectorcall;
}
return (PyObject *)descr;
}
......
Objects/methodobject.c
View file @ 0d722f3c
......@@ -19,6 +19,17 @@ static int numfree = 0;
/* undefine macro trampoline to PyCFunction_NewEx */
#undef PyCFunction_New
/* Forward declarations */
static PyObject * cfunction_vectorcall_FASTCALL(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames);
static PyObject * cfunction_vectorcall_FASTCALL_KEYWORDS(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames);
static PyObject * cfunction_vectorcall_NOARGS(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames);
static PyObject * cfunction_vectorcall_O(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames);
PyObject *
PyCFunction_New(PyMethodDef *ml, PyObject *self)
{
......@@ -28,6 +39,33 @@ PyCFunction_New(PyMethodDef *ml, PyObject *self)
PyObject *
PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module)
{
/* Figure out correct vectorcall function to use */
vectorcallfunc vectorcall;
switch (ml->ml_flags & (METH_VARARGS | METH_FASTCALL | METH_NOARGS | METH_O | METH_KEYWORDS))
{
case METH_VARARGS:
case METH_VARARGS | METH_KEYWORDS:
/* For METH_VARARGS functions, it's more efficient to use tp_call
* instead of vectorcall. */
vectorcall = NULL;
break;
case METH_FASTCALL:
vectorcall = cfunction_vectorcall_FASTCALL;
break;
case METH_FASTCALL | METH_KEYWORDS:
vectorcall = cfunction_vectorcall_FASTCALL_KEYWORDS;
break;
case METH_NOARGS:
vectorcall = cfunction_vectorcall_NOARGS;
break;
case METH_O:
vectorcall = cfunction_vectorcall_O;
break;
default:
PyErr_SetString(PyExc_SystemError, "bad call flags");
return NULL;
}
PyCFunctionObject *op;
op = free_list;
if (op != NULL) {
......@@ -46,14 +84,7 @@ PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module)
op->m_self = self;
Py_XINCREF(module);
op->m_module = module;
if (ml->ml_flags & METH_VARARGS) {
/* For METH_VARARGS functions, it's more efficient to use tp_call
* instead of vectorcall. */
op->vectorcall = NULL;
}
else {
op->vectorcall = _PyCFunction_Vectorcall;
}
op->vectorcall = vectorcall;
_PyObject_GC_TRACK(op);
return (PyObject *)op;
}
......@@ -333,3 +364,121 @@ _PyCFunction_DebugMallocStats(FILE *out)
"free PyCFunctionObject",
numfree, sizeof(PyCFunctionObject));
}
/* Vectorcall functions for each of the PyCFunction calling conventions,
* except for METH_VARARGS (possibly combined with METH_KEYWORDS) which
* doesn't use vectorcall.
*
* First, common helpers
*/
static const char *
get_name(PyObject *func)
{
assert(PyCFunction_Check(func));
PyMethodDef *method = ((PyCFunctionObject *)func)->m_ml;
return method->ml_name;
}
typedef void (*funcptr)(void);
static inline int
cfunction_check_kwargs(PyObject *func, PyObject *kwnames)
{
assert(!PyErr_Occurred());
assert(PyCFunction_Check(func));
if (kwnames && PyTuple_GET_SIZE(kwnames)) {
PyErr_Format(PyExc_TypeError,
"%.200s() takes no keyword arguments", get_name(func));
return -1;
}
return 0;
}
static inline funcptr
cfunction_enter_call(PyObject *func)
{
if (Py_EnterRecursiveCall(" while calling a Python object")) {
return NULL;
}
return (funcptr)PyCFunction_GET_FUNCTION(func);
}
/* Now the actual vectorcall functions */
static PyObject *
cfunction_vectorcall_FASTCALL(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
{
if (cfunction_check_kwargs(func, kwnames)) {
return NULL;
}
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
_PyCFunctionFast meth = (_PyCFunctionFast)
cfunction_enter_call(func);
if (meth == NULL) {
return NULL;
}
PyObject *result = meth(PyCFunction_GET_SELF(func), args, nargs);
Py_LeaveRecursiveCall();
return result;
}
static PyObject *
cfunction_vectorcall_FASTCALL_KEYWORDS(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
{
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
_PyCFunctionFastWithKeywords meth = (_PyCFunctionFastWithKeywords)
cfunction_enter_call(func);
if (meth == NULL) {
return NULL;
}
PyObject *result = meth(PyCFunction_GET_SELF(func), args, nargs, kwnames);
Py_LeaveRecursiveCall();
return result;
}
static PyObject *
cfunction_vectorcall_NOARGS(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
{
if (cfunction_check_kwargs(func, kwnames)) {
return NULL;
}
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
if (nargs != 0) {
PyErr_Format(PyExc_TypeError,
"%.200s() takes no arguments (%zd given)", get_name(func), nargs);
return NULL;
}
PyCFunction meth = (PyCFunction)cfunction_enter_call(func);
if (meth == NULL) {
return NULL;
}
PyObject *result = meth(PyCFunction_GET_SELF(func), NULL);
Py_LeaveRecursiveCall();
return result;
}
static PyObject *
cfunction_vectorcall_O(
PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames)
{
if (cfunction_check_kwargs(func, kwnames)) {
return NULL;
}
Py_ssize_t nargs = PyVectorcall_NARGS(nargsf);
if (nargs != 1) {
PyErr_Format(PyExc_TypeError,
"%.200s() takes exactly one argument (%zd given)",
get_name(func), nargs);
return NULL;
}
PyCFunction meth = (PyCFunction)cfunction_enter_call(func);
if (meth == NULL) {
return NULL;
}
PyObject *result = meth(PyCFunction_GET_SELF(func), args[0]);
Py_LeaveRecursiveCall();
return result;
}
Python/ceval.c
View file @ 0d722f3c
......@@ -4936,7 +4936,7 @@ trace_call_function(PyThreadState *tstate,
{
PyObject *x;
if (PyCFunction_Check(func)) {
C_TRACE(x, _PyCFunction_Vectorcall(func, args, nargs, kwnames));
C_TRACE(x, _PyObject_Vectorcall(func, args, nargs, kwnames));
return x;
}
else if (Py_TYPE(func) == &PyMethodDescr_Type && nargs > 0) {
......@@ -4952,9 +4952,9 @@ trace_call_function(PyThreadState *tstate,
if (func == NULL) {
return NULL;
}
C_TRACE(x, _PyCFunction_Vectorcall(func,
args+1, nargs-1,
kwnames));
C_TRACE(x, _PyObject_Vectorcall(func,
args+1, nargs-1,
kwnames));
Py_DECREF(func);
return x;
}
......
Tools/gdb/libpython.py
View file @ 0d722f3c
......@@ -1563,8 +1563,8 @@ class Frame(object):
if not caller:
return False
if caller in ('_PyCFunction_Vectorcall',
'cfunction_call_varargs'):
if (caller.startswith('cfunction_vectorcall_') or
caller == 'cfunction_call_varargs'):
arg_name = 'func'
# Within that frame:
# "func" is the local containing the PyObject* of the
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7