Commit 03e3c340 authored by Ivan Levkivskyi's avatar Ivan Levkivskyi Committed by GitHub

bpo-31333: Re-implement ABCMeta in C (#5273)

This adds C versions of methods used by ABCMeta that
improve performance of various ABC operations.
parent 667b91a5
......@@ -853,6 +853,12 @@ Optimizations
* Constant folding is moved from peephole optimizer to new AST optimizer.
(Contributed by Eugene Toder and INADA Naoki in :issue:`29469`)
* Most functions and methods in :mod:`abc` have been rewrittent in C.
This makes creation of abstract base classes, and calling :func:`isinstance`
and :func:`issubclass` on them 1.5x faster. This also reduces Python
start-up time by up to 10%. (Contributed by Ivan Levkivskyi and INADA Naoki
in :issue:`31333`)
Build and C API Changes
=======================
......
from _weakrefset import WeakSet
def get_cache_token():
"""Returns the current ABC cache token.
The token is an opaque object (supporting equality testing) identifying the
current version of the ABC cache for virtual subclasses. The token changes
with every call to ``register()`` on any ABC.
"""
return ABCMeta._abc_invalidation_counter
class ABCMeta(type):
"""Metaclass for defining Abstract Base Classes (ABCs).
Use this metaclass to create an ABC. An ABC can be subclassed
directly, and then acts as a mix-in class. You can also register
unrelated concrete classes (even built-in classes) and unrelated
ABCs as 'virtual subclasses' -- these and their descendants will
be considered subclasses of the registering ABC by the built-in
issubclass() function, but the registering ABC won't show up in
their MRO (Method Resolution Order) nor will method
implementations defined by the registering ABC be callable (not
even via super()).
"""
# A global counter that is incremented each time a class is
# registered as a virtual subclass of anything. It forces the
# negative cache to be cleared before its next use.
# Note: this counter is private. Use `abc.get_cache_token()` for
# external code.
_abc_invalidation_counter = 0
def __new__(mcls, name, bases, namespace, **kwargs):
cls = super().__new__(mcls, name, bases, namespace, **kwargs)
# Compute set of abstract method names
abstracts = {name
for name, value in namespace.items()
if getattr(value, "__isabstractmethod__", False)}
for base in bases:
for name in getattr(base, "__abstractmethods__", set()):
value = getattr(cls, name, None)
if getattr(value, "__isabstractmethod__", False):
abstracts.add(name)
cls.__abstractmethods__ = frozenset(abstracts)
# Set up inheritance registry
cls._abc_registry = WeakSet()
cls._abc_cache = WeakSet()
cls._abc_negative_cache = WeakSet()
cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
return cls
def register(cls, subclass):
"""Register a virtual subclass of an ABC.
Returns the subclass, to allow usage as a class decorator.
"""
if not isinstance(subclass, type):
raise TypeError("Can only register classes")
if issubclass(subclass, cls):
return subclass # Already a subclass
# Subtle: test for cycles *after* testing for "already a subclass";
# this means we allow X.register(X) and interpret it as a no-op.
if issubclass(cls, subclass):
# This would create a cycle, which is bad for the algorithm below
raise RuntimeError("Refusing to create an inheritance cycle")
cls._abc_registry.add(subclass)
ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache
return subclass
def _dump_registry(cls, file=None):
"""Debug helper to print the ABC registry."""
print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file)
print(f"Inv. counter: {get_cache_token()}", file=file)
for name in cls.__dict__:
if name.startswith("_abc_"):
value = getattr(cls, name)
if isinstance(value, WeakSet):
value = set(value)
print(f"{name}: {value!r}", file=file)
def _abc_registry_clear(cls):
"""Clear the registry (for debugging or testing)."""
cls._abc_registry.clear()
def _abc_caches_clear(cls):
"""Clear the caches (for debugging or testing)."""
cls._abc_cache.clear()
cls._abc_negative_cache.clear()
def __instancecheck__(cls, instance):
"""Override for isinstance(instance, cls)."""
# Inline the cache checking
subclass = instance.__class__
if subclass in cls._abc_cache:
return True
subtype = type(instance)
if subtype is subclass:
if (cls._abc_negative_cache_version ==
ABCMeta._abc_invalidation_counter and
subclass in cls._abc_negative_cache):
return False
# Fall back to the subclass check.
return cls.__subclasscheck__(subclass)
return any(cls.__subclasscheck__(c) for c in (subclass, subtype))
def __subclasscheck__(cls, subclass):
"""Override for issubclass(subclass, cls)."""
# Check cache
if subclass in cls._abc_cache:
return True
# Check negative cache; may have to invalidate
if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter:
# Invalidate the negative cache
cls._abc_negative_cache = WeakSet()
cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
elif subclass in cls._abc_negative_cache:
return False
# Check the subclass hook
ok = cls.__subclasshook__(subclass)
if ok is not NotImplemented:
assert isinstance(ok, bool)
if ok:
cls._abc_cache.add(subclass)
else:
cls._abc_negative_cache.add(subclass)
return ok
# Check if it's a direct subclass
if cls in getattr(subclass, '__mro__', ()):
cls._abc_cache.add(subclass)
return True
# Check if it's a subclass of a registered class (recursive)
for rcls in cls._abc_registry:
if issubclass(subclass, rcls):
cls._abc_cache.add(subclass)
return True
# Check if it's a subclass of a subclass (recursive)
for scls in cls.__subclasses__():
if issubclass(subclass, scls):
cls._abc_cache.add(subclass)
return True
# No dice; update negative cache
cls._abc_negative_cache.add(subclass)
return False
......@@ -3,8 +3,6 @@
"""Abstract Base Classes (ABCs) according to PEP 3119."""
from _weakrefset import WeakSet
def abstractmethod(funcobj):
"""A decorator indicating abstract methods.
......@@ -27,8 +25,7 @@ def abstractmethod(funcobj):
class abstractclassmethod(classmethod):
"""
A decorator indicating abstract classmethods.
"""A decorator indicating abstract classmethods.
Similar to abstractmethod.
......@@ -51,8 +48,7 @@ class abstractclassmethod(classmethod):
class abstractstaticmethod(staticmethod):
"""
A decorator indicating abstract staticmethods.
"""A decorator indicating abstract staticmethods.
Similar to abstractmethod.
......@@ -75,8 +71,7 @@ class abstractstaticmethod(staticmethod):
class abstractproperty(property):
"""
A decorator indicating abstract properties.
"""A decorator indicating abstract properties.
Requires that the metaclass is ABCMeta or derived from it. A
class that has a metaclass derived from ABCMeta cannot be
......@@ -106,131 +101,66 @@ class abstractproperty(property):
__isabstractmethod__ = True
class ABCMeta(type):
"""Metaclass for defining Abstract Base Classes (ABCs).
Use this metaclass to create an ABC. An ABC can be subclassed
directly, and then acts as a mix-in class. You can also register
unrelated concrete classes (even built-in classes) and unrelated
ABCs as 'virtual subclasses' -- these and their descendants will
be considered subclasses of the registering ABC by the built-in
issubclass() function, but the registering ABC won't show up in
their MRO (Method Resolution Order) nor will method
implementations defined by the registering ABC be callable (not
even via super()).
"""
# A global counter that is incremented each time a class is
# registered as a virtual subclass of anything. It forces the
# negative cache to be cleared before its next use.
# Note: this counter is private. Use `abc.get_cache_token()` for
# external code.
_abc_invalidation_counter = 0
def __new__(mcls, name, bases, namespace, **kwargs):
cls = super().__new__(mcls, name, bases, namespace, **kwargs)
# Compute set of abstract method names
abstracts = {name
for name, value in namespace.items()
if getattr(value, "__isabstractmethod__", False)}
for base in bases:
for name in getattr(base, "__abstractmethods__", set()):
value = getattr(cls, name, None)
if getattr(value, "__isabstractmethod__", False):
abstracts.add(name)
cls.__abstractmethods__ = frozenset(abstracts)
# Set up inheritance registry
cls._abc_registry = WeakSet()
cls._abc_cache = WeakSet()
cls._abc_negative_cache = WeakSet()
cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
return cls
def register(cls, subclass):
"""Register a virtual subclass of an ABC.
Returns the subclass, to allow usage as a class decorator.
try:
from _abc import (get_cache_token, _abc_init, _abc_register,
_abc_instancecheck, _abc_subclasscheck, _get_dump,
_reset_registry, _reset_caches)
except ImportError:
from _py_abc import ABCMeta, get_cache_token
ABCMeta.__module__ = 'abc'
else:
class ABCMeta(type):
"""Metaclass for defining Abstract Base Classes (ABCs).
Use this metaclass to create an ABC. An ABC can be subclassed
directly, and then acts as a mix-in class. You can also register
unrelated concrete classes (even built-in classes) and unrelated
ABCs as 'virtual subclasses' -- these and their descendants will
be considered subclasses of the registering ABC by the built-in
issubclass() function, but the registering ABC won't show up in
their MRO (Method Resolution Order) nor will method
implementations defined by the registering ABC be callable (not
even via super()).
"""
if not isinstance(subclass, type):
raise TypeError("Can only register classes")
if issubclass(subclass, cls):
return subclass # Already a subclass
# Subtle: test for cycles *after* testing for "already a subclass";
# this means we allow X.register(X) and interpret it as a no-op.
if issubclass(cls, subclass):
# This would create a cycle, which is bad for the algorithm below
raise RuntimeError("Refusing to create an inheritance cycle")
cls._abc_registry.add(subclass)
ABCMeta._abc_invalidation_counter += 1 # Invalidate negative cache
return subclass
def _dump_registry(cls, file=None):
"""Debug helper to print the ABC registry."""
print("Class: %s.%s" % (cls.__module__, cls.__qualname__), file=file)
print("Inv.counter: %s" % ABCMeta._abc_invalidation_counter, file=file)
for name in cls.__dict__:
if name.startswith("_abc_"):
value = getattr(cls, name)
if isinstance(value, WeakSet):
value = set(value)
print("%s: %r" % (name, value), file=file)
def __instancecheck__(cls, instance):
"""Override for isinstance(instance, cls)."""
# Inline the cache checking
subclass = instance.__class__
if subclass in cls._abc_cache:
return True
subtype = type(instance)
if subtype is subclass:
if (cls._abc_negative_cache_version ==
ABCMeta._abc_invalidation_counter and
subclass in cls._abc_negative_cache):
return False
# Fall back to the subclass check.
return cls.__subclasscheck__(subclass)
return any(cls.__subclasscheck__(c) for c in {subclass, subtype})
def __subclasscheck__(cls, subclass):
"""Override for issubclass(subclass, cls)."""
# Check cache
if subclass in cls._abc_cache:
return True
# Check negative cache; may have to invalidate
if cls._abc_negative_cache_version < ABCMeta._abc_invalidation_counter:
# Invalidate the negative cache
cls._abc_negative_cache = WeakSet()
cls._abc_negative_cache_version = ABCMeta._abc_invalidation_counter
elif subclass in cls._abc_negative_cache:
return False
# Check the subclass hook
ok = cls.__subclasshook__(subclass)
if ok is not NotImplemented:
assert isinstance(ok, bool)
if ok:
cls._abc_cache.add(subclass)
else:
cls._abc_negative_cache.add(subclass)
return ok
# Check if it's a direct subclass
if cls in getattr(subclass, '__mro__', ()):
cls._abc_cache.add(subclass)
return True
# Check if it's a subclass of a registered class (recursive)
for rcls in cls._abc_registry:
if issubclass(subclass, rcls):
cls._abc_cache.add(subclass)
return True
# Check if it's a subclass of a subclass (recursive)
for scls in cls.__subclasses__():
if issubclass(subclass, scls):
cls._abc_cache.add(subclass)
return True
# No dice; update negative cache
cls._abc_negative_cache.add(subclass)
return False
def __new__(mcls, name, bases, namespace, **kwargs):
cls = super().__new__(mcls, name, bases, namespace, **kwargs)
_abc_init(cls)
return cls
def register(cls, subclass):
"""Register a virtual subclass of an ABC.
Returns the subclass, to allow usage as a class decorator.
"""
return _abc_register(cls, subclass)
def __instancecheck__(cls, instance):
"""Override for isinstance(instance, cls)."""
return _abc_instancecheck(cls, instance)
def __subclasscheck__(cls, subclass):
"""Override for issubclass(subclass, cls)."""
return _abc_subclasscheck(cls, subclass)
def _dump_registry(cls, file=None):
"""Debug helper to print the ABC registry."""
print(f"Class: {cls.__module__}.{cls.__qualname__}", file=file)
print(f"Inv. counter: {get_cache_token()}", file=file)
(_abc_registry, _abc_cache, _abc_negative_cache,
_abc_negative_cache_version) = _get_dump(cls)
print(f"_abc_registry: {_abc_registry!r}", file=file)
print(f"_abc_cache: {_abc_cache!r}", file=file)
print(f"_abc_negative_cache: {_abc_negative_cache!r}", file=file)
print(f"_abc_negative_cache_version: {_abc_negative_cache_version!r}",
file=file)
def _abc_registry_clear(cls):
"""Clear the registry (for debugging or testing)."""
_reset_registry(cls)
def _abc_caches_clear(cls):
"""Clear the caches (for debugging or testing)."""
_reset_caches(cls)
class ABC(metaclass=ABCMeta):
......@@ -238,13 +168,3 @@ class ABC(metaclass=ABCMeta):
inheritance.
"""
__slots__ = ()
def get_cache_token():
"""Returns the current ABC cache token.
The token is an opaque object (supporting equality testing) identifying the
current version of the ABC cache for virtual subclasses. The token changes
with every call to ``register()`` on any ABC.
"""
return ABCMeta._abc_invalidation_counter
......@@ -5,6 +5,13 @@ import sys
import warnings
from inspect import isabstract
from test import support
try:
from _abc import _get_dump
except ImportError:
def _get_dump(cls):
# For legacy Python version
return (cls._abc_registry, cls._abc_cache,
cls._abc_negative_cache, cls._abc_negative_cache_version)
def dash_R(the_module, test, indirect_test, huntrleaks):
......@@ -36,7 +43,7 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
if not isabstract(abc):
continue
for obj in abc.__subclasses__() + [abc]:
abcs[obj] = obj._abc_registry.copy()
abcs[obj] = _get_dump(obj)[0]
# bpo-31217: Integer pool to get a single integer object for the same
# value. The pool is used to prevent false alarm when checking for memory
......@@ -113,7 +120,6 @@ def dash_R(the_module, test, indirect_test, huntrleaks):
def dash_R_cleanup(fs, ps, pic, zdc, abcs):
import gc, copyreg
import collections.abc
from weakref import WeakSet
# Restore some original values.
warnings.filters[:] = fs
......@@ -137,9 +143,10 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs):
abs_classes = filter(isabstract, abs_classes)
for abc in abs_classes:
for obj in abc.__subclasses__() + [abc]:
obj._abc_registry = abcs.get(obj, WeakSet()).copy()
obj._abc_cache.clear()
obj._abc_negative_cache.clear()
for ref in abcs.get(obj, set()):
if ref() is not None:
obj.register(ref())
obj._abc_caches_clear()
clear_caches()
......
This diff is collapsed.
......@@ -761,8 +761,8 @@ class GenericTests(BaseTestCase):
self.assertIsInstance(1, C)
C[int]
self.assertIsInstance(1, C)
C._abc_registry.clear()
C._abc_cache.clear() # To keep refleak hunting mode clean
C._abc_registry_clear()
C._abc_caches_clear() # To keep refleak hunting mode clean
def test_false_subclasses(self):
class MyMapping(MutableMapping[str, str]): pass
......
``_abc`` module is added. It is a speedup module with C implementations for
various functions and methods in ``abc``. Creating an ABC subclass and calling
``isinstance`` or ``issubclass`` with an ABC subclass are up to 1.5x faster.
In addition, this makes Python start-up up to 10% faster.
Note that the new implementation hides internal registry and caches, previously
accessible via private attributes ``_abc_registry``, ``_abc_cache``, and
``_abc_negative_cache``. There are three debugging helper methods that can be
used instead ``_dump_registry``, ``_abc_registry_clear``, and
``_abc_caches_clear``.
......@@ -114,6 +114,7 @@ _weakref _weakref.c # weak references
_functools _functoolsmodule.c # Tools for working with functions and callable objects
_operator _operator.c # operator.add() and similar goodies
_collections _collectionsmodule.c # Container types
_abc _abc.c # Abstract base classes
itertools itertoolsmodule.c # Functions creating iterators for efficient looping
atexit atexitmodule.c # Register functions to be run at interpreter-shutdown
_signal signalmodule.c
......
This diff is collapsed.
/*[clinic input]
preserve
[clinic start generated code]*/
PyDoc_STRVAR(_abc__reset_registry__doc__,
"_reset_registry($module, self, /)\n"
"--\n"
"\n"
"Internal ABC helper to reset registry of a given class.\n"
"\n"
"Should be only used by refleak.py");
#define _ABC__RESET_REGISTRY_METHODDEF \
{"_reset_registry", (PyCFunction)_abc__reset_registry, METH_O, _abc__reset_registry__doc__},
PyDoc_STRVAR(_abc__reset_caches__doc__,
"_reset_caches($module, self, /)\n"
"--\n"
"\n"
"Internal ABC helper to reset both caches of a given class.\n"
"\n"
"Should be only used by refleak.py");
#define _ABC__RESET_CACHES_METHODDEF \
{"_reset_caches", (PyCFunction)_abc__reset_caches, METH_O, _abc__reset_caches__doc__},
PyDoc_STRVAR(_abc__get_dump__doc__,
"_get_dump($module, self, /)\n"
"--\n"
"\n"
"Internal ABC helper for cache and registry debugging.\n"
"\n"
"Return shallow copies of registry, of both caches, and\n"
"negative cache version. Don\'t call this function directly,\n"
"instead use ABC._dump_registry() for a nice repr.");
#define _ABC__GET_DUMP_METHODDEF \
{"_get_dump", (PyCFunction)_abc__get_dump, METH_O, _abc__get_dump__doc__},
PyDoc_STRVAR(_abc__abc_init__doc__,
"_abc_init($module, self, /)\n"
"--\n"
"\n"
"Internal ABC helper for class set-up. Should be never used outside abc module.");
#define _ABC__ABC_INIT_METHODDEF \
{"_abc_init", (PyCFunction)_abc__abc_init, METH_O, _abc__abc_init__doc__},
PyDoc_STRVAR(_abc__abc_register__doc__,
"_abc_register($module, self, subclass, /)\n"
"--\n"
"\n"
"Internal ABC helper for subclasss registration. Should be never used outside abc module.");
#define _ABC__ABC_REGISTER_METHODDEF \
{"_abc_register", (PyCFunction)_abc__abc_register, METH_FASTCALL, _abc__abc_register__doc__},
static PyObject *
_abc__abc_register_impl(PyObject *module, PyObject *self, PyObject *subclass);
static PyObject *
_abc__abc_register(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
{
PyObject *return_value = NULL;
PyObject *self;
PyObject *subclass;
if (!_PyArg_UnpackStack(args, nargs, "_abc_register",
2, 2,
&self, &subclass)) {
goto exit;
}
return_value = _abc__abc_register_impl(module, self, subclass);
exit:
return return_value;
}
PyDoc_STRVAR(_abc__abc_instancecheck__doc__,
"_abc_instancecheck($module, self, instance, /)\n"
"--\n"
"\n"
"Internal ABC helper for instance checks. Should be never used outside abc module.");
#define _ABC__ABC_INSTANCECHECK_METHODDEF \
{"_abc_instancecheck", (PyCFunction)_abc__abc_instancecheck, METH_FASTCALL, _abc__abc_instancecheck__doc__},
static PyObject *
_abc__abc_instancecheck_impl(PyObject *module, PyObject *self,
PyObject *instance);
static PyObject *
_abc__abc_instancecheck(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
{
PyObject *return_value = NULL;
PyObject *self;
PyObject *instance;
if (!_PyArg_UnpackStack(args, nargs, "_abc_instancecheck",
2, 2,
&self, &instance)) {
goto exit;
}
return_value = _abc__abc_instancecheck_impl(module, self, instance);
exit:
return return_value;
}
PyDoc_STRVAR(_abc__abc_subclasscheck__doc__,
"_abc_subclasscheck($module, self, subclass, /)\n"
"--\n"
"\n"
"Internal ABC helper for subclasss checks. Should be never used outside abc module.");
#define _ABC__ABC_SUBCLASSCHECK_METHODDEF \
{"_abc_subclasscheck", (PyCFunction)_abc__abc_subclasscheck, METH_FASTCALL, _abc__abc_subclasscheck__doc__},
static PyObject *
_abc__abc_subclasscheck_impl(PyObject *module, PyObject *self,
PyObject *subclass);
static PyObject *
_abc__abc_subclasscheck(PyObject *module, PyObject *const *args, Py_ssize_t nargs)
{
PyObject *return_value = NULL;
PyObject *self;
PyObject *subclass;
if (!_PyArg_UnpackStack(args, nargs, "_abc_subclasscheck",
2, 2,
&self, &subclass)) {
goto exit;
}
return_value = _abc__abc_subclasscheck_impl(module, self, subclass);
exit:
return return_value;
}
PyDoc_STRVAR(_abc_get_cache_token__doc__,
"get_cache_token($module, /)\n"
"--\n"
"\n"
"Returns the current ABC cache token.\n"
"\n"
"The token is an opaque object (supporting equality testing) identifying the\n"
"current version of the ABC cache for virtual subclasses. The token changes\n"
"with every call to register() on any ABC.");
#define _ABC_GET_CACHE_TOKEN_METHODDEF \
{"get_cache_token", (PyCFunction)_abc_get_cache_token, METH_NOARGS, _abc_get_cache_token__doc__},
static PyObject *
_abc_get_cache_token_impl(PyObject *module);
static PyObject *
_abc_get_cache_token(PyObject *module, PyObject *Py_UNUSED(ignored))
{
return _abc_get_cache_token_impl(module);
}
/*[clinic end generated code: output=9d6f861a8f45bc6f input=a9049054013a1b77]*/
......@@ -5,6 +5,7 @@
#include "Python.h"
extern PyObject* PyInit__abc(void);
extern PyObject* PyInit_array(void);
extern PyObject* PyInit_audioop(void);
extern PyObject* PyInit_binascii(void);
......@@ -80,6 +81,7 @@ extern PyObject* PyInit__imp(void);
struct _inittab _PyImport_Inittab[] = {
{"_abc", PyInit__abc},
{"array", PyInit_array},
{"_ast", PyInit__ast},
{"audioop", PyInit_audioop},
......
......@@ -228,6 +228,7 @@
<ClInclude Include="$(zlibDir)\zutil.h" />
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\Modules\_abc.c" />
<ClCompile Include="..\Modules\_asynciomodule.c" />
<ClCompile Include="..\Modules\_bisectmodule.c" />
<ClCompile Include="..\Modules\_blake2\blake2module.c" />
......
......@@ -473,6 +473,9 @@
</ClInclude>
</ItemGroup>
<ItemGroup>
<ClCompile Include="..\Modules\_abc.c">
<Filter>Modules</Filter>
</ClCompile>
<ClCompile Include="..\Modules\_bisectmodule.c">
<Filter>Modules</Filter>
</ClCompile>
......
......@@ -712,6 +712,8 @@ class PyBuildExt(build_ext):
exts.append( Extension('_opcode', ['_opcode.c']) )
# asyncio speedups
exts.append( Extension("_asyncio", ["_asynciomodule.c"]) )
# _abc speedups
exts.append( Extension("_abc", ["_abc.c"]) )
# _queue module
exts.append( Extension("_queue", ["_queuemodule.c"]) )
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment