Commit 012ea948 authored by Jason Madden's avatar Jason Madden

Add a non-CFFI implementation of the Ring abstraction based on the deque work,...

Add a non-CFFI implementation of the Ring abstraction based on the deque work, and add a tox environment to test it. Only one ZODB test fails.
parent 38d86ef7
......@@ -69,9 +69,7 @@ def _sweeping_ring(f):
self._is_sweeping_ring = False
return locked
from collections import deque
from . import ring
from .ring import Ring
@implementer(IPickleCache)
class PickleCache(object):
......@@ -102,8 +100,7 @@ class PickleCache(object):
self.non_ghost_count = 0
self.persistent_classes = {}
self.data = weakref.WeakValueDictionary()
self.ring_home = ring.CPersistentRingHead()
self.ring_nodes = set()
self.ring = Ring()
self.cache_size_bytes = cache_size_bytes
# IPickleCache API
......@@ -163,11 +160,8 @@ class PickleCache(object):
else:
self.data[oid] = value
_gc_monitor(value)
node = ring.CPersistentRing(value)
object.__setattr__(value, '_Persistent__ring', node)
if _OGA(value, '_p_state') != GHOST:
ring.add(self.ring_home, node)
self.ring_nodes.add(node)
if _OGA(value, '_p_state') != GHOST and value not in self.ring:
self.ring.add(value)
self.non_ghost_count += 1
def __delitem__(self, oid):
......@@ -179,7 +173,7 @@ class PickleCache(object):
del self.persistent_classes[oid]
else:
value = self.data.pop(oid)
self._remove_from_ring(value)
self.ring.delete(value)
def get(self, oid, default=None):
""" See IPickleCache.
......@@ -200,24 +194,19 @@ class PickleCache(object):
return False # marker return for tests
value = self.data[oid]
node = _OGA(value, '_Persistent__ring')
if node is None: # tests
return
was_in_ring = bool(node.r_next)
was_in_ring = value in self.ring
if not was_in_ring:
if _OGA(value, '_p_state') != GHOST:
ring.add(self.ring_home, node)
self.ring_nodes.add(node)
self.ring.add(value)
self.non_ghost_count += 1
else:
ring.move_to_head(self.ring_home, node)
self.ring.move_to_head(value)
def ringlen(self):
""" See IPickleCache.
"""
return ring.ringlen(self.ring_home)
return len(self.ring)
def items(self):
""" See IPickleCache.
......@@ -228,8 +217,7 @@ class PickleCache(object):
""" See IPickleCache.
"""
result = []
for item in ring.iteritems(self.ring_home):
obj = ring.get_object(item)
for obj in self.ring:
result.append((obj._p_oid, obj))
return result
......@@ -356,15 +344,17 @@ class PickleCache(object):
@_sweeping_ring
def _sweep(self, target, target_size_bytes=0):
# lock
ejected = 0
for here in list(ring.iteritems(self.ring_home)):
# To avoid mutating datastructures in place or making a copy,
# and to work efficiently with both the CFFI ring and the
# deque-based ring, we collect the objects and their indexes
# up front and then hand them off for ejection.
# We don't use enumerate because that's slow under PyPy
i = -1
to_eject = []
for value in self.ring:
if self.non_ghost_count <= target and (self.total_estimated_size <= target_size_bytes or not target_size_bytes):
break
value = ring.get_object(here)
here = here.r_next
i += 1
if value._p_state == UPTODATE:
# The C implementation will only evict things that are specifically
# in the up-to-date state
......@@ -387,9 +377,13 @@ class PickleCache(object):
# they don't cooperate (without this check a bunch of test_picklecache
# breaks)
or not isinstance(value, _SWEEPABLE_TYPES)):
self._remove_from_ring(value)
to_eject.append((i, value))
self.non_ghost_count -= 1
ejected += 1
ejected = len(to_eject)
if ejected:
self.ring.delete_all(to_eject)
del to_eject # Got to clear our local if we want the GC to get the weak refs
if ejected and _SWEEP_NEEDS_GC:
# See comments on _SWEEP_NEEDS_GC
......@@ -401,7 +395,7 @@ class PickleCache(object):
value = self.data.get(oid)
if value is not None and value._p_state != GHOST:
value._p_invalidate()
self._remove_from_ring(value)
self.ring.delete(value)
elif oid in self.persistent_classes:
persistent_class = self.persistent_classes[oid]
del self.persistent_classes[oid]
......@@ -412,8 +406,3 @@ class PickleCache(object):
persistent_class._p_invalidate()
except AttributeError:
pass
def _remove_from_ring(self, value):
if value._Persistent__ring.r_next:
ring.del_(value._Persistent__ring)
self.ring_nodes.discard(value._Persistent__ring)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
##############################################################################
#
# Copyright (c) 2009 Zope Foundation and Contributors.
# All Rights Reserved.
#
# This software is subject to the provisions of the Zope Public License,
# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
#pylint: disable=W0212
try:
from cffi import FFI
import os
this_dir = os.path.dirname(os.path.abspath(__file__))
"""
from cffi import FFI
import pkg_resources
import os
ffi = FFI()
with open(os.path.join(this_dir, 'ring.h')) as f:
ffi.cdef(f.read())
ffi = FFI()
ring = ffi.verify("""
#include "ring.c"
""", include_dirs=[os.path.dirname(os.path.abspath(__file__))])
ffi.cdef("""
typedef struct CPersistentRingEx_struct
{
struct CPersistentRingEx_struct *r_prev;
struct CPersistentRingEx_struct *r_next;
void* object;
} CPersistentRingEx;
""")
_OGA = object.__getattribute__
_OSA = object.__setattr__
ffi.cdef(pkg_resources.resource_string('persistent', 'ring.h'))
#pylint: disable=E1101
class _CFFIRing(object):
ring = ffi.verify("""
typedef struct CPersistentRingEx_struct
{
struct CPersistentRingEx_struct *r_prev;
struct CPersistentRingEx_struct *r_next;
void* object;
} CPersistentRingEx;
#include "ring.c"
""", include_dirs=[os.path.dirname(os.path.abspath(__file__))])
__slots__ = ('ring_home', 'ring_to_obj')
def __init__(self):
node = self.ring_home = ffi.new("CPersistentRing*")
node.r_next = node
node.r_prev = node
class CPersistentRing(object):
self.ring_to_obj = dict()
def __init__(self, obj=None):
self.handle = None
self.node = ffi.new("CPersistentRingEx*")
if obj is not None:
self.handle = self.node.object = ffi.new_handle(obj)
self._object = obj # Circular reference
def __len__(self):
return len(self.ring_to_obj)
def __getattr__(self, name):
return getattr(self.node, name)
def __contains__(self, pobj):
return getattr(pobj, '_Persistent__ring', self) in self.ring_to_obj
def get_object(self):
return get_object(self.node)
def add(self, pobj):
node = ffi.new("CPersistentRing*")
ring.ring_add(self.ring_home, node)
self.ring_to_obj[node] = pobj
object.__setattr__(pobj, '_Persistent__ring', node)
def CPersistentRingHead():
head = CPersistentRing()
head.node.r_next = head.node
head.node.r_prev = head.node
return head
def delete(self, pobj):
node = getattr(pobj, '_Persistent__ring', None)
if node is not None and node.r_next:
ring.ring_del(node)
self.ring_to_obj.pop(node, None)
def _c(node):
return ffi.cast("CPersistentRing*", node.node)
def move_to_head(self, pobj):
node = pobj._Persistent__ring
ring.ring_move_to_head(self.ring_home, node)
def add(head, elt):
ring.ring_add(_c(head), _c(elt))
def delete_all(self, indexes_and_values):
for _, value in indexes_and_values:
self.delete(value)
def del_(elt):
ring.ring_del(_c(elt))
def iteritems(self):
head = self.ring_home
here = head.r_next
while here != head:
yield here
here = here.r_next
def move_to_head(head, elt):
ring.ring_move_to_head(_c(head), _c(elt))
def __iter__(self):
ring_to_obj = self.ring_to_obj
for node in self.iteritems():
yield ring_to_obj[node]
def iteritems(head):
here = head.r_next
while here != head.node:
yield here
here = here.r_next
Ring = _CFFIRing
def ringlen(head):
count = 0
for _ in iteritems(head):
count += 1
return count
except ImportError:
def get_object(node):
return ffi.from_handle(node.object)
from collections import deque
print CPersistentRing()
class _DequeRing(object):
__slots__ = ('ring', 'ring_oids')
def __init__(self):
self.ring = deque()
self.ring_oids = set()
def __len__(self):
return len(self.ring)
def __contains__(self, pobj):
return pobj._p_oid in self.ring_oids
def add(self, pobj):
self.ring.append(pobj)
self.ring_oids.add(pobj._p_oid)
def delete(self, pobj):
# Note that we do not use self.ring.remove() because that
# uses equality semantics and we don't want to call the persistent
# object's __eq__ method (which might wake it up just after we
# tried to ghost it)
i = 0 # Using a manual numeric counter instead of enumerate() is much faster on PyPy
for o in self.ring:
if o is pobj:
del self.ring[i]
self.ring_oids.discard(pobj._p_oid)
return 1
i += 1
def move_to_head(self, pobj):
self.delete(pobj)
self.add(pobj)
def delete_all(self, indexes_and_values):
for ix, value in reversed(indexes_and_values):
del self.ring[ix]
self.ring_oids.discard(value._p_oid)
def __iter__(self):
return iter(self.ring)
Ring = _DequeRing
......@@ -11,9 +11,16 @@
# FOR A PARTICULAR PURPOSE.
#
##############################################################################
import unittest
import gc
_is_jython = hasattr(gc, 'getJythonGCFlags')
import os
import platform
import sys
import unittest
_py_impl = getattr(platform, 'python_implementation', lambda: None)
_is_pypy = _py_impl() == 'PyPy'
_is_jython = 'java' in sys.platform
_marker = object()
class PickleCacheTests(unittest.TestCase):
......@@ -970,22 +977,6 @@ class PickleCacheTests(unittest.TestCase):
finally:
persistent.picklecache._SWEEPABLE_TYPES = sweep_types
def test_invalidate_not_in_cache(self):
# A contrived test of corruption
cache = self._makeOne()
p = self._makePersist(jar=cache.jar)
p._p_state = 0 # non-ghost, get in the ring
cache[p._p_oid] = p
self.assertEqual(cache.cache_non_ghost_count, 1)
from ..ring import get_object,ffi
self.assertEqual(get_object(cache.ring_home.r_next), p)
cache.ring_home.r_next.object = ffi.NULL
# Nothing to test, just that it doesn't break
cache._invalidate(p._p_oid)
if _is_jython:
def with_deterministic_gc(f):
def test(self):
......@@ -1066,6 +1057,14 @@ class PickleCacheTests(unittest.TestCase):
self.assertTrue(pclass.invalidated)
def test_ring_impl(self):
from .. import ring
if _is_pypy or os.getenv('USING_CFFI'):
self.assertEqual(ring.Ring, ring._CFFIRing)
else:
self.assertEqual(ring.Ring, ring._DequeRing)
class DummyPersistent(object):
def _p_invalidate(self):
......
......@@ -3,7 +3,7 @@ envlist =
# Jython 2.7rc2 does work, but unfortunately has an issue running
# with Tox 1.9.2 (http://bugs.jython.org/issue2325)
# py26,py27,py27-pure,pypy,py32,py33,py34,pypy3,jython,coverage,docs
py26,py27,py27-pure,pypy,py32,py33,py34,pypy3,coverage,docs
py26,py27,py27-pure,py27-pure-cffi,pypy,py32,py33,py34,pypy3,coverage,docs
[testenv]
deps =
......@@ -25,6 +25,19 @@ deps =
commands =
python setup.py test -q
[testenv:py27-pure-cffi]
basepython =
python2.7
setenv =
PURE_PYTHON = 1
USING_CFFI = 1
deps =
{[testenv]deps}
cffi
commands =
python setup.py test -q
[testenv:coverage]
basepython =
python2.6
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment