Skip to content

Z
ZEO
Commit 226ee421 authored by Toby Dickenson's avatar Toby Dickenson
Browse files
Options

merged toby-stiff-cache-branch and toby-unicode-branch

parent 87c6d4a6
Hide whitespace changes
Inline Side-by-side
Showing with 2669 additions and 1429 deletions
src/BTrees/BTreeTemplate.c
View file @ 226ee421
......@@ -12,7 +12,7 @@
****************************************************************************/
#define BTREETEMPLATE_C "$Id: BTreeTemplate.c,v 1.25 2002/03/08 18:33:01 jeremy Exp $\n"
#define BTREETEMPLATE_C "$Id: BTreeTemplate.c,v 1.26 2002/03/27 10:14:01 htrd Exp $\n"
/*
** _BTree_get
......@@ -540,7 +540,7 @@ _BTree_clear(BTree *self)
if (self->firstbucket)
{
ASSERT(self->firstbucket->ob_refcnt > 1,
ASSERT(self->firstbucket->ob_refcnt > 0,
"Invalid firstbucket pointer", -1);
Py_DECREF(self->firstbucket);
self->firstbucket=NULL;
......@@ -573,7 +573,7 @@ BTree__p_deactivate(BTree *self, PyObject *args)
if (self->state==cPersistent_UPTODATE_STATE && self->jar)
{
if (_BTree_clear(self) < 0) return NULL;
self->state=cPersistent_GHOST_STATE;
PER_GHOSTIFY(self);
}
Py_INCREF(Py_None);
......
src/BTrees/BucketTemplate.c
View file @ 226ee421
......@@ -12,7 +12,7 @@
****************************************************************************/
#define BUCKETTEMPLATE_C "$Id: BucketTemplate.c,v 1.28 2002/03/08 18:33:01 jeremy Exp $\n"
#define BUCKETTEMPLATE_C "$Id: BucketTemplate.c,v 1.29 2002/03/27 10:14:01 htrd Exp $\n"
/*
** _bucket_get
......@@ -809,7 +809,7 @@ bucket__p_deactivate(Bucket *self, PyObject *args)
if (self->state==cPersistent_UPTODATE_STATE && self->jar)
{
if (_bucket_clear(self) < 0) return NULL;
self->state=cPersistent_GHOST_STATE;
PER_GHOSTIFY(self);
}
Py_INCREF(Py_None);
......
src/Persistence/cPersistence.c
View file @ 226ee421
......@@ -14,11 +14,21 @@
static char cPersistence_doc_string[] =
"Defines Persistent mixin class for persistent objects.\n"
"\n"
"$Id: cPersistence.c,v 1.50 2002/03/08 18:36:13 jeremy Exp $\n";
"$Id: cPersistence.c,v 1.51 2002/03/27 10:14:04 htrd Exp $\n";
#include <string.h>
#include "cPersistence.h"
/* the layout of this struct is the same as the start of ccobject in cPickleCache.c */
struct ccobject_head_struct {
PyObject_HEAD
CPersistentRing ring_home;
int non_ghost_count;
};
#define HOME(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->ring_home)) )
#define NON_GHOST_COUNT(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->non_ghost_count)) )
#define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
#define UNLESS(E) if(!(E))
#define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
......@@ -112,21 +122,82 @@ if(self->state < 0 && self->jar) \
{ \
PyObject *r; \
\
int *count = NON_GHOST_COUNT(self); \
if(count) \
{ \
(*count)++; \
self->ring.next = HOME(self); \
self->ring.prev = HOME(self)->prev; \
HOME(self)->prev->next = &self->ring; \
HOME(self)->prev = &self->ring; \
Py_INCREF(self); \
} \
self->state=cPersistent_CHANGED_STATE; \
UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self)) \
{ \
self->state=cPersistent_GHOST_STATE; \
ghostify(self); \
return ER; \
} \
self->state=cPersistent_UPTODATE_STATE; \
Py_DECREF(r); \
}
#define KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self) \
if(HOME(self) && self->state>=0) { \
self->ring.prev->next = self->ring.next; \
self->ring.next->prev = self->ring.prev; \
self->ring.next = HOME(self); \
self->ring.prev = HOME(self)->prev; \
HOME(self)->prev->next = &self->ring; \
HOME(self)->prev = &self->ring; }
/****************************************************************************/
staticforward PyExtensionClass Pertype;
static void
accessed(cPersistentObject *self)
{
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
}
static void
ghostify(cPersistentObject *self)
{
int *count;
count = NON_GHOST_COUNT(self);
if(count && (self->state>=0))
{
(*count)--;
self->ring.next->prev = self->ring.prev;
self->ring.prev->next = self->ring.next;
self->ring.prev = NULL;
self->ring.next = NULL;
self->state = cPersistent_GHOST_STATE;
Py_DECREF(self);
}
else
{
self->state = cPersistent_GHOST_STATE;
}
}
static void
deallocated(cPersistentObject *self)
{
if(self->state>=0) ghostify(self);
if(self->cache)
{
PyObject *v=PyObject_CallMethod(self->cache,"_oid_unreferenced","O",self->oid);
if(!v) PyErr_Clear(); /* and explode later */
Py_XDECREF(v);
}
Py_XDECREF(self->jar);
Py_XDECREF(self->oid);
}
static int
changed(cPersistentObject *self)
{
......@@ -185,7 +256,7 @@ Per___changed__(cPersistentObject *self, PyObject *args)
static PyObject *
Per__p_deactivate(cPersistentObject *self, PyObject *args)
{
PyObject *dict;
PyObject *dict,*dict2=NULL;
#ifdef DEBUG_LOG
if (idebug_log < 0) call_debug("reinit",self);
......@@ -197,13 +268,22 @@ Per__p_deactivate(cPersistentObject *self, PyObject *args)
if (self->state==cPersistent_UPTODATE_STATE && self->jar &&
HasInstDict(self) && (dict=INSTANCE_DICT(self)))
{
dict2 = PyDict_Copy(dict);
PyDict_Clear(dict);
/* Note that we need to set to ghost state unless we are
called directly. Methods that override this need to
do the same! */
self->state=cPersistent_GHOST_STATE;
ghostify(self);
}
/* need to delay releasing the last reference on instance attributes
until after we have finished accounting for losing our state */
if(dict2)
{
PyDict_Clear(dict2);
Py_DECREF(dict2);
}
Py_INCREF(Py_None);
return Py_None;
}
......@@ -333,8 +413,8 @@ Per_dealloc(cPersistentObject *self)
#ifdef DEBUG_LOG
if(idebug_log < 0) call_debug("del",self);
#endif
Py_XDECREF(self->jar);
Py_XDECREF(self->oid);
deallocated(self);
Py_XDECREF(self->cache);
Py_DECREF(self->ob_type);
PyObject_DEL(self);
}
......@@ -387,7 +467,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
{
UPDATE_STATE_IF_NECESSARY(self, NULL);
self->atime=((long)(time(NULL)/3))%65536;
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
if (self->serial[7]=='\0' && self->serial[6]=='\0' &&
self->serial[5]=='\0' && self->serial[4]=='\0' &&
......@@ -419,7 +499,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
{
UPDATE_STATE_IF_NECESSARY(self, NULL);
self->atime=((long)(time(NULL)/3))%65536;
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
}
return getattrf((PyObject *)self, oname);
......@@ -466,6 +546,21 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
{
if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6])
{
if(HOME(self))
{
int result;
if(!v)
{
PyErr_SetString(PyExc_ValueError,"can not delete the oid of a cached object");
return -1;
}
if(PyObject_Cmp(self->oid,v,&result)<0) return -1;
if(result)
{
PyErr_SetString(PyExc_ValueError,"can not change the oid of a cached object");
return -1;
}
}
Py_XINCREF(v);
ASSIGN(self->oid, v);
return 0;
......@@ -509,7 +604,6 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
v=PyObject_GetAttr(OBJECT(self), py__p_deactivate);
if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); }
if (v) { Py_DECREF(v); }
self->state=cPersistent_GHOST_STATE;
return 0;
}
if (PyObject_IsTrue(v)) return changed(self);
......@@ -521,8 +615,7 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
{
UPDATE_STATE_IF_NECESSARY(self, -1);
/* Record access times */
self->atime=((long)(time(NULL)/3))%65536;
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
if((! (*name=='_' && name[1]=='v' && name[2]=='_'))
&& (self->state != cPersistent_CHANGED_STATE && self->jar)
......@@ -680,9 +773,11 @@ truecPersistenceCAPI = {
(getattrofunc)Per_getattro, /*tp_getattr with object key*/
(setattrofunc)Per_setattro, /*tp_setattr with object key*/
changed,
accessed,
ghostify,
deallocated,
(intfunctionwithpythonarg)Per_setstate,
(pergetattr)Per_getattr,
(persetattr)_setattro,
};
void
......
src/Persistence/cPersistence.h
View file @ 226ee421
......@@ -18,12 +18,21 @@
#include "ExtensionClass.h"
#include <time.h>
#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid; char serial[8]; unsigned short atime; signed char state; unsigned char reserved;
#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid, *cache; CPersistentRing ring; char serial[8]; signed char state; unsigned char reserved[3];
#define cPersistent_GHOST_STATE -1
#define cPersistent_UPTODATE_STATE 0
#define cPersistent_CHANGED_STATE 1
#define cPersistent_STICKY_STATE 2
struct ccobject_head_struct;
typedef struct CPersistentRing_struct
{
struct CPersistentRing_struct *prev;
struct CPersistentRing_struct *next;
} CPersistentRing;
typedef struct {
cPersistent_HEAD
} cPersistentObject;
......@@ -36,6 +45,9 @@ typedef struct {
getattrofunc getattro;
setattrofunc setattro;
int (*changed)(cPersistentObject*);
void (*accessed)(cPersistentObject*);
void (*ghostify)(cPersistentObject*);
void (*deallocated)(cPersistentObject*);
int (*setstate)(PyObject*);
pergetattr pergetattro;
persetattr persetattro;
......@@ -59,11 +71,13 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
#define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O)))
#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
#define PER_GHOSTIFY(O) (cPersistenceCAPI->ghostify((cPersistentObject*)(O)))
#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
#define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
#define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
#define PER_DEL(O) Py_XDECREF((O)->jar); Py_XDECREF((O)->oid);
#define PER_DEL(O) (cPersistenceCAPI->deallocated((cPersistentObject*)(O)))
#define PER_USE(O) \
(((O)->state != cPersistent_GHOST_STATE \
......@@ -71,7 +85,7 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
? (((O)->state==cPersistent_UPTODATE_STATE) \
? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0)
#define PER_ACCESSED(O) ((O)->atime=((long)(time(NULL)/3))%65536)
#define PER_ACCESSED(O) (cPersistenceCAPI->accessed((cPersistentObject*)(O)))
#endif
......
src/Persistence/cPickleCache.c
View file @ 226ee421
......@@ -2,448 +2,531 @@
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
static char cPickleCache_doc_string[] =
static char cPickleCache_doc_string[] =
"Defines the PickleCache used by ZODB Connection objects.\n"
"\n"
"$Id: cPickleCache.c,v 1.41 2002/03/08 18:36:14 jeremy Exp $\n";
"$Id: cPickleCache.c,v 1.42 2002/03/27 10:14:04 htrd Exp $\n";
/* Compute the current time in the units and range used for peristent
objects. */
#define PER_TIME() ((long)(time(NULL) / 3)) % 65536
#define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
#define UNLESS(E) if(!(E))
#define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
#define OBJECT(O) ((PyObject*)O)
#define DONT_USE_CPERSISTENCECAPI
#include "cPersistence.h"
#include <time.h>
#include <stddef.h>
#undef Py_FindMethod
static PyObject *py_reload, *py__p_jar, *py__p_changed;
typedef struct {
PyObject_HEAD
PyObject *data;
PyObject *jar;
PyObject *setklassstate;
int position;
int cache_size;
int cache_age;
/* Cache statistics */
int sum_deal;
int sum_deac;
double sum_age;
int n, na;
time_t last_check; /* Time of last gc */
double mean_age;
double mean_deal;
double mean_deac;
double df, dfa; /* Degees of freedom for above stats */
} ccobject;
static PyObject *py__p_oid, *py_reload, *py__p_jar, *py__p_changed;
#define WEIGHTING_PERIOD 600
/* define this for extra debugging checks, and lousy performance */
#define MUCH_RING_CHECKING 1
/*
How to compute weighted means?
/* Do we want 'engine noise'.... abstract debugging output useful for
visualizing cache behavior */
#if 0
#define ENGINE_NOISE(A) printf(A)
#else
#define ENGINE_NOISE(A) ((void)A)
#endif
Assume we have two means, a current mean, M, and a mean as of some
time d seconds in the past, Md. The means have effective degrees
of freedom, N, and Nd. Where Nd is adjusted by d is some fashion.
The combined mean is (M*N+Md*Nd)/(N+Nd). The degrees of freedom
of the combined mean, Nc, is N+Nd. Nd is computed by weighting
an old degree of freedom with the weight: I/(I+d), where I is some
suitably chosen constant, which we will call a "weighting period".
*/
/* the layout of this struct is the same as the start of ccobject_head in cPersistence.c */
typedef struct {
PyObject_HEAD
CPersistentRing ring_home;
int non_ghost_count;
int klass_count;
PyObject *data;
PyObject *jar;
PyObject *setklassstate;
int cache_size;
int ring_lock;
int cache_drain_resistance;
} ccobject;
staticforward PyTypeObject Cctype;
staticforward int present_in_ring(ccobject *self,CPersistentRing *target);
staticforward int check_ring(ccobject *self,const char *context);
staticforward int cc_ass_sub(ccobject *self, PyObject *key, PyObject *v);
/* ---------------------------------------------------------------- */
static int
gc_item(ccobject *self, PyObject *key, PyObject *v, long now, int dt)
static PyObject *object_from_oid(ccobject *self,PyObject *key)
/* somewhat of a replacement for PyDict_GetItem(self->data....
however this returns a *new* reference */
{
if (!(v && key))
return 0;
self->n++;
/* If there is at most one reference to this object, then the
cache has the only reference. It can be removed. */
if (v->ob_refcnt <= 1) {
self->sum_deal++;
/* XXX The fact that this works will iterating over
self->data with PyDict_Next() is an accident of the
current Python dictionary implementation. */
return PyDict_DelItem(self->data, key);
}
PyObject *v = PyDict_GetItem(self->data, key);
if(!v) return NULL;
if (dt >= 0 &&
(!PyExtensionClass_Check(v)) &&
((cPersistentObject*)v)->jar == self->jar /* I'm paranoid */ &&
((cPersistentObject*)v)->state == cPersistent_UPTODATE_STATE) {
now -= ((cPersistentObject*)v)->atime;
if (now < 0)
now += 65536;
self->na++;
self->sum_age += now;
if (now > dt) {
/* We have a cPersistent object that hasn't been used in
a while. Reinitialize it, hopefully freeing it's
state.
*/
self->sum_deac++;
if (PyObject_SetAttr(v, py__p_changed, Py_None) < 0)
PyErr_Clear();
}
}
return 0;
Py_INCREF(v);
return v;
}
static void
update_stats(ccobject *self, time_t now)
static cPersistentObject *object_from_ring(ccobject *self,CPersistentRing *here,const char *context)
{
double d, deal, deac;
d = now - self->last_check;
if(d < 1)
return;
self->df *= WEIGHTING_PERIOD / (WEIGHTING_PERIOD + d);
self->dfa *= WEIGHTING_PERIOD / (WEIGHTING_PERIOD + d);
self->mean_age = ((self->mean_age * self->dfa + self->sum_age)/
(self->dfa + self->na)) * 3;
self->sum_age = 0;
deac = self->sum_deac / d;
self->sum_deac = 0;
self->mean_deac = ((self->mean_deac * self->dfa+deac)/
(self->dfa + self->na));
self->sum_deac = 0;
self->dfa += self->na;
self->na = 0;
deal=self->sum_deal/d;
self->sum_deal = 0;
self->mean_deal = ((self->mean_deal * self->df + deal)/
(self->df +self->n));
self->sum_deal = 0;
self->df += self->n;
self->n = 0;
self->last_check = now;
/* Given a position in the LRU ring, return a borrowed
reference to the object at that point in the ring. The caller is
responsible for ensuring that this ring position really does
correspond to a persistent object, although the debugging
version will double-check this. */
PyObject *object = (PyObject *)(((char *)here)-offsetof(cPersistentObject,ring));
#ifdef MUCH_RING_CHECKING
if(!PyExtensionInstance_Check(object))
{
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered non-ExtensionClass object in %s",context);
return NULL;
}
if(!(((PyExtensionClass*)(object->ob_type))->class_flags & PERSISTENT_TYPE_FLAG))
{
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered non-persistent object in %s",context);
return NULL;
}
if(((cPersistentObject*)object)->jar!=self->jar)
{
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered object from a different jar in %s",context);
return NULL;
}
if(((cPersistentObject*)object)->cache!=(PyObject *)self)
{
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered broken ring in %s",context);
return NULL;
}
#endif
return (cPersistentObject *)object;
}
static int
check_size(ccobject *self)
scan_gc_items(ccobject *self,int target)
{
if (self->cache_size < 1)
return 0;
return PyDict_Size(self->data);
cPersistentObject *object;
int error;
CPersistentRing placeholder;
CPersistentRing *here = self->ring_home.next;
#ifdef MUCH_RING_CHECKING
int safety_counter = self->cache_size*10;
if(safety_counter<10000) safety_counter = 10000;
#endif
while(1)
{
if(check_ring(self,"mid-gc")) return -1;
#ifdef MUCH_RING_CHECKING
if(!safety_counter--)
{
/* This loop has been running for a very long time.
It is possible that someone loaded a very large number of objects,
and now wants us to blow them all away. However it may
also indicate a logic error. If the loop has been running this
long then you really have to doubt it will ever terminate.
In the MUCH_RING_CHECKING build we prefer to raise an exception
here */
PyErr_SetString(PyExc_RuntimeError,"scan_gc_items safety counter exceeded");
return -1;
}
if(!present_in_ring(self,here))
{
/* Our current working position is no longer in the ring. Thats bad. */
PyErr_SetString(PyExc_RuntimeError,"working position fell out the ring, in scan_gc_items");
return -1;
}
#endif
if(here==&self->ring_home)
{
/* back to the home position. stop looking */
return 0;
}
/* At this point we know that the ring only contains nodes from
persistent objects, plus our own home node. We can safely
assume this is a persistent object now we know it is not the home */
object = object_from_ring(self,here,"scan_gc_items");
if(!object) return -1;
if(self->non_ghost_count<=target)
{
/* we are small enough */
return 0;
}
else if(object->state==cPersistent_UPTODATE_STATE)
{
/* deactivate it. This is the main memory saver. */
ENGINE_NOISE("G");
/* add a placeholder */
placeholder.next = here->next;
placeholder.prev = here;
here->next->prev = &placeholder;
here->next = &placeholder;
error = PyObject_SetAttr((PyObject *)object,py__p_changed,Py_None);
/* unlink the placeholder */
placeholder.next->prev=placeholder.prev;
placeholder.prev->next=placeholder.next;
here = placeholder.next;
if(error)
return -1; /* problem */
}
else
{
ENGINE_NOISE(".");
here = here->next;
}
}
}
static int
gc_all_items(ccobject *self, int now, int dt)
static PyObject *
lockgc(ccobject *self,int target_size)
{
PyObject *key, *v;
int i;
if(self->ring_lock)
{
Py_INCREF(Py_None);
return Py_None;
}
for(i = 0; PyDict_Next(self->data, &i, &key, &v); )
if (gc_item(self, key, v, now, dt) < 0)
return -1;
return 0;
if(check_ring(self,"pre-gc")) return NULL;
ENGINE_NOISE("<");
self->ring_lock = 1;
if(scan_gc_items(self,target_size))
{
self->ring_lock = 0;
return NULL;
}
self->ring_lock = 0;
ENGINE_NOISE(">\n");
if(check_ring(self,"post-gc")) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static int
fullgc(ccobject *self, int dt)
static PyObject *
cc_incrgc(ccobject *self, PyObject *args)
{
long now;
int n=1;
if (check_size(self) <= 0)
return 0;
int starting_size = self->non_ghost_count;
now = PER_TIME();
dt /= 3;
int target_size = self->cache_size;
if (gc_all_items(self, now, dt) < 0)
return -1;
self->position = 0;
if(self->cache_drain_resistance>=1)
{
/* This cache will gradually drain down to a small size. Check
a (small) number of objects proportional to the current size */
if (now - self->last_check > 1)
update_stats(self, now);
return 0;
}
static int
reallyfullgc(ccobject *self, int dt)
{
int l, last;
time_t now;
last = check_size(self);
if (last <= 0)
return 0;
now = PER_TIME();
/* Units are 3 seconds */
dt /= 3;
/* First time through should get refcounts to 1 */
if (gc_all_items(self, now, dt) < 0)
return -1;
l = PyDict_Size(self->data);
if (l < 0)
return -1;
/* Now continue to collect until the size of the cache stops
decreasing. */
while (l < last) {
if (gc_all_items(self, now, dt) < 0)
return -1;
last = l;
l = PyDict_Size(self->data);
if (l < 0)
return -1;
int target_size_2 = starting_size - 1 - starting_size/self->cache_drain_resistance;
if(target_size_2<target_size)
target_size = target_size_2;
}
if (now - self->last_check > 1)
update_stats(self, now);
self->position = 0;
return 0;
}
static int
maybegc(ccobject *self, PyObject *thisv)
{
int n, s, size, dt;
long now;
PyObject *key=0, *v=0;
s = check_size(self);
if (s <= 0)
return 0;
now = PER_TIME();
size = self->cache_size;
self->cache_size = 0;
/* Decide how many objects to look at */
n = (s - size) / 10;
if (n < 3)
n = 3;
/* Decide how much time to give them before deactivating them */
s = 8 * size / s;
if (s > 100)
s = 100;
dt = (long)(self->cache_age * (0.2 + 0.1 * s));
/* Units are 3 seconds */
dt /= 3;
while (--n >= 0) {
if (PyDict_Next(self->data, &(self->position), &key, &v)) {
if (v != thisv && gc_item(self, key, v, now, dt) < 0) {
self->cache_size=size;
return -1;
}
}
else
self->position = 0;
}
self->cache_size = size;
if (now - self->last_check > 1)
update_stats(self, now);
UNLESS (PyArg_ParseTuple(args, "|i",&n)) return NULL;
return 0;
return lockgc(self,target_size);
}
static PyObject *
cc_full_sweep(ccobject *self, PyObject *args)
{
int dt = self->cache_age;
if (!PyArg_ParseTuple(args, "|i:full_sweep", &dt))
return NULL;
if (dt < -1) {
PyErr_SetString(PyExc_ValueError, "age must be >= -1");
return NULL;
}
if (fullgc(self, dt) == -1)
return NULL;
Py_INCREF(Py_None);
return Py_None;
int dt=0;
UNLESS(PyArg_ParseTuple(args, "|i", &dt)) return NULL;
return lockgc(self,0);
}
static PyObject *
cc_reallyfull_sweep(ccobject *self, PyObject *args)
{
int dt = self->cache_age;
if (!PyArg_ParseTuple(args, "|i:minimize", &dt))
return NULL;
if (dt < -1) {
PyErr_SetString(PyExc_ValueError, "age must be >= -1");
return NULL;
}
if (reallyfullgc(self, dt) == -1)
return NULL;
Py_INCREF(Py_None);
return Py_None;
int dt=0;
UNLESS(PyArg_ParseTuple(args, "|i", &dt)) return NULL;
return lockgc(self,0);
}
static PyObject *
cc_incrgc(ccobject *self, PyObject *args)
static void
_invalidate(ccobject *self, PyObject *key)
{
int n = 1;
PyObject *v=object_from_oid(self, key);
if (!PyArg_ParseTuple(args, "|i:incrgr", &n))
return NULL;
if(!v)
{
/* shouldnt this be an error? for now Ill follow Jims lead */
PyErr_Clear();
}
else
{
if (PyExtensionClass_Check(v))
{
if(v->ob_refcnt <= 1)
{
self->klass_count--;
if (PyDict_DelItem(self->data, key) < 0)
PyErr_Clear();
}
else
{
v=PyObject_CallFunction(self->setklassstate,
"O", v);
if (v) Py_DECREF(v);
else PyErr_Clear();
}
}
else
{
if(PyObject_DelAttr(v,py__p_changed) < 0)
PyErr_Clear();
}
Py_DECREF(v);
}
}
for (; --n >= 0;)
if (maybegc(self, NULL) < 0)
return NULL;
static PyObject *
cc_invalidate(ccobject *self, PyObject *args)
{
PyObject *inv, *key, *v;
int i;
if (PyArg_ParseTuple(args, "O!", &PyDict_Type, &inv)) {
for (i=0; PyDict_Next(inv, &i, &key, &v); )
if (key==Py_None)
{ /* Eek some nitwit invalidated everything! */
for (i=0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
break;
}
else
_invalidate(self, key);
PyDict_Clear(inv);
}
else {
PyErr_Clear();
UNLESS (PyArg_ParseTuple(args, "O", &inv)) return NULL;
if (PyString_Check(inv))
_invalidate(self, inv);
else if (inv==Py_None) /* All */
for (i=0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
else {
int l;
PyErr_Clear();
if ((l=PyObject_Length(inv)) < 0) return NULL;
for(i=l; --i >= 0; )
{
UNLESS (key=PySequence_GetItem(inv, i)) return NULL;
_invalidate(self, key);
Py_DECREF(key);
}
PySequence_DelSlice(inv, 0, l);
}
}
Py_INCREF(Py_None);
return Py_None;
}
static void
_invalidate(ccobject *self, PyObject *key)
static PyObject *
cc_get(ccobject *self, PyObject *args)
{
PyObject *v = PyDict_GetItem(self->data, key);
PyObject *r, *key, *d=0;
if (!v)
return;
if (PyExtensionClass_Check(v))
if (v->ob_refcnt <= 1) {
self->sum_deal++;
if (PyDict_DelItem(self->data, key) < 0)
PyErr_Clear();
} else {
PyObject *t = PyTuple_New(1);
if (t) {
PyTuple_SET_ITEM(t, 0, v);
v = PyObject_CallObject(self->setklassstate, t);
/* Set tuple element to NULL so that deallocating the
tuple does not decref t.
*/
PyTuple_SET_ITEM(t, 0, NULL);
Py_DECREF(t);
} else
v = t;
if (v)
Py_DECREF(v);
else
PyErr_Clear();
UNLESS (PyArg_ParseTuple(args,"O|O", &key, &d)) return NULL;
UNLESS (r=(PyObject *)object_from_oid(self, key))
{
if (d)
{
PyErr_Clear();
r=d;
Py_INCREF(r);
}
else
{
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
else if (PyObject_DelAttr(v, py__p_changed) < 0)
PyErr_Clear();
}
return r;
}
static void
_invalidate_all(ccobject *self)
static PyObject *
cc_klass_items(ccobject *self, PyObject *args)
{
PyObject *key, *v;
int i;
PyObject *l,*k,*v;
int p = 0;
if(!PyArg_ParseTuple(args,"")) return NULL;
l = PyList_New(0);
if(!l) return NULL;
while(PyDict_Next(self->data, &p, &k, &v))
{
if(PyExtensionClass_Check(v))
{
v=PyObject_CallMethod(l,"append","((OO))",k,v);
if(!v)
{
Py_DECREF(l);
return NULL;
}
}
}
for (i = 0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
return l;
}
static PyObject *
cc_invalidate(ccobject *self, PyObject *args)
cc_lru_items(ccobject *self, PyObject *args)
{
PyObject *inv, *key, *v;
int i;
PyObject *l;
CPersistentRing *here;
if (!PyArg_ParseTuple(args, "O:invalidate", &inv))
return NULL;
if (PyDict_Check(inv)) {
for (i = 0; PyDict_Next(inv, &i, &key, &v); )
if (key == Py_None) {
/* Eek some nitwit invalidated everything! */
_invalidate_all(self);
break;
}
else
_invalidate(self, key);
PyDict_Clear(inv);
} else if (PyString_Check(inv))
_invalidate(self, inv);
else if (inv == Py_None) /* All */
_invalidate_all(self);
else {
int l = PyObject_Length(inv);
if (l < 0)
return NULL;
for (i = l; --i >= 0; ) {
key = PySequence_GetItem(inv, i);
if (!key)
return NULL;
_invalidate(self, key);
Py_DECREF(key);
}
PySequence_DelSlice(inv, 0, l);
if(!PyArg_ParseTuple(args,"")) return NULL;
if(self->ring_lock)
{
PyErr_SetString(PyExc_ValueError,".lru_items() is unavailable during garbage collection");
return NULL;
}
Py_INCREF(Py_None);
return Py_None;
if(check_ring(self,"pre-cc_items")) return NULL;
l = PyList_New(0);
if(!l) return NULL;
here = self->ring_home.next;
while(here!=&self->ring_home)
{
cPersistentObject *object = object_from_ring(self,here,"cc_items");
PyObject *v;
if(!object)
{
Py_DECREF(l);
return NULL;
}
v=PyObject_CallMethod(l,"append","((OO))",object->oid,object);
if(!v)
{
Py_DECREF(l);
return NULL;
}
Py_DECREF(v);
here = here->next;
}
return l;
}
static PyObject *
cc_get(ccobject *self, PyObject *args)
cc_oid_unreferenced(ccobject *self, PyObject *args)
{
PyObject *r, *key, *d = NULL;
PyObject *oid,*v;
if(!PyArg_ParseTuple(args,"O",&oid)) return NULL;
if (!PyArg_ParseTuple(args, "O|O:get", &key, &d))
return NULL;
r = PyDict_GetItem(self->data, key);
if (!r) {
if (d)
r = d;
else {
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
v = PyDict_GetItem(self->data, oid);
if(!v) return NULL;
if(v->ob_refcnt)
{
PyErr_Format(PyExc_ValueError,"object has reference count of %d, should be zero",v->ob_refcnt);
return NULL;
}
Py_INCREF(r);
return r;
/* Need to be very hairy here because a dictionary is about
to decref an already deleted object */
#ifdef Py_TRACE_REFS
#error "this code path has not been tested - Toby Dickenson"
_Py_NewReference(v);
/* it may be a problem that v->ob_type is still NULL? */
#else
Py_INCREF(v);
#endif
if(v->ob_refcnt!=1)
{
PyErr_SetString(PyExc_ValueError,"refcount is not 1 after resurrection");
return NULL;
}
/* return the stolen reference */
Py_INCREF(v);
PyDict_DelItem(self->data, oid);
if(v->ob_refcnt!=1)
{
PyErr_SetString(PyExc_ValueError,"refcount is not 1 after removal from dict");
return NULL;
}
/* undo the temporary resurrection */
#ifdef Py_TRACE_REFS
_Py_ForgetReference(v);
#else
v->ob_refcnt=0;
#endif
Py_INCREF(Py_None);
return Py_None;
}
static struct PyMethodDef cc_methods[] = {
{"_oid_unreferenced", (PyCFunction)cc_oid_unreferenced, METH_VARARGS,
NULL
},
{"lru_items", (PyCFunction)cc_lru_items, METH_VARARGS,
"List (oid, object) pairs from the lru list, as 2-tuples.\n"
},
{"klass_items", (PyCFunction)cc_klass_items, METH_VARARGS,
"List (oid, object) pairs of cached persistent classes.\n"
},
{"full_sweep", (PyCFunction)cc_full_sweep, METH_VARARGS,
"full_sweep([age]) -- Perform a full sweep of the cache\n\n"
"Make a single pass through the cache, removing any objects that are no\n"
"longer referenced, and deactivating objects that have not been\n"
"accessed in the number of seconds given by 'age'. "
"'age defaults to the cache age.\n"
"longer referenced, and deactivating enough objects to bring\n"
"the cache under its size limit\n"
"The optional 'age' parameter is ignored.\n"
},
{"minimize", (PyCFunction)cc_reallyfull_sweep, METH_VARARGS,
"minimize([age]) -- Remove as many objects as possible\n\n"
"Make multiple passes through the cache, removing any objects that are no\n"
"longer referenced, and deactivating objects that have not been\n"
"accessed in the number of seconds given by 'age'. 'age defaults to 0.\n"
"longer referenced, and deactivating enough objects to bring the"
" cache under its size limit\n"
"The option 'age' parameter is ignored.\n"
},
{"incrgc", (PyCFunction)cc_incrgc, METH_VARARGS,
"incrgc() -- Perform incremental garbage collection"},
"incrgc([n]) -- Perform incremental garbage collection\n\n"
"Some other implementations support an optional parameter 'n' which\n"
"indicates a repetition count; this value is ignored.\n"},
{"invalidate", (PyCFunction)cc_invalidate, METH_VARARGS,
"invalidate(oids) -- invalidate one, many, or all ids"},
{"get", (PyCFunction)cc_get, METH_VARARGS,
......@@ -454,115 +537,102 @@ static struct PyMethodDef cc_methods[] = {
static ccobject *
newccobject(PyObject *jar, int cache_size, int cache_age)
{
ccobject *self;
ccobject *self;
self = PyObject_NEW(ccobject, &Cctype);
if (!self)
UNLESS(self = PyObject_NEW(ccobject, &Cctype)) return NULL;
self->setklassstate=self->jar=NULL;
if((self->data=PyDict_New()))
{
self->jar=jar;
Py_INCREF(jar);
UNLESS (self->setklassstate=PyObject_GetAttrString(jar, "setklassstate"))
return NULL;
self->setklassstate = self->jar = NULL;
self->data = PyDict_New();
if (self->data) {
self->jar=jar;
Py_INCREF(jar);
self->setklassstate = PyObject_GetAttrString(jar, "setklassstate");
if (!self->setklassstate) {
Py_DECREF(jar);
Py_DECREF(self->data);
goto error;
}
self->position = 0;
self->cache_size = cache_size;
self->cache_age = cache_age < 1 ? 1 : cache_age;
self->sum_deal = 0;
self->sum_deac = 0;
self->sum_age = 0;
self->mean_deal = 0;
self->mean_deac = 0;
self->mean_age = 0;
self->df = 1;
self->dfa = 1;
self->n = 0;
self->na = 0;
self->last_check = time(NULL);
return self;
self->cache_size=cache_size;
self->non_ghost_count=0;
self->klass_count=0;
self->cache_drain_resistance=0;
self->ring_lock=0;
self->ring_home.next = &self->ring_home;
self->ring_home.prev = &self->ring_home;
return self;
}
error:
Py_DECREF(self);
return NULL;
Py_DECREF(self);
return NULL;
}
static void
cc_dealloc(ccobject *self)
{
Py_XDECREF(self->data);
Py_XDECREF(self->jar);
Py_XDECREF(self->setklassstate);
PyObject_DEL(self);
Py_XDECREF(self->data);
Py_XDECREF(self->jar);
Py_XDECREF(self->setklassstate);
PyMem_DEL(self);
}
static PyObject *
cc_getattr(ccobject *self, char *name)
{
PyObject *r;
if (*name == 'c') {
if(strcmp(name, "cache_age") == 0)
return PyInt_FromLong(self->cache_age);
if(strcmp(name, "cache_size") == 0)
return PyInt_FromLong(self->cache_size);
if(strcmp(name, "cache_mean_age") == 0)
return PyFloat_FromDouble(self->mean_age);
if(strcmp(name, "cache_mean_deal") == 0)
return PyFloat_FromDouble(self->mean_deal);
if(strcmp(name, "cache_mean_deac") == 0)
return PyFloat_FromDouble(self->mean_deac);
if(strcmp(name, "cache_df") == 0)
return PyFloat_FromDouble(self->df);
if(strcmp(name, "cache_dfa") == 0)
return PyFloat_FromDouble(self->dfa);
if(strcmp(name, "cache_last_gc_time") == 0)
return PyFloat_FromDouble(self->last_check);
if(strcmp(name, "cache_data") == 0) {
Py_INCREF(self->data);
return self->data;
PyObject *r;
if(check_ring(self,"getattr")) return NULL;
if(*name=='c')
{
if(strcmp(name,"cache_age")==0)
return PyInt_FromLong(0); /* this cache does not use this value */
if(strcmp(name,"cache_size")==0)
return PyInt_FromLong(self->cache_size);
if(strcmp(name,"cache_drain_resistance")==0)
return PyInt_FromLong(self->cache_drain_resistance);
if(strcmp(name,"cache_non_ghost_count")==0)
return PyInt_FromLong(self->non_ghost_count);
if(strcmp(name,"cache_klass_count")==0)
return PyInt_FromLong(self->klass_count);
if(strcmp(name,"cache_data")==0)
{
/* now a copy of our data; the ring is too fragile */
return PyDict_Copy(self->data);
}
}
if ((strcmp(name, "has_key") == 0)
|| (strcmp(name, "items") == 0)
|| (strcmp(name, "keys") == 0))
return PyObject_GetAttrString(self->data, name);
r = Py_FindMethod(cc_methods, (PyObject *)self, name);
if (!r) {
PyErr_Clear();
return PyObject_GetAttrString(self->data, name);
}
if((*name=='h' && strcmp(name, "has_key")==0) ||
(*name=='i' && strcmp(name, "items")==0) ||
(*name=='k' && strcmp(name, "keys")==0)
)
return PyObject_GetAttrString(self->data, name);
if((r=Py_FindMethod(cc_methods, (PyObject *)self, name)))
return r;
PyErr_Clear();
return PyObject_GetAttrString(self->data, name);
}
static int
cc_setattr(ccobject *self, char *name, PyObject *value)
{
if (value) {
if(value)
{
int v;
if (strcmp(name, "cache_age") == 0) {
v = PyInt_AsLong(value);
if (v == -1 && PyErr_Occurred())
return -1;
if (v > 0)
self->cache_age = v;
if(strcmp(name,"cache_age")==0)
{
/* this cache doesnt use the age */
return 0;
}
}
if (strcmp(name, "cache_size") == 0) {
v = PyInt_AsLong(value);
if (v == -1 && PyErr_Occurred())
return -1;
self->cache_size = v;
if(strcmp(name,"cache_size")==0)
{
UNLESS(PyArg_Parse(value,"i",&v)) return -1;
self->cache_size=v;
return 0;
}
}
}
if(strcmp(name,"cache_drain_resistance")==0)
{
UNLESS(PyArg_Parse(value,"i",&v)) return -1;
self->cache_drain_resistance=v;
return 0;
}
}
PyErr_SetString(PyExc_AttributeError, name);
return -1;
}
......@@ -570,7 +640,7 @@ cc_setattr(ccobject *self, char *name, PyObject *value)
static int
cc_length(ccobject *self)
{
return PyDict_Size(self->data);
return PyObject_Length(self->data);
}
static PyObject *
......@@ -578,39 +648,245 @@ cc_subscript(ccobject *self, PyObject *key)
{
PyObject *r;
r = PyDict_GetItem(self->data, key);
if (!r) {
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
if(check_ring(self,"__getitem__")) return NULL;
UNLESS (r=(PyObject *)object_from_oid(self, key))
{
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
Py_INCREF(r);
return r;
}
static int
cc_ass_sub(ccobject *self, PyObject *key, PyObject *v)
{
if (v) {
if (PyExtensionClass_Check(v)
||
(PyExtensionInstance_Check(v)
&&
(((PyExtensionClass*)(v->ob_type))->class_flags
& PERSISTENT_TYPE_FLAG)
&&
(v->ob_type->tp_basicsize >= sizeof(cPersistentObject))
)
)
return PyDict_SetItem(self->data, key, v);
PyErr_SetString(PyExc_ValueError,
"Cache values must be persistent objects or classes.");
return -1;
int result;
if(v)
{
if( ( PyExtensionInstance_Check(v) &&
(((PyExtensionClass*)(v->ob_type))->class_flags & PERSISTENT_TYPE_FLAG) &&
(v->ob_type->tp_basicsize >= sizeof(cPersistentObject))
)
||
PyExtensionClass_Check(v)
)
{
PyObject *oid = PyObject_GetAttr(v,py__p_oid);
PyObject *object_again;
if(!oid)
{
return -1;
}
if(PyObject_Cmp(key,oid,&result))
{
Py_DECREF(oid);
return -1;
}
Py_DECREF(oid);
if(result)
{
PyErr_SetString(PyExc_ValueError,"key must be the same as the object's oid attribute");
return -1;
}
object_again = object_from_oid(self, key);
if(object_again)
{
if(object_again!=v)
{
Py_DECREF(object_again);
PyErr_SetString(PyExc_ValueError,"Can not re-register object under a different oid");
return -1;
}
else
{
/* re-register under the same oid - no work needed */
Py_DECREF(object_again);
return 0;
}
}
if(PyExtensionClass_Check(v))
{
if(PyDict_SetItem(self->data, key, v)) return -1;
self->klass_count++;
return 0;
}
else
{
if(((cPersistentObject*)v)->cache)
{
if(((cPersistentObject*)v)->cache==(PyObject *)self)
{
/* This object is already one of ours, which is ok.
It would be very strange if someone was trying to register the
same object under a different key */
}
else
{
/* This object is already in a different cache. */
PyErr_SetString(PyExc_ValueError, "Cache values may only be in one cache.");
return -1;
}
}
if(check_ring(self,"pre-setitem")) return -1;
if(PyDict_SetItem(self->data, key, v)) return -1;
Py_INCREF(self);
((cPersistentObject*)v)->cache = (PyObject *)self;
if(((cPersistentObject*)v)->state>=0)
{
/* insert this non-ghost object into the ring just behind the home position */
self->non_ghost_count++;
((cPersistentObject*)v)->ring.next = &self->ring_home;
((cPersistentObject*)v)->ring.prev = self->ring_home.prev;
self->ring_home.prev->next = &((cPersistentObject*)v)->ring;
self->ring_home.prev = &((cPersistentObject*)v)->ring;
}
else
{
/* steal a reference from the dictionary; ghosts have a weak reference */
Py_DECREF(v);
}
if(check_ring(self,"post-setitem")) return -1;
return 0;
}
}
else
{
PyErr_SetString(PyExc_ValueError, "Cache values must be persistent objects.");
return -1;
}
}
else
{
/* unlink this item from the ring */
if(check_ring(self,"pre-delitem")) return -1;
v = (PyObject *)object_from_oid(self,key);
if(!v) return -1;
if(PyExtensionClass_Check(v))
{
self->klass_count--;
}
else
{
if(((cPersistentObject*)v)->state>=0)
{
self->non_ghost_count--;
((cPersistentObject*)v)->ring.next->prev = ((cPersistentObject*)v)->ring.prev;
((cPersistentObject*)v)->ring.prev->next = ((cPersistentObject*)v)->ring.next;
((cPersistentObject*)v)->ring.prev = NULL;
((cPersistentObject*)v)->ring.next = NULL;
}
else
{
/* This is a ghost object, so we havent kept a reference count on it.
For it have stayed alive this long someone else must be keeping a reference
to it. Therefore we need to temporarily give it back a reference count
before calling DelItem below */
Py_INCREF(v);
}
Py_DECREF(((cPersistentObject*)v)->cache);
((cPersistentObject*)v)->cache = NULL;
}
Py_DECREF(v);
if(PyDict_DelItem(self->data, key))
{
PyErr_SetString(PyExc_RuntimeError,
"unexpectedly couldnt remove key in cc_ass_sub");
return -1;
}
if(check_ring(self,"post-delitem")) return -1;
return 0;
}
return PyDict_DelItem(self->data, key);
}
static int _check_ring(ccobject *self,const char *context)
{
CPersistentRing *here = &(self->ring_home);
int expected = 1+self->non_ghost_count;
int total = 0;
do
{
if(++total>(expected+10)) return 3; /* ring too big, by a large margin */
if(!here->next) return 4; /* various linking problems */
if(!here->prev) return 5;
if(!here->next->prev) return 7;
if(!here->prev->next) return 8;
if(here->prev->next!=here) return 9;
if(here->next->prev!=here) return 10;
if(!self->ring_lock)
{
/* if the ring must be locked then it only contains object other than persistent instances */
if(here!=&self->ring_home)
{
cPersistentObject *object = object_from_ring(self,here,context);
if(!object) return 12;
if(object->state==cPersistent_GHOST_STATE)
return 13;
}
}
here = here->next;
}
while(here!=&self->ring_home);
if(self->ring_lock)
{
if(total<expected) return 6; /* ring too small; too big is ok when locked */
}
else
{
if(total!=expected) return 14; /* ring size wrong, or bad ghost accounting */
}
return 0;
}
static int check_ring(ccobject *self,const char *context)
{
#ifdef MUCH_RING_CHECKING
int code=_check_ring(self,context);
if(code)
{
/*printf(stderr,"BROKEN RING (code %d) in %s, size %d\n",code,context,PyDict_Size(self->data));*/
PyErr_Format(PyExc_RuntimeError,"broken ring (code %d) in %s, size %d",code,context,PyDict_Size(self->data));
return code;
}
#endif
return 0;
}
static int
present_in_ring(ccobject *self,CPersistentRing *target)
{
CPersistentRing *here = self->ring_home.next;
while(1)
{
if(here==target)
{
return 1;
}
if(here==&self->ring_home)
{
/* back to the home position, and we didnt find it */
return 0;
}
here = here->next;
}
}
static PyMappingMethods cc_as_mapping = {
(inquiry)cc_length, /*mp_length*/
(binaryfunc)cc_subscript, /*mp_subscript*/
......@@ -636,6 +912,10 @@ static PyTypeObject Cctype = {
(hashfunc)0, /*tp_hash*/
(ternaryfunc)0, /*tp_call*/
(reprfunc)0, /*tp_str*/
/* Space for future expansion */
0L,0L,0L,0L,
""
};
static PyObject *
......@@ -644,9 +924,9 @@ cCM_new(PyObject *self, PyObject *args)
int cache_size=100, cache_age=1000;
PyObject *jar;
if (!PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
UNLESS(PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
return NULL;
return (PyObject *)newccobject(jar, cache_size, cache_age);
return (PyObject*)newccobject(jar, cache_size,cache_age);
}
static struct PyMethodDef cCM_methods[] = {
......@@ -657,12 +937,11 @@ static struct PyMethodDef cCM_methods[] = {
void
initcPickleCache(void)
{
PyObject *m;
PyObject *m, *d;
Cctype.ob_type = &PyType_Type;
Cctype.ob_type=&PyType_Type;
if (!ExtensionClassImported)
return;
UNLESS(ExtensionClassImported) return;
m = Py_InitModule4("cPickleCache", cCM_methods, cPickleCache_doc_string,
(PyObject*)NULL, PYTHON_API_VERSION);
......@@ -670,4 +949,15 @@ initcPickleCache(void)
py_reload = PyString_InternFromString("reload");
py__p_jar = PyString_InternFromString("_p_jar");
py__p_changed = PyString_InternFromString("_p_changed");
py__p_oid = PyString_InternFromString("_p_oid");
d = PyModule_GetDict(m);
PyDict_SetItemString(d,"cache_variant",PyString_FromString("stiff/c"));
#ifdef MUCH_RING_CHECKING
PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(1));
#else
PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(0));
#endif
}
src/ZODB/Connection.py
View file @ 226ee421
......@@ -13,14 +13,14 @@
##############################################################################
"""Database connection support
$Id: Connection.py,v 1.63 2002/02/11 23:40:42 gvanrossum Exp $"""
__version__='$Revision: 1.63 $'[11:-2]
$Id: Connection.py,v 1.64 2002/03/27 10:14:03 htrd Exp $"""
__version__='$Revision: 1.64 $'[11:-2]
from cPickleCache import PickleCache
from cPickleCache import PickleCache, MUCH_RING_CHECKING
from POSException import ConflictError, ReadConflictError
from ExtensionClass import Base
import ExportImport, TmpStore
from zLOG import LOG, ERROR, BLATHER
from zLOG import LOG, ERROR, BLATHER, WARNING
from coptimizations import new_persistent_id
from ConflictResolution import ResolvedSerial
......@@ -32,6 +32,11 @@ from types import StringType, ClassType
global_code_timestamp = 0
if MUCH_RING_CHECKING:
# To get rid of this warning, change the define inside cPickleCache.c and recompile.
LOG('ZODB',WARNING, 'Using cPickleCache with low performance (but extra debugging checks)')
del MUCH_RING_CHECKING
def updateCodeTimestamp():
'''
Called after changes are made to persistence-based classes.
......@@ -65,12 +70,35 @@ class Connection(ExportImport.ExportImport):
"""Create a new Connection"""
self._version=version
self._cache=cache=PickleCache(self, cache_size, cache_deactivate_after)
if version:
# Caches for versions end up empty if the version
# is not used for a while. Non-version caches
# keep their content indefinitely.
self._cache.cache_drain_resistance = 100
self._incrgc=self.cacheGC=cache.incrgc
self._invalidated=d={}
self._invalid=d.has_key
self._committed=[]
self._code_timestamp = global_code_timestamp
def _cache_items(self):
# find all items on the lru list
items = self._cache.lru_items()
# fine everything. some on the lru list, some not
everything = self._cache.cache_data
# remove those items that are on the lru list
for k,v in items:
del everything[k]
# return a list of [ghosts....not recently used.....recently used]
return everything.items() + items
def __repr__(self):
if self._version:
ver = ' (in version %s)' % `self._version`
else:
ver = ''
return '<Connection at %08x%s>' % (id(self),ver)
def _breakcr(self):
try: del self._cache
except: pass
......@@ -414,9 +442,9 @@ class Connection(ExportImport.ExportImport):
for oid in creating:
o=cache_get(oid, None)
if o is not None:
del cache[oid]
del o._p_jar
del o._p_oid
del cache[oid]
#XXX
......@@ -441,9 +469,14 @@ class Connection(ExportImport.ExportImport):
def root(self): return self['\0\0\0\0\0\0\0\0']
def setstate(self, object):
oid=object._p_oid
if self._storage is None:
msg = "Shouldn't load state for %s when the connection is closed" % `oid`
LOG('ZODB',ERROR, msg)
raise RuntimeError(msg)
try:
oid=object._p_oid
p, serial = self._storage.load(oid, self._version)
# XXX this is quite conservative!
......
src/ZODB/DB.py
View file @ 226ee421
......@@ -13,8 +13,8 @@
##############################################################################
"""Database objects
$Id: DB.py,v 1.39 2002/02/11 23:40:42 gvanrossum Exp $"""
__version__='$Revision: 1.39 $'[11:-2]
$Id: DB.py,v 1.40 2002/03/27 10:14:04 htrd Exp $"""
__version__='$Revision: 1.40 $'[11:-2]
import cPickle, cStringIO, sys, POSException, UndoLogCompatible
from Connection import Connection
......@@ -177,7 +177,7 @@ class DB(UndoLogCompatible.UndoLogCompatible):
def f(con, detail=detail, rc=sys.getrefcount, conn_no=conn_no):
conn_no[0] = conn_no[0] + 1
cn = conn_no[0]
for oid, ob in con._cache.items():
for oid, ob in con._cache_items():
id=''
if hasattr(ob,'__dict__'):
d=ob.__dict__
......@@ -224,11 +224,21 @@ class DB(UndoLogCompatible.UndoLogCompatible):
def cacheSize(self):
m=[0]
def f(con, m=m):
m[0]=m[0]+len(con._cache)
m[0]=m[0]+con._cache.cache_non_ghost_count
self._connectionMap(f)
return m[0]
def cacheDetailSize(self):
m=[]
def f(con, m=m):
m.append({'connection':repr(con),
'ngsize':con._cache.cache_non_ghost_count,
'size':len(con._cache)})
self._connectionMap(f)
m.sort()
return m
def close(self): self._storage.close()
def commitVersion(self, source, destination=''):
......
src/ZODB/cPersistence.c
View file @ 226ee421
......@@ -14,11 +14,21 @@
static char cPersistence_doc_string[] =
"Defines Persistent mixin class for persistent objects.\n"
"\n"
"$Id: cPersistence.c,v 1.50 2002/03/08 18:36:13 jeremy Exp $\n";
"$Id: cPersistence.c,v 1.51 2002/03/27 10:14:04 htrd Exp $\n";
#include <string.h>
#include "cPersistence.h"
/* the layout of this struct is the same as the start of ccobject in cPickleCache.c */
struct ccobject_head_struct {
PyObject_HEAD
CPersistentRing ring_home;
int non_ghost_count;
};
#define HOME(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->ring_home)) )
#define NON_GHOST_COUNT(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->non_ghost_count)) )
#define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
#define UNLESS(E) if(!(E))
#define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
......@@ -112,21 +122,82 @@ if(self->state < 0 && self->jar) \
{ \
PyObject *r; \
\
int *count = NON_GHOST_COUNT(self); \
if(count) \
{ \
(*count)++; \
self->ring.next = HOME(self); \
self->ring.prev = HOME(self)->prev; \
HOME(self)->prev->next = &self->ring; \
HOME(self)->prev = &self->ring; \
Py_INCREF(self); \
} \
self->state=cPersistent_CHANGED_STATE; \
UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self)) \
{ \
self->state=cPersistent_GHOST_STATE; \
ghostify(self); \
return ER; \
} \
self->state=cPersistent_UPTODATE_STATE; \
Py_DECREF(r); \
}
#define KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self) \
if(HOME(self) && self->state>=0) { \
self->ring.prev->next = self->ring.next; \
self->ring.next->prev = self->ring.prev; \
self->ring.next = HOME(self); \
self->ring.prev = HOME(self)->prev; \
HOME(self)->prev->next = &self->ring; \
HOME(self)->prev = &self->ring; }
/****************************************************************************/
staticforward PyExtensionClass Pertype;
static void
accessed(cPersistentObject *self)
{
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
}
static void
ghostify(cPersistentObject *self)
{
int *count;
count = NON_GHOST_COUNT(self);
if(count && (self->state>=0))
{
(*count)--;
self->ring.next->prev = self->ring.prev;
self->ring.prev->next = self->ring.next;
self->ring.prev = NULL;
self->ring.next = NULL;
self->state = cPersistent_GHOST_STATE;
Py_DECREF(self);
}
else
{
self->state = cPersistent_GHOST_STATE;
}
}
static void
deallocated(cPersistentObject *self)
{
if(self->state>=0) ghostify(self);
if(self->cache)
{
PyObject *v=PyObject_CallMethod(self->cache,"_oid_unreferenced","O",self->oid);
if(!v) PyErr_Clear(); /* and explode later */
Py_XDECREF(v);
}
Py_XDECREF(self->jar);
Py_XDECREF(self->oid);
}
static int
changed(cPersistentObject *self)
{
......@@ -185,7 +256,7 @@ Per___changed__(cPersistentObject *self, PyObject *args)
static PyObject *
Per__p_deactivate(cPersistentObject *self, PyObject *args)
{
PyObject *dict;
PyObject *dict,*dict2=NULL;
#ifdef DEBUG_LOG
if (idebug_log < 0) call_debug("reinit",self);
......@@ -197,13 +268,22 @@ Per__p_deactivate(cPersistentObject *self, PyObject *args)
if (self->state==cPersistent_UPTODATE_STATE && self->jar &&
HasInstDict(self) && (dict=INSTANCE_DICT(self)))
{
dict2 = PyDict_Copy(dict);
PyDict_Clear(dict);
/* Note that we need to set to ghost state unless we are
called directly. Methods that override this need to
do the same! */
self->state=cPersistent_GHOST_STATE;
ghostify(self);
}
/* need to delay releasing the last reference on instance attributes
until after we have finished accounting for losing our state */
if(dict2)
{
PyDict_Clear(dict2);
Py_DECREF(dict2);
}
Py_INCREF(Py_None);
return Py_None;
}
......@@ -333,8 +413,8 @@ Per_dealloc(cPersistentObject *self)
#ifdef DEBUG_LOG
if(idebug_log < 0) call_debug("del",self);
#endif
Py_XDECREF(self->jar);
Py_XDECREF(self->oid);
deallocated(self);
Py_XDECREF(self->cache);
Py_DECREF(self->ob_type);
PyObject_DEL(self);
}
......@@ -387,7 +467,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
{
UPDATE_STATE_IF_NECESSARY(self, NULL);
self->atime=((long)(time(NULL)/3))%65536;
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
if (self->serial[7]=='\0' && self->serial[6]=='\0' &&
self->serial[5]=='\0' && self->serial[4]=='\0' &&
......@@ -419,7 +499,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
{
UPDATE_STATE_IF_NECESSARY(self, NULL);
self->atime=((long)(time(NULL)/3))%65536;
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
}
return getattrf((PyObject *)self, oname);
......@@ -466,6 +546,21 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
{
if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6])
{
if(HOME(self))
{
int result;
if(!v)
{
PyErr_SetString(PyExc_ValueError,"can not delete the oid of a cached object");
return -1;
}
if(PyObject_Cmp(self->oid,v,&result)<0) return -1;
if(result)
{
PyErr_SetString(PyExc_ValueError,"can not change the oid of a cached object");
return -1;
}
}
Py_XINCREF(v);
ASSIGN(self->oid, v);
return 0;
......@@ -509,7 +604,6 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
v=PyObject_GetAttr(OBJECT(self), py__p_deactivate);
if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); }
if (v) { Py_DECREF(v); }
self->state=cPersistent_GHOST_STATE;
return 0;
}
if (PyObject_IsTrue(v)) return changed(self);
......@@ -521,8 +615,7 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
{
UPDATE_STATE_IF_NECESSARY(self, -1);
/* Record access times */
self->atime=((long)(time(NULL)/3))%65536;
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
if((! (*name=='_' && name[1]=='v' && name[2]=='_'))
&& (self->state != cPersistent_CHANGED_STATE && self->jar)
......@@ -680,9 +773,11 @@ truecPersistenceCAPI = {
(getattrofunc)Per_getattro, /*tp_getattr with object key*/
(setattrofunc)Per_setattro, /*tp_setattr with object key*/
changed,
accessed,
ghostify,
deallocated,
(intfunctionwithpythonarg)Per_setstate,
(pergetattr)Per_getattr,
(persetattr)_setattro,
};
void
......
src/ZODB/cPersistence.h
View file @ 226ee421
......@@ -18,12 +18,21 @@
#include "ExtensionClass.h"
#include <time.h>
#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid; char serial[8]; unsigned short atime; signed char state; unsigned char reserved;
#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid, *cache; CPersistentRing ring; char serial[8]; signed char state; unsigned char reserved[3];
#define cPersistent_GHOST_STATE -1
#define cPersistent_UPTODATE_STATE 0
#define cPersistent_CHANGED_STATE 1
#define cPersistent_STICKY_STATE 2
struct ccobject_head_struct;
typedef struct CPersistentRing_struct
{
struct CPersistentRing_struct *prev;
struct CPersistentRing_struct *next;
} CPersistentRing;
typedef struct {
cPersistent_HEAD
} cPersistentObject;
......@@ -36,6 +45,9 @@ typedef struct {
getattrofunc getattro;
setattrofunc setattro;
int (*changed)(cPersistentObject*);
void (*accessed)(cPersistentObject*);
void (*ghostify)(cPersistentObject*);
void (*deallocated)(cPersistentObject*);
int (*setstate)(PyObject*);
pergetattr pergetattro;
persetattr persetattro;
......@@ -59,11 +71,13 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
#define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O)))
#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
#define PER_GHOSTIFY(O) (cPersistenceCAPI->ghostify((cPersistentObject*)(O)))
#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
#define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
#define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
#define PER_DEL(O) Py_XDECREF((O)->jar); Py_XDECREF((O)->oid);
#define PER_DEL(O) (cPersistenceCAPI->deallocated((cPersistentObject*)(O)))
#define PER_USE(O) \
(((O)->state != cPersistent_GHOST_STATE \
......@@ -71,7 +85,7 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
? (((O)->state==cPersistent_UPTODATE_STATE) \
? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0)
#define PER_ACCESSED(O) ((O)->atime=((long)(time(NULL)/3))%65536)
#define PER_ACCESSED(O) (cPersistenceCAPI->accessed((cPersistentObject*)(O)))
#endif
......
src/ZODB/cPickleCache.c
View file @ 226ee421
......@@ -2,448 +2,531 @@
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
static char cPickleCache_doc_string[] =
static char cPickleCache_doc_string[] =
"Defines the PickleCache used by ZODB Connection objects.\n"
"\n"
"$Id: cPickleCache.c,v 1.41 2002/03/08 18:36:14 jeremy Exp $\n";
"$Id: cPickleCache.c,v 1.42 2002/03/27 10:14:04 htrd Exp $\n";
/* Compute the current time in the units and range used for peristent
objects. */
#define PER_TIME() ((long)(time(NULL) / 3)) % 65536
#define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
#define UNLESS(E) if(!(E))
#define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
#define OBJECT(O) ((PyObject*)O)
#define DONT_USE_CPERSISTENCECAPI
#include "cPersistence.h"
#include <time.h>
#include <stddef.h>
#undef Py_FindMethod
static PyObject *py_reload, *py__p_jar, *py__p_changed;
typedef struct {
PyObject_HEAD
PyObject *data;
PyObject *jar;
PyObject *setklassstate;
int position;
int cache_size;
int cache_age;
/* Cache statistics */
int sum_deal;
int sum_deac;
double sum_age;
int n, na;
time_t last_check; /* Time of last gc */
double mean_age;
double mean_deal;
double mean_deac;
double df, dfa; /* Degees of freedom for above stats */
} ccobject;
static PyObject *py__p_oid, *py_reload, *py__p_jar, *py__p_changed;
#define WEIGHTING_PERIOD 600
/* define this for extra debugging checks, and lousy performance */
#define MUCH_RING_CHECKING 1
/*
How to compute weighted means?
/* Do we want 'engine noise'.... abstract debugging output useful for
visualizing cache behavior */
#if 0
#define ENGINE_NOISE(A) printf(A)
#else
#define ENGINE_NOISE(A) ((void)A)
#endif
Assume we have two means, a current mean, M, and a mean as of some
time d seconds in the past, Md. The means have effective degrees
of freedom, N, and Nd. Where Nd is adjusted by d is some fashion.
The combined mean is (M*N+Md*Nd)/(N+Nd). The degrees of freedom
of the combined mean, Nc, is N+Nd. Nd is computed by weighting
an old degree of freedom with the weight: I/(I+d), where I is some
suitably chosen constant, which we will call a "weighting period".
*/
/* the layout of this struct is the same as the start of ccobject_head in cPersistence.c */
typedef struct {
PyObject_HEAD
CPersistentRing ring_home;
int non_ghost_count;
int klass_count;
PyObject *data;
PyObject *jar;
PyObject *setklassstate;
int cache_size;
int ring_lock;
int cache_drain_resistance;
} ccobject;
staticforward PyTypeObject Cctype;
staticforward int present_in_ring(ccobject *self,CPersistentRing *target);
staticforward int check_ring(ccobject *self,const char *context);
staticforward int cc_ass_sub(ccobject *self, PyObject *key, PyObject *v);
/* ---------------------------------------------------------------- */
static int
gc_item(ccobject *self, PyObject *key, PyObject *v, long now, int dt)
static PyObject *object_from_oid(ccobject *self,PyObject *key)
/* somewhat of a replacement for PyDict_GetItem(self->data....
however this returns a *new* reference */
{
if (!(v && key))
return 0;
self->n++;
/* If there is at most one reference to this object, then the
cache has the only reference. It can be removed. */
if (v->ob_refcnt <= 1) {
self->sum_deal++;
/* XXX The fact that this works will iterating over
self->data with PyDict_Next() is an accident of the
current Python dictionary implementation. */
return PyDict_DelItem(self->data, key);
}
PyObject *v = PyDict_GetItem(self->data, key);
if(!v) return NULL;
if (dt >= 0 &&
(!PyExtensionClass_Check(v)) &&
((cPersistentObject*)v)->jar == self->jar /* I'm paranoid */ &&
((cPersistentObject*)v)->state == cPersistent_UPTODATE_STATE) {
now -= ((cPersistentObject*)v)->atime;
if (now < 0)
now += 65536;
self->na++;
self->sum_age += now;
if (now > dt) {
/* We have a cPersistent object that hasn't been used in
a while. Reinitialize it, hopefully freeing it's
state.
*/
self->sum_deac++;
if (PyObject_SetAttr(v, py__p_changed, Py_None) < 0)
PyErr_Clear();
}
}
return 0;
Py_INCREF(v);
return v;
}
static void
update_stats(ccobject *self, time_t now)
static cPersistentObject *object_from_ring(ccobject *self,CPersistentRing *here,const char *context)
{
double d, deal, deac;
d = now - self->last_check;
if(d < 1)
return;
self->df *= WEIGHTING_PERIOD / (WEIGHTING_PERIOD + d);
self->dfa *= WEIGHTING_PERIOD / (WEIGHTING_PERIOD + d);
self->mean_age = ((self->mean_age * self->dfa + self->sum_age)/
(self->dfa + self->na)) * 3;
self->sum_age = 0;
deac = self->sum_deac / d;
self->sum_deac = 0;
self->mean_deac = ((self->mean_deac * self->dfa+deac)/
(self->dfa + self->na));
self->sum_deac = 0;
self->dfa += self->na;
self->na = 0;
deal=self->sum_deal/d;
self->sum_deal = 0;
self->mean_deal = ((self->mean_deal * self->df + deal)/
(self->df +self->n));
self->sum_deal = 0;
self->df += self->n;
self->n = 0;
self->last_check = now;
/* Given a position in the LRU ring, return a borrowed
reference to the object at that point in the ring. The caller is
responsible for ensuring that this ring position really does
correspond to a persistent object, although the debugging
version will double-check this. */
PyObject *object = (PyObject *)(((char *)here)-offsetof(cPersistentObject,ring));
#ifdef MUCH_RING_CHECKING
if(!PyExtensionInstance_Check(object))
{
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered non-ExtensionClass object in %s",context);
return NULL;
}
if(!(((PyExtensionClass*)(object->ob_type))->class_flags & PERSISTENT_TYPE_FLAG))
{
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered non-persistent object in %s",context);
return NULL;
}
if(((cPersistentObject*)object)->jar!=self->jar)
{
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered object from a different jar in %s",context);
return NULL;
}
if(((cPersistentObject*)object)->cache!=(PyObject *)self)
{
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered broken ring in %s",context);
return NULL;
}
#endif
return (cPersistentObject *)object;
}
static int
check_size(ccobject *self)
scan_gc_items(ccobject *self,int target)
{
if (self->cache_size < 1)
return 0;
return PyDict_Size(self->data);
cPersistentObject *object;
int error;
CPersistentRing placeholder;
CPersistentRing *here = self->ring_home.next;
#ifdef MUCH_RING_CHECKING
int safety_counter = self->cache_size*10;
if(safety_counter<10000) safety_counter = 10000;
#endif
while(1)
{
if(check_ring(self,"mid-gc")) return -1;
#ifdef MUCH_RING_CHECKING
if(!safety_counter--)
{
/* This loop has been running for a very long time.
It is possible that someone loaded a very large number of objects,
and now wants us to blow them all away. However it may
also indicate a logic error. If the loop has been running this
long then you really have to doubt it will ever terminate.
In the MUCH_RING_CHECKING build we prefer to raise an exception
here */
PyErr_SetString(PyExc_RuntimeError,"scan_gc_items safety counter exceeded");
return -1;
}
if(!present_in_ring(self,here))
{
/* Our current working position is no longer in the ring. Thats bad. */
PyErr_SetString(PyExc_RuntimeError,"working position fell out the ring, in scan_gc_items");
return -1;
}
#endif
if(here==&self->ring_home)
{
/* back to the home position. stop looking */
return 0;
}
/* At this point we know that the ring only contains nodes from
persistent objects, plus our own home node. We can safely
assume this is a persistent object now we know it is not the home */
object = object_from_ring(self,here,"scan_gc_items");
if(!object) return -1;
if(self->non_ghost_count<=target)
{
/* we are small enough */
return 0;
}
else if(object->state==cPersistent_UPTODATE_STATE)
{
/* deactivate it. This is the main memory saver. */
ENGINE_NOISE("G");
/* add a placeholder */
placeholder.next = here->next;
placeholder.prev = here;
here->next->prev = &placeholder;
here->next = &placeholder;
error = PyObject_SetAttr((PyObject *)object,py__p_changed,Py_None);
/* unlink the placeholder */
placeholder.next->prev=placeholder.prev;
placeholder.prev->next=placeholder.next;
here = placeholder.next;
if(error)
return -1; /* problem */
}
else
{
ENGINE_NOISE(".");
here = here->next;
}
}
}
static int
gc_all_items(ccobject *self, int now, int dt)
static PyObject *
lockgc(ccobject *self,int target_size)
{
PyObject *key, *v;
int i;
if(self->ring_lock)
{
Py_INCREF(Py_None);
return Py_None;
}
for(i = 0; PyDict_Next(self->data, &i, &key, &v); )
if (gc_item(self, key, v, now, dt) < 0)
return -1;
return 0;
if(check_ring(self,"pre-gc")) return NULL;
ENGINE_NOISE("<");
self->ring_lock = 1;
if(scan_gc_items(self,target_size))
{
self->ring_lock = 0;
return NULL;
}
self->ring_lock = 0;
ENGINE_NOISE(">\n");
if(check_ring(self,"post-gc")) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static int
fullgc(ccobject *self, int dt)
static PyObject *
cc_incrgc(ccobject *self, PyObject *args)
{
long now;
int n=1;
if (check_size(self) <= 0)
return 0;
int starting_size = self->non_ghost_count;
now = PER_TIME();
dt /= 3;
int target_size = self->cache_size;
if (gc_all_items(self, now, dt) < 0)
return -1;
self->position = 0;
if(self->cache_drain_resistance>=1)
{
/* This cache will gradually drain down to a small size. Check
a (small) number of objects proportional to the current size */
if (now - self->last_check > 1)
update_stats(self, now);
return 0;
}
static int
reallyfullgc(ccobject *self, int dt)
{
int l, last;
time_t now;
last = check_size(self);
if (last <= 0)
return 0;
now = PER_TIME();
/* Units are 3 seconds */
dt /= 3;
/* First time through should get refcounts to 1 */
if (gc_all_items(self, now, dt) < 0)
return -1;
l = PyDict_Size(self->data);
if (l < 0)
return -1;
/* Now continue to collect until the size of the cache stops
decreasing. */
while (l < last) {
if (gc_all_items(self, now, dt) < 0)
return -1;
last = l;
l = PyDict_Size(self->data);
if (l < 0)
return -1;
int target_size_2 = starting_size - 1 - starting_size/self->cache_drain_resistance;
if(target_size_2<target_size)
target_size = target_size_2;
}
if (now - self->last_check > 1)
update_stats(self, now);
self->position = 0;
return 0;
}
static int
maybegc(ccobject *self, PyObject *thisv)
{
int n, s, size, dt;
long now;
PyObject *key=0, *v=0;
s = check_size(self);
if (s <= 0)
return 0;
now = PER_TIME();
size = self->cache_size;
self->cache_size = 0;
/* Decide how many objects to look at */
n = (s - size) / 10;
if (n < 3)
n = 3;
/* Decide how much time to give them before deactivating them */
s = 8 * size / s;
if (s > 100)
s = 100;
dt = (long)(self->cache_age * (0.2 + 0.1 * s));
/* Units are 3 seconds */
dt /= 3;
while (--n >= 0) {
if (PyDict_Next(self->data, &(self->position), &key, &v)) {
if (v != thisv && gc_item(self, key, v, now, dt) < 0) {
self->cache_size=size;
return -1;
}
}
else
self->position = 0;
}
self->cache_size = size;
if (now - self->last_check > 1)
update_stats(self, now);
UNLESS (PyArg_ParseTuple(args, "|i",&n)) return NULL;
return 0;
return lockgc(self,target_size);
}
static PyObject *
cc_full_sweep(ccobject *self, PyObject *args)
{
int dt = self->cache_age;
if (!PyArg_ParseTuple(args, "|i:full_sweep", &dt))
return NULL;
if (dt < -1) {
PyErr_SetString(PyExc_ValueError, "age must be >= -1");
return NULL;
}
if (fullgc(self, dt) == -1)
return NULL;
Py_INCREF(Py_None);
return Py_None;
int dt=0;
UNLESS(PyArg_ParseTuple(args, "|i", &dt)) return NULL;
return lockgc(self,0);
}
static PyObject *
cc_reallyfull_sweep(ccobject *self, PyObject *args)
{
int dt = self->cache_age;
if (!PyArg_ParseTuple(args, "|i:minimize", &dt))
return NULL;
if (dt < -1) {
PyErr_SetString(PyExc_ValueError, "age must be >= -1");
return NULL;
}
if (reallyfullgc(self, dt) == -1)
return NULL;
Py_INCREF(Py_None);
return Py_None;
int dt=0;
UNLESS(PyArg_ParseTuple(args, "|i", &dt)) return NULL;
return lockgc(self,0);
}
static PyObject *
cc_incrgc(ccobject *self, PyObject *args)
static void
_invalidate(ccobject *self, PyObject *key)
{
int n = 1;
PyObject *v=object_from_oid(self, key);
if (!PyArg_ParseTuple(args, "|i:incrgr", &n))
return NULL;
if(!v)
{
/* shouldnt this be an error? for now Ill follow Jims lead */
PyErr_Clear();
}
else
{
if (PyExtensionClass_Check(v))
{
if(v->ob_refcnt <= 1)
{
self->klass_count--;
if (PyDict_DelItem(self->data, key) < 0)
PyErr_Clear();
}
else
{
v=PyObject_CallFunction(self->setklassstate,
"O", v);
if (v) Py_DECREF(v);
else PyErr_Clear();
}
}
else
{
if(PyObject_DelAttr(v,py__p_changed) < 0)
PyErr_Clear();
}
Py_DECREF(v);
}
}
for (; --n >= 0;)
if (maybegc(self, NULL) < 0)
return NULL;
static PyObject *
cc_invalidate(ccobject *self, PyObject *args)
{
PyObject *inv, *key, *v;
int i;
if (PyArg_ParseTuple(args, "O!", &PyDict_Type, &inv)) {
for (i=0; PyDict_Next(inv, &i, &key, &v); )
if (key==Py_None)
{ /* Eek some nitwit invalidated everything! */
for (i=0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
break;
}
else
_invalidate(self, key);
PyDict_Clear(inv);
}
else {
PyErr_Clear();
UNLESS (PyArg_ParseTuple(args, "O", &inv)) return NULL;
if (PyString_Check(inv))
_invalidate(self, inv);
else if (inv==Py_None) /* All */
for (i=0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
else {
int l;
PyErr_Clear();
if ((l=PyObject_Length(inv)) < 0) return NULL;
for(i=l; --i >= 0; )
{
UNLESS (key=PySequence_GetItem(inv, i)) return NULL;
_invalidate(self, key);
Py_DECREF(key);
}
PySequence_DelSlice(inv, 0, l);
}
}
Py_INCREF(Py_None);
return Py_None;
}
static void
_invalidate(ccobject *self, PyObject *key)
static PyObject *
cc_get(ccobject *self, PyObject *args)
{
PyObject *v = PyDict_GetItem(self->data, key);
PyObject *r, *key, *d=0;
if (!v)
return;
if (PyExtensionClass_Check(v))
if (v->ob_refcnt <= 1) {
self->sum_deal++;
if (PyDict_DelItem(self->data, key) < 0)
PyErr_Clear();
} else {
PyObject *t = PyTuple_New(1);
if (t) {
PyTuple_SET_ITEM(t, 0, v);
v = PyObject_CallObject(self->setklassstate, t);
/* Set tuple element to NULL so that deallocating the
tuple does not decref t.
*/
PyTuple_SET_ITEM(t, 0, NULL);
Py_DECREF(t);
} else
v = t;
if (v)
Py_DECREF(v);
else
PyErr_Clear();
UNLESS (PyArg_ParseTuple(args,"O|O", &key, &d)) return NULL;
UNLESS (r=(PyObject *)object_from_oid(self, key))
{
if (d)
{
PyErr_Clear();
r=d;
Py_INCREF(r);
}
else
{
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
else if (PyObject_DelAttr(v, py__p_changed) < 0)
PyErr_Clear();
}
return r;
}
static void
_invalidate_all(ccobject *self)
static PyObject *
cc_klass_items(ccobject *self, PyObject *args)
{
PyObject *key, *v;
int i;
PyObject *l,*k,*v;
int p = 0;
if(!PyArg_ParseTuple(args,"")) return NULL;
l = PyList_New(0);
if(!l) return NULL;
while(PyDict_Next(self->data, &p, &k, &v))
{
if(PyExtensionClass_Check(v))
{
v=PyObject_CallMethod(l,"append","((OO))",k,v);
if(!v)
{
Py_DECREF(l);
return NULL;
}
}
}
for (i = 0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
return l;
}
static PyObject *
cc_invalidate(ccobject *self, PyObject *args)
cc_lru_items(ccobject *self, PyObject *args)
{
PyObject *inv, *key, *v;
int i;
PyObject *l;
CPersistentRing *here;
if (!PyArg_ParseTuple(args, "O:invalidate", &inv))
return NULL;
if (PyDict_Check(inv)) {
for (i = 0; PyDict_Next(inv, &i, &key, &v); )
if (key == Py_None) {
/* Eek some nitwit invalidated everything! */
_invalidate_all(self);
break;
}
else
_invalidate(self, key);
PyDict_Clear(inv);
} else if (PyString_Check(inv))
_invalidate(self, inv);
else if (inv == Py_None) /* All */
_invalidate_all(self);
else {
int l = PyObject_Length(inv);
if (l < 0)
return NULL;
for (i = l; --i >= 0; ) {
key = PySequence_GetItem(inv, i);
if (!key)
return NULL;
_invalidate(self, key);
Py_DECREF(key);
}
PySequence_DelSlice(inv, 0, l);
if(!PyArg_ParseTuple(args,"")) return NULL;
if(self->ring_lock)
{
PyErr_SetString(PyExc_ValueError,".lru_items() is unavailable during garbage collection");
return NULL;
}
Py_INCREF(Py_None);
return Py_None;
if(check_ring(self,"pre-cc_items")) return NULL;
l = PyList_New(0);
if(!l) return NULL;
here = self->ring_home.next;
while(here!=&self->ring_home)
{
cPersistentObject *object = object_from_ring(self,here,"cc_items");
PyObject *v;
if(!object)
{
Py_DECREF(l);
return NULL;
}
v=PyObject_CallMethod(l,"append","((OO))",object->oid,object);
if(!v)
{
Py_DECREF(l);
return NULL;
}
Py_DECREF(v);
here = here->next;
}
return l;
}
static PyObject *
cc_get(ccobject *self, PyObject *args)
cc_oid_unreferenced(ccobject *self, PyObject *args)
{
PyObject *r, *key, *d = NULL;
PyObject *oid,*v;
if(!PyArg_ParseTuple(args,"O",&oid)) return NULL;
if (!PyArg_ParseTuple(args, "O|O:get", &key, &d))
return NULL;
r = PyDict_GetItem(self->data, key);
if (!r) {
if (d)
r = d;
else {
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
v = PyDict_GetItem(self->data, oid);
if(!v) return NULL;
if(v->ob_refcnt)
{
PyErr_Format(PyExc_ValueError,"object has reference count of %d, should be zero",v->ob_refcnt);
return NULL;
}
Py_INCREF(r);
return r;
/* Need to be very hairy here because a dictionary is about
to decref an already deleted object */
#ifdef Py_TRACE_REFS
#error "this code path has not been tested - Toby Dickenson"
_Py_NewReference(v);
/* it may be a problem that v->ob_type is still NULL? */
#else
Py_INCREF(v);
#endif
if(v->ob_refcnt!=1)
{
PyErr_SetString(PyExc_ValueError,"refcount is not 1 after resurrection");
return NULL;
}
/* return the stolen reference */
Py_INCREF(v);
PyDict_DelItem(self->data, oid);
if(v->ob_refcnt!=1)
{
PyErr_SetString(PyExc_ValueError,"refcount is not 1 after removal from dict");
return NULL;
}
/* undo the temporary resurrection */
#ifdef Py_TRACE_REFS
_Py_ForgetReference(v);
#else
v->ob_refcnt=0;
#endif
Py_INCREF(Py_None);
return Py_None;
}
static struct PyMethodDef cc_methods[] = {
{"_oid_unreferenced", (PyCFunction)cc_oid_unreferenced, METH_VARARGS,
NULL
},
{"lru_items", (PyCFunction)cc_lru_items, METH_VARARGS,
"List (oid, object) pairs from the lru list, as 2-tuples.\n"
},
{"klass_items", (PyCFunction)cc_klass_items, METH_VARARGS,
"List (oid, object) pairs of cached persistent classes.\n"
},
{"full_sweep", (PyCFunction)cc_full_sweep, METH_VARARGS,
"full_sweep([age]) -- Perform a full sweep of the cache\n\n"
"Make a single pass through the cache, removing any objects that are no\n"
"longer referenced, and deactivating objects that have not been\n"
"accessed in the number of seconds given by 'age'. "
"'age defaults to the cache age.\n"
"longer referenced, and deactivating enough objects to bring\n"
"the cache under its size limit\n"
"The optional 'age' parameter is ignored.\n"
},
{"minimize", (PyCFunction)cc_reallyfull_sweep, METH_VARARGS,
"minimize([age]) -- Remove as many objects as possible\n\n"
"Make multiple passes through the cache, removing any objects that are no\n"
"longer referenced, and deactivating objects that have not been\n"
"accessed in the number of seconds given by 'age'. 'age defaults to 0.\n"
"longer referenced, and deactivating enough objects to bring the"
" cache under its size limit\n"
"The option 'age' parameter is ignored.\n"
},
{"incrgc", (PyCFunction)cc_incrgc, METH_VARARGS,
"incrgc() -- Perform incremental garbage collection"},
"incrgc([n]) -- Perform incremental garbage collection\n\n"
"Some other implementations support an optional parameter 'n' which\n"
"indicates a repetition count; this value is ignored.\n"},
{"invalidate", (PyCFunction)cc_invalidate, METH_VARARGS,
"invalidate(oids) -- invalidate one, many, or all ids"},
{"get", (PyCFunction)cc_get, METH_VARARGS,
......@@ -454,115 +537,102 @@ static struct PyMethodDef cc_methods[] = {
static ccobject *
newccobject(PyObject *jar, int cache_size, int cache_age)
{
ccobject *self;
ccobject *self;
self = PyObject_NEW(ccobject, &Cctype);
if (!self)
UNLESS(self = PyObject_NEW(ccobject, &Cctype)) return NULL;
self->setklassstate=self->jar=NULL;
if((self->data=PyDict_New()))
{
self->jar=jar;
Py_INCREF(jar);
UNLESS (self->setklassstate=PyObject_GetAttrString(jar, "setklassstate"))
return NULL;
self->setklassstate = self->jar = NULL;
self->data = PyDict_New();
if (self->data) {
self->jar=jar;
Py_INCREF(jar);
self->setklassstate = PyObject_GetAttrString(jar, "setklassstate");
if (!self->setklassstate) {
Py_DECREF(jar);
Py_DECREF(self->data);
goto error;
}
self->position = 0;
self->cache_size = cache_size;
self->cache_age = cache_age < 1 ? 1 : cache_age;
self->sum_deal = 0;
self->sum_deac = 0;
self->sum_age = 0;
self->mean_deal = 0;
self->mean_deac = 0;
self->mean_age = 0;
self->df = 1;
self->dfa = 1;
self->n = 0;
self->na = 0;
self->last_check = time(NULL);
return self;
self->cache_size=cache_size;
self->non_ghost_count=0;
self->klass_count=0;
self->cache_drain_resistance=0;
self->ring_lock=0;
self->ring_home.next = &self->ring_home;
self->ring_home.prev = &self->ring_home;
return self;
}
error:
Py_DECREF(self);
return NULL;
Py_DECREF(self);
return NULL;
}
static void
cc_dealloc(ccobject *self)
{
Py_XDECREF(self->data);
Py_XDECREF(self->jar);
Py_XDECREF(self->setklassstate);
PyObject_DEL(self);
Py_XDECREF(self->data);
Py_XDECREF(self->jar);
Py_XDECREF(self->setklassstate);
PyMem_DEL(self);
}
static PyObject *
cc_getattr(ccobject *self, char *name)
{
PyObject *r;
if (*name == 'c') {
if(strcmp(name, "cache_age") == 0)
return PyInt_FromLong(self->cache_age);
if(strcmp(name, "cache_size") == 0)
return PyInt_FromLong(self->cache_size);
if(strcmp(name, "cache_mean_age") == 0)
return PyFloat_FromDouble(self->mean_age);
if(strcmp(name, "cache_mean_deal") == 0)
return PyFloat_FromDouble(self->mean_deal);
if(strcmp(name, "cache_mean_deac") == 0)
return PyFloat_FromDouble(self->mean_deac);
if(strcmp(name, "cache_df") == 0)
return PyFloat_FromDouble(self->df);
if(strcmp(name, "cache_dfa") == 0)
return PyFloat_FromDouble(self->dfa);
if(strcmp(name, "cache_last_gc_time") == 0)
return PyFloat_FromDouble(self->last_check);
if(strcmp(name, "cache_data") == 0) {
Py_INCREF(self->data);
return self->data;
PyObject *r;
if(check_ring(self,"getattr")) return NULL;
if(*name=='c')
{
if(strcmp(name,"cache_age")==0)
return PyInt_FromLong(0); /* this cache does not use this value */
if(strcmp(name,"cache_size")==0)
return PyInt_FromLong(self->cache_size);
if(strcmp(name,"cache_drain_resistance")==0)
return PyInt_FromLong(self->cache_drain_resistance);
if(strcmp(name,"cache_non_ghost_count")==0)
return PyInt_FromLong(self->non_ghost_count);
if(strcmp(name,"cache_klass_count")==0)
return PyInt_FromLong(self->klass_count);
if(strcmp(name,"cache_data")==0)
{
/* now a copy of our data; the ring is too fragile */
return PyDict_Copy(self->data);
}
}
if ((strcmp(name, "has_key") == 0)
|| (strcmp(name, "items") == 0)
|| (strcmp(name, "keys") == 0))
return PyObject_GetAttrString(self->data, name);
r = Py_FindMethod(cc_methods, (PyObject *)self, name);
if (!r) {
PyErr_Clear();
return PyObject_GetAttrString(self->data, name);
}
if((*name=='h' && strcmp(name, "has_key")==0) ||
(*name=='i' && strcmp(name, "items")==0) ||
(*name=='k' && strcmp(name, "keys")==0)
)
return PyObject_GetAttrString(self->data, name);
if((r=Py_FindMethod(cc_methods, (PyObject *)self, name)))
return r;
PyErr_Clear();
return PyObject_GetAttrString(self->data, name);
}
static int
cc_setattr(ccobject *self, char *name, PyObject *value)
{
if (value) {
if(value)
{
int v;
if (strcmp(name, "cache_age") == 0) {
v = PyInt_AsLong(value);
if (v == -1 && PyErr_Occurred())
return -1;
if (v > 0)
self->cache_age = v;
if(strcmp(name,"cache_age")==0)
{
/* this cache doesnt use the age */
return 0;
}
}
if (strcmp(name, "cache_size") == 0) {
v = PyInt_AsLong(value);
if (v == -1 && PyErr_Occurred())
return -1;
self->cache_size = v;
if(strcmp(name,"cache_size")==0)
{
UNLESS(PyArg_Parse(value,"i",&v)) return -1;
self->cache_size=v;
return 0;
}
}
}
if(strcmp(name,"cache_drain_resistance")==0)
{
UNLESS(PyArg_Parse(value,"i",&v)) return -1;
self->cache_drain_resistance=v;
return 0;
}
}
PyErr_SetString(PyExc_AttributeError, name);
return -1;
}
......@@ -570,7 +640,7 @@ cc_setattr(ccobject *self, char *name, PyObject *value)
static int
cc_length(ccobject *self)
{
return PyDict_Size(self->data);
return PyObject_Length(self->data);
}
static PyObject *
......@@ -578,39 +648,245 @@ cc_subscript(ccobject *self, PyObject *key)
{
PyObject *r;
r = PyDict_GetItem(self->data, key);
if (!r) {
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
if(check_ring(self,"__getitem__")) return NULL;
UNLESS (r=(PyObject *)object_from_oid(self, key))
{
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
Py_INCREF(r);
return r;
}
static int
cc_ass_sub(ccobject *self, PyObject *key, PyObject *v)
{
if (v) {
if (PyExtensionClass_Check(v)
||
(PyExtensionInstance_Check(v)
&&
(((PyExtensionClass*)(v->ob_type))->class_flags
& PERSISTENT_TYPE_FLAG)
&&
(v->ob_type->tp_basicsize >= sizeof(cPersistentObject))
)
)
return PyDict_SetItem(self->data, key, v);
PyErr_SetString(PyExc_ValueError,
"Cache values must be persistent objects or classes.");
return -1;
int result;
if(v)
{
if( ( PyExtensionInstance_Check(v) &&
(((PyExtensionClass*)(v->ob_type))->class_flags & PERSISTENT_TYPE_FLAG) &&
(v->ob_type->tp_basicsize >= sizeof(cPersistentObject))
)
||
PyExtensionClass_Check(v)
)
{
PyObject *oid = PyObject_GetAttr(v,py__p_oid);
PyObject *object_again;
if(!oid)
{
return -1;
}
if(PyObject_Cmp(key,oid,&result))
{
Py_DECREF(oid);
return -1;
}
Py_DECREF(oid);
if(result)
{
PyErr_SetString(PyExc_ValueError,"key must be the same as the object's oid attribute");
return -1;
}
object_again = object_from_oid(self, key);
if(object_again)
{
if(object_again!=v)
{
Py_DECREF(object_again);
PyErr_SetString(PyExc_ValueError,"Can not re-register object under a different oid");
return -1;
}
else
{
/* re-register under the same oid - no work needed */
Py_DECREF(object_again);
return 0;
}
}
if(PyExtensionClass_Check(v))
{
if(PyDict_SetItem(self->data, key, v)) return -1;
self->klass_count++;
return 0;
}
else
{
if(((cPersistentObject*)v)->cache)
{
if(((cPersistentObject*)v)->cache==(PyObject *)self)
{
/* This object is already one of ours, which is ok.
It would be very strange if someone was trying to register the
same object under a different key */
}
else
{
/* This object is already in a different cache. */
PyErr_SetString(PyExc_ValueError, "Cache values may only be in one cache.");
return -1;
}
}
if(check_ring(self,"pre-setitem")) return -1;
if(PyDict_SetItem(self->data, key, v)) return -1;
Py_INCREF(self);
((cPersistentObject*)v)->cache = (PyObject *)self;
if(((cPersistentObject*)v)->state>=0)
{
/* insert this non-ghost object into the ring just behind the home position */
self->non_ghost_count++;
((cPersistentObject*)v)->ring.next = &self->ring_home;
((cPersistentObject*)v)->ring.prev = self->ring_home.prev;
self->ring_home.prev->next = &((cPersistentObject*)v)->ring;
self->ring_home.prev = &((cPersistentObject*)v)->ring;
}
else
{
/* steal a reference from the dictionary; ghosts have a weak reference */
Py_DECREF(v);
}
if(check_ring(self,"post-setitem")) return -1;
return 0;
}
}
else
{
PyErr_SetString(PyExc_ValueError, "Cache values must be persistent objects.");
return -1;
}
}
else
{
/* unlink this item from the ring */
if(check_ring(self,"pre-delitem")) return -1;
v = (PyObject *)object_from_oid(self,key);
if(!v) return -1;
if(PyExtensionClass_Check(v))
{
self->klass_count--;
}
else
{
if(((cPersistentObject*)v)->state>=0)
{
self->non_ghost_count--;
((cPersistentObject*)v)->ring.next->prev = ((cPersistentObject*)v)->ring.prev;
((cPersistentObject*)v)->ring.prev->next = ((cPersistentObject*)v)->ring.next;
((cPersistentObject*)v)->ring.prev = NULL;
((cPersistentObject*)v)->ring.next = NULL;
}
else
{
/* This is a ghost object, so we havent kept a reference count on it.
For it have stayed alive this long someone else must be keeping a reference
to it. Therefore we need to temporarily give it back a reference count
before calling DelItem below */
Py_INCREF(v);
}
Py_DECREF(((cPersistentObject*)v)->cache);
((cPersistentObject*)v)->cache = NULL;
}
Py_DECREF(v);
if(PyDict_DelItem(self->data, key))
{
PyErr_SetString(PyExc_RuntimeError,
"unexpectedly couldnt remove key in cc_ass_sub");
return -1;
}
if(check_ring(self,"post-delitem")) return -1;
return 0;
}
return PyDict_DelItem(self->data, key);
}
static int _check_ring(ccobject *self,const char *context)
{
CPersistentRing *here = &(self->ring_home);
int expected = 1+self->non_ghost_count;
int total = 0;
do
{
if(++total>(expected+10)) return 3; /* ring too big, by a large margin */
if(!here->next) return 4; /* various linking problems */
if(!here->prev) return 5;
if(!here->next->prev) return 7;
if(!here->prev->next) return 8;
if(here->prev->next!=here) return 9;
if(here->next->prev!=here) return 10;
if(!self->ring_lock)
{
/* if the ring must be locked then it only contains object other than persistent instances */
if(here!=&self->ring_home)
{
cPersistentObject *object = object_from_ring(self,here,context);
if(!object) return 12;
if(object->state==cPersistent_GHOST_STATE)
return 13;
}
}
here = here->next;
}
while(here!=&self->ring_home);
if(self->ring_lock)
{
if(total<expected) return 6; /* ring too small; too big is ok when locked */
}
else
{
if(total!=expected) return 14; /* ring size wrong, or bad ghost accounting */
}
return 0;
}
static int check_ring(ccobject *self,const char *context)
{
#ifdef MUCH_RING_CHECKING
int code=_check_ring(self,context);
if(code)
{
/*printf(stderr,"BROKEN RING (code %d) in %s, size %d\n",code,context,PyDict_Size(self->data));*/
PyErr_Format(PyExc_RuntimeError,"broken ring (code %d) in %s, size %d",code,context,PyDict_Size(self->data));
return code;
}
#endif
return 0;
}
static int
present_in_ring(ccobject *self,CPersistentRing *target)
{
CPersistentRing *here = self->ring_home.next;
while(1)
{
if(here==target)
{
return 1;
}
if(here==&self->ring_home)
{
/* back to the home position, and we didnt find it */
return 0;
}
here = here->next;
}
}
static PyMappingMethods cc_as_mapping = {
(inquiry)cc_length, /*mp_length*/
(binaryfunc)cc_subscript, /*mp_subscript*/
......@@ -636,6 +912,10 @@ static PyTypeObject Cctype = {
(hashfunc)0, /*tp_hash*/
(ternaryfunc)0, /*tp_call*/
(reprfunc)0, /*tp_str*/
/* Space for future expansion */
0L,0L,0L,0L,
""
};
static PyObject *
......@@ -644,9 +924,9 @@ cCM_new(PyObject *self, PyObject *args)
int cache_size=100, cache_age=1000;
PyObject *jar;
if (!PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
UNLESS(PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
return NULL;
return (PyObject *)newccobject(jar, cache_size, cache_age);
return (PyObject*)newccobject(jar, cache_size,cache_age);
}
static struct PyMethodDef cCM_methods[] = {
......@@ -657,12 +937,11 @@ static struct PyMethodDef cCM_methods[] = {
void
initcPickleCache(void)
{
PyObject *m;
PyObject *m, *d;
Cctype.ob_type = &PyType_Type;
Cctype.ob_type=&PyType_Type;
if (!ExtensionClassImported)
return;
UNLESS(ExtensionClassImported) return;
m = Py_InitModule4("cPickleCache", cCM_methods, cPickleCache_doc_string,
(PyObject*)NULL, PYTHON_API_VERSION);
......@@ -670,4 +949,15 @@ initcPickleCache(void)
py_reload = PyString_InternFromString("reload");
py__p_jar = PyString_InternFromString("_p_jar");
py__p_changed = PyString_InternFromString("_p_changed");
py__p_oid = PyString_InternFromString("_p_oid");
d = PyModule_GetDict(m);
PyDict_SetItemString(d,"cache_variant",PyString_FromString("stiff/c"));
#ifdef MUCH_RING_CHECKING
PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(1));
#else
PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(0));
#endif
}
src/persistent/cPersistence.c
View file @ 226ee421
......@@ -14,11 +14,21 @@
static char cPersistence_doc_string[] =
"Defines Persistent mixin class for persistent objects.\n"
"\n"
"$Id: cPersistence.c,v 1.50 2002/03/08 18:36:13 jeremy Exp $\n";
"$Id: cPersistence.c,v 1.51 2002/03/27 10:14:04 htrd Exp $\n";
#include <string.h>
#include "cPersistence.h"
/* the layout of this struct is the same as the start of ccobject in cPickleCache.c */
struct ccobject_head_struct {
PyObject_HEAD
CPersistentRing ring_home;
int non_ghost_count;
};
#define HOME(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->ring_home)) )
#define NON_GHOST_COUNT(O) ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->non_ghost_count)) )
#define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
#define UNLESS(E) if(!(E))
#define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
......@@ -112,21 +122,82 @@ if(self->state < 0 && self->jar) \
{ \
PyObject *r; \
\
int *count = NON_GHOST_COUNT(self); \
if(count) \
{ \
(*count)++; \
self->ring.next = HOME(self); \
self->ring.prev = HOME(self)->prev; \
HOME(self)->prev->next = &self->ring; \
HOME(self)->prev = &self->ring; \
Py_INCREF(self); \
} \
self->state=cPersistent_CHANGED_STATE; \
UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self)) \
{ \
self->state=cPersistent_GHOST_STATE; \
ghostify(self); \
return ER; \
} \
self->state=cPersistent_UPTODATE_STATE; \
Py_DECREF(r); \
}
#define KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self) \
if(HOME(self) && self->state>=0) { \
self->ring.prev->next = self->ring.next; \
self->ring.next->prev = self->ring.prev; \
self->ring.next = HOME(self); \
self->ring.prev = HOME(self)->prev; \
HOME(self)->prev->next = &self->ring; \
HOME(self)->prev = &self->ring; }
/****************************************************************************/
staticforward PyExtensionClass Pertype;
static void
accessed(cPersistentObject *self)
{
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
}
static void
ghostify(cPersistentObject *self)
{
int *count;
count = NON_GHOST_COUNT(self);
if(count && (self->state>=0))
{
(*count)--;
self->ring.next->prev = self->ring.prev;
self->ring.prev->next = self->ring.next;
self->ring.prev = NULL;
self->ring.next = NULL;
self->state = cPersistent_GHOST_STATE;
Py_DECREF(self);
}
else
{
self->state = cPersistent_GHOST_STATE;
}
}
static void
deallocated(cPersistentObject *self)
{
if(self->state>=0) ghostify(self);
if(self->cache)
{
PyObject *v=PyObject_CallMethod(self->cache,"_oid_unreferenced","O",self->oid);
if(!v) PyErr_Clear(); /* and explode later */
Py_XDECREF(v);
}
Py_XDECREF(self->jar);
Py_XDECREF(self->oid);
}
static int
changed(cPersistentObject *self)
{
......@@ -185,7 +256,7 @@ Per___changed__(cPersistentObject *self, PyObject *args)
static PyObject *
Per__p_deactivate(cPersistentObject *self, PyObject *args)
{
PyObject *dict;
PyObject *dict,*dict2=NULL;
#ifdef DEBUG_LOG
if (idebug_log < 0) call_debug("reinit",self);
......@@ -197,13 +268,22 @@ Per__p_deactivate(cPersistentObject *self, PyObject *args)
if (self->state==cPersistent_UPTODATE_STATE && self->jar &&
HasInstDict(self) && (dict=INSTANCE_DICT(self)))
{
dict2 = PyDict_Copy(dict);
PyDict_Clear(dict);
/* Note that we need to set to ghost state unless we are
called directly. Methods that override this need to
do the same! */
self->state=cPersistent_GHOST_STATE;
ghostify(self);
}
/* need to delay releasing the last reference on instance attributes
until after we have finished accounting for losing our state */
if(dict2)
{
PyDict_Clear(dict2);
Py_DECREF(dict2);
}
Py_INCREF(Py_None);
return Py_None;
}
......@@ -333,8 +413,8 @@ Per_dealloc(cPersistentObject *self)
#ifdef DEBUG_LOG
if(idebug_log < 0) call_debug("del",self);
#endif
Py_XDECREF(self->jar);
Py_XDECREF(self->oid);
deallocated(self);
Py_XDECREF(self->cache);
Py_DECREF(self->ob_type);
PyObject_DEL(self);
}
......@@ -387,7 +467,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
{
UPDATE_STATE_IF_NECESSARY(self, NULL);
self->atime=((long)(time(NULL)/3))%65536;
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
if (self->serial[7]=='\0' && self->serial[6]=='\0' &&
self->serial[5]=='\0' && self->serial[4]=='\0' &&
......@@ -419,7 +499,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
{
UPDATE_STATE_IF_NECESSARY(self, NULL);
self->atime=((long)(time(NULL)/3))%65536;
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
}
return getattrf((PyObject *)self, oname);
......@@ -466,6 +546,21 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
{
if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6])
{
if(HOME(self))
{
int result;
if(!v)
{
PyErr_SetString(PyExc_ValueError,"can not delete the oid of a cached object");
return -1;
}
if(PyObject_Cmp(self->oid,v,&result)<0) return -1;
if(result)
{
PyErr_SetString(PyExc_ValueError,"can not change the oid of a cached object");
return -1;
}
}
Py_XINCREF(v);
ASSIGN(self->oid, v);
return 0;
......@@ -509,7 +604,6 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
v=PyObject_GetAttr(OBJECT(self), py__p_deactivate);
if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); }
if (v) { Py_DECREF(v); }
self->state=cPersistent_GHOST_STATE;
return 0;
}
if (PyObject_IsTrue(v)) return changed(self);
......@@ -521,8 +615,7 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
{
UPDATE_STATE_IF_NECESSARY(self, -1);
/* Record access times */
self->atime=((long)(time(NULL)/3))%65536;
KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
if((! (*name=='_' && name[1]=='v' && name[2]=='_'))
&& (self->state != cPersistent_CHANGED_STATE && self->jar)
......@@ -680,9 +773,11 @@ truecPersistenceCAPI = {
(getattrofunc)Per_getattro, /*tp_getattr with object key*/
(setattrofunc)Per_setattro, /*tp_setattr with object key*/
changed,
accessed,
ghostify,
deallocated,
(intfunctionwithpythonarg)Per_setstate,
(pergetattr)Per_getattr,
(persetattr)_setattro,
};
void
......
src/persistent/cPersistence.h
View file @ 226ee421
......@@ -18,12 +18,21 @@
#include "ExtensionClass.h"
#include <time.h>
#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid; char serial[8]; unsigned short atime; signed char state; unsigned char reserved;
#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid, *cache; CPersistentRing ring; char serial[8]; signed char state; unsigned char reserved[3];
#define cPersistent_GHOST_STATE -1
#define cPersistent_UPTODATE_STATE 0
#define cPersistent_CHANGED_STATE 1
#define cPersistent_STICKY_STATE 2
struct ccobject_head_struct;
typedef struct CPersistentRing_struct
{
struct CPersistentRing_struct *prev;
struct CPersistentRing_struct *next;
} CPersistentRing;
typedef struct {
cPersistent_HEAD
} cPersistentObject;
......@@ -36,6 +45,9 @@ typedef struct {
getattrofunc getattro;
setattrofunc setattro;
int (*changed)(cPersistentObject*);
void (*accessed)(cPersistentObject*);
void (*ghostify)(cPersistentObject*);
void (*deallocated)(cPersistentObject*);
int (*setstate)(PyObject*);
pergetattr pergetattro;
persetattr persetattro;
......@@ -59,11 +71,13 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
#define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O)))
#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
#define PER_GHOSTIFY(O) (cPersistenceCAPI->ghostify((cPersistentObject*)(O)))
#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
#define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
#define PER_PREVENT_DEACTIVATION(O) ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
#define PER_DEL(O) Py_XDECREF((O)->jar); Py_XDECREF((O)->oid);
#define PER_DEL(O) (cPersistenceCAPI->deallocated((cPersistentObject*)(O)))
#define PER_USE(O) \
(((O)->state != cPersistent_GHOST_STATE \
......@@ -71,7 +85,7 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
? (((O)->state==cPersistent_UPTODATE_STATE) \
? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0)
#define PER_ACCESSED(O) ((O)->atime=((long)(time(NULL)/3))%65536)
#define PER_ACCESSED(O) (cPersistenceCAPI->accessed((cPersistentObject*)(O)))
#endif
......
src/persistent/cPickleCache.c
View file @ 226ee421
......@@ -2,448 +2,531 @@
Copyright (c) 2001, 2002 Zope Corporation and Contributors.
All Rights Reserved.
This software is subject to the provisions of the Zope Public License,
Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution.
THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
FOR A PARTICULAR PURPOSE
****************************************************************************/
static char cPickleCache_doc_string[] =
static char cPickleCache_doc_string[] =
"Defines the PickleCache used by ZODB Connection objects.\n"
"\n"
"$Id: cPickleCache.c,v 1.41 2002/03/08 18:36:14 jeremy Exp $\n";
"$Id: cPickleCache.c,v 1.42 2002/03/27 10:14:04 htrd Exp $\n";
/* Compute the current time in the units and range used for peristent
objects. */
#define PER_TIME() ((long)(time(NULL) / 3)) % 65536
#define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
#define UNLESS(E) if(!(E))
#define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
#define OBJECT(O) ((PyObject*)O)
#define DONT_USE_CPERSISTENCECAPI
#include "cPersistence.h"
#include <time.h>
#include <stddef.h>
#undef Py_FindMethod
static PyObject *py_reload, *py__p_jar, *py__p_changed;
typedef struct {
PyObject_HEAD
PyObject *data;
PyObject *jar;
PyObject *setklassstate;
int position;
int cache_size;
int cache_age;
/* Cache statistics */
int sum_deal;
int sum_deac;
double sum_age;
int n, na;
time_t last_check; /* Time of last gc */
double mean_age;
double mean_deal;
double mean_deac;
double df, dfa; /* Degees of freedom for above stats */
} ccobject;
static PyObject *py__p_oid, *py_reload, *py__p_jar, *py__p_changed;
#define WEIGHTING_PERIOD 600
/* define this for extra debugging checks, and lousy performance */
#define MUCH_RING_CHECKING 1
/*
How to compute weighted means?
/* Do we want 'engine noise'.... abstract debugging output useful for
visualizing cache behavior */
#if 0
#define ENGINE_NOISE(A) printf(A)
#else
#define ENGINE_NOISE(A) ((void)A)
#endif
Assume we have two means, a current mean, M, and a mean as of some
time d seconds in the past, Md. The means have effective degrees
of freedom, N, and Nd. Where Nd is adjusted by d is some fashion.
The combined mean is (M*N+Md*Nd)/(N+Nd). The degrees of freedom
of the combined mean, Nc, is N+Nd. Nd is computed by weighting
an old degree of freedom with the weight: I/(I+d), where I is some
suitably chosen constant, which we will call a "weighting period".
*/
/* the layout of this struct is the same as the start of ccobject_head in cPersistence.c */
typedef struct {
PyObject_HEAD
CPersistentRing ring_home;
int non_ghost_count;
int klass_count;
PyObject *data;
PyObject *jar;
PyObject *setklassstate;
int cache_size;
int ring_lock;
int cache_drain_resistance;
} ccobject;
staticforward PyTypeObject Cctype;
staticforward int present_in_ring(ccobject *self,CPersistentRing *target);
staticforward int check_ring(ccobject *self,const char *context);
staticforward int cc_ass_sub(ccobject *self, PyObject *key, PyObject *v);
/* ---------------------------------------------------------------- */
static int
gc_item(ccobject *self, PyObject *key, PyObject *v, long now, int dt)
static PyObject *object_from_oid(ccobject *self,PyObject *key)
/* somewhat of a replacement for PyDict_GetItem(self->data....
however this returns a *new* reference */
{
if (!(v && key))
return 0;
self->n++;
/* If there is at most one reference to this object, then the
cache has the only reference. It can be removed. */
if (v->ob_refcnt <= 1) {
self->sum_deal++;
/* XXX The fact that this works will iterating over
self->data with PyDict_Next() is an accident of the
current Python dictionary implementation. */
return PyDict_DelItem(self->data, key);
}
PyObject *v = PyDict_GetItem(self->data, key);
if(!v) return NULL;
if (dt >= 0 &&
(!PyExtensionClass_Check(v)) &&
((cPersistentObject*)v)->jar == self->jar /* I'm paranoid */ &&
((cPersistentObject*)v)->state == cPersistent_UPTODATE_STATE) {
now -= ((cPersistentObject*)v)->atime;
if (now < 0)
now += 65536;
self->na++;
self->sum_age += now;
if (now > dt) {
/* We have a cPersistent object that hasn't been used in
a while. Reinitialize it, hopefully freeing it's
state.
*/
self->sum_deac++;
if (PyObject_SetAttr(v, py__p_changed, Py_None) < 0)
PyErr_Clear();
}
}
return 0;
Py_INCREF(v);
return v;
}
static void
update_stats(ccobject *self, time_t now)
static cPersistentObject *object_from_ring(ccobject *self,CPersistentRing *here,const char *context)
{
double d, deal, deac;
d = now - self->last_check;
if(d < 1)
return;
self->df *= WEIGHTING_PERIOD / (WEIGHTING_PERIOD + d);
self->dfa *= WEIGHTING_PERIOD / (WEIGHTING_PERIOD + d);
self->mean_age = ((self->mean_age * self->dfa + self->sum_age)/
(self->dfa + self->na)) * 3;
self->sum_age = 0;
deac = self->sum_deac / d;
self->sum_deac = 0;
self->mean_deac = ((self->mean_deac * self->dfa+deac)/
(self->dfa + self->na));
self->sum_deac = 0;
self->dfa += self->na;
self->na = 0;
deal=self->sum_deal/d;
self->sum_deal = 0;
self->mean_deal = ((self->mean_deal * self->df + deal)/
(self->df +self->n));
self->sum_deal = 0;
self->df += self->n;
self->n = 0;
self->last_check = now;
/* Given a position in the LRU ring, return a borrowed
reference to the object at that point in the ring. The caller is
responsible for ensuring that this ring position really does
correspond to a persistent object, although the debugging
version will double-check this. */
PyObject *object = (PyObject *)(((char *)here)-offsetof(cPersistentObject,ring));
#ifdef MUCH_RING_CHECKING
if(!PyExtensionInstance_Check(object))
{
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered non-ExtensionClass object in %s",context);
return NULL;
}
if(!(((PyExtensionClass*)(object->ob_type))->class_flags & PERSISTENT_TYPE_FLAG))
{
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered non-persistent object in %s",context);
return NULL;
}
if(((cPersistentObject*)object)->jar!=self->jar)
{
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered object from a different jar in %s",context);
return NULL;
}
if(((cPersistentObject*)object)->cache!=(PyObject *)self)
{
PyErr_Format(PyExc_RuntimeError,"Unexpectedly encountered broken ring in %s",context);
return NULL;
}
#endif
return (cPersistentObject *)object;
}
static int
check_size(ccobject *self)
scan_gc_items(ccobject *self,int target)
{
if (self->cache_size < 1)
return 0;
return PyDict_Size(self->data);
cPersistentObject *object;
int error;
CPersistentRing placeholder;
CPersistentRing *here = self->ring_home.next;
#ifdef MUCH_RING_CHECKING
int safety_counter = self->cache_size*10;
if(safety_counter<10000) safety_counter = 10000;
#endif
while(1)
{
if(check_ring(self,"mid-gc")) return -1;
#ifdef MUCH_RING_CHECKING
if(!safety_counter--)
{
/* This loop has been running for a very long time.
It is possible that someone loaded a very large number of objects,
and now wants us to blow them all away. However it may
also indicate a logic error. If the loop has been running this
long then you really have to doubt it will ever terminate.
In the MUCH_RING_CHECKING build we prefer to raise an exception
here */
PyErr_SetString(PyExc_RuntimeError,"scan_gc_items safety counter exceeded");
return -1;
}
if(!present_in_ring(self,here))
{
/* Our current working position is no longer in the ring. Thats bad. */
PyErr_SetString(PyExc_RuntimeError,"working position fell out the ring, in scan_gc_items");
return -1;
}
#endif
if(here==&self->ring_home)
{
/* back to the home position. stop looking */
return 0;
}
/* At this point we know that the ring only contains nodes from
persistent objects, plus our own home node. We can safely
assume this is a persistent object now we know it is not the home */
object = object_from_ring(self,here,"scan_gc_items");
if(!object) return -1;
if(self->non_ghost_count<=target)
{
/* we are small enough */
return 0;
}
else if(object->state==cPersistent_UPTODATE_STATE)
{
/* deactivate it. This is the main memory saver. */
ENGINE_NOISE("G");
/* add a placeholder */
placeholder.next = here->next;
placeholder.prev = here;
here->next->prev = &placeholder;
here->next = &placeholder;
error = PyObject_SetAttr((PyObject *)object,py__p_changed,Py_None);
/* unlink the placeholder */
placeholder.next->prev=placeholder.prev;
placeholder.prev->next=placeholder.next;
here = placeholder.next;
if(error)
return -1; /* problem */
}
else
{
ENGINE_NOISE(".");
here = here->next;
}
}
}
static int
gc_all_items(ccobject *self, int now, int dt)
static PyObject *
lockgc(ccobject *self,int target_size)
{
PyObject *key, *v;
int i;
if(self->ring_lock)
{
Py_INCREF(Py_None);
return Py_None;
}
for(i = 0; PyDict_Next(self->data, &i, &key, &v); )
if (gc_item(self, key, v, now, dt) < 0)
return -1;
return 0;
if(check_ring(self,"pre-gc")) return NULL;
ENGINE_NOISE("<");
self->ring_lock = 1;
if(scan_gc_items(self,target_size))
{
self->ring_lock = 0;
return NULL;
}
self->ring_lock = 0;
ENGINE_NOISE(">\n");
if(check_ring(self,"post-gc")) return NULL;
Py_INCREF(Py_None);
return Py_None;
}
static int
fullgc(ccobject *self, int dt)
static PyObject *
cc_incrgc(ccobject *self, PyObject *args)
{
long now;
int n=1;
if (check_size(self) <= 0)
return 0;
int starting_size = self->non_ghost_count;
now = PER_TIME();
dt /= 3;
int target_size = self->cache_size;
if (gc_all_items(self, now, dt) < 0)
return -1;
self->position = 0;
if(self->cache_drain_resistance>=1)
{
/* This cache will gradually drain down to a small size. Check
a (small) number of objects proportional to the current size */
if (now - self->last_check > 1)
update_stats(self, now);
return 0;
}
static int
reallyfullgc(ccobject *self, int dt)
{
int l, last;
time_t now;
last = check_size(self);
if (last <= 0)
return 0;
now = PER_TIME();
/* Units are 3 seconds */
dt /= 3;
/* First time through should get refcounts to 1 */
if (gc_all_items(self, now, dt) < 0)
return -1;
l = PyDict_Size(self->data);
if (l < 0)
return -1;
/* Now continue to collect until the size of the cache stops
decreasing. */
while (l < last) {
if (gc_all_items(self, now, dt) < 0)
return -1;
last = l;
l = PyDict_Size(self->data);
if (l < 0)
return -1;
int target_size_2 = starting_size - 1 - starting_size/self->cache_drain_resistance;
if(target_size_2<target_size)
target_size = target_size_2;
}
if (now - self->last_check > 1)
update_stats(self, now);
self->position = 0;
return 0;
}
static int
maybegc(ccobject *self, PyObject *thisv)
{
int n, s, size, dt;
long now;
PyObject *key=0, *v=0;
s = check_size(self);
if (s <= 0)
return 0;
now = PER_TIME();
size = self->cache_size;
self->cache_size = 0;
/* Decide how many objects to look at */
n = (s - size) / 10;
if (n < 3)
n = 3;
/* Decide how much time to give them before deactivating them */
s = 8 * size / s;
if (s > 100)
s = 100;
dt = (long)(self->cache_age * (0.2 + 0.1 * s));
/* Units are 3 seconds */
dt /= 3;
while (--n >= 0) {
if (PyDict_Next(self->data, &(self->position), &key, &v)) {
if (v != thisv && gc_item(self, key, v, now, dt) < 0) {
self->cache_size=size;
return -1;
}
}
else
self->position = 0;
}
self->cache_size = size;
if (now - self->last_check > 1)
update_stats(self, now);
UNLESS (PyArg_ParseTuple(args, "|i",&n)) return NULL;
return 0;
return lockgc(self,target_size);
}
static PyObject *
cc_full_sweep(ccobject *self, PyObject *args)
{
int dt = self->cache_age;
if (!PyArg_ParseTuple(args, "|i:full_sweep", &dt))
return NULL;
if (dt < -1) {
PyErr_SetString(PyExc_ValueError, "age must be >= -1");
return NULL;
}
if (fullgc(self, dt) == -1)
return NULL;
Py_INCREF(Py_None);
return Py_None;
int dt=0;
UNLESS(PyArg_ParseTuple(args, "|i", &dt)) return NULL;
return lockgc(self,0);
}
static PyObject *
cc_reallyfull_sweep(ccobject *self, PyObject *args)
{
int dt = self->cache_age;
if (!PyArg_ParseTuple(args, "|i:minimize", &dt))
return NULL;
if (dt < -1) {
PyErr_SetString(PyExc_ValueError, "age must be >= -1");
return NULL;
}
if (reallyfullgc(self, dt) == -1)
return NULL;
Py_INCREF(Py_None);
return Py_None;
int dt=0;
UNLESS(PyArg_ParseTuple(args, "|i", &dt)) return NULL;
return lockgc(self,0);
}
static PyObject *
cc_incrgc(ccobject *self, PyObject *args)
static void
_invalidate(ccobject *self, PyObject *key)
{
int n = 1;
PyObject *v=object_from_oid(self, key);
if (!PyArg_ParseTuple(args, "|i:incrgr", &n))
return NULL;
if(!v)
{
/* shouldnt this be an error? for now Ill follow Jims lead */
PyErr_Clear();
}
else
{
if (PyExtensionClass_Check(v))
{
if(v->ob_refcnt <= 1)
{
self->klass_count--;
if (PyDict_DelItem(self->data, key) < 0)
PyErr_Clear();
}
else
{
v=PyObject_CallFunction(self->setklassstate,
"O", v);
if (v) Py_DECREF(v);
else PyErr_Clear();
}
}
else
{
if(PyObject_DelAttr(v,py__p_changed) < 0)
PyErr_Clear();
}
Py_DECREF(v);
}
}
for (; --n >= 0;)
if (maybegc(self, NULL) < 0)
return NULL;
static PyObject *
cc_invalidate(ccobject *self, PyObject *args)
{
PyObject *inv, *key, *v;
int i;
if (PyArg_ParseTuple(args, "O!", &PyDict_Type, &inv)) {
for (i=0; PyDict_Next(inv, &i, &key, &v); )
if (key==Py_None)
{ /* Eek some nitwit invalidated everything! */
for (i=0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
break;
}
else
_invalidate(self, key);
PyDict_Clear(inv);
}
else {
PyErr_Clear();
UNLESS (PyArg_ParseTuple(args, "O", &inv)) return NULL;
if (PyString_Check(inv))
_invalidate(self, inv);
else if (inv==Py_None) /* All */
for (i=0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
else {
int l;
PyErr_Clear();
if ((l=PyObject_Length(inv)) < 0) return NULL;
for(i=l; --i >= 0; )
{
UNLESS (key=PySequence_GetItem(inv, i)) return NULL;
_invalidate(self, key);
Py_DECREF(key);
}
PySequence_DelSlice(inv, 0, l);
}
}
Py_INCREF(Py_None);
return Py_None;
}
static void
_invalidate(ccobject *self, PyObject *key)
static PyObject *
cc_get(ccobject *self, PyObject *args)
{
PyObject *v = PyDict_GetItem(self->data, key);
PyObject *r, *key, *d=0;
if (!v)
return;
if (PyExtensionClass_Check(v))
if (v->ob_refcnt <= 1) {
self->sum_deal++;
if (PyDict_DelItem(self->data, key) < 0)
PyErr_Clear();
} else {
PyObject *t = PyTuple_New(1);
if (t) {
PyTuple_SET_ITEM(t, 0, v);
v = PyObject_CallObject(self->setklassstate, t);
/* Set tuple element to NULL so that deallocating the
tuple does not decref t.
*/
PyTuple_SET_ITEM(t, 0, NULL);
Py_DECREF(t);
} else
v = t;
if (v)
Py_DECREF(v);
else
PyErr_Clear();
UNLESS (PyArg_ParseTuple(args,"O|O", &key, &d)) return NULL;
UNLESS (r=(PyObject *)object_from_oid(self, key))
{
if (d)
{
PyErr_Clear();
r=d;
Py_INCREF(r);
}
else
{
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
else if (PyObject_DelAttr(v, py__p_changed) < 0)
PyErr_Clear();
}
return r;
}
static void
_invalidate_all(ccobject *self)
static PyObject *
cc_klass_items(ccobject *self, PyObject *args)
{
PyObject *key, *v;
int i;
PyObject *l,*k,*v;
int p = 0;
if(!PyArg_ParseTuple(args,"")) return NULL;
l = PyList_New(0);
if(!l) return NULL;
while(PyDict_Next(self->data, &p, &k, &v))
{
if(PyExtensionClass_Check(v))
{
v=PyObject_CallMethod(l,"append","((OO))",k,v);
if(!v)
{
Py_DECREF(l);
return NULL;
}
}
}
for (i = 0; PyDict_Next(self->data, &i, &key, &v); )
_invalidate(self, key);
return l;
}
static PyObject *
cc_invalidate(ccobject *self, PyObject *args)
cc_lru_items(ccobject *self, PyObject *args)
{
PyObject *inv, *key, *v;
int i;
PyObject *l;
CPersistentRing *here;
if (!PyArg_ParseTuple(args, "O:invalidate", &inv))
return NULL;
if (PyDict_Check(inv)) {
for (i = 0; PyDict_Next(inv, &i, &key, &v); )
if (key == Py_None) {
/* Eek some nitwit invalidated everything! */
_invalidate_all(self);
break;
}
else
_invalidate(self, key);
PyDict_Clear(inv);
} else if (PyString_Check(inv))
_invalidate(self, inv);
else if (inv == Py_None) /* All */
_invalidate_all(self);
else {
int l = PyObject_Length(inv);
if (l < 0)
return NULL;
for (i = l; --i >= 0; ) {
key = PySequence_GetItem(inv, i);
if (!key)
return NULL;
_invalidate(self, key);
Py_DECREF(key);
}
PySequence_DelSlice(inv, 0, l);
if(!PyArg_ParseTuple(args,"")) return NULL;
if(self->ring_lock)
{
PyErr_SetString(PyExc_ValueError,".lru_items() is unavailable during garbage collection");
return NULL;
}
Py_INCREF(Py_None);
return Py_None;
if(check_ring(self,"pre-cc_items")) return NULL;
l = PyList_New(0);
if(!l) return NULL;
here = self->ring_home.next;
while(here!=&self->ring_home)
{
cPersistentObject *object = object_from_ring(self,here,"cc_items");
PyObject *v;
if(!object)
{
Py_DECREF(l);
return NULL;
}
v=PyObject_CallMethod(l,"append","((OO))",object->oid,object);
if(!v)
{
Py_DECREF(l);
return NULL;
}
Py_DECREF(v);
here = here->next;
}
return l;
}
static PyObject *
cc_get(ccobject *self, PyObject *args)
cc_oid_unreferenced(ccobject *self, PyObject *args)
{
PyObject *r, *key, *d = NULL;
PyObject *oid,*v;
if(!PyArg_ParseTuple(args,"O",&oid)) return NULL;
if (!PyArg_ParseTuple(args, "O|O:get", &key, &d))
return NULL;
r = PyDict_GetItem(self->data, key);
if (!r) {
if (d)
r = d;
else {
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
v = PyDict_GetItem(self->data, oid);
if(!v) return NULL;
if(v->ob_refcnt)
{
PyErr_Format(PyExc_ValueError,"object has reference count of %d, should be zero",v->ob_refcnt);
return NULL;
}
Py_INCREF(r);
return r;
/* Need to be very hairy here because a dictionary is about
to decref an already deleted object */
#ifdef Py_TRACE_REFS
#error "this code path has not been tested - Toby Dickenson"
_Py_NewReference(v);
/* it may be a problem that v->ob_type is still NULL? */
#else
Py_INCREF(v);
#endif
if(v->ob_refcnt!=1)
{
PyErr_SetString(PyExc_ValueError,"refcount is not 1 after resurrection");
return NULL;
}
/* return the stolen reference */
Py_INCREF(v);
PyDict_DelItem(self->data, oid);
if(v->ob_refcnt!=1)
{
PyErr_SetString(PyExc_ValueError,"refcount is not 1 after removal from dict");
return NULL;
}
/* undo the temporary resurrection */
#ifdef Py_TRACE_REFS
_Py_ForgetReference(v);
#else
v->ob_refcnt=0;
#endif
Py_INCREF(Py_None);
return Py_None;
}
static struct PyMethodDef cc_methods[] = {
{"_oid_unreferenced", (PyCFunction)cc_oid_unreferenced, METH_VARARGS,
NULL
},
{"lru_items", (PyCFunction)cc_lru_items, METH_VARARGS,
"List (oid, object) pairs from the lru list, as 2-tuples.\n"
},
{"klass_items", (PyCFunction)cc_klass_items, METH_VARARGS,
"List (oid, object) pairs of cached persistent classes.\n"
},
{"full_sweep", (PyCFunction)cc_full_sweep, METH_VARARGS,
"full_sweep([age]) -- Perform a full sweep of the cache\n\n"
"Make a single pass through the cache, removing any objects that are no\n"
"longer referenced, and deactivating objects that have not been\n"
"accessed in the number of seconds given by 'age'. "
"'age defaults to the cache age.\n"
"longer referenced, and deactivating enough objects to bring\n"
"the cache under its size limit\n"
"The optional 'age' parameter is ignored.\n"
},
{"minimize", (PyCFunction)cc_reallyfull_sweep, METH_VARARGS,
"minimize([age]) -- Remove as many objects as possible\n\n"
"Make multiple passes through the cache, removing any objects that are no\n"
"longer referenced, and deactivating objects that have not been\n"
"accessed in the number of seconds given by 'age'. 'age defaults to 0.\n"
"longer referenced, and deactivating enough objects to bring the"
" cache under its size limit\n"
"The option 'age' parameter is ignored.\n"
},
{"incrgc", (PyCFunction)cc_incrgc, METH_VARARGS,
"incrgc() -- Perform incremental garbage collection"},
"incrgc([n]) -- Perform incremental garbage collection\n\n"
"Some other implementations support an optional parameter 'n' which\n"
"indicates a repetition count; this value is ignored.\n"},
{"invalidate", (PyCFunction)cc_invalidate, METH_VARARGS,
"invalidate(oids) -- invalidate one, many, or all ids"},
{"get", (PyCFunction)cc_get, METH_VARARGS,
......@@ -454,115 +537,102 @@ static struct PyMethodDef cc_methods[] = {
static ccobject *
newccobject(PyObject *jar, int cache_size, int cache_age)
{
ccobject *self;
ccobject *self;
self = PyObject_NEW(ccobject, &Cctype);
if (!self)
UNLESS(self = PyObject_NEW(ccobject, &Cctype)) return NULL;
self->setklassstate=self->jar=NULL;
if((self->data=PyDict_New()))
{
self->jar=jar;
Py_INCREF(jar);
UNLESS (self->setklassstate=PyObject_GetAttrString(jar, "setklassstate"))
return NULL;
self->setklassstate = self->jar = NULL;
self->data = PyDict_New();
if (self->data) {
self->jar=jar;
Py_INCREF(jar);
self->setklassstate = PyObject_GetAttrString(jar, "setklassstate");
if (!self->setklassstate) {
Py_DECREF(jar);
Py_DECREF(self->data);
goto error;
}
self->position = 0;
self->cache_size = cache_size;
self->cache_age = cache_age < 1 ? 1 : cache_age;
self->sum_deal = 0;
self->sum_deac = 0;
self->sum_age = 0;
self->mean_deal = 0;
self->mean_deac = 0;
self->mean_age = 0;
self->df = 1;
self->dfa = 1;
self->n = 0;
self->na = 0;
self->last_check = time(NULL);
return self;
self->cache_size=cache_size;
self->non_ghost_count=0;
self->klass_count=0;
self->cache_drain_resistance=0;
self->ring_lock=0;
self->ring_home.next = &self->ring_home;
self->ring_home.prev = &self->ring_home;
return self;
}
error:
Py_DECREF(self);
return NULL;
Py_DECREF(self);
return NULL;
}
static void
cc_dealloc(ccobject *self)
{
Py_XDECREF(self->data);
Py_XDECREF(self->jar);
Py_XDECREF(self->setklassstate);
PyObject_DEL(self);
Py_XDECREF(self->data);
Py_XDECREF(self->jar);
Py_XDECREF(self->setklassstate);
PyMem_DEL(self);
}
static PyObject *
cc_getattr(ccobject *self, char *name)
{
PyObject *r;
if (*name == 'c') {
if(strcmp(name, "cache_age") == 0)
return PyInt_FromLong(self->cache_age);
if(strcmp(name, "cache_size") == 0)
return PyInt_FromLong(self->cache_size);
if(strcmp(name, "cache_mean_age") == 0)
return PyFloat_FromDouble(self->mean_age);
if(strcmp(name, "cache_mean_deal") == 0)
return PyFloat_FromDouble(self->mean_deal);
if(strcmp(name, "cache_mean_deac") == 0)
return PyFloat_FromDouble(self->mean_deac);
if(strcmp(name, "cache_df") == 0)
return PyFloat_FromDouble(self->df);
if(strcmp(name, "cache_dfa") == 0)
return PyFloat_FromDouble(self->dfa);
if(strcmp(name, "cache_last_gc_time") == 0)
return PyFloat_FromDouble(self->last_check);
if(strcmp(name, "cache_data") == 0) {
Py_INCREF(self->data);
return self->data;
PyObject *r;
if(check_ring(self,"getattr")) return NULL;
if(*name=='c')
{
if(strcmp(name,"cache_age")==0)
return PyInt_FromLong(0); /* this cache does not use this value */
if(strcmp(name,"cache_size")==0)
return PyInt_FromLong(self->cache_size);
if(strcmp(name,"cache_drain_resistance")==0)
return PyInt_FromLong(self->cache_drain_resistance);
if(strcmp(name,"cache_non_ghost_count")==0)
return PyInt_FromLong(self->non_ghost_count);
if(strcmp(name,"cache_klass_count")==0)
return PyInt_FromLong(self->klass_count);
if(strcmp(name,"cache_data")==0)
{
/* now a copy of our data; the ring is too fragile */
return PyDict_Copy(self->data);
}
}
if ((strcmp(name, "has_key") == 0)
|| (strcmp(name, "items") == 0)
|| (strcmp(name, "keys") == 0))
return PyObject_GetAttrString(self->data, name);
r = Py_FindMethod(cc_methods, (PyObject *)self, name);
if (!r) {
PyErr_Clear();
return PyObject_GetAttrString(self->data, name);
}
if((*name=='h' && strcmp(name, "has_key")==0) ||
(*name=='i' && strcmp(name, "items")==0) ||
(*name=='k' && strcmp(name, "keys")==0)
)
return PyObject_GetAttrString(self->data, name);
if((r=Py_FindMethod(cc_methods, (PyObject *)self, name)))
return r;
PyErr_Clear();
return PyObject_GetAttrString(self->data, name);
}
static int
cc_setattr(ccobject *self, char *name, PyObject *value)
{
if (value) {
if(value)
{
int v;
if (strcmp(name, "cache_age") == 0) {
v = PyInt_AsLong(value);
if (v == -1 && PyErr_Occurred())
return -1;
if (v > 0)
self->cache_age = v;
if(strcmp(name,"cache_age")==0)
{
/* this cache doesnt use the age */
return 0;
}
}
if (strcmp(name, "cache_size") == 0) {
v = PyInt_AsLong(value);
if (v == -1 && PyErr_Occurred())
return -1;
self->cache_size = v;
if(strcmp(name,"cache_size")==0)
{
UNLESS(PyArg_Parse(value,"i",&v)) return -1;
self->cache_size=v;
return 0;
}
}
}
if(strcmp(name,"cache_drain_resistance")==0)
{
UNLESS(PyArg_Parse(value,"i",&v)) return -1;
self->cache_drain_resistance=v;
return 0;
}
}
PyErr_SetString(PyExc_AttributeError, name);
return -1;
}
......@@ -570,7 +640,7 @@ cc_setattr(ccobject *self, char *name, PyObject *value)
static int
cc_length(ccobject *self)
{
return PyDict_Size(self->data);
return PyObject_Length(self->data);
}
static PyObject *
......@@ -578,39 +648,245 @@ cc_subscript(ccobject *self, PyObject *key)
{
PyObject *r;
r = PyDict_GetItem(self->data, key);
if (!r) {
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
if(check_ring(self,"__getitem__")) return NULL;
UNLESS (r=(PyObject *)object_from_oid(self, key))
{
PyErr_SetObject(PyExc_KeyError, key);
return NULL;
}
Py_INCREF(r);
return r;
}
static int
cc_ass_sub(ccobject *self, PyObject *key, PyObject *v)
{
if (v) {
if (PyExtensionClass_Check(v)
||
(PyExtensionInstance_Check(v)
&&
(((PyExtensionClass*)(v->ob_type))->class_flags
& PERSISTENT_TYPE_FLAG)
&&
(v->ob_type->tp_basicsize >= sizeof(cPersistentObject))
)
)
return PyDict_SetItem(self->data, key, v);
PyErr_SetString(PyExc_ValueError,
"Cache values must be persistent objects or classes.");
return -1;
int result;
if(v)
{
if( ( PyExtensionInstance_Check(v) &&
(((PyExtensionClass*)(v->ob_type))->class_flags & PERSISTENT_TYPE_FLAG) &&
(v->ob_type->tp_basicsize >= sizeof(cPersistentObject))
)
||
PyExtensionClass_Check(v)
)
{
PyObject *oid = PyObject_GetAttr(v,py__p_oid);
PyObject *object_again;
if(!oid)
{
return -1;
}
if(PyObject_Cmp(key,oid,&result))
{
Py_DECREF(oid);
return -1;
}
Py_DECREF(oid);
if(result)
{
PyErr_SetString(PyExc_ValueError,"key must be the same as the object's oid attribute");
return -1;
}
object_again = object_from_oid(self, key);
if(object_again)
{
if(object_again!=v)
{
Py_DECREF(object_again);
PyErr_SetString(PyExc_ValueError,"Can not re-register object under a different oid");
return -1;
}
else
{
/* re-register under the same oid - no work needed */
Py_DECREF(object_again);
return 0;
}
}
if(PyExtensionClass_Check(v))
{
if(PyDict_SetItem(self->data, key, v)) return -1;
self->klass_count++;
return 0;
}
else
{
if(((cPersistentObject*)v)->cache)
{
if(((cPersistentObject*)v)->cache==(PyObject *)self)
{
/* This object is already one of ours, which is ok.
It would be very strange if someone was trying to register the
same object under a different key */
}
else
{
/* This object is already in a different cache. */
PyErr_SetString(PyExc_ValueError, "Cache values may only be in one cache.");
return -1;
}
}
if(check_ring(self,"pre-setitem")) return -1;
if(PyDict_SetItem(self->data, key, v)) return -1;
Py_INCREF(self);
((cPersistentObject*)v)->cache = (PyObject *)self;
if(((cPersistentObject*)v)->state>=0)
{
/* insert this non-ghost object into the ring just behind the home position */
self->non_ghost_count++;
((cPersistentObject*)v)->ring.next = &self->ring_home;
((cPersistentObject*)v)->ring.prev = self->ring_home.prev;
self->ring_home.prev->next = &((cPersistentObject*)v)->ring;
self->ring_home.prev = &((cPersistentObject*)v)->ring;
}
else
{
/* steal a reference from the dictionary; ghosts have a weak reference */
Py_DECREF(v);
}
if(check_ring(self,"post-setitem")) return -1;
return 0;
}
}
else
{
PyErr_SetString(PyExc_ValueError, "Cache values must be persistent objects.");
return -1;
}
}
else
{
/* unlink this item from the ring */
if(check_ring(self,"pre-delitem")) return -1;
v = (PyObject *)object_from_oid(self,key);
if(!v) return -1;
if(PyExtensionClass_Check(v))
{
self->klass_count--;
}
else
{
if(((cPersistentObject*)v)->state>=0)
{
self->non_ghost_count--;
((cPersistentObject*)v)->ring.next->prev = ((cPersistentObject*)v)->ring.prev;
((cPersistentObject*)v)->ring.prev->next = ((cPersistentObject*)v)->ring.next;
((cPersistentObject*)v)->ring.prev = NULL;
((cPersistentObject*)v)->ring.next = NULL;
}
else
{
/* This is a ghost object, so we havent kept a reference count on it.
For it have stayed alive this long someone else must be keeping a reference
to it. Therefore we need to temporarily give it back a reference count
before calling DelItem below */
Py_INCREF(v);
}
Py_DECREF(((cPersistentObject*)v)->cache);
((cPersistentObject*)v)->cache = NULL;
}
Py_DECREF(v);
if(PyDict_DelItem(self->data, key))
{
PyErr_SetString(PyExc_RuntimeError,
"unexpectedly couldnt remove key in cc_ass_sub");
return -1;
}
if(check_ring(self,"post-delitem")) return -1;
return 0;
}
return PyDict_DelItem(self->data, key);
}
static int _check_ring(ccobject *self,const char *context)
{
CPersistentRing *here = &(self->ring_home);
int expected = 1+self->non_ghost_count;
int total = 0;
do
{
if(++total>(expected+10)) return 3; /* ring too big, by a large margin */
if(!here->next) return 4; /* various linking problems */
if(!here->prev) return 5;
if(!here->next->prev) return 7;
if(!here->prev->next) return 8;
if(here->prev->next!=here) return 9;
if(here->next->prev!=here) return 10;
if(!self->ring_lock)
{
/* if the ring must be locked then it only contains object other than persistent instances */
if(here!=&self->ring_home)
{
cPersistentObject *object = object_from_ring(self,here,context);
if(!object) return 12;
if(object->state==cPersistent_GHOST_STATE)
return 13;
}
}
here = here->next;
}
while(here!=&self->ring_home);
if(self->ring_lock)
{
if(total<expected) return 6; /* ring too small; too big is ok when locked */
}
else
{
if(total!=expected) return 14; /* ring size wrong, or bad ghost accounting */
}
return 0;
}
static int check_ring(ccobject *self,const char *context)
{
#ifdef MUCH_RING_CHECKING
int code=_check_ring(self,context);
if(code)
{
/*printf(stderr,"BROKEN RING (code %d) in %s, size %d\n",code,context,PyDict_Size(self->data));*/
PyErr_Format(PyExc_RuntimeError,"broken ring (code %d) in %s, size %d",code,context,PyDict_Size(self->data));
return code;
}
#endif
return 0;
}
static int
present_in_ring(ccobject *self,CPersistentRing *target)
{
CPersistentRing *here = self->ring_home.next;
while(1)
{
if(here==target)
{
return 1;
}
if(here==&self->ring_home)
{
/* back to the home position, and we didnt find it */
return 0;
}
here = here->next;
}
}
static PyMappingMethods cc_as_mapping = {
(inquiry)cc_length, /*mp_length*/
(binaryfunc)cc_subscript, /*mp_subscript*/
......@@ -636,6 +912,10 @@ static PyTypeObject Cctype = {
(hashfunc)0, /*tp_hash*/
(ternaryfunc)0, /*tp_call*/
(reprfunc)0, /*tp_str*/
/* Space for future expansion */
0L,0L,0L,0L,
""
};
static PyObject *
......@@ -644,9 +924,9 @@ cCM_new(PyObject *self, PyObject *args)
int cache_size=100, cache_age=1000;
PyObject *jar;
if (!PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
UNLESS(PyArg_ParseTuple(args, "O|ii", &jar, &cache_size, &cache_age))
return NULL;
return (PyObject *)newccobject(jar, cache_size, cache_age);
return (PyObject*)newccobject(jar, cache_size,cache_age);
}
static struct PyMethodDef cCM_methods[] = {
......@@ -657,12 +937,11 @@ static struct PyMethodDef cCM_methods[] = {
void
initcPickleCache(void)
{
PyObject *m;
PyObject *m, *d;
Cctype.ob_type = &PyType_Type;
Cctype.ob_type=&PyType_Type;
if (!ExtensionClassImported)
return;
UNLESS(ExtensionClassImported) return;
m = Py_InitModule4("cPickleCache", cCM_methods, cPickleCache_doc_string,
(PyObject*)NULL, PYTHON_API_VERSION);
......@@ -670,4 +949,15 @@ initcPickleCache(void)
py_reload = PyString_InternFromString("reload");
py__p_jar = PyString_InternFromString("_p_jar");
py__p_changed = PyString_InternFromString("_p_changed");
py__p_oid = PyString_InternFromString("_p_oid");
d = PyModule_GetDict(m);
PyDict_SetItemString(d,"cache_variant",PyString_FromString("stiff/c"));
#ifdef MUCH_RING_CHECKING
PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(1));
#else
PyDict_SetItemString(d,"MUCH_RING_CHECKING",PyInt_FromLong(0));
#endif
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7