merged toby-stiff-cache-branch and toby-unicode-branch

src/BTrees/BTreeTemplate.c

@@ -12,7 +12,7 @@
   
  ****************************************************************************/
 
-#define BTREETEMPLATE_C "$Id: BTreeTemplate.c,v 1.25 2002/03/08 18:33:01 jeremy Exp $\n"
+#define BTREETEMPLATE_C "$Id: BTreeTemplate.c,v 1.26 2002/03/27 10:14:01 htrd Exp $\n"
 
 /*
 ** _BTree_get
@@ -540,7 +540,7 @@ _BTree_clear(BTree *self)
 
   if (self->firstbucket)
     {
-      ASSERT(self->firstbucket->ob_refcnt > 1, 
+      ASSERT(self->firstbucket->ob_refcnt > 0, 
              "Invalid firstbucket pointer", -1);
       Py_DECREF(self->firstbucket);
       self->firstbucket=NULL;
@@ -573,7 +573,7 @@ BTree__p_deactivate(BTree *self, PyObject *args)
   if (self->state==cPersistent_UPTODATE_STATE && self->jar)
     {
       if (_BTree_clear(self) < 0) return NULL;
-      self->state=cPersistent_GHOST_STATE;
+      PER_GHOSTIFY(self);
     }
 
   Py_INCREF(Py_None);

src/BTrees/BucketTemplate.c

@@ -12,7 +12,7 @@
   
  ****************************************************************************/
 
-#define BUCKETTEMPLATE_C "$Id: BucketTemplate.c,v 1.28 2002/03/08 18:33:01 jeremy Exp $\n"
+#define BUCKETTEMPLATE_C "$Id: BucketTemplate.c,v 1.29 2002/03/27 10:14:01 htrd Exp $\n"
 
 /*
 ** _bucket_get
@@ -809,7 +809,7 @@ bucket__p_deactivate(Bucket *self, PyObject *args)
   if (self->state==cPersistent_UPTODATE_STATE && self->jar)
     {
       if (_bucket_clear(self) < 0) return NULL;
-      self->state=cPersistent_GHOST_STATE;
+      PER_GHOSTIFY(self);
     }
 
   Py_INCREF(Py_None);

src/Persistence/cPersistence.c

@@ -14,11 +14,21 @@
 static char cPersistence_doc_string[] = 
 "Defines Persistent mixin class for persistent objects.\n"
 "\n"
-"$Id: cPersistence.c,v 1.50 2002/03/08 18:36:13 jeremy Exp $\n";
+"$Id: cPersistence.c,v 1.51 2002/03/27 10:14:04 htrd Exp $\n";
 
 #include <string.h>
 #include "cPersistence.h"
 
+/* the layout of this struct is the same as the start of ccobject in cPickleCache.c */
+struct ccobject_head_struct {
+  PyObject_HEAD
+  CPersistentRing ring_home;
+  int non_ghost_count;
+};
+
+#define HOME(O)             ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->ring_home)) )
+#define NON_GHOST_COUNT(O)  ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->non_ghost_count)) )
+
 #define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
 #define UNLESS(E) if(!(E))
 #define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
@@ -112,21 +122,82 @@ if(self->state < 0 && self->jar)                                 \
 {								 \
   PyObject *r;							 \
       								 \
+  int *count = NON_GHOST_COUNT(self);                            \
+  if(count)                                                      \
+  {                                                              \
+      (*count)++;                                                \
+      self->ring.next = HOME(self);                              \
+      self->ring.prev = HOME(self)->prev;                        \
+      HOME(self)->prev->next = &self->ring;                      \
+      HOME(self)->prev = &self->ring;                            \
+      Py_INCREF(self);                                           \
+  }                                                              \
   self->state=cPersistent_CHANGED_STATE; 	                 \
   UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self))   \
     {                                                            \
-      self->state=cPersistent_GHOST_STATE;                       \
+      ghostify(self);                                            \
       return ER;                                                 \
     }								 \
   self->state=cPersistent_UPTODATE_STATE;			 \
   Py_DECREF(r);							 \
 }
 
+#define KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self) \
+ if(HOME(self) && self->state>=0) {            \
+ self->ring.prev->next = self->ring.next;      \
+ self->ring.next->prev = self->ring.prev;      \
+ self->ring.next = HOME(self);           \
+ self->ring.prev = HOME(self)->prev;       \
+ HOME(self)->prev->next = &self->ring;    \
+ HOME(self)->prev = &self->ring; }
+
+
 
 /****************************************************************************/
 
 staticforward PyExtensionClass Pertype;
 
+static void
+accessed(cPersistentObject *self)
+{
+    KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
+}
+
+static void
+ghostify(cPersistentObject *self)
+{
+    int *count;
+    count = NON_GHOST_COUNT(self);
+    if(count && (self->state>=0))
+    {
+        (*count)--;
+        self->ring.next->prev = self->ring.prev;
+        self->ring.prev->next = self->ring.next;
+        self->ring.prev = NULL;
+        self->ring.next = NULL;
+        self->state = cPersistent_GHOST_STATE;
+        Py_DECREF(self);
+    }
+    else
+    {
+        self->state = cPersistent_GHOST_STATE;
+    }
+}
+
+static void
+deallocated(cPersistentObject *self)
+{
+    if(self->state>=0) ghostify(self);
+    if(self->cache)
+    {
+        PyObject *v=PyObject_CallMethod(self->cache,"_oid_unreferenced","O",self->oid);
+        if(!v) PyErr_Clear(); /* and explode later */
+        Py_XDECREF(v);
+    }
+    Py_XDECREF(self->jar);
+    Py_XDECREF(self->oid);
+}
+
 static int
 changed(cPersistentObject *self)
 {
@@ -185,7 +256,7 @@ Per___changed__(cPersistentObject *self, PyObject *args)
 static PyObject *
 Per__p_deactivate(cPersistentObject *self, PyObject *args)
 {
-  PyObject *dict;
+  PyObject *dict,*dict2=NULL;
 
 #ifdef DEBUG_LOG
   if (idebug_log < 0) call_debug("reinit",self);
@@ -197,13 +268,22 @@ Per__p_deactivate(cPersistentObject *self, PyObject *args)
   if (self->state==cPersistent_UPTODATE_STATE && self->jar &&
       HasInstDict(self) && (dict=INSTANCE_DICT(self)))
     {
+      dict2 = PyDict_Copy(dict);
       PyDict_Clear(dict);
       /* Note that we need to set to ghost state unless we are 
 	 called directly. Methods that override this need to
          do the same! */
-      self->state=cPersistent_GHOST_STATE;
+      ghostify(self);
     }
 
+  /* need to delay releasing the last reference on instance attributes
+  until after we have finished accounting for losing our state */
+  if(dict2)
+  {
+      PyDict_Clear(dict2);
+      Py_DECREF(dict2);
+  }
+
   Py_INCREF(Py_None);
   return Py_None;
 }
@@ -333,8 +413,8 @@ Per_dealloc(cPersistentObject *self)
 #ifdef DEBUG_LOG
   if(idebug_log < 0) call_debug("del",self);
 #endif
-  Py_XDECREF(self->jar);
-  Py_XDECREF(self->oid);
+  deallocated(self);
+  Py_XDECREF(self->cache);
   Py_DECREF(self->ob_type);
   PyObject_DEL(self);
 }
@@ -387,7 +467,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
 	      {
 		UPDATE_STATE_IF_NECESSARY(self, NULL);
 
-		self->atime=((long)(time(NULL)/3))%65536;
+		KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
 
 		if (self->serial[7]=='\0' && self->serial[6]=='\0' &&
 		    self->serial[5]=='\0' && self->serial[4]=='\0' &&
@@ -419,7 +499,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
     {
       UPDATE_STATE_IF_NECESSARY(self, NULL);
 
-      self->atime=((long)(time(NULL)/3))%65536;
+      KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
     }
 
   return getattrf((PyObject *)self, oname);
@@ -466,6 +546,21 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
     {
       if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6])
 	{
+	  if(HOME(self))
+	  {
+	    int result;
+	    if(!v)
+	    {
+              PyErr_SetString(PyExc_ValueError,"can not delete the oid of a cached object");
+              return -1;
+	    }
+	    if(PyObject_Cmp(self->oid,v,&result)<0) return -1;
+	    if(result)
+	    {
+              PyErr_SetString(PyExc_ValueError,"can not change the oid of a cached object");
+              return -1;
+            }
+	  }
 	  Py_XINCREF(v);
 	  ASSIGN(self->oid, v);
 	  return 0;
@@ -509,7 +604,6 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
 	      v=PyObject_GetAttr(OBJECT(self), py__p_deactivate);
 	      if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); }
 	      if (v) { Py_DECREF(v); }
-	      self->state=cPersistent_GHOST_STATE;
 	      return 0;
 	    }
 	  if (PyObject_IsTrue(v)) return changed(self);
@@ -521,8 +615,7 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
     {
       UPDATE_STATE_IF_NECESSARY(self, -1);
       
-      /* Record access times */
-      self->atime=((long)(time(NULL)/3))%65536;
+      KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
 
       if((! (*name=='_' && name[1]=='v' && name[2]=='_'))
 	 && (self->state != cPersistent_CHANGED_STATE && self->jar)
@@ -680,9 +773,11 @@ truecPersistenceCAPI = {
   (getattrofunc)Per_getattro,	/*tp_getattr with object key*/
   (setattrofunc)Per_setattro,	/*tp_setattr with object key*/
   changed,
+  accessed,
+  ghostify,
+  deallocated,
   (intfunctionwithpythonarg)Per_setstate,
   (pergetattr)Per_getattr,
-  (persetattr)_setattro,
 };
 
 void

src/Persistence/cPersistence.h

@@ -18,12 +18,21 @@
 #include "ExtensionClass.h"
 #include <time.h>
 
-#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid; char serial[8]; unsigned short atime;  signed char state;  unsigned char reserved; 
+
+#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid, *cache; CPersistentRing ring; char serial[8]; signed char state; unsigned char reserved[3];
 #define cPersistent_GHOST_STATE -1
 #define cPersistent_UPTODATE_STATE 0
 #define cPersistent_CHANGED_STATE 1
 #define cPersistent_STICKY_STATE 2
 
+struct ccobject_head_struct;
+
+typedef struct CPersistentRing_struct
+{
+    struct CPersistentRing_struct *prev;
+    struct CPersistentRing_struct *next;
+} CPersistentRing;
+
 typedef struct {
   cPersistent_HEAD
 } cPersistentObject;
@@ -36,6 +45,9 @@ typedef struct {
   getattrofunc getattro;
   setattrofunc setattro;
   int (*changed)(cPersistentObject*);
+  void (*accessed)(cPersistentObject*);
+  void (*ghostify)(cPersistentObject*);
+  void (*deallocated)(cPersistentObject*);
   int (*setstate)(PyObject*);
   pergetattr pergetattro;
   persetattr persetattro;
@@ -59,11 +71,13 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
 
 #define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O)))
 
-#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE)) 
+#define PER_GHOSTIFY(O) (cPersistenceCAPI->ghostify((cPersistentObject*)(O)))
+
+#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
 
-#define PER_PREVENT_DEACTIVATION(O)  ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE)) 
+#define PER_PREVENT_DEACTIVATION(O)  ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
 
-#define PER_DEL(O) Py_XDECREF((O)->jar); Py_XDECREF((O)->oid);
+#define PER_DEL(O) (cPersistenceCAPI->deallocated((cPersistentObject*)(O)))
 
 #define PER_USE(O) \
 (((O)->state != cPersistent_GHOST_STATE \
@@ -71,7 +85,7 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
  ? (((O)->state==cPersistent_UPTODATE_STATE) \
     ? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0)
 
-#define PER_ACCESSED(O) ((O)->atime=((long)(time(NULL)/3))%65536) 
+#define PER_ACCESSED(O)  (cPersistenceCAPI->accessed((cPersistentObject*)(O)))
 
 #endif
 

src/ZODB/Connection.py

@@ -13,14 +13,14 @@
 ##############################################################################
 """Database connection support
 
-$Id: Connection.py,v 1.63 2002/02/11 23:40:42 gvanrossum Exp $"""
-__version__='$Revision: 1.63 $'[11:-2]
+$Id: Connection.py,v 1.64 2002/03/27 10:14:03 htrd Exp $"""
+__version__='$Revision: 1.64 $'[11:-2]
 
-from cPickleCache import PickleCache
+from cPickleCache import PickleCache, MUCH_RING_CHECKING
 from POSException import ConflictError, ReadConflictError
 from ExtensionClass import Base
 import ExportImport, TmpStore
-from zLOG import LOG, ERROR, BLATHER
+from zLOG import LOG, ERROR, BLATHER, WARNING
 from coptimizations import new_persistent_id
 from ConflictResolution import ResolvedSerial
 
@@ -32,6 +32,11 @@ from types import StringType, ClassType
 
 global_code_timestamp = 0
 
+if MUCH_RING_CHECKING:
+  # To get rid of this warning, change the define inside cPickleCache.c and recompile.
+  LOG('ZODB',WARNING, 'Using cPickleCache with low performance (but extra debugging checks)')
+del MUCH_RING_CHECKING
+
 def updateCodeTimestamp():
     '''
     Called after changes are made to persistence-based classes.
@@ -65,12 +70,35 @@ class Connection(ExportImport.ExportImport):
         """Create a new Connection"""
         self._version=version
         self._cache=cache=PickleCache(self, cache_size, cache_deactivate_after)
+        if version:
+            # Caches for versions end up empty if the version
+            # is not used for a while. Non-version caches
+            # keep their content indefinitely.
+            self._cache.cache_drain_resistance = 100
         self._incrgc=self.cacheGC=cache.incrgc
         self._invalidated=d={}
         self._invalid=d.has_key
         self._committed=[]
         self._code_timestamp = global_code_timestamp
 
+    def _cache_items(self):
+        # find all items on the lru list
+        items = self._cache.lru_items()
+        # fine everything. some on the lru list, some not
+        everything = self._cache.cache_data
+        # remove those items that are on the lru list
+        for k,v in items:
+            del everything[k]
+        # return a list of [ghosts....not recently used.....recently used]
+        return everything.items() + items
+
+    def __repr__(self):
+        if self._version:
+            ver = ' (in version %s)' % `self._version`
+        else:
+            ver = ''
+        return '<Connection at %08x%s>' % (id(self),ver)
+
     def _breakcr(self):
         try: del self._cache
         except: pass
@@ -414,9 +442,9 @@ class Connection(ExportImport.ExportImport):
         for oid in creating:
             o=cache_get(oid, None)
             if o is not None:
+                del cache[oid]
                 del o._p_jar
                 del o._p_oid
-                del cache[oid]
 
     #XXX
 
@@ -441,9 +469,14 @@ class Connection(ExportImport.ExportImport):
     def root(self): return self['\0\0\0\0\0\0\0\0']
 
     def setstate(self, object):
+        oid=object._p_oid
+
+        if self._storage is None:
+            msg = "Shouldn't load state for %s when the connection is closed" % `oid`
+            LOG('ZODB',ERROR, msg)
+            raise RuntimeError(msg)
+
         try:
-            oid=object._p_oid
-             
             p, serial = self._storage.load(oid, self._version)
 
             # XXX this is quite conservative!

src/ZODB/DB.py

@@ -13,8 +13,8 @@
 ##############################################################################
 """Database objects
 
-$Id: DB.py,v 1.39 2002/02/11 23:40:42 gvanrossum Exp $"""
-__version__='$Revision: 1.39 $'[11:-2]
+$Id: DB.py,v 1.40 2002/03/27 10:14:04 htrd Exp $"""
+__version__='$Revision: 1.40 $'[11:-2]
 
 import cPickle, cStringIO, sys, POSException, UndoLogCompatible
 from Connection import Connection
@@ -177,7 +177,7 @@ class DB(UndoLogCompatible.UndoLogCompatible):
         def f(con, detail=detail, rc=sys.getrefcount, conn_no=conn_no):
             conn_no[0] = conn_no[0] + 1
             cn = conn_no[0]
-            for oid, ob in con._cache.items():
+            for oid, ob in con._cache_items():
                 id=''
                 if hasattr(ob,'__dict__'):
                     d=ob.__dict__
@@ -224,11 +224,21 @@ class DB(UndoLogCompatible.UndoLogCompatible):
     def cacheSize(self):
         m=[0]
         def f(con, m=m):
-            m[0]=m[0]+len(con._cache)
+            m[0]=m[0]+con._cache.cache_non_ghost_count
 
         self._connectionMap(f)
         return m[0]
 
+    def cacheDetailSize(self):
+        m=[]
+        def f(con, m=m):
+            m.append({'connection':repr(con),
+                      'ngsize':con._cache.cache_non_ghost_count,
+                      'size':len(con._cache)})
+        self._connectionMap(f)
+        m.sort()
+        return m
+
     def close(self): self._storage.close()
 
     def commitVersion(self, source, destination=''):

src/ZODB/cPersistence.c

@@ -14,11 +14,21 @@
 static char cPersistence_doc_string[] = 
 "Defines Persistent mixin class for persistent objects.\n"
 "\n"
-"$Id: cPersistence.c,v 1.50 2002/03/08 18:36:13 jeremy Exp $\n";
+"$Id: cPersistence.c,v 1.51 2002/03/27 10:14:04 htrd Exp $\n";
 
 #include <string.h>
 #include "cPersistence.h"
 
+/* the layout of this struct is the same as the start of ccobject in cPickleCache.c */
+struct ccobject_head_struct {
+  PyObject_HEAD
+  CPersistentRing ring_home;
+  int non_ghost_count;
+};
+
+#define HOME(O)             ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->ring_home)) )
+#define NON_GHOST_COUNT(O)  ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->non_ghost_count)) )
+
 #define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
 #define UNLESS(E) if(!(E))
 #define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
@@ -112,21 +122,82 @@ if(self->state < 0 && self->jar)                                 \
 {								 \
   PyObject *r;							 \
       								 \
+  int *count = NON_GHOST_COUNT(self);                            \
+  if(count)                                                      \
+  {                                                              \
+      (*count)++;                                                \
+      self->ring.next = HOME(self);                              \
+      self->ring.prev = HOME(self)->prev;                        \
+      HOME(self)->prev->next = &self->ring;                      \
+      HOME(self)->prev = &self->ring;                            \
+      Py_INCREF(self);                                           \
+  }                                                              \
   self->state=cPersistent_CHANGED_STATE; 	                 \
   UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self))   \
     {                                                            \
-      self->state=cPersistent_GHOST_STATE;                       \
+      ghostify(self);                                            \
       return ER;                                                 \
     }								 \
   self->state=cPersistent_UPTODATE_STATE;			 \
   Py_DECREF(r);							 \
 }
 
+#define KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self) \
+ if(HOME(self) && self->state>=0) {            \
+ self->ring.prev->next = self->ring.next;      \
+ self->ring.next->prev = self->ring.prev;      \
+ self->ring.next = HOME(self);           \
+ self->ring.prev = HOME(self)->prev;       \
+ HOME(self)->prev->next = &self->ring;    \
+ HOME(self)->prev = &self->ring; }
+
+
 
 /****************************************************************************/
 
 staticforward PyExtensionClass Pertype;
 
+static void
+accessed(cPersistentObject *self)
+{
+    KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
+}
+
+static void
+ghostify(cPersistentObject *self)
+{
+    int *count;
+    count = NON_GHOST_COUNT(self);
+    if(count && (self->state>=0))
+    {
+        (*count)--;
+        self->ring.next->prev = self->ring.prev;
+        self->ring.prev->next = self->ring.next;
+        self->ring.prev = NULL;
+        self->ring.next = NULL;
+        self->state = cPersistent_GHOST_STATE;
+        Py_DECREF(self);
+    }
+    else
+    {
+        self->state = cPersistent_GHOST_STATE;
+    }
+}
+
+static void
+deallocated(cPersistentObject *self)
+{
+    if(self->state>=0) ghostify(self);
+    if(self->cache)
+    {
+        PyObject *v=PyObject_CallMethod(self->cache,"_oid_unreferenced","O",self->oid);
+        if(!v) PyErr_Clear(); /* and explode later */
+        Py_XDECREF(v);
+    }
+    Py_XDECREF(self->jar);
+    Py_XDECREF(self->oid);
+}
+
 static int
 changed(cPersistentObject *self)
 {
@@ -185,7 +256,7 @@ Per___changed__(cPersistentObject *self, PyObject *args)
 static PyObject *
 Per__p_deactivate(cPersistentObject *self, PyObject *args)
 {
-  PyObject *dict;
+  PyObject *dict,*dict2=NULL;
 
 #ifdef DEBUG_LOG
   if (idebug_log < 0) call_debug("reinit",self);
@@ -197,13 +268,22 @@ Per__p_deactivate(cPersistentObject *self, PyObject *args)
   if (self->state==cPersistent_UPTODATE_STATE && self->jar &&
       HasInstDict(self) && (dict=INSTANCE_DICT(self)))
     {
+      dict2 = PyDict_Copy(dict);
       PyDict_Clear(dict);
       /* Note that we need to set to ghost state unless we are 
 	 called directly. Methods that override this need to
          do the same! */
-      self->state=cPersistent_GHOST_STATE;
+      ghostify(self);
     }
 
+  /* need to delay releasing the last reference on instance attributes
+  until after we have finished accounting for losing our state */
+  if(dict2)
+  {
+      PyDict_Clear(dict2);
+      Py_DECREF(dict2);
+  }
+
   Py_INCREF(Py_None);
   return Py_None;
 }
@@ -333,8 +413,8 @@ Per_dealloc(cPersistentObject *self)
 #ifdef DEBUG_LOG
   if(idebug_log < 0) call_debug("del",self);
 #endif
-  Py_XDECREF(self->jar);
-  Py_XDECREF(self->oid);
+  deallocated(self);
+  Py_XDECREF(self->cache);
   Py_DECREF(self->ob_type);
   PyObject_DEL(self);
 }
@@ -387,7 +467,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
 	      {
 		UPDATE_STATE_IF_NECESSARY(self, NULL);
 
-		self->atime=((long)(time(NULL)/3))%65536;
+		KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
 
 		if (self->serial[7]=='\0' && self->serial[6]=='\0' &&
 		    self->serial[5]=='\0' && self->serial[4]=='\0' &&
@@ -419,7 +499,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
     {
       UPDATE_STATE_IF_NECESSARY(self, NULL);
 
-      self->atime=((long)(time(NULL)/3))%65536;
+      KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
     }
 
   return getattrf((PyObject *)self, oname);
@@ -466,6 +546,21 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
     {
       if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6])
 	{
+	  if(HOME(self))
+	  {
+	    int result;
+	    if(!v)
+	    {
+              PyErr_SetString(PyExc_ValueError,"can not delete the oid of a cached object");
+              return -1;
+	    }
+	    if(PyObject_Cmp(self->oid,v,&result)<0) return -1;
+	    if(result)
+	    {
+              PyErr_SetString(PyExc_ValueError,"can not change the oid of a cached object");
+              return -1;
+            }
+	  }
 	  Py_XINCREF(v);
 	  ASSIGN(self->oid, v);
 	  return 0;
@@ -509,7 +604,6 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
 	      v=PyObject_GetAttr(OBJECT(self), py__p_deactivate);
 	      if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); }
 	      if (v) { Py_DECREF(v); }
-	      self->state=cPersistent_GHOST_STATE;
 	      return 0;
 	    }
 	  if (PyObject_IsTrue(v)) return changed(self);
@@ -521,8 +615,7 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
     {
       UPDATE_STATE_IF_NECESSARY(self, -1);
       
-      /* Record access times */
-      self->atime=((long)(time(NULL)/3))%65536;
+      KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
 
       if((! (*name=='_' && name[1]=='v' && name[2]=='_'))
 	 && (self->state != cPersistent_CHANGED_STATE && self->jar)
@@ -680,9 +773,11 @@ truecPersistenceCAPI = {
   (getattrofunc)Per_getattro,	/*tp_getattr with object key*/
   (setattrofunc)Per_setattro,	/*tp_setattr with object key*/
   changed,
+  accessed,
+  ghostify,
+  deallocated,
   (intfunctionwithpythonarg)Per_setstate,
   (pergetattr)Per_getattr,
-  (persetattr)_setattro,
 };
 
 void

src/ZODB/cPersistence.h

@@ -18,12 +18,21 @@
 #include "ExtensionClass.h"
 #include <time.h>
 
-#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid; char serial[8]; unsigned short atime;  signed char state;  unsigned char reserved; 
+
+#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid, *cache; CPersistentRing ring; char serial[8]; signed char state; unsigned char reserved[3];
 #define cPersistent_GHOST_STATE -1
 #define cPersistent_UPTODATE_STATE 0
 #define cPersistent_CHANGED_STATE 1
 #define cPersistent_STICKY_STATE 2
 
+struct ccobject_head_struct;
+
+typedef struct CPersistentRing_struct
+{
+    struct CPersistentRing_struct *prev;
+    struct CPersistentRing_struct *next;
+} CPersistentRing;
+
 typedef struct {
   cPersistent_HEAD
 } cPersistentObject;
@@ -36,6 +45,9 @@ typedef struct {
   getattrofunc getattro;
   setattrofunc setattro;
   int (*changed)(cPersistentObject*);
+  void (*accessed)(cPersistentObject*);
+  void (*ghostify)(cPersistentObject*);
+  void (*deallocated)(cPersistentObject*);
   int (*setstate)(PyObject*);
   pergetattr pergetattro;
   persetattr persetattro;
@@ -59,11 +71,13 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
 
 #define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O)))
 
-#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE)) 
+#define PER_GHOSTIFY(O) (cPersistenceCAPI->ghostify((cPersistentObject*)(O)))
+
+#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
 
-#define PER_PREVENT_DEACTIVATION(O)  ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE)) 
+#define PER_PREVENT_DEACTIVATION(O)  ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
 
-#define PER_DEL(O) Py_XDECREF((O)->jar); Py_XDECREF((O)->oid);
+#define PER_DEL(O) (cPersistenceCAPI->deallocated((cPersistentObject*)(O)))
 
 #define PER_USE(O) \
 (((O)->state != cPersistent_GHOST_STATE \
@@ -71,7 +85,7 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
  ? (((O)->state==cPersistent_UPTODATE_STATE) \
     ? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0)
 
-#define PER_ACCESSED(O) ((O)->atime=((long)(time(NULL)/3))%65536) 
+#define PER_ACCESSED(O)  (cPersistenceCAPI->accessed((cPersistentObject*)(O)))
 
 #endif
 

src/persistent/cPersistence.c

@@ -14,11 +14,21 @@
 static char cPersistence_doc_string[] = 
 "Defines Persistent mixin class for persistent objects.\n"
 "\n"
-"$Id: cPersistence.c,v 1.50 2002/03/08 18:36:13 jeremy Exp $\n";
+"$Id: cPersistence.c,v 1.51 2002/03/27 10:14:04 htrd Exp $\n";
 
 #include <string.h>
 #include "cPersistence.h"
 
+/* the layout of this struct is the same as the start of ccobject in cPickleCache.c */
+struct ccobject_head_struct {
+  PyObject_HEAD
+  CPersistentRing ring_home;
+  int non_ghost_count;
+};
+
+#define HOME(O)             ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->ring_home)) )
+#define NON_GHOST_COUNT(O)  ((!((O)->cache))?(NULL): (&(((struct ccobject_head_struct *)((O)->cache))->non_ghost_count)) )
+
 #define ASSIGN(V,E) {PyObject *__e; __e=(E); Py_XDECREF(V); (V)=__e;}
 #define UNLESS(E) if(!(E))
 #define UNLESS_ASSIGN(V,E) ASSIGN(V,E) UNLESS(V)
@@ -112,21 +122,82 @@ if(self->state < 0 && self->jar)                                 \
 {								 \
   PyObject *r;							 \
       								 \
+  int *count = NON_GHOST_COUNT(self);                            \
+  if(count)                                                      \
+  {                                                              \
+      (*count)++;                                                \
+      self->ring.next = HOME(self);                              \
+      self->ring.prev = HOME(self)->prev;                        \
+      HOME(self)->prev->next = &self->ring;                      \
+      HOME(self)->prev = &self->ring;                            \
+      Py_INCREF(self);                                           \
+  }                                                              \
   self->state=cPersistent_CHANGED_STATE; 	                 \
   UNLESS(r=callmethod1(self->jar,py_setstate,(PyObject*)self))   \
     {                                                            \
-      self->state=cPersistent_GHOST_STATE;                       \
+      ghostify(self);                                            \
       return ER;                                                 \
     }								 \
   self->state=cPersistent_UPTODATE_STATE;			 \
   Py_DECREF(r);							 \
 }
 
+#define KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self) \
+ if(HOME(self) && self->state>=0) {            \
+ self->ring.prev->next = self->ring.next;      \
+ self->ring.next->prev = self->ring.prev;      \
+ self->ring.next = HOME(self);           \
+ self->ring.prev = HOME(self)->prev;       \
+ HOME(self)->prev->next = &self->ring;    \
+ HOME(self)->prev = &self->ring; }
+
+
 
 /****************************************************************************/
 
 staticforward PyExtensionClass Pertype;
 
+static void
+accessed(cPersistentObject *self)
+{
+    KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
+}
+
+static void
+ghostify(cPersistentObject *self)
+{
+    int *count;
+    count = NON_GHOST_COUNT(self);
+    if(count && (self->state>=0))
+    {
+        (*count)--;
+        self->ring.next->prev = self->ring.prev;
+        self->ring.prev->next = self->ring.next;
+        self->ring.prev = NULL;
+        self->ring.next = NULL;
+        self->state = cPersistent_GHOST_STATE;
+        Py_DECREF(self);
+    }
+    else
+    {
+        self->state = cPersistent_GHOST_STATE;
+    }
+}
+
+static void
+deallocated(cPersistentObject *self)
+{
+    if(self->state>=0) ghostify(self);
+    if(self->cache)
+    {
+        PyObject *v=PyObject_CallMethod(self->cache,"_oid_unreferenced","O",self->oid);
+        if(!v) PyErr_Clear(); /* and explode later */
+        Py_XDECREF(v);
+    }
+    Py_XDECREF(self->jar);
+    Py_XDECREF(self->oid);
+}
+
 static int
 changed(cPersistentObject *self)
 {
@@ -185,7 +256,7 @@ Per___changed__(cPersistentObject *self, PyObject *args)
 static PyObject *
 Per__p_deactivate(cPersistentObject *self, PyObject *args)
 {
-  PyObject *dict;
+  PyObject *dict,*dict2=NULL;
 
 #ifdef DEBUG_LOG
   if (idebug_log < 0) call_debug("reinit",self);
@@ -197,13 +268,22 @@ Per__p_deactivate(cPersistentObject *self, PyObject *args)
   if (self->state==cPersistent_UPTODATE_STATE && self->jar &&
       HasInstDict(self) && (dict=INSTANCE_DICT(self)))
     {
+      dict2 = PyDict_Copy(dict);
       PyDict_Clear(dict);
       /* Note that we need to set to ghost state unless we are 
 	 called directly. Methods that override this need to
          do the same! */
-      self->state=cPersistent_GHOST_STATE;
+      ghostify(self);
     }
 
+  /* need to delay releasing the last reference on instance attributes
+  until after we have finished accounting for losing our state */
+  if(dict2)
+  {
+      PyDict_Clear(dict2);
+      Py_DECREF(dict2);
+  }
+
   Py_INCREF(Py_None);
   return Py_None;
 }
@@ -333,8 +413,8 @@ Per_dealloc(cPersistentObject *self)
 #ifdef DEBUG_LOG
   if(idebug_log < 0) call_debug("del",self);
 #endif
-  Py_XDECREF(self->jar);
-  Py_XDECREF(self->oid);
+  deallocated(self);
+  Py_XDECREF(self->cache);
   Py_DECREF(self->ob_type);
   PyObject_DEL(self);
 }
@@ -387,7 +467,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
 	      {
 		UPDATE_STATE_IF_NECESSARY(self, NULL);
 
-		self->atime=((long)(time(NULL)/3))%65536;
+		KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
 
 		if (self->serial[7]=='\0' && self->serial[6]=='\0' &&
 		    self->serial[5]=='\0' && self->serial[4]=='\0' &&
@@ -419,7 +499,7 @@ Per_getattr(cPersistentObject *self, PyObject *oname, char *name,
     {
       UPDATE_STATE_IF_NECESSARY(self, NULL);
 
-      self->atime=((long)(time(NULL)/3))%65536;
+      KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
     }
 
   return getattrf((PyObject *)self, oname);
@@ -466,6 +546,21 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
     {
       if(name[3]=='o' && name[4]=='i' && name[5]=='d' && ! name[6])
 	{
+	  if(HOME(self))
+	  {
+	    int result;
+	    if(!v)
+	    {
+              PyErr_SetString(PyExc_ValueError,"can not delete the oid of a cached object");
+              return -1;
+	    }
+	    if(PyObject_Cmp(self->oid,v,&result)<0) return -1;
+	    if(result)
+	    {
+              PyErr_SetString(PyExc_ValueError,"can not change the oid of a cached object");
+              return -1;
+            }
+	  }
 	  Py_XINCREF(v);
 	  ASSIGN(self->oid, v);
 	  return 0;
@@ -509,7 +604,6 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
 	      v=PyObject_GetAttr(OBJECT(self), py__p_deactivate);
 	      if (v) { ASSIGN(v, PyObject_CallObject(v, NULL)); }
 	      if (v) { Py_DECREF(v); }
-	      self->state=cPersistent_GHOST_STATE;
 	      return 0;
 	    }
 	  if (PyObject_IsTrue(v)) return changed(self);
@@ -521,8 +615,7 @@ _setattro(cPersistentObject *self, PyObject *oname, PyObject *v,
     {
       UPDATE_STATE_IF_NECESSARY(self, -1);
       
-      /* Record access times */
-      self->atime=((long)(time(NULL)/3))%65536;
+      KEEP_THIS_ONE_AROUND_FOR_A_WHILE(self);
 
       if((! (*name=='_' && name[1]=='v' && name[2]=='_'))
 	 && (self->state != cPersistent_CHANGED_STATE && self->jar)
@@ -680,9 +773,11 @@ truecPersistenceCAPI = {
   (getattrofunc)Per_getattro,	/*tp_getattr with object key*/
   (setattrofunc)Per_setattro,	/*tp_setattr with object key*/
   changed,
+  accessed,
+  ghostify,
+  deallocated,
   (intfunctionwithpythonarg)Per_setstate,
   (pergetattr)Per_getattr,
-  (persetattr)_setattro,
 };
 
 void

src/persistent/cPersistence.h

@@ -18,12 +18,21 @@
 #include "ExtensionClass.h"
 #include <time.h>
 
-#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid; char serial[8]; unsigned short atime;  signed char state;  unsigned char reserved; 
+
+#define cPersistent_HEAD PyObject_HEAD PyObject *jar, *oid, *cache; CPersistentRing ring; char serial[8]; signed char state; unsigned char reserved[3];
 #define cPersistent_GHOST_STATE -1
 #define cPersistent_UPTODATE_STATE 0
 #define cPersistent_CHANGED_STATE 1
 #define cPersistent_STICKY_STATE 2
 
+struct ccobject_head_struct;
+
+typedef struct CPersistentRing_struct
+{
+    struct CPersistentRing_struct *prev;
+    struct CPersistentRing_struct *next;
+} CPersistentRing;
+
 typedef struct {
   cPersistent_HEAD
 } cPersistentObject;
@@ -36,6 +45,9 @@ typedef struct {
   getattrofunc getattro;
   setattrofunc setattro;
   int (*changed)(cPersistentObject*);
+  void (*accessed)(cPersistentObject*);
+  void (*ghostify)(cPersistentObject*);
+  void (*deallocated)(cPersistentObject*);
   int (*setstate)(PyObject*);
   pergetattr pergetattro;
   persetattr persetattro;
@@ -59,11 +71,13 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
 
 #define PER_CHANGED(O) (cPersistenceCAPI->changed((cPersistentObject*)(O)))
 
-#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE)) 
+#define PER_GHOSTIFY(O) (cPersistenceCAPI->ghostify((cPersistentObject*)(O)))
+
+#define PER_ALLOW_DEACTIVATION(O) ((O)->state==cPersistent_STICKY_STATE && ((O)->state=cPersistent_UPTODATE_STATE))
 
-#define PER_PREVENT_DEACTIVATION(O)  ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE)) 
+#define PER_PREVENT_DEACTIVATION(O)  ((O)->state==cPersistent_UPTODATE_STATE && ((O)->state=cPersistent_STICKY_STATE))
 
-#define PER_DEL(O) Py_XDECREF((O)->jar); Py_XDECREF((O)->oid);
+#define PER_DEL(O) (cPersistenceCAPI->deallocated((cPersistentObject*)(O)))
 
 #define PER_USE(O) \
 (((O)->state != cPersistent_GHOST_STATE \
@@ -71,7 +85,7 @@ static cPersistenceCAPIstruct *cPersistenceCAPI;
  ? (((O)->state==cPersistent_UPTODATE_STATE) \
     ? ((O)->state=cPersistent_STICKY_STATE) : 1) : 0)
 
-#define PER_ACCESSED(O) ((O)->atime=((long)(time(NULL)/3))%65536) 
+#define PER_ACCESSED(O)  (cPersistenceCAPI->accessed((cPersistentObject*)(O)))
 
 #endif