Skip to content

Z
Zope
Commit 621094f8 authored by Jim Fulton's avatar Jim Fulton
Browse files
Options

Began to add reference counting. 

Not done yet.
parent 13057caa
Hide whitespace changes
Inline Side-by-side
Showing with 190 additions and 34 deletions
lib/python/BDBStorage/BDBFullStorage.py
View file @ 621094f8
...@@ -10,11 +10,15 @@ class Full(Base): ...@@ -10,11 +10,15 @@ class Full(Base):
def _setupDbs(self): def _setupDbs(self):
# Supports Base framework # Supports Base framework
self._index=self._setupDB('current') self._index=self._setupDB('current')
for name in 'pickle', 'record', 'transactions', 'vids', 'versions': for name in (
'pickle', 'record', 'transactions', 'vids', 'versions',
'referenceCount', 'pickleReferenceCount',
):
self._setupDB(name) self._setupDB(name)
self._setupDB('currentVersions', flags=db.DB_DUP) self._setupDB('currentVersions', flags=db.DB_DUP)
self._setupDB('transaction_oids', flags=db.DB_DUP) self._setupDB('transaction_oids', flags=db.DB_DUP)
self._setupDB('references', flags=db.DB_DUP)
c=self._vids.cursor() c=self._vids.cursor()
v=c.get(db.DB_LAST) v=c.get(db.DB_LAST)
...@@ -26,7 +30,10 @@ class Full(Base): ...@@ -26,7 +30,10 @@ class Full(Base):
# Supports Base framework # Supports Base framework
return ('current', 'pickle', 'record', return ('current', 'pickle', 'record',
'transactions', 'transaction_oids', 'transactions', 'transaction_oids',
'vids', 'versions', 'currentVersions') 'vids', 'versions', 'currentVersions',
'referenceCount', 'pickleReferenceCount',
'references',
)
def abortVersion(self, src, transaction): def abortVersion(self, src, transaction):
...@@ -45,8 +52,7 @@ class Full(Base): ...@@ -45,8 +52,7 @@ class Full(Base):
get=c.get get=c.get
current=self._current current=self._current
records=self._record records=self._record
tmp=self._tmp storeNV=self._tmp.storeNV
dump=marshal.dump
zero="\0\0\0\0\0\0\0\0" zero="\0\0\0\0\0\0\0\0"
while i: while i:
v, oid = i v, oid = i
...@@ -56,21 +62,23 @@ class Full(Base): ...@@ -56,21 +62,23 @@ class Full(Base):
record=records[oid+tid] record=records[oid+tid]
rvid, nv, data = unpack("8s8s8s", record[:24]) rvid, nv, data = unpack("8s8s8s", record[:24])
if rvid != vid: raise "vid inconsistent with currentVersions" if rvid != vid: raise "vid inconsistent with currentVersions"
if nv == zero: continue if nv == zero:
# This object was created in the version, so there's
# nothing to do. We can skip it.
continue
# Get non-version data # Get non-version data
record=records[oid+nv] record=records[oid+nv]
rvid, nv, data = unpack("8s8s8s", record[:24]) rvid, nv, data = unpack("8s8s8s", record[:24])
if rvid: raise "expected non-version data" if rvid: raise "expected non-version data"
dump(('s',(oid,zero,zero,data,'',tid)), tmp) storeNV(oid, data, tid)
save_oid(oid) save_oid(oid)
i=get(db.DB_NEXT_DUP) i=get(db.DB_NEXT_DUP)
dump(('v',vid),tmp) self._tmp.versionDiscard(vid)
self._vtmp[vid]='a'
return oids return oids
finally: finally:
...@@ -94,14 +102,13 @@ class Full(Base): ...@@ -94,14 +102,13 @@ class Full(Base):
get=c.get get=c.get
current=self._current current=self._current
records=self._record records=self._record
tmp=self._tmp store=self._tmp.store
dump=marshal.dump
zero="\0\0\0\0\0\0\0\0" zero="\0\0\0\0\0\0\0\0"
try: dvid=self._vids[dest] try: dvid=self._vids[dest]
except KeyError: except KeyError:
dvid=self._newvid() dvid=self._newvid()
dump(('v',(vid, version)), tmp) self._tmp.newVersion(version, vid)
while i: while i:
v, oid = i v, oid = i
...@@ -113,15 +120,13 @@ class Full(Base): ...@@ -113,15 +120,13 @@ class Full(Base):
if rvid != vid: raise "vid inconsistent with currentVersions" if rvid != vid: raise "vid inconsistent with currentVersions"
if not dest: nv=zero if not dest: nv=zero
dump(('s',(oid,dvid,nv,data,'',tid)), tmp) store(oid,dvid,nv,data,'',tid)
save_oid(oid) save_oid(oid)
i=get(db.DB_NEXT_DUP) i=get(db.DB_NEXT_DUP)
dump(('d',vid),tmp) self._tmp.versionDiscard(vid)
self._vtmp[vid]='c'
if dest: self._vtmp[dvid]='d'
return oids return oids
finally: finally:
...@@ -170,7 +175,7 @@ class Full(Base): ...@@ -170,7 +175,7 @@ class Full(Base):
try: vid=self._vids[version] try: vid=self._vids[version]
except: except:
vid=self._newvid() vid=self._newvid()
dump(('v',(vid, version)), self._tmp) self._tmp.newVersion(version, vid)
else: else:
vid=nv='\0\0\0\0\0\0\0\0' vid=nv='\0\0\0\0\0\0\0\0'
...@@ -187,7 +192,7 @@ class Full(Base): ...@@ -187,7 +192,7 @@ class Full(Base):
else: else:
nv='\0\0\0\0\0\0\0\0' nv='\0\0\0\0\0\0\0\0'
dump(('s',(oid, vid, nv, '', data, old)), self._tmp) self._tmp.store(oid, vid, nv, '', data, old)
finally: self._lock_release() finally: self._lock_release()
...@@ -265,6 +270,17 @@ class Full(Base): ...@@ -265,6 +270,17 @@ class Full(Base):
currentVersions=self._currentVersions currentVersions=self._currentVersions
unpack=struct.unpack unpack=struct.unpack
try:
for oid in dups(self._transaction_oids raise POSException.UndoError, 'Undoable transaction'
txn=self._env.txn_begin()
current=self._current
record=self._record
pickle=self._pickle
currentVersions=self._currentVersions
unpack=struct.unpack
try: try:
for oid in dups(self._transaction_oids, tid, txn): for oid in dups(self._transaction_oids, tid, txn):
if current.get(oid, txn) != tid: self._undoable(txn) if current.get(oid, txn) != tid: self._undoable(txn)
...@@ -412,3 +428,65 @@ def del_dup(database, key, value, txn): ...@@ -412,3 +428,65 @@ def del_dup(database, key, value, txn):
c.close() c.close()
class Log:
def __init__(self, file):
self._file=file
file.seek(0)
h=file.read(5)
size=0
if len(h) == 5:
state=h[0]
if state=='c':
size=unpack(">i", h[1:])
else:
state='s'
file.seek(0)
file.write('s\0\0\0\0')
self._state=state
self._size=size
def clear(self):
if self._state=='p':
raise "Can't clear state with uncommitted promised log"
self._file.seek(0)
self._file.write('s')
self._size=0
def promise(self):
self._state='p'
size=self._file.tell()-5
self._file.seek(0)
self._file.write('p'+pack(">i", size))
self._size=size
def commit(self):
file=self._file
l=file.tell()-1
if l:
self._file.seek(0)
self._file.write('c')
self._state='c'
def store(self, oid, vid, nv, dataptr, pickle, previous,
dump=marshal.dump):
dump(('s',(oid,vid,nv,data,pickle,previous)), self._file)
def storeNV(self, oid, data, tid,
dump=marshal.dump, zero='\0\0\0\0\0\0\0\0'):
dump(('s',(oid,zero,zero,data,'',tid)), self._file)
def versionDiscard(self, vid,
dump=marshal.dump):
dump(('d',(vid)), self._file)
def newVersion(self, version, vid,
dump=marshal.dump):
dump(('v',(version, vid)), self._file)
lib/python/BDBStorage/Full.py
View file @ 621094f8
...@@ -10,11 +10,15 @@ class Full(Base): ...@@ -10,11 +10,15 @@ class Full(Base):
def _setupDbs(self): def _setupDbs(self):
# Supports Base framework # Supports Base framework
self._index=self._setupDB('current') self._index=self._setupDB('current')
for name in 'pickle', 'record', 'transactions', 'vids', 'versions': for name in (
'pickle', 'record', 'transactions', 'vids', 'versions',
'referenceCount', 'pickleReferenceCount',
):
self._setupDB(name) self._setupDB(name)
self._setupDB('currentVersions', flags=db.DB_DUP) self._setupDB('currentVersions', flags=db.DB_DUP)
self._setupDB('transaction_oids', flags=db.DB_DUP) self._setupDB('transaction_oids', flags=db.DB_DUP)
self._setupDB('references', flags=db.DB_DUP)
c=self._vids.cursor() c=self._vids.cursor()
v=c.get(db.DB_LAST) v=c.get(db.DB_LAST)
...@@ -26,7 +30,10 @@ class Full(Base): ...@@ -26,7 +30,10 @@ class Full(Base):
# Supports Base framework # Supports Base framework
return ('current', 'pickle', 'record', return ('current', 'pickle', 'record',
'transactions', 'transaction_oids', 'transactions', 'transaction_oids',
'vids', 'versions', 'currentVersions') 'vids', 'versions', 'currentVersions',
'referenceCount', 'pickleReferenceCount',
'references',
)
def abortVersion(self, src, transaction): def abortVersion(self, src, transaction):
...@@ -45,8 +52,7 @@ class Full(Base): ...@@ -45,8 +52,7 @@ class Full(Base):
get=c.get get=c.get
current=self._current current=self._current
records=self._record records=self._record
tmp=self._tmp storeNV=self._tmp.storeNV
dump=marshal.dump
zero="\0\0\0\0\0\0\0\0" zero="\0\0\0\0\0\0\0\0"
while i: while i:
v, oid = i v, oid = i
...@@ -56,21 +62,23 @@ class Full(Base): ...@@ -56,21 +62,23 @@ class Full(Base):
record=records[oid+tid] record=records[oid+tid]
rvid, nv, data = unpack("8s8s8s", record[:24]) rvid, nv, data = unpack("8s8s8s", record[:24])
if rvid != vid: raise "vid inconsistent with currentVersions" if rvid != vid: raise "vid inconsistent with currentVersions"
if nv == zero: continue if nv == zero:
# This object was created in the version, so there's
# nothing to do. We can skip it.
continue
# Get non-version data # Get non-version data
record=records[oid+nv] record=records[oid+nv]
rvid, nv, data = unpack("8s8s8s", record[:24]) rvid, nv, data = unpack("8s8s8s", record[:24])
if rvid: raise "expected non-version data" if rvid: raise "expected non-version data"
dump(('s',(oid,zero,zero,data,'',tid)), tmp) storeNV(oid, data, tid)
save_oid(oid) save_oid(oid)
i=get(db.DB_NEXT_DUP) i=get(db.DB_NEXT_DUP)
dump(('v',vid),tmp) self._tmp.versionDiscard(vid)
self._vtmp[vid]='a'
return oids return oids
finally: finally:
...@@ -94,14 +102,13 @@ class Full(Base): ...@@ -94,14 +102,13 @@ class Full(Base):
get=c.get get=c.get
current=self._current current=self._current
records=self._record records=self._record
tmp=self._tmp store=self._tmp.store
dump=marshal.dump
zero="\0\0\0\0\0\0\0\0" zero="\0\0\0\0\0\0\0\0"
try: dvid=self._vids[dest] try: dvid=self._vids[dest]
except KeyError: except KeyError:
dvid=self._newvid() dvid=self._newvid()
dump(('v',(vid, version)), tmp) self._tmp.newVersion(version, vid)
while i: while i:
v, oid = i v, oid = i
...@@ -113,15 +120,13 @@ class Full(Base): ...@@ -113,15 +120,13 @@ class Full(Base):
if rvid != vid: raise "vid inconsistent with currentVersions" if rvid != vid: raise "vid inconsistent with currentVersions"
if not dest: nv=zero if not dest: nv=zero
dump(('s',(oid,dvid,nv,data,'',tid)), tmp) store(oid,dvid,nv,data,'',tid)
save_oid(oid) save_oid(oid)
i=get(db.DB_NEXT_DUP) i=get(db.DB_NEXT_DUP)
dump(('d',vid),tmp) self._tmp.versionDiscard(vid)
self._vtmp[vid]='c'
if dest: self._vtmp[dvid]='d'
return oids return oids
finally: finally:
...@@ -170,7 +175,7 @@ class Full(Base): ...@@ -170,7 +175,7 @@ class Full(Base):
try: vid=self._vids[version] try: vid=self._vids[version]
except: except:
vid=self._newvid() vid=self._newvid()
dump(('v',(vid, version)), self._tmp) self._tmp.newVersion(version, vid)
else: else:
vid=nv='\0\0\0\0\0\0\0\0' vid=nv='\0\0\0\0\0\0\0\0'
...@@ -187,7 +192,7 @@ class Full(Base): ...@@ -187,7 +192,7 @@ class Full(Base):
else: else:
nv='\0\0\0\0\0\0\0\0' nv='\0\0\0\0\0\0\0\0'
dump(('s',(oid, vid, nv, '', data, old)), self._tmp) self._tmp.store(oid, vid, nv, '', data, old)
finally: self._lock_release() finally: self._lock_release()
...@@ -265,6 +270,17 @@ class Full(Base): ...@@ -265,6 +270,17 @@ class Full(Base):
currentVersions=self._currentVersions currentVersions=self._currentVersions
unpack=struct.unpack unpack=struct.unpack
try:
for oid in dups(self._transaction_oids raise POSException.UndoError, 'Undoable transaction'
txn=self._env.txn_begin()
current=self._current
record=self._record
pickle=self._pickle
currentVersions=self._currentVersions
unpack=struct.unpack
try: try:
for oid in dups(self._transaction_oids, tid, txn): for oid in dups(self._transaction_oids, tid, txn):
if current.get(oid, txn) != tid: self._undoable(txn) if current.get(oid, txn) != tid: self._undoable(txn)
...@@ -412,3 +428,65 @@ def del_dup(database, key, value, txn): ...@@ -412,3 +428,65 @@ def del_dup(database, key, value, txn):
c.close() c.close()
class Log:
def __init__(self, file):
self._file=file
file.seek(0)
h=file.read(5)
size=0
if len(h) == 5:
state=h[0]
if state=='c':
size=unpack(">i", h[1:])
else:
state='s'
file.seek(0)
file.write('s\0\0\0\0')
self._state=state
self._size=size
def clear(self):
if self._state=='p':
raise "Can't clear state with uncommitted promised log"
self._file.seek(0)
self._file.write('s')
self._size=0
def promise(self):
self._state='p'
size=self._file.tell()-5
self._file.seek(0)
self._file.write('p'+pack(">i", size))
self._size=size
def commit(self):
file=self._file
l=file.tell()-1
if l:
self._file.seek(0)
self._file.write('c')
self._state='c'
def store(self, oid, vid, nv, dataptr, pickle, previous,
dump=marshal.dump):
dump(('s',(oid,vid,nv,data,pickle,previous)), self._file)
def storeNV(self, oid, data, tid,
dump=marshal.dump, zero='\0\0\0\0\0\0\0\0'):
dump(('s',(oid,zero,zero,data,'',tid)), self._file)
def versionDiscard(self, vid,
dump=marshal.dump):
dump(('d',(vid)), self._file)
def newVersion(self, version, vid,
dump=marshal.dump):
dump(('v',(version, vid)), self._file)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7