Commit 3dc5f44e authored by Jim Fulton's avatar Jim Fulton

Fixed a bug that caused packing to fail in data sets with

lots of version commits.

Fixed a bug that caused bogus error logs saying that there were
bad references for objects created in uncommitted versions.
parent b8efc6d2
...@@ -184,7 +184,7 @@ ...@@ -184,7 +184,7 @@
# may have a back pointer to a version record or to a non-version # may have a back pointer to a version record or to a non-version
# record. # record.
# #
__version__='$Revision: 1.24 $'[11:-2] __version__='$Revision: 1.25 $'[11:-2]
import struct, time, os, bpthread, string, base64, sys import struct, time, os, bpthread, string, base64, sys
from struct import pack, unpack from struct import pack, unpack
...@@ -213,9 +213,9 @@ def panic(message, *data): ...@@ -213,9 +213,9 @@ def panic(message, *data):
LOG('ZODB FS',PANIC,"%s ERROR: %s\n" % (packed_version, message)) LOG('ZODB FS',PANIC,"%s ERROR: %s\n" % (packed_version, message))
raise CorruptedTransactionError, message raise CorruptedTransactionError, message
class FileStorageError: pass class FileStorageError(POSException.StorageError): pass
class FileStorageFormatError(FileStorageError, POSException.StorageError): class FileStorageFormatError(FileStorageError):
"""Invalid file format """Invalid file format
The format of the given file is not valid The format of the given file is not valid
...@@ -425,13 +425,15 @@ class FileStorage(BaseStorage.BaseStorage): ...@@ -425,13 +425,15 @@ class FileStorage(BaseStorage.BaseStorage):
h=read(42) h=read(42)
doid,serial,prev,tloc,vlen,plen = unpack(">8s8s8s8sH8s", h) doid,serial,prev,tloc,vlen,plen = unpack(">8s8s8s8sH8s", h)
if vlen: if vlen:
file.seek(16,1) nv = read(8) != z64
file.seek(8,1) # Skip previous version record pointer
version=read(vlen) version=read(vlen)
else: else:
version='' version=''
nv=0
if plen != z64: return read(u64(plen)), version if plen != z64: return read(u64(plen)), version, nv
return _loadBack(file, oid, read(8))[0], version return _loadBack(file, oid, read(8))[0], version, nv
def _load(self, oid, version, _index, file): def _load(self, oid, version, _index, file):
pos=_index[oid] pos=_index[oid]
...@@ -752,6 +754,7 @@ class FileStorage(BaseStorage.BaseStorage): ...@@ -752,6 +754,7 @@ class FileStorage(BaseStorage.BaseStorage):
name=self.__name__ name=self.__name__
file=open(name, 'r+b') file=open(name, 'r+b')
stop=`apply(TimeStamp, time.gmtime(t)[:5]+(t%60,))` stop=`apply(TimeStamp, time.gmtime(t)[:5]+(t%60,))`
if stop==z64: raise FileStorageError, 'Invalid pack time'
try: try:
################################################################## ##################################################################
...@@ -760,8 +763,12 @@ class FileStorage(BaseStorage.BaseStorage): ...@@ -760,8 +763,12 @@ class FileStorage(BaseStorage.BaseStorage):
# Record pack time so we don't undo while packing # Record pack time so we don't undo while packing
_lock_acquire() _lock_acquire()
locked=1
if self._packt != z64:
raise FileStorageError, 'Already packing'
self._packt=stop self._packt=stop
_lock_release() _lock_release()
locked=0
packpos, maxoid, ltid = read_index( packpos, maxoid, ltid = read_index(
file, name, index, vindex, tindex, stop) file, name, index, vindex, tindex, stop)
...@@ -777,9 +784,9 @@ class FileStorage(BaseStorage.BaseStorage): ...@@ -777,9 +784,9 @@ class FileStorage(BaseStorage.BaseStorage):
oid=pop() oid=pop()
if referenced(oid): continue if referenced(oid): continue
try: try:
p, v = _loada(oid, index, file) p, v, nv = _loada(oid, index, file)
referencesf(p, rootl) referencesf(p, rootl)
if v: if nv:
p, serial = _load(oid, '', index, file) p, serial = _load(oid, '', index, file)
referencesf(p, rootl) referencesf(p, rootl)
...@@ -829,6 +836,8 @@ class FileStorage(BaseStorage.BaseStorage): ...@@ -829,6 +836,8 @@ class FileStorage(BaseStorage.BaseStorage):
pnv=None pnv=None
while 1: while 1:
# Check for end of packed records # Check for end of packed records
if packing and pos >= packpos: if packing and pos >= packpos:
# OK, we're done with the old stuff, now we have # OK, we're done with the old stuff, now we have
...@@ -889,8 +898,6 @@ class FileStorage(BaseStorage.BaseStorage): ...@@ -889,8 +898,6 @@ class FileStorage(BaseStorage.BaseStorage):
plen=u64(splen) plen=u64(splen)
dlen=42+(plen or 8) dlen=42+(plen or 8)
# print u64(oid), pos, vlen, plen, pindex.get(oid,'?')
if vlen: if vlen:
dlen=dlen+16+vlen dlen=dlen+16+vlen
if packing and pindex_get(oid,0) != pos: if packing and pindex_get(oid,0) != pos:
...@@ -900,7 +907,7 @@ class FileStorage(BaseStorage.BaseStorage): ...@@ -900,7 +907,7 @@ class FileStorage(BaseStorage.BaseStorage):
continue continue
pnv=u64(read(8)) pnv=u64(read(8))
# skip pos prev ver rec # skip position of previous version record
seek(8,1) seek(8,1)
version=read(vlen) version=read(vlen)
pv=p64(vindex_get(version, 0)) pv=p64(vindex_get(version, 0))
...@@ -936,6 +943,14 @@ class FileStorage(BaseStorage.BaseStorage): ...@@ -936,6 +943,14 @@ class FileStorage(BaseStorage.BaseStorage):
pos=pos+dlen pos=pos+dlen
continue continue
# Ok, we've gotten this far, so we have
# the current record and we're ready to
# read the pickle, but we're in the wrong
# place, after wandering around to figure
# out is we were current. Seek back
# to pickle data:
seek(pos+42)
nvindex[oid]=opos nvindex[oid]=opos
tappend((oid,opos)) tappend((oid,opos))
...@@ -996,8 +1011,6 @@ class FileStorage(BaseStorage.BaseStorage): ...@@ -996,8 +1011,6 @@ class FileStorage(BaseStorage.BaseStorage):
write(p) write(p)
# print 'current', opos
# skip the (intentionally redundant) transaction length # skip the (intentionally redundant) transaction length
pos=pos+8 pos=pos+8
...@@ -1179,8 +1192,6 @@ def read_index(file, name, index, vindex, tindex, stop='\377'*8): ...@@ -1179,8 +1192,6 @@ def read_index(file, name, index, vindex, tindex, stop='\377'*8):
dlen=42+(plen or 8) dlen=42+(plen or 8)
tappend((oid,pos)) tappend((oid,pos))
# print u64(oid), pos, vlen, plen, index.get(oid,'?')
if vlen: if vlen:
dlen=dlen+16+vlen dlen=dlen+16+vlen
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment