Fixed metadata diff regressing

git-svn-id: http://svn.savannah.nongnu.org/svn/rdiff-backup@675 2b77aa54-bcbc-44c9-a7ec-4f6cf2b41109

trunk/rdiff-backup/CHANGELOG

@@ -17,6 +17,8 @@ Alec Berryman's fs_abilities patch is supposed to help with AFS.
 
 Fixed filename-too-long crash when quoting.
 
+Patched carbonfile support, re-enabled it by default.
+
 
 New in v1.1.0 (2005/10/24)
 --------------------------

trunk/rdiff-backup/TODO

@@ -2,8 +2,6 @@ For comparing, check source filesystem's abilities
 
 Clean up connection dropped message
 
-Make sure regress handles metadata diffs
-
 Clean up compare reports
 
 Test comparing of single files, and files/directories specified by
@@ -22,4 +20,3 @@ Think about adding Gaudet's idea for keeping track of renamed files.
 Look into different inode generation techniques (see treescan, Dean
 Gaudet's other post).
 
-

trunk/rdiff-backup/rdiff_backup/metadata.py

@@ -345,17 +345,19 @@ class FlatFile:
 	_extractor = FlatExtractor # Override to class that iterates objects
 	_object_to_record = None # Set to function converting object to record
 	_prefix = None # Set to required prefix
-	def __init__(self, rp, mode):
+	def __init__(self, rp, mode, check_path = 1, compress = 1):
 		"""Open rp for reading ('r') or writing ('w')"""
 		self.rp = rp
 		self.mode = mode
 		self._record_buffer = []
-		assert rp.isincfile() and rp.getincbase_str() == self._prefix, rp
+		if check_path:
+			assert rp.isincfile() and rp.getincbase_str() == self._prefix, rp
+			compress = rp.isinccompressed()
 		if mode == 'r':
-			self.fileobj = self.rp.open("rb", rp.isinccompressed())
+			self.fileobj = self.rp.open("rb", compress)
 		else:
 			assert mode == 'w' and not self.rp.lstat(), (mode, rp)
-			self.fileobj = self.rp.open("wb", rp.isinccompressed())
+			self.fileobj = self.rp.open("wb", compress)
 
 	def write_record(self, record):
 		"""Write a (text) record into the file"""
@@ -556,18 +558,17 @@ class PatchDiffMan(Manager):
 				# exact compare here, can't use == on rorps
 				yield old_rorp
 
-	def sorted_meta_inclist(self, min_time = 0):
-		"""Return list of mirror_metadata incs, reverse sorted by time"""
-		if not self.prefixmap.has_key('mirror_metadata'): return []
-		sortlist = [(rp.getinctime(), rp)
-					for rp in self.prefixmap['mirror_metadata']]
+	def sorted_prefix_inclist(self, prefix, min_time = 0):
+		"""Return reverse sorted (by time) list of incs with given prefix"""
+		if not self.prefixmap.has_key(prefix): return []
+		sortlist = [(rp.getinctime(), rp) for rp in self.prefixmap[prefix]]
 		sortlist.sort()
 		sortlist.reverse()
 		return [rp for (time, rp) in sortlist if time >= min_time]
 
 	def check_needs_diff(self):
 		"""Check if we should diff, returns (new, old) rps, or (None, None)"""
-		inclist = self.sorted_meta_inclist()
+		inclist = self.sorted_prefix_inclist('mirror_metadata')
 		assert len(inclist) >= 1
 		if len(inclist) == 1: return (None, None)
 		newrp, oldrp = inclist[:2]
@@ -604,7 +605,7 @@ class PatchDiffMan(Manager):
 
 	def relevant_meta_incs(self, time):
 		"""Return list [snapshotrp, diffrps ...] time sorted"""
-		inclist = self.sorted_meta_inclist(min_time = time)
+		inclist = self.sorted_prefix_inclist('mirror_metadata', min_time=time)
 		if not inclist: return inclist
 		assert inclist[-1].getinctime() == time, inclist[-1]
 		for i in range(len(inclist)-1, -1, -1):
@@ -631,6 +632,7 @@ ManagerObj = None # Set this later to Manager instance
 def SetManager():
 	global ManagerObj
 	ManagerObj = PatchDiffMan()
+	return ManagerObj
 
 
 import eas_acls # put at bottom to avoid python circularity bug

trunk/rdiff-backup/rdiff_backup/regress.py

@@ -63,9 +63,9 @@ def Regress(mirror_rp):
 	assert mirror_rp.index == () and inc_rpath.index == ()
 	assert mirror_rp.isdir() and inc_rpath.isdir()
 	assert mirror_rp.conn is inc_rpath.conn is Globals.local_connection
-	set_regress_time()
+	manager, former_current_mirror_rp = set_regress_time()
 	set_restore_times()
-	former_current_mirror_rp = remove_rbdir_increments()
+	regress_rbdir(manager)
 	ITR = rorpiter.IterTreeReducer(RegressITRB, [])
 	for rf in iterate_meta_rfs(mirror_rp, inc_rpath): ITR(rf.index, rf)
 	ITR.Finish()
@@ -81,14 +81,15 @@ def set_regress_time():
 
 	"""
 	global regress_time, unsuccessful_backup_time
-	curmir_incs = restore.get_inclist(Globals.rbdir.append("current_mirror"))
+	manager = metadata.SetManager()
+	curmir_incs = manager.sorted_prefix_inclist('current_mirror')
 	assert len(curmir_incs) == 2, \
 		   "Found %s current_mirror flags, expected 2" % len(curmir_incs)
-	inctimes = [inc.getinctime() for inc in curmir_incs]
-	inctimes.sort()
-	regress_time = inctimes[0]
-	unsuccessful_backup_time = inctimes[-1]
+	mirror_rp_to_delete = curmir_incs[0]
+	regress_time = curmir_incs[1].getinctime()
+	unsuccessful_backup_time = mirror_rp_to_delete.getinctime()
 	log.Log("Regressing to " + Time.timetopretty(regress_time), 4)
+	return manager, mirror_rp_to_delete
 
 def set_restore_times():
 	"""Set _rest_time and _mirror_time in the restore module
@@ -100,24 +101,51 @@ def set_restore_times():
 	restore.MirrorStruct._mirror_time = unsuccessful_backup_time
 	restore.MirrorStruct._rest_time = regress_time
 
-def remove_rbdir_increments():
+def regress_rbdir(meta_manager):
 	"""Delete the increments in the rdiff-backup-data directory
 
 	Returns the former current mirror rp so we can delete it later.
 	All of the other rp's should be deleted before the actual regress,
 	to clear up disk space the rest of the procedure may need.
 
+	Also, in case the previous session failed while diffing the
+	metadata file, either recreate the mirror_metadata snapshot, or
+	delete the extra regress_time diff.
+
 	"""
-	former_current_mirror = None
-	for filename in Globals.rbdir.listdir():
-		rp = Globals.rbdir.append(filename)
-		if rp.isincfile() and rp.getinctime() == unsuccessful_backup_time:
-			if rp.getincbase_str() == "current_mirror":
-				former_current_mirror = rp
+	has_meta_diff, has_meta_snap = 0, 0
+	for old_rp in meta_manager.timerpmap[regress_time]:
+		if old_rp.getincbase_str() == 'mirror_metadata':
+			if old_rp.getinctype() == 'snapshot': has_meta_snap = 1
 			else:
-				log.Log("Removing rdiff-backup-data increment " + rp.path, 5)
-				rp.delete()
-	return former_current_mirror
+				assert old_rp.getinctype() == 'diff', old_rp
+				has_meta_diff = 1
+	if has_meta_diff and not has_meta_snap: recreate_meta(meta_manager)
+
+	for new_rp in meta_manager.timerpmap[unsuccessful_backup_time]:
+		if new_rp.getincbase_str() != 'current_mirror':
+			log.Log("Deleting old diff at " + new_rp.path, 5)
+			new_rp.delete()
+
+def recreate_meta(meta_manager):
+	"""Make regress_time mirror_metadata snapshot by patching
+
+	We write to a tempfile first.  Otherwise, in case of a crash, it
+	would seem we would have an intact snapshot and partial diff, not
+	the reverse.
+
+	"""
+	temprp = TempFile.new_in_dir(Globals.rbdir)
+	writer = metadata.MetadataFile(temprp, 'w', check_path = 0)
+	for rorp in meta_manager.get_meta_at_time(regress_time, None):
+		writer.write_object(rorp)
+	writer.close()
+
+	finalrp = Globals.rbdir.append("mirror_metadata.%s.snapshot.gz" %
+								   Time.timetostring(regress_time))
+	assert not finalrp.lstat(), finalrp
+	rpath.rename(temprp, finalrp)
+	if Globals.fsync_directories: Globals.rbdir.fsync()
 
 def iterate_raw_rfs(mirror_rp, inc_rp):
 	"""Iterate all RegressFile objects in mirror/inc directory
@@ -146,8 +174,8 @@ def yield_metadata():
 	metadata.SetManager()
 	metadata_iter = metadata.ManagerObj.GetAtTime(regress_time)
 	if metadata_iter: return metadata_iter
-	log.Log.FatalError("No metadata for time %s found, cannot regress"
-					   % Time.timetopretty(regress_time))
+	log.Log.FatalError("No metadata for time %s (%s) found,\ncannot regress"
+					   % (Time.timetopretty(regress_time), regress_time))
 
 def iterate_meta_rfs(mirror_rp, inc_rp):
 	"""Yield RegressFile objects with extra metadata information added

trunk/rdiff-backup/testing/metadatatest.py

@@ -199,7 +199,7 @@ class MetadataTest(unittest.TestCase):
 		man.ConvertMetaToDiff()
 
 		man = PatchDiffMan()
-		l = man.sorted_meta_inclist()
+		l = man.sorted_prefix_inclist('mirror_metadata')
 		assert l[0].getinctype() == 'snapshot'
 		assert l[0].getinctime() == 40000
 		assert l[1].getinctype() == 'snapshot'