From 9bf030ef2ca4ef1c5821210d4b46af89404f83f7 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 25 Jul 2020 19:36:08 +0200 Subject: [PATCH 1/2] add a test for hints persistence and behaviour, see #4830 --- src/borg/testsuite/repository.py | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index 9c3183557..3a2d92368 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -792,6 +792,46 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase): self.assert_equal(self.repository.get(H(0)), b'data2') +class RepositoryHintsTestCase(RepositoryTestCaseBase): + + def test_hints_persistence(self): + self.repository.put(H(0), b'data') + self.repository.delete(H(0)) + self.repository.commit(compact=False) + shadow_index_expected = self.repository.shadow_index + compact_expected = self.repository.compact + segments_expected = self.repository.segments + # close and re-open the repository (create fresh Repository instance) to + # check whether hints were persisted to / reloaded from disk + self.reopen() + with self.repository: + # see also do_compact() + self.repository.put(H(42), b'foobar') # this will call prepare_txn() and load the hints data + # check if hints persistence worked: + self.assert_equal(shadow_index_expected, self.repository.shadow_index) + self.assert_equal(compact_expected, self.repository.compact) + del self.repository.segments[2] # ignore the segment created by put(H(42), ...) + self.assert_equal(segments_expected, self.repository.segments) + + def test_hints_behaviour(self): + self.repository.put(H(0), b'data') + self.assert_equal(self.repository.shadow_index, {}) + self.assert_true(len(self.repository.compact) == 0) + self.repository.delete(H(0)) + self.repository.commit(compact=False) + # now there should be an entry for H(0) in shadow_index + self.assert_in(H(0), self.repository.shadow_index) + self.assert_equal(len(self.repository.shadow_index[H(0)]), 1) + self.assert_in(0, self.repository.compact) # segment 0 can be compacted + self.repository.put(H(42), b'foobar') # see also do_compact() + self.repository.commit(compact=True, threshold=0.0) # compact completely! + # nothing to compact any more! no info left about stuff that does not exist any more: + self.assert_equal(self.repository.shadow_index[H(0)], []) + # segment 0 was compacted away, no info about it left: + self.assert_not_in(0, self.repository.compact) + self.assert_not_in(0, self.repository.segments) + + class RemoteRepositoryTestCase(RepositoryTestCase): repository = None # type: RemoteRepository From 7bfa7661923ea44ba77a6700d92a9f68f73eb1ce Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 25 Jul 2020 16:13:03 +0200 Subject: [PATCH 2/2] persist shadow_index in between borg runs, fixes #4830 in borg 1.1, compact_segments() was always run directly after some repo writing operation (in same borg process). but now, only "borg compact" is used to compact segments and it is a separate borg invocation (new process), so we need to persist the shadow_index so we do not lose that information. --- src/borg/repository.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/borg/repository.py b/src/borg/repository.py index ad9fa66f4..b3489c1ae 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -160,6 +160,7 @@ class Repository: # This is an index of shadowed log entries during this transaction. Consider the following sequence: # segment_n PUT A, segment_x DELETE A # After the "DELETE A" in segment_x the shadow index will contain "A -> [n]". + # .delete() is updating this index, it is persisted into "hints" file and is later used by .compact_segments(). self.shadow_index = {} self._active_txn = False self.lock_wait = lock_wait @@ -570,6 +571,7 @@ class Repository: self.segments = hints[b'segments'] self.compact = FreeSpace() self.storage_quota_use = 0 + self.shadow_index = {} for segment in sorted(hints[b'compact']): logger.debug('Rebuilding sparse info for segment %d', segment) self._rebuild_sparse(segment) @@ -580,6 +582,7 @@ class Repository: self.segments = hints[b'segments'] self.compact = FreeSpace(hints[b'compact']) self.storage_quota_use = hints.get(b'storage_quota_use', 0) + self.shadow_index = hints.get(b'shadow_index', {}) self.log_storage_quota() # Drop uncommitted segments in the shadow index for key, shadowed_segments in self.shadow_index.items(): @@ -600,6 +603,7 @@ class Repository: b'segments': self.segments, b'compact': self.compact, b'storage_quota_use': self.storage_quota_use, + b'shadow_index': self.shadow_index, } integrity = { # Integrity version started at 2, the current hints version. @@ -975,7 +979,7 @@ class Repository: segments_transaction_id = self.io.get_segments_transaction_id() logger.debug('Segment transaction is %s', segments_transaction_id) logger.debug('Determined transaction is %s', transaction_id) - self.prepare_txn(None) # self.index, self.compact, self.segments all empty now! + self.prepare_txn(None) # self.index, self.compact, self.segments, self.shadow_index all empty now! segment_count = sum(1 for _ in self.io.segment_iterator()) logger.debug('Found %d segments', segment_count)