From efe60bd640641ee3a7a21787a53d4dc45401acc6 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Sun, 24 Jun 2018 04:35:52 +0200
Subject: [PATCH] test fixes

---
 src/borg/testsuite/archiver.py   | 34 +++++------
 src/borg/testsuite/repository.py | 98 ++++++++++++++++----------------
 2 files changed, 67 insertions(+), 65 deletions(-)

diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
index 3745cfffa..75a49657c 100644
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -1463,7 +1463,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
                 if 'chunks' in item:
                     first_chunk_id = item.chunks[0].id
                     repository.delete(first_chunk_id)
-                    repository.commit()
+                    repository.commit(compact=False)
                     break
         output = self.cmd('delete', '--force', self.repository_location + '::test')
         self.assert_in('deleted archive was corrupted', output)
@@ -1479,7 +1479,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
             archive = Archive(repository, key, manifest, 'test')
             id = archive.metadata.items[0]
             repository.put(id, b'corrupted items metadata stream chunk')
-            repository.commit()
+            repository.commit(compact=False)
         self.cmd('delete', '--force', '--force', self.repository_location + '::test')
         self.cmd('check', '--repair', self.repository_location)
         output = self.cmd('list', self.repository_location)
@@ -1533,7 +1533,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
             manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
             manifest.config[b'feature_flags'] = {operation.value.encode(): {b'mandatory': [b'unknown-feature']}}
             manifest.write()
-            repository.commit()
+            repository.commit(compact=False)
 
     def cmd_raises_unknown_feature(self, args):
         if self.FORK_DEFAULT:
@@ -2249,7 +2249,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
                     break
             else:
                 assert False  # missed the file
-            repository.commit()
+            repository.commit(compact=False)
         self.cmd('check', '--repair', self.repository_location, exit_code=0)
 
         mountpoint = os.path.join(self.tmpdir, 'mountpoint')
@@ -2970,7 +2970,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
                     break
             else:
                 self.fail('should not happen')
-            repository.commit()
+            repository.commit(compact=False)
         self.cmd('check', self.repository_location, exit_code=1)
         output = self.cmd('check', '--repair', self.repository_location, exit_code=0)
         self.assert_in('New missing file chunk detected', output)
@@ -3013,7 +3013,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
         archive, repository = self.open_archive('archive1')
         with repository:
             repository.delete(archive.metadata.items[0])
-            repository.commit()
+            repository.commit(compact=False)
         self.cmd('check', self.repository_location, exit_code=1)
         self.cmd('check', '--repair', self.repository_location, exit_code=0)
         self.cmd('check', self.repository_location, exit_code=0)
@@ -3022,7 +3022,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
         archive, repository = self.open_archive('archive1')
         with repository:
             repository.delete(archive.id)
-            repository.commit()
+            repository.commit(compact=False)
         self.cmd('check', self.repository_location, exit_code=1)
         self.cmd('check', '--repair', self.repository_location, exit_code=0)
         self.cmd('check', self.repository_location, exit_code=0)
@@ -3031,7 +3031,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
         archive, repository = self.open_archive('archive1')
         with repository:
             repository.delete(Manifest.MANIFEST_ID)
-            repository.commit()
+            repository.commit(compact=False)
         self.cmd('check', self.repository_location, exit_code=1)
         output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0)
         self.assert_in('archive1', output)
@@ -3044,7 +3044,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
             manifest = repository.get(Manifest.MANIFEST_ID)
             corrupted_manifest = manifest + b'corrupted!'
             repository.put(Manifest.MANIFEST_ID, corrupted_manifest)
-            repository.commit()
+            repository.commit(compact=False)
         self.cmd('check', self.repository_location, exit_code=1)
         output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0)
         self.assert_in('archive1', output)
@@ -3061,7 +3061,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
             chunk = repository.get(archive.id)
             corrupted_chunk = chunk + b'corrupted!'
             repository.put(archive.id, corrupted_chunk)
-            repository.commit()
+            repository.commit(compact=False)
         self.cmd('check', self.repository_location, exit_code=1)
         output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0)
         self.assert_in('archive2', output)
@@ -3086,7 +3086,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
             })
             archive_id = key.id_hash(archive)
             repository.put(archive_id, key.encrypt(archive))
-            repository.commit()
+            repository.commit(compact=False)
         self.cmd('check', self.repository_location, exit_code=1)
         self.cmd('check', '--repair', self.repository_location, exit_code=0)
         output = self.cmd('list', self.repository_location)
@@ -3098,7 +3098,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
         self.cmd('check', self.repository_location, exit_code=0)
         with Repository(self.repository_location, exclusive=True) as repository:
             repository.put(b'01234567890123456789012345678901', b'xxxx')
-            repository.commit()
+            repository.commit(compact=False)
         self.cmd('check', self.repository_location, exit_code=1)
         self.cmd('check', self.repository_location, exit_code=1)
         self.cmd('check', '--repair', self.repository_location, exit_code=0)
@@ -3117,7 +3117,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
                     data = repository.get(chunk.id) + b'1234'
                     repository.put(chunk.id, data)
                     break
-            repository.commit()
+            repository.commit(compact=False)
         self.cmd('check', self.repository_location, exit_code=0)
         output = self.cmd('check', '--verify-data', self.repository_location, exit_code=1)
         assert bin_to_hex(chunk.id) + ', integrity error' in output
@@ -3136,7 +3136,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
         with Repository(self.repository_location, exclusive=True) as repository:
             for id_ in repository.list():
                 repository.delete(id_)
-            repository.commit()
+            repository.commit(compact=False)
         self.cmd('check', self.repository_location, exit_code=1)
 
     def test_attic013_acl_bug(self):
@@ -3179,7 +3179,7 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
                 'config': {},
                 'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT),
             })))
-            repository.commit()
+            repository.commit(compact=False)
 
     def test_fresh_init_tam_required(self):
         self.cmd('init', '--encryption=repokey', self.repository_location)
@@ -3191,7 +3191,7 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
                 'archives': {},
                 'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT),
             })))
-            repository.commit()
+            repository.commit(compact=False)
 
         with pytest.raises(TAMRequiredError):
             self.cmd('list', self.repository_location)
@@ -3209,7 +3209,7 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
             manifest = msgpack.unpackb(key.decrypt(None, repository.get(Manifest.MANIFEST_ID)))
             del manifest[b'tam']
             repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb(manifest)))
-            repository.commit()
+            repository.commit(compact=False)
         output = self.cmd('list', '--debug', self.repository_location)
         assert 'archive1234' in output
         assert 'TAM not found and not required' in output
diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py
index 49f2e1f10..6af9da6bb 100644
--- a/src/borg/testsuite/repository.py
+++ b/src/borg/testsuite/repository.py
@@ -50,7 +50,7 @@ class RepositoryTestCaseBase(BaseTestCase):
         self.repository.put(H(0), b'foo')
         self.repository.put(H(1), b'bar')
         self.repository.put(H(3), b'bar')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         self.repository.put(H(1), b'bar2')
         self.repository.put(H(2), b'boo')
         self.repository.delete(H(3))
@@ -65,7 +65,7 @@ class RepositoryTestCase(RepositoryTestCaseBase):
         self.assert_equal(self.repository.get(key50), b'SOMEDATA')
         self.repository.delete(key50)
         self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(key50))
-        self.repository.commit()
+        self.repository.commit(compact=False)
         self.repository.close()
         with self.open() as repository2:
             self.assert_raises(Repository.ObjectNotFound, lambda: repository2.get(key50))
@@ -79,10 +79,10 @@ class RepositoryTestCase(RepositoryTestCaseBase):
         """
         self.repository.put(H(0), b'foo')
         self.repository.put(H(1), b'foo')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         self.repository.delete(H(0))
         self.repository.put(H(1), b'bar')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         self.assert_equal(self.repository.get(H(1)), b'bar')
 
     def test_consistency(self):
@@ -102,7 +102,7 @@ class RepositoryTestCase(RepositoryTestCaseBase):
         """
         self.repository.put(H(0), b'foo')
         self.assert_equal(self.repository.get(H(0)), b'foo')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         self.repository.put(H(0), b'foo2')
         self.assert_equal(self.repository.get(H(0)), b'foo2')
         self.repository.rollback()
@@ -113,29 +113,29 @@ class RepositoryTestCase(RepositoryTestCaseBase):
         """
         self.repository.put(H(0), b'foo')
         self.repository.put(H(0), b'foo2')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         self.assert_equal(self.repository.get(H(0)), b'foo2')
 
     def test_single_kind_transactions(self):
         # put
         self.repository.put(H(0), b'foo')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         self.repository.close()
         # replace
         self.repository = self.open()
         with self.repository:
             self.repository.put(H(0), b'bar')
-            self.repository.commit()
+            self.repository.commit(compact=False)
         # delete
         self.repository = self.open()
         with self.repository:
             self.repository.delete(H(0))
-            self.repository.commit()
+            self.repository.commit(compact=False)
 
     def test_list(self):
         for x in range(100):
             self.repository.put(H(x), b'SOMEDATA')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         all = self.repository.list()
         self.assert_equal(len(all), 100)
         first_half = self.repository.list(limit=50)
@@ -149,7 +149,7 @@ class RepositoryTestCase(RepositoryTestCaseBase):
     def test_scan(self):
         for x in range(100):
             self.repository.put(H(x), b'SOMEDATA')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         all = self.repository.scan()
         assert len(all) == 100
         first_half = self.repository.scan(limit=50)
@@ -177,6 +177,8 @@ class LocalRepositoryTestCase(RepositoryTestCaseBase):
     def _assert_sparse(self):
         # The superseded 123456... PUT
         assert self.repository.compact[0] == 41 + 9
+        # a COMMIT
+        assert self.repository.compact[1] == 9
         # The DELETE issued by the superseding PUT (or issued directly)
         assert self.repository.compact[2] == 41
         self.repository._rebuild_sparse(0)
@@ -185,14 +187,14 @@ class LocalRepositoryTestCase(RepositoryTestCaseBase):
     def test_sparse1(self):
         self.repository.put(H(0), b'foo')
         self.repository.put(H(1), b'123456789')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         self.repository.put(H(1), b'bar')
         self._assert_sparse()
 
     def test_sparse2(self):
         self.repository.put(H(0), b'foo')
         self.repository.put(H(1), b'123456789')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         self.repository.delete(H(1))
         self._assert_sparse()
 
@@ -207,14 +209,14 @@ class LocalRepositoryTestCase(RepositoryTestCaseBase):
         # ...while _rebuild_sparse can mark whole segments as completely sparse (which then includes the segment magic)
         assert self.repository.compact[0] == 41 + 41 + 4 + len(MAGIC)
 
-        self.repository.commit()
+        self.repository.commit(compact=True)
         assert 0 not in [segment for segment, _ in self.repository.io.segment_iterator()]
 
     def test_uncommitted_garbage(self):
         # uncommitted garbage should be no problem, it is cleaned up automatically.
         # we just have to be careful with invalidation of cached FDs in LoggedIO.
         self.repository.put(H(0), b'foo')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         # write some crap to a uncommitted segment file
         last_segment = self.repository.io.get_latest_segment()
         with open(self.repository.io.segment_filename(last_segment + 1), 'wb') as f:
@@ -224,7 +226,7 @@ class LocalRepositoryTestCase(RepositoryTestCaseBase):
         self.repository = self.open()
         with self.repository:
             self.repository.put(H(0), b'bar')  # this may trigger compact_segments()
-            self.repository.commit()
+            self.repository.commit(compact=True)
         # the point here is that nothing blows up with an exception.
 
 
@@ -244,7 +246,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
         self.add_keys()
         self.repository.compact_segments = None
         try:
-            self.repository.commit()
+            self.repository.commit(compact=True)
         except TypeError:
             pass
         self.reopen()
@@ -256,7 +258,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
         self.add_keys()
         self.repository.write_index = None
         try:
-            self.repository.commit()
+            self.repository.commit(compact=False)
         except TypeError:
             pass
         self.reopen()
@@ -294,7 +296,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
         self.add_keys()
         self.repository.io.delete_segment = None
         try:
-            self.repository.commit()
+            self.repository.commit(compact=False)
         except TypeError:
             pass
         self.reopen()
@@ -313,9 +315,9 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
     def test_moved_deletes_are_tracked(self):
         self.repository.put(H(1), b'1')
         self.repository.put(H(2), b'2')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         self.repository.delete(H(1))
-        self.repository.commit()
+        self.repository.commit(compact=True)
         last_segment = self.repository.io.get_latest_segment() - 1
         num_deletes = 0
         for tag, key, offset, size in self.repository.io.iter_objects(last_segment):
@@ -325,7 +327,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
         assert num_deletes == 1
         assert last_segment in self.repository.compact
         self.repository.put(H(3), b'3')
-        self.repository.commit()
+        self.repository.commit(compact=True)
         assert last_segment not in self.repository.compact
         assert not self.repository.io.segment_exists(last_segment)
         for segment, _ in self.repository.io.segment_iterator():
@@ -337,7 +339,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
         self.repository.put(H(1), b'1')
         # This is the segment with our original PUT of interest
         put_segment = get_latest_segment()
-        self.repository.commit()
+        self.repository.commit(compact=False)
 
         # We now delete H(1), and force this segment to not be compacted, which can happen
         # if it's not sparse enough (symbolized by H(2) here).
@@ -349,12 +351,12 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
         del self.repository.compact[put_segment]
         del self.repository.compact[delete_segment]
 
-        self.repository.commit()
+        self.repository.commit(compact=True)
 
         # Now we perform an unrelated operation on the segment containing the DELETE,
         # causing it to be compacted.
         self.repository.delete(H(2))
-        self.repository.commit()
+        self.repository.commit(compact=True)
 
         assert self.repository.io.segment_exists(put_segment)
         assert not self.repository.io.segment_exists(delete_segment)
@@ -370,7 +372,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
         self.repository.put(H(1), b'1')
         self.repository.delete(H(1))
         assert self.repository.shadow_index[H(1)] == [0]
-        self.repository.commit()
+        self.repository.commit(compact=True)
         # note how an empty list means that nothing is shadowed for sure
         assert self.repository.shadow_index[H(1)] == []
         self.repository.put(H(1), b'1')
@@ -397,21 +399,21 @@ class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
         def segments_in_repository():
             return len(list(self.repository.io.segment_iterator()))
         self.repository.put(H(0), b'foo')
-        self.repository.commit()
+        self.repository.commit(compact=False)
 
         self.repository.append_only = False
         assert segments_in_repository() == 2
         self.repository.put(H(0), b'foo')
-        self.repository.commit()
+        self.repository.commit(compact=True)
         # normal: compact squashes the data together, only one segment
-        assert segments_in_repository() == 4
+        assert segments_in_repository() == 2
 
         self.repository.append_only = True
-        assert segments_in_repository() == 4
+        assert segments_in_repository() == 2
         self.repository.put(H(0), b'foo')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         # append only: does not compact, only new segments written
-        assert segments_in_repository() == 6
+        assert segments_in_repository() == 4
 
 
 class RepositoryFreeSpaceTestCase(RepositoryTestCaseBase):
@@ -424,7 +426,7 @@ class RepositoryFreeSpaceTestCase(RepositoryTestCaseBase):
         with self.repository:
             self.repository.put(H(0), b'foobar')
             with pytest.raises(Repository.InsufficientFreeSpaceError):
-                self.repository.commit()
+                self.repository.commit(compact=False)
         assert os.path.exists(self.repository.path)
 
     def test_create_free_space(self):
@@ -443,7 +445,7 @@ class QuotaTestCase(RepositoryTestCaseBase):
         assert self.repository.storage_quota_use == 1234 + 5678 + 2 * 41
         self.repository.delete(H(1))
         assert self.repository.storage_quota_use == 5678 + 41
-        self.repository.commit()
+        self.repository.commit(compact=False)
         self.reopen()
         with self.repository:
             # Open new transaction; hints and thus quota data is not loaded unless needed.
@@ -456,12 +458,12 @@ class QuotaTestCase(RepositoryTestCaseBase):
         self.repository.storage_quota = 50
         self.repository.put(H(1), b'')
         assert self.repository.storage_quota_use == 41
-        self.repository.commit()
+        self.repository.commit(compact=False)
         with pytest.raises(Repository.StorageQuotaExceeded):
             self.repository.put(H(2), b'')
         assert self.repository.storage_quota_use == 82
         with pytest.raises(Repository.StorageQuotaExceeded):
-            self.repository.commit()
+            self.repository.commit(compact=False)
         assert self.repository.storage_quota_use == 82
         self.reopen()
         with self.repository:
@@ -517,13 +519,13 @@ class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
     def setUp(self):
         super().setUp()
         self.repository.put(H(0), b'foo')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         self.repository.close()
 
     def do_commit(self):
         with self.repository:
             self.repository.put(H(0), b'fox')
-            self.repository.commit()
+            self.repository.commit(compact=False)
 
     def test_corrupted_hints(self):
         with open(os.path.join(self.repository.path, 'hints.1'), 'ab') as fd:
@@ -620,9 +622,9 @@ class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
             assert self.repository.get(H(0)) == b'foo'
             self.repository.put(H(1), b'bar')
             self.repository.put(H(2), b'baz')
-            self.repository.commit()
+            self.repository.commit(compact=False)
             self.repository.put(H(2), b'bazz')
-            self.repository.commit()
+            self.repository.commit(compact=False)
 
         hints_path = os.path.join(self.repository.path, 'hints.5')
         with open(hints_path, 'r+b') as fd:
@@ -640,7 +642,7 @@ class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
             self.repository.append_only = False
             self.repository.put(H(3), b'1234')
             # Do a compaction run. Succeeds, since the failed checksum prompted a rebuild of the index+hints.
-            self.repository.commit()
+            self.repository.commit(compact=True)
 
             assert len(self.repository) == 4
             assert self.repository.get(H(0)) == b'foo'
@@ -656,7 +658,7 @@ class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
             self.repository.put(H(3), b'1234')
             # Do a compaction run. Fails, since the corrupted refcount was not detected and leads to an assertion failure.
             with pytest.raises(AssertionError) as exc_info:
-                self.repository.commit()
+                self.repository.commit(compact=True)
             assert 'Corrupted segment reference count' in str(exc_info.value)
 
 
@@ -678,7 +680,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
         for ids in segments:
             for id_ in ids:
                 self.repository.put(H(id_), b'data')
-            self.repository.commit()
+            self.repository.commit(compact=False)
 
     def get_head(self):
         return sorted(int(n) for n in os.listdir(os.path.join(self.tmppath, 'repository', 'data', '0')) if n.isdigit())[-1]
@@ -757,7 +759,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
         self.check(status=False)
         self.assert_equal(self.list_indices(), ['index.1'])
         self.check(repair=True, status=True)
-        self.assert_equal(self.list_indices(), ['index.3'])
+        self.assert_equal(self.list_indices(), ['index.2'])
         self.check(status=True)
         self.get_objects(3)
         self.assert_equal(set([1, 2, 3]), self.list_objects())
@@ -783,7 +785,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
         self.repository.put(H(0), b'data2')
         # Simulate a crash before compact
         with patch.object(Repository, 'compact_segments') as compact:
-            self.repository.commit()
+            self.repository.commit(compact=True)
             compact.assert_called_once_with()
         self.reopen()
         with self.repository:
@@ -903,18 +905,18 @@ class RemoteLegacyFree(RepositoryTestCaseBase):
     def test_legacy_free(self):
         # put
         self.repository.put(H(0), b'foo')
-        self.repository.commit()
+        self.repository.commit(compact=False)
         self.repository.close()
         # replace
         self.repository = self.open()
         with self.repository:
             self.repository.put(H(0), b'bar')
-            self.repository.commit()
+            self.repository.commit(compact=False)
         # delete
         self.repository = self.open()
         with self.repository:
             self.repository.delete(H(0))
-            self.repository.commit()
+            self.repository.commit(compact=False)
 
 
 class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase):