upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-03-27 12:54:05 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

test fixes

Browse source
This commit is contained in:
Thomas Waldmann 2018-06-24 04:35:52 +02:00
parent aeef082483
commit efe60bd640
2 changed files with 67 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
src/borg/testsuite/archiver.py
View file

@ -1463,7 +1463,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
if 'chunks' in item:
first_chunk_id = item.chunks[0].id
repository.delete(first_chunk_id)
repository.commit()
repository.commit(compact=False)
break
output = self.cmd('delete', '--force', self.repository_location + '::test')
self.assert_in('deleted archive was corrupted', output)
@ -1479,7 +1479,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
archive = Archive(repository, key, manifest, 'test')
id = archive.metadata.items[0]
repository.put(id, b'corrupted items metadata stream chunk')
repository.commit()
repository.commit(compact=False)
self.cmd('delete', '--force', '--force', self.repository_location + '::test')
self.cmd('check', '--repair', self.repository_location)
output = self.cmd('list', self.repository_location)
@ -1533,7 +1533,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
manifest.config[b'feature_flags'] = {operation.value.encode(): {b'mandatory': [b'unknown-feature']}}
manifest.write()
repository.commit()
repository.commit(compact=False)
def cmd_raises_unknown_feature(self, args):
if self.FORK_DEFAULT:
@ -2249,7 +2249,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
break
else:
assert False # missed the file
repository.commit()
repository.commit(compact=False)
self.cmd('check', '--repair', self.repository_location, exit_code=0)
mountpoint = os.path.join(self.tmpdir, 'mountpoint')
@ -2970,7 +2970,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
break
else:
self.fail('should not happen')
repository.commit()
repository.commit(compact=False)
self.cmd('check', self.repository_location, exit_code=1)
output = self.cmd('check', '--repair', self.repository_location, exit_code=0)
self.assert_in('New missing file chunk detected', output)
@ -3013,7 +3013,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
archive, repository = self.open_archive('archive1')
with repository:
repository.delete(archive.metadata.items[0])
repository.commit()
repository.commit(compact=False)
self.cmd('check', self.repository_location, exit_code=1)
self.cmd('check', '--repair', self.repository_location, exit_code=0)
self.cmd('check', self.repository_location, exit_code=0)
@ -3022,7 +3022,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
archive, repository = self.open_archive('archive1')
with repository:
repository.delete(archive.id)
repository.commit()
repository.commit(compact=False)
self.cmd('check', self.repository_location, exit_code=1)
self.cmd('check', '--repair', self.repository_location, exit_code=0)
self.cmd('check', self.repository_location, exit_code=0)
@ -3031,7 +3031,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
archive, repository = self.open_archive('archive1')
with repository:
repository.delete(Manifest.MANIFEST_ID)
repository.commit()
repository.commit(compact=False)
self.cmd('check', self.repository_location, exit_code=1)
output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0)
self.assert_in('archive1', output)
@ -3044,7 +3044,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
manifest = repository.get(Manifest.MANIFEST_ID)
corrupted_manifest = manifest + b'corrupted!'
repository.put(Manifest.MANIFEST_ID, corrupted_manifest)
repository.commit()
repository.commit(compact=False)
self.cmd('check', self.repository_location, exit_code=1)
output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0)
self.assert_in('archive1', output)
@ -3061,7 +3061,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
chunk = repository.get(archive.id)
corrupted_chunk = chunk + b'corrupted!'
repository.put(archive.id, corrupted_chunk)
repository.commit()
repository.commit(compact=False)
self.cmd('check', self.repository_location, exit_code=1)
output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0)
self.assert_in('archive2', output)
@ -3086,7 +3086,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
})
archive_id = key.id_hash(archive)
repository.put(archive_id, key.encrypt(archive))
repository.commit()
repository.commit(compact=False)
self.cmd('check', self.repository_location, exit_code=1)
self.cmd('check', '--repair', self.repository_location, exit_code=0)
output = self.cmd('list', self.repository_location)
@ -3098,7 +3098,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
self.cmd('check', self.repository_location, exit_code=0)
with Repository(self.repository_location, exclusive=True) as repository:
repository.put(b'01234567890123456789012345678901', b'xxxx')
repository.commit()
repository.commit(compact=False)
self.cmd('check', self.repository_location, exit_code=1)
self.cmd('check', self.repository_location, exit_code=1)
self.cmd('check', '--repair', self.repository_location, exit_code=0)
@ -3117,7 +3117,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
data = repository.get(chunk.id) + b'1234'
repository.put(chunk.id, data)
break
repository.commit()
repository.commit(compact=False)
self.cmd('check', self.repository_location, exit_code=0)
output = self.cmd('check', '--verify-data', self.repository_location, exit_code=1)
assert bin_to_hex(chunk.id) + ', integrity error' in output
@ -3136,7 +3136,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
with Repository(self.repository_location, exclusive=True) as repository:
for id_ in repository.list():
repository.delete(id_)
repository.commit()
repository.commit(compact=False)
self.cmd('check', self.repository_location, exit_code=1)
def test_attic013_acl_bug(self):
@ -3179,7 +3179,7 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
'config': {},
'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT),
})))
repository.commit()
repository.commit(compact=False)
def test_fresh_init_tam_required(self):
self.cmd('init', '--encryption=repokey', self.repository_location)
@ -3191,7 +3191,7 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
'archives': {},
'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT),
})))
repository.commit()
repository.commit(compact=False)
with pytest.raises(TAMRequiredError):
self.cmd('list', self.repository_location)
@ -3209,7 +3209,7 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
manifest = msgpack.unpackb(key.decrypt(None, repository.get(Manifest.MANIFEST_ID)))
del manifest[b'tam']
repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb(manifest)))
repository.commit()
repository.commit(compact=False)
output = self.cmd('list', '--debug', self.repository_location)
assert 'archive1234' in output
assert 'TAM not found and not required' in output

98
src/borg/testsuite/repository.py
View file

@ -50,7 +50,7 @@ class RepositoryTestCaseBase(BaseTestCase):
self.repository.put(H(0), b'foo')
self.repository.put(H(1), b'bar')
self.repository.put(H(3), b'bar')
self.repository.commit()
self.repository.commit(compact=False)
self.repository.put(H(1), b'bar2')
self.repository.put(H(2), b'boo')
self.repository.delete(H(3))
@ -65,7 +65,7 @@ class RepositoryTestCase(RepositoryTestCaseBase):
self.assert_equal(self.repository.get(key50), b'SOMEDATA')
self.repository.delete(key50)
self.assert_raises(Repository.ObjectNotFound, lambda: self.repository.get(key50))
self.repository.commit()
self.repository.commit(compact=False)
self.repository.close()
with self.open() as repository2:
self.assert_raises(Repository.ObjectNotFound, lambda: repository2.get(key50))
@ -79,10 +79,10 @@ class RepositoryTestCase(RepositoryTestCaseBase):
"""
self.repository.put(H(0), b'foo')
self.repository.put(H(1), b'foo')
self.repository.commit()
self.repository.commit(compact=False)
self.repository.delete(H(0))
self.repository.put(H(1), b'bar')
self.repository.commit()
self.repository.commit(compact=False)
self.assert_equal(self.repository.get(H(1)), b'bar')
def test_consistency(self):
@ -102,7 +102,7 @@ class RepositoryTestCase(RepositoryTestCaseBase):
"""
self.repository.put(H(0), b'foo')
self.assert_equal(self.repository.get(H(0)), b'foo')
self.repository.commit()
self.repository.commit(compact=False)
self.repository.put(H(0), b'foo2')
self.assert_equal(self.repository.get(H(0)), b'foo2')
self.repository.rollback()
@ -113,29 +113,29 @@ class RepositoryTestCase(RepositoryTestCaseBase):
"""
self.repository.put(H(0), b'foo')
self.repository.put(H(0), b'foo2')
self.repository.commit()
self.repository.commit(compact=False)
self.assert_equal(self.repository.get(H(0)), b'foo2')
def test_single_kind_transactions(self):
# put
self.repository.put(H(0), b'foo')
self.repository.commit()
self.repository.commit(compact=False)
self.repository.close()
# replace
self.repository = self.open()
with self.repository:
self.repository.put(H(0), b'bar')
self.repository.commit()
self.repository.commit(compact=False)
# delete
self.repository = self.open()
with self.repository:
self.repository.delete(H(0))
self.repository.commit()
self.repository.commit(compact=False)
def test_list(self):
for x in range(100):
self.repository.put(H(x), b'SOMEDATA')
self.repository.commit()
self.repository.commit(compact=False)
all = self.repository.list()
self.assert_equal(len(all), 100)
first_half = self.repository.list(limit=50)
@ -149,7 +149,7 @@ class RepositoryTestCase(RepositoryTestCaseBase):
def test_scan(self):
for x in range(100):
self.repository.put(H(x), b'SOMEDATA')
self.repository.commit()
self.repository.commit(compact=False)
all = self.repository.scan()
assert len(all) == 100
first_half = self.repository.scan(limit=50)
@ -177,6 +177,8 @@ class LocalRepositoryTestCase(RepositoryTestCaseBase):
def _assert_sparse(self):
# The superseded 123456... PUT
assert self.repository.compact[0] == 41 + 9
# a COMMIT
assert self.repository.compact[1] == 9
# The DELETE issued by the superseding PUT (or issued directly)
assert self.repository.compact[2] == 41
self.repository._rebuild_sparse(0)
@ -185,14 +187,14 @@ class LocalRepositoryTestCase(RepositoryTestCaseBase):
def test_sparse1(self):
self.repository.put(H(0), b'foo')
self.repository.put(H(1), b'123456789')
self.repository.commit()
self.repository.commit(compact=False)
self.repository.put(H(1), b'bar')
self._assert_sparse()
def test_sparse2(self):
self.repository.put(H(0), b'foo')
self.repository.put(H(1), b'123456789')
self.repository.commit()
self.repository.commit(compact=False)
self.repository.delete(H(1))
self._assert_sparse()
@ -207,14 +209,14 @@ class LocalRepositoryTestCase(RepositoryTestCaseBase):
# ...while _rebuild_sparse can mark whole segments as completely sparse (which then includes the segment magic)
assert self.repository.compact[0] == 41 + 41 + 4 + len(MAGIC)
self.repository.commit()
self.repository.commit(compact=True)
assert 0 not in [segment for segment, _ in self.repository.io.segment_iterator()]
def test_uncommitted_garbage(self):
# uncommitted garbage should be no problem, it is cleaned up automatically.
# we just have to be careful with invalidation of cached FDs in LoggedIO.
self.repository.put(H(0), b'foo')
self.repository.commit()
self.repository.commit(compact=False)
# write some crap to a uncommitted segment file
last_segment = self.repository.io.get_latest_segment()
with open(self.repository.io.segment_filename(last_segment + 1), 'wb') as f:
@ -224,7 +226,7 @@ class LocalRepositoryTestCase(RepositoryTestCaseBase):
self.repository = self.open()
with self.repository:
self.repository.put(H(0), b'bar') # this may trigger compact_segments()
self.repository.commit()
self.repository.commit(compact=True)
# the point here is that nothing blows up with an exception.
@ -244,7 +246,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
self.add_keys()
self.repository.compact_segments = None
try:
self.repository.commit()
self.repository.commit(compact=True)
except TypeError:
pass
self.reopen()
@ -256,7 +258,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
self.add_keys()
self.repository.write_index = None
try:
self.repository.commit()
self.repository.commit(compact=False)
except TypeError:
pass
self.reopen()
@ -294,7 +296,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
self.add_keys()
self.repository.io.delete_segment = None
try:
self.repository.commit()
self.repository.commit(compact=False)
except TypeError:
pass
self.reopen()
@ -313,9 +315,9 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
def test_moved_deletes_are_tracked(self):
self.repository.put(H(1), b'1')
self.repository.put(H(2), b'2')
self.repository.commit()
self.repository.commit(compact=False)
self.repository.delete(H(1))
self.repository.commit()
self.repository.commit(compact=True)
last_segment = self.repository.io.get_latest_segment() - 1
num_deletes = 0
for tag, key, offset, size in self.repository.io.iter_objects(last_segment):
@ -325,7 +327,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
assert num_deletes == 1
assert last_segment in self.repository.compact
self.repository.put(H(3), b'3')
self.repository.commit()
self.repository.commit(compact=True)
assert last_segment not in self.repository.compact
assert not self.repository.io.segment_exists(last_segment)
for segment, _ in self.repository.io.segment_iterator():
@ -337,7 +339,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
self.repository.put(H(1), b'1')
# This is the segment with our original PUT of interest
put_segment = get_latest_segment()
self.repository.commit()
self.repository.commit(compact=False)
# We now delete H(1), and force this segment to not be compacted, which can happen
# if it's not sparse enough (symbolized by H(2) here).
@ -349,12 +351,12 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
del self.repository.compact[put_segment]
del self.repository.compact[delete_segment]
self.repository.commit()
self.repository.commit(compact=True)
# Now we perform an unrelated operation on the segment containing the DELETE,
# causing it to be compacted.
self.repository.delete(H(2))
self.repository.commit()
self.repository.commit(compact=True)
assert self.repository.io.segment_exists(put_segment)
assert not self.repository.io.segment_exists(delete_segment)
@ -370,7 +372,7 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
self.repository.put(H(1), b'1')
self.repository.delete(H(1))
assert self.repository.shadow_index[H(1)] == [0]
self.repository.commit()
self.repository.commit(compact=True)
# note how an empty list means that nothing is shadowed for sure
assert self.repository.shadow_index[H(1)] == []
self.repository.put(H(1), b'1')
@ -397,21 +399,21 @@ class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
def segments_in_repository():
return len(list(self.repository.io.segment_iterator()))
self.repository.put(H(0), b'foo')
self.repository.commit()
self.repository.commit(compact=False)
self.repository.append_only = False
assert segments_in_repository() == 2
self.repository.put(H(0), b'foo')
self.repository.commit()
self.repository.commit(compact=True)
# normal: compact squashes the data together, only one segment
assert segments_in_repository() == 4
assert segments_in_repository() == 2
self.repository.append_only = True
assert segments_in_repository() == 4
assert segments_in_repository() == 2
self.repository.put(H(0), b'foo')
self.repository.commit()
self.repository.commit(compact=False)
# append only: does not compact, only new segments written
assert segments_in_repository() == 6
assert segments_in_repository() == 4
class RepositoryFreeSpaceTestCase(RepositoryTestCaseBase):
@ -424,7 +426,7 @@ class RepositoryFreeSpaceTestCase(RepositoryTestCaseBase):
with self.repository:
self.repository.put(H(0), b'foobar')
with pytest.raises(Repository.InsufficientFreeSpaceError):
self.repository.commit()
self.repository.commit(compact=False)
assert os.path.exists(self.repository.path)
def test_create_free_space(self):
@ -443,7 +445,7 @@ class QuotaTestCase(RepositoryTestCaseBase):
assert self.repository.storage_quota_use == 1234 + 5678 + 2 * 41
self.repository.delete(H(1))
assert self.repository.storage_quota_use == 5678 + 41
self.repository.commit()
self.repository.commit(compact=False)
self.reopen()
with self.repository:
# Open new transaction; hints and thus quota data is not loaded unless needed.
@ -456,12 +458,12 @@ class QuotaTestCase(RepositoryTestCaseBase):
self.repository.storage_quota = 50
self.repository.put(H(1), b'')
assert self.repository.storage_quota_use == 41
self.repository.commit()
self.repository.commit(compact=False)
with pytest.raises(Repository.StorageQuotaExceeded):
self.repository.put(H(2), b'')
assert self.repository.storage_quota_use == 82
with pytest.raises(Repository.StorageQuotaExceeded):
self.repository.commit()
self.repository.commit(compact=False)
assert self.repository.storage_quota_use == 82
self.reopen()
with self.repository:
@ -517,13 +519,13 @@ class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
def setUp(self):
super().setUp()
self.repository.put(H(0), b'foo')
self.repository.commit()
self.repository.commit(compact=False)
self.repository.close()
def do_commit(self):
with self.repository:
self.repository.put(H(0), b'fox')
self.repository.commit()
self.repository.commit(compact=False)
def test_corrupted_hints(self):
with open(os.path.join(self.repository.path, 'hints.1'), 'ab') as fd:
@ -620,9 +622,9 @@ class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
assert self.repository.get(H(0)) == b'foo'
self.repository.put(H(1), b'bar')
self.repository.put(H(2), b'baz')
self.repository.commit()
self.repository.commit(compact=False)
self.repository.put(H(2), b'bazz')
self.repository.commit()
self.repository.commit(compact=False)
hints_path = os.path.join(self.repository.path, 'hints.5')
with open(hints_path, 'r+b') as fd:
@ -640,7 +642,7 @@ class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
self.repository.append_only = False
self.repository.put(H(3), b'1234')
# Do a compaction run. Succeeds, since the failed checksum prompted a rebuild of the index+hints.
self.repository.commit()
self.repository.commit(compact=True)
assert len(self.repository) == 4
assert self.repository.get(H(0)) == b'foo'
@ -656,7 +658,7 @@ class RepositoryAuxiliaryCorruptionTestCase(RepositoryTestCaseBase):
self.repository.put(H(3), b'1234')
# Do a compaction run. Fails, since the corrupted refcount was not detected and leads to an assertion failure.
with pytest.raises(AssertionError) as exc_info:
self.repository.commit()
self.repository.commit(compact=True)
assert 'Corrupted segment reference count' in str(exc_info.value)
@ -678,7 +680,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
for ids in segments:
for id_ in ids:
self.repository.put(H(id_), b'data')
self.repository.commit()
self.repository.commit(compact=False)
def get_head(self):
return sorted(int(n) for n in os.listdir(os.path.join(self.tmppath, 'repository', 'data', '0')) if n.isdigit())[-1]
@ -757,7 +759,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
self.check(status=False)
self.assert_equal(self.list_indices(), ['index.1'])
self.check(repair=True, status=True)
self.assert_equal(self.list_indices(), ['index.3'])
self.assert_equal(self.list_indices(), ['index.2'])
self.check(status=True)
self.get_objects(3)
self.assert_equal(set([1, 2, 3]), self.list_objects())
@ -783,7 +785,7 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
self.repository.put(H(0), b'data2')
# Simulate a crash before compact
with patch.object(Repository, 'compact_segments') as compact:
self.repository.commit()
self.repository.commit(compact=True)
compact.assert_called_once_with()
self.reopen()
with self.repository:
@ -903,18 +905,18 @@ class RemoteLegacyFree(RepositoryTestCaseBase):
def test_legacy_free(self):
# put
self.repository.put(H(0), b'foo')
self.repository.commit()
self.repository.commit(compact=False)
self.repository.close()
# replace
self.repository = self.open()
with self.repository:
self.repository.put(H(0), b'bar')
self.repository.commit()
self.repository.commit(compact=False)
# delete
self.repository = self.open()
with self.repository:
self.repository.delete(H(0))
self.repository.commit()
self.repository.commit(compact=False)
class RemoteRepositoryCheckTestCase(RepositoryCheckTestCase):