upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-05-28 04:03:21 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #5693 from ThomasWaldmann/repo-fixes-1.1

Browse source 
Repo changes backport to 1.1-maint
This commit is contained in:
TW 2021-02-15 21:41:44 +01:00 committed by GitHub
commit 641418d06d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 64 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
src/borg/repository.py
View file

@ -774,6 +774,8 @@ class Repository:
try:
self.shadow_index[key].remove(segment)
except (KeyError, ValueError):
# do not remove entry with empty shadowed_segments list here,
# it is needed for shadowed_put_exists code (see below)!
pass
elif tag == TAG_DELETE and not in_index:
# If the shadow index doesn't contain this key, then we can't say if there's a shadowed older tag,
@ -825,6 +827,11 @@ class Repository:
new_segment, size = self.io.write_delete(key)
self.compact[new_segment] += size
segments.setdefault(new_segment, 0)
else:
# we did not keep the delete tag for key (see if-branch)
if not self.shadow_index[key]:
# shadowed segments list is empty -> remove it
del self.shadow_index[key]
assert segments[segment] == 0, 'Corrupted segment reference count - corrupted index or hints'
unused.append(segment)
pi.show()
@ -1135,13 +1142,11 @@ class Repository:
except KeyError:
pass
else:
self.segments[segment] -= 1
size = self.io.read(segment, offset, id, read_data=False)
self.storage_quota_use -= size
self.compact[segment] += size
segment, size = self.io.write_delete(id)
self.compact[segment] += size
self.segments.setdefault(segment, 0)
# note: doing a delete first will do some bookkeeping.
# we do not want to update the shadow_index here, because
# we know already that we will PUT to this id, so it will
# be in the repo index (and we won't need it in the shadow_index).
self._delete(id, segment, offset, update_shadow_index=False)
segment, offset = self.io.write_put(id, data)
self.storage_quota_use += len(data) + self.io.put_header_fmt.size
self.segments.setdefault(segment, 0)
@ -1164,7 +1169,16 @@ class Repository:
segment, offset = self.index.pop(id)
except KeyError:
raise self.ObjectNotFound(id, self.path) from None
self.shadow_index.setdefault(id, []).append(segment)
# if we get here, there is an object with this id in the repo,
# we write a DEL here that shadows the respective PUT.
# after the delete, the object is not in the repo index any more,
# for the compaction code, we need to update the shadow_index in this case.
self._delete(id, segment, offset, update_shadow_index=True)
def _delete(self, id, segment, offset, *, update_shadow_index):
# common code used by put and delete
if update_shadow_index:
self.shadow_index.setdefault(id, []).append(segment)
self.segments[segment] -= 1
size = self.io.read(segment, offset, id, read_data=False)
self.storage_quota_use -= size

45
src/borg/testsuite/repository.py
View file

@ -14,7 +14,7 @@ from ..helpers import IntegrityError
from ..helpers import msgpack
from ..locking import Lock, LockFailed
from ..remote import RemoteRepository, InvalidRPCMethod, PathNotAllowed, ConnectionClosedWithHint, handle_remote_line
from ..repository import Repository, LoggedIO, MAGIC, MAX_DATA_SIZE, TAG_DELETE
from ..repository import Repository, LoggedIO, MAGIC, MAX_DATA_SIZE, TAG_DELETE, TAG_PUT, TAG_COMMIT
from . import BaseTestCase
from .hashindex import H
@ -54,6 +54,16 @@ class RepositoryTestCaseBase(BaseTestCase):
self.repository.put(H(2), b'boo')
self.repository.delete(H(3))
def repo_dump(self, label=None):
label = label + ': ' if label is not None else ''
H_trans = {H(i): i for i in range(10)}
H_trans[None] = -1 # key == None appears in commits
tag_trans = {TAG_PUT: 'put', TAG_DELETE: 'del', TAG_COMMIT: 'comm'}
for segment, fn in self.repository.io.segment_iterator():
for tag, key, offset, size in self.repository.io.iter_objects(segment):
print("%s%s H(%d) -> %s[%d..+%d]" % (label, tag_trans[tag], H_trans[key], fn, offset, size))
print()
class RepositoryTestCase(RepositoryTestCaseBase):
@ -313,8 +323,10 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
self.repository.put(H(1), b'1')
self.repository.put(H(2), b'2')
self.repository.commit()
self.repo_dump('p1 p2 cc')
self.repository.delete(H(1))
self.repository.commit()
self.repo_dump('d1 cc')
last_segment = self.repository.io.get_latest_segment() - 1
num_deletes = 0
for tag, key, offset, size in self.repository.io.iter_objects(last_segment):
@ -325,11 +337,16 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
assert last_segment in self.repository.compact
self.repository.put(H(3), b'3')
self.repository.commit()
self.repo_dump('p3 cc')
assert last_segment not in self.repository.compact
assert not self.repository.io.segment_exists(last_segment)
for segment, _ in self.repository.io.segment_iterator():
for tag, key, offset, size in self.repository.io.iter_objects(segment):
assert tag != TAG_DELETE
assert key != H(1)
# after compaction, there should be no empty shadowed_segments lists left over.
# we have no put or del any more for H(1), so we lost knowledge about H(1).
assert H(1) not in self.repository.shadow_index
def test_shadowed_entries_are_preserved(self):
get_latest_segment = self.repository.io.get_latest_segment
@ -370,16 +387,19 @@ class RepositoryCommitTestCase(RepositoryTestCaseBase):
self.repository.delete(H(1))
assert self.repository.shadow_index[H(1)] == [0]
self.repository.commit()
self.repo_dump('p1 d1 cc')
# note how an empty list means that nothing is shadowed for sure
assert self.repository.shadow_index[H(1)] == []
assert self.repository.shadow_index[H(1)] == [] # because the delete is considered unstable
self.repository.put(H(1), b'1')
self.repository.delete(H(1))
self.repo_dump('p1 d1')
# 0 put/delete; 1 commit; 2 compacted; 3 commit; 4 put/delete
assert self.repository.shadow_index[H(1)] == [4]
self.repository.rollback()
self.repo_dump('r')
self.repository.put(H(2), b'1')
# After the rollback segment 4 shouldn't be considered anymore
assert self.repository.shadow_index[H(1)] == []
assert self.repository.shadow_index[H(1)] == [] # because the delete is considered unstable
class RepositoryAppendOnlyTestCase(RepositoryTestCaseBase):
@ -790,6 +810,25 @@ class RepositoryCheckTestCase(RepositoryTestCaseBase):
self.assert_equal(self.repository.get(H(0)), b'data2')
class RepositoryHintsTestCase(RepositoryTestCaseBase):
def test_hints_behaviour(self):
self.repository.put(H(0), b'data')
self.assert_equal(self.repository.shadow_index, {})
self.assert_true(len(self.repository.compact) == 0)
self.repository.delete(H(0))
self.repository.commit()
# this is to make the previous delete "stable", see delete_is_not_stable:
self.repository.put(H(1), b'data')
self.repository.commit()
# in 1.1-maint, commit() always implicitly compacts.
# nothing to compact any more! no info left about stuff that does not exist any more:
self.assert_not_in(H(0), self.repository.shadow_index)
# segment 0 was compacted away, no info about it left:
self.assert_not_in(0, self.repository.compact)
self.assert_not_in(0, self.repository.segments)
class RemoteRepositoryTestCase(RepositoryTestCase):
repository = None # type: RemoteRepository