From 639eba16354008ac7d97d966084eac2fb930a262 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 13 Nov 2016 15:22:51 +0100 Subject: [PATCH 1/2] Fix check incorrectly reporting attic 0.13 and earlier archives as corrupt --- borg/archive.py | 3 +++ borg/testsuite/archiver.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/borg/archive.py b/borg/archive.py index 654a1da9a..b2f9df70f 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -1020,6 +1020,9 @@ class ArchiveChecker: def valid_item(obj): if not isinstance(obj, StableDict): return False + # A bug in Attic up to and including release 0.13 added a (meaningless) b'acl' key to every item. + # We ignore it here, should it exist. See test_attic013_acl_bug for details. + obj.pop(b'acl', None) keys = set(obj) return REQUIRED_ITEM_KEYS.issubset(keys) and keys.issubset(item_keys) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index b50304b5a..a7d8ff3e6 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1451,6 +1451,25 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): repository.commit() self.cmd('check', self.repository_location, exit_code=1) + def test_attic013_acl_bug(self): + # Attic up to release 0.13 contained a bug where every item unintentionally received + # a b'acl'=None key-value pair. + # This bug can still live on in Borg repositories (through borg upgrade). + archive, repository = self.open_archive('archive1') + with repository: + manifest, key = Manifest.load(repository) + with Cache(repository, key, manifest) as cache: + archive = Archive(repository, key, manifest, '0.13', cache=cache, create=True) + archive.items_buffer.add({ + # path and mtime are required. + b'path': '1234', + b'mtime': 0, + # acl is the offending key. + b'acl': None + }) + archive.save() + self.cmd('check', self.repository_location, exit_code=0) + @pytest.mark.skipif(sys.platform == 'cygwin', reason='remote is broken on cygwin and hangs') class RemoteArchiverTestCase(ArchiverTestCase): From a898297669213fb95eab4f71a838121993775e53 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 13 Nov 2016 02:00:39 +0100 Subject: [PATCH 2/2] check: improve "did not get expected metadata dict" diagnostic --- borg/archive.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index b2f9df70f..da8a0e70a 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -1017,14 +1017,21 @@ class ArchiveChecker: self.error_found = True logger.error(msg) + def list_keys_safe(keys): + return ', '.join((k.decode() if isinstance(k, bytes) else str(k) for k in keys)) + def valid_item(obj): if not isinstance(obj, StableDict): - return False + return False, 'not a dictionary' # A bug in Attic up to and including release 0.13 added a (meaningless) b'acl' key to every item. # We ignore it here, should it exist. See test_attic013_acl_bug for details. obj.pop(b'acl', None) keys = set(obj) - return REQUIRED_ITEM_KEYS.issubset(keys) and keys.issubset(item_keys) + if not REQUIRED_ITEM_KEYS.issubset(keys): + return False, 'missing required keys: ' + list_keys_safe(REQUIRED_ITEM_KEYS - keys) + if not keys.issubset(item_keys): + return False, 'invalid keys: ' + list_keys_safe(keys - item_keys) + return True, '' i = 0 for state, items in groupby(archive[b'items'], missing_chunk_detector): @@ -1040,10 +1047,11 @@ class ArchiveChecker: unpacker.feed(self.key.decrypt(chunk_id, cdata)) try: for item in unpacker: - if valid_item(item): + valid, reason = valid_item(item) + if valid: yield item else: - report('Did not get expected metadata dict when unpacking item metadata', chunk_id, i) + report('Did not get expected metadata dict when unpacking item metadata (%s)' % reason, chunk_id, i) except RobustUnpacker.UnpackerCrashed as err: report('Unpacker crashed while unpacking item metadata, trying to resync...', chunk_id, i) unpacker.resync()