diff --git a/borg/archive.py b/borg/archive.py index 370134671..1894ad7a7 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -594,9 +594,15 @@ ITEM_KEYS = frozenset([b'path', b'source', b'rdev', b'chunks', # this is the set of keys that are always present in items: REQUIRED_ITEM_KEYS = frozenset([b'path', b'mtime', ]) +# this set must be kept complete, otherwise rebuild_manifest might malfunction: +ARCHIVE_KEYS = frozenset([b'version', b'name', b'items', b'cmdline', b'hostname', b'username', b'time', b'time_end', ]) -def valid_msgpacked_item(d, item_keys_serialized): - """check if the data looks like a msgpacked item metadata dict""" +# this is the set of keys that are always present in archives: +REQUIRED_ARCHIVE_KEYS = frozenset([b'version', b'name', b'items', b'cmdline', b'time', ]) + + +def valid_msgpacked_dict(d, keys_serialized): + """check if the data looks like a msgpacked dict""" d_len = len(d) if d_len == 0: return False @@ -606,7 +612,7 @@ def valid_msgpacked_item(d, item_keys_serialized): offs = 3 else: # object is not a map (dict) - # note: we must not have item dicts with > 2^16-1 elements + # note: we must not have dicts with > 2^16-1 elements return False if d_len <= offs: return False @@ -620,7 +626,7 @@ def valid_msgpacked_item(d, item_keys_serialized): return False # is the bytestring any of the expected key names? key_serialized = d[offs:] - return any(key_serialized.startswith(pattern) for pattern in item_keys_serialized) + return any(key_serialized.startswith(pattern) for pattern in keys_serialized) class RobustUnpacker: @@ -654,7 +660,7 @@ class RobustUnpacker: if not data: raise StopIteration # Abort early if the data does not look like a serialized item dict - if not valid_msgpacked_item(data, self.item_keys): + if not valid_msgpacked_dict(data, self.item_keys): data = data[1:] continue self._unpacker = msgpack.Unpacker(object_hook=StableDict) @@ -731,6 +737,12 @@ class ArchiveChecker: Iterates through all objects in the repository looking for archive metadata blocks. """ + def valid_archive(obj): + if not isinstance(obj, dict): + return False + keys = set(obj) + return REQUIRED_ARCHIVE_KEYS.issubset(keys) + logger.info('Rebuilding missing manifest, this might take some time...') # as we have lost the manifest, we do not know any more what valid item keys we had. # collecting any key we encounter in a damaged repo seems unwise, thus we just use @@ -738,11 +750,11 @@ class ArchiveChecker: # lost manifest on a older borg version than the most recent one that was ever used # within this repository (assuming that newer borg versions support more item keys). manifest = Manifest(self.key, self.repository) + archive_keys_serialized = [msgpack.packb(name) for name in ARCHIVE_KEYS] for chunk_id, _ in self.chunks.iteritems(): cdata = self.repository.get(chunk_id) data = self.key.decrypt(chunk_id, cdata) - # Some basic sanity checks of the payload before feeding it into msgpack - if len(data) < 2 or ((data[0] & 0xf0) != 0x80) or ((data[1] & 0xe0) != 0xa0): + if not valid_msgpacked_dict(data, archive_keys_serialized): continue if b'cmdline' not in data or b'\xa7version\x01' not in data: continue @@ -752,7 +764,7 @@ class ArchiveChecker: # msgpack with invalid data except (TypeError, ValueError, StopIteration): continue - if isinstance(archive, dict) and b'items' in archive and b'cmdline' in archive: + if valid_archive(archive): logger.info('Found archive %s', archive[b'name'].decode('utf-8')) manifest.archives[archive[b'name'].decode('utf-8')] = {b'id': chunk_id, b'time': archive[b'time']} logger.info('Manifest rebuild complete.') diff --git a/borg/testsuite/archive.py b/borg/testsuite/archive.py index 66676f334..919d57a0c 100644 --- a/borg/testsuite/archive.py +++ b/borg/testsuite/archive.py @@ -4,7 +4,7 @@ from unittest.mock import Mock import msgpack import pytest -from ..archive import Archive, CacheChunkBuffer, RobustUnpacker, valid_msgpacked_item, ITEM_KEYS +from ..archive import Archive, CacheChunkBuffer, RobustUnpacker, valid_msgpacked_dict, ITEM_KEYS from ..key import PlaintextKey from ..helpers import Manifest from . import BaseTestCase @@ -127,7 +127,7 @@ def item_keys_serialized(): [42, 23.42, True, b'foobar', {b'foo': b'bar'}, [b'foo', b'bar'], (b'foo', b'bar')] )]) def test_invalid_msgpacked_item(packed, item_keys_serialized): - assert not valid_msgpacked_item(packed, item_keys_serialized) + assert not valid_msgpacked_dict(packed, item_keys_serialized) @pytest.mark.parametrize('packed', @@ -137,11 +137,11 @@ def test_invalid_msgpacked_item(packed, item_keys_serialized): dict((k, b'x' * 1000) for k in ITEM_KEYS), # as big (key count and volume) as it gets ]]) def test_valid_msgpacked_items(packed, item_keys_serialized): - assert valid_msgpacked_item(packed, item_keys_serialized) + assert valid_msgpacked_dict(packed, item_keys_serialized) def test_key_length_msgpacked_items(): key = b'x' * 32 # 31 bytes is the limit for fixstr msgpack type data = {key: b''} item_keys_serialized = [msgpack.packb(key), ] - assert valid_msgpacked_item(msgpack.packb(data), item_keys_serialized) + assert valid_msgpacked_dict(msgpack.packb(data), item_keys_serialized)