From 619cb123e56e1ceb043ff56044ecb4d372972657 Mon Sep 17 00:00:00 2001 From: enkore Date: Mon, 28 Nov 2016 22:51:01 +0100 Subject: [PATCH 1/7] 1.0 maint AUTHORS +me --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 9749b1c52..9111a4506 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,6 +7,7 @@ Borg authors ("The Borg Collective") - Yuri D'Elia - Michael Hanselmann - Teemu Toivanen +- Marian Beermann Borg is a fork of Attic. From 9e760a69a29f7ebc055c4adf6f81b0a4de6aba52 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 29 Nov 2016 14:08:58 +0100 Subject: [PATCH 2/7] test_get_(cache|keys)_dir: clean env state, fixes #1897 make sure the BORG_(CACHE|KEYS)_DIR env var is not set initially. --- borg/testsuite/helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 1087b552d..d0fa86b6b 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -627,6 +627,7 @@ class TestParseTimestamp(BaseTestCase): def test_get_cache_dir(monkeypatch): """test that get_cache_dir respects environment""" + monkeypatch.delenv('BORG_CACHE_DIR', raising=False) monkeypatch.delenv('XDG_CACHE_HOME', raising=False) assert get_cache_dir() == os.path.join(os.path.expanduser('~'), '.cache', 'borg') monkeypatch.setenv('XDG_CACHE_HOME', '/var/tmp/.cache') @@ -637,6 +638,7 @@ def test_get_cache_dir(monkeypatch): def test_get_keys_dir(monkeypatch): """test that get_keys_dir respects environment""" + monkeypatch.delenv('BORG_KEYS_DIR', raising=False) monkeypatch.delenv('XDG_CONFIG_HOME', raising=False) assert get_keys_dir() == os.path.join(os.path.expanduser('~'), '.config', 'borg', 'keys') monkeypatch.setenv('XDG_CONFIG_HOME', '/var/tmp/.config') From cd50e286f712242fbbcc5c2143ab26c9ecf0d7d4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 29 Nov 2016 01:34:11 +0100 Subject: [PATCH 3/7] fix traceback in Error handler if id is None, fixes #1894 --- borg/key.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borg/key.py b/borg/key.py index 7b65d9093..944e17d72 100644 --- a/borg/key.py +++ b/borg/key.py @@ -105,7 +105,8 @@ class PlaintextKey(KeyBase): def decrypt(self, id, data): if data[0] != self.TYPE: - raise IntegrityError('Chunk %s: Invalid encryption envelope' % bin_to_hex(id)) + id_str = bin_to_hex(id) if id is not None else '(unknown)' + raise IntegrityError('Chunk %s: Invalid encryption envelope' % id_str) data = self.compressor.decompress(memoryview(data)[1:]) if id and sha256(data).digest() != id: raise IntegrityError('Chunk %s: id verification failed' % bin_to_hex(id)) From 989b2286ff171ecae4ed3b821acb4a1a14090766 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 30 Nov 2016 00:37:30 +0100 Subject: [PATCH 4/7] fix TypeError in errorhandler, fixes #1903 --- borg/key.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/borg/key.py b/borg/key.py index 944e17d72..e4fcd03d9 100644 --- a/borg/key.py +++ b/borg/key.py @@ -143,11 +143,13 @@ class AESKeyBase(KeyBase): def decrypt(self, id, data): if not (data[0] == self.TYPE or data[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)): - raise IntegrityError('Chunk %s: Invalid encryption envelope' % bin_to_hex(id)) + id_str = bin_to_hex(id) if id is not None else '(unknown)' + raise IntegrityError('Chunk %s: Invalid encryption envelope' % id_str) hmac_given = memoryview(data)[1:33] hmac_computed = memoryview(HMAC(self.enc_hmac_key, memoryview(data)[33:], sha256).digest()) if not compare_digest(hmac_computed, hmac_given): - raise IntegrityError('Chunk %s: Encryption envelope checksum mismatch' % bin_to_hex(id)) + id_str = bin_to_hex(id) if id is not None else '(unknown)' + raise IntegrityError('Chunk %s: Encryption envelope checksum mismatch' % id_str) self.dec_cipher.reset(iv=PREFIX + data[33:41]) data = self.compressor.decompress(self.dec_cipher.decrypt(data[41:])) if id: From 71775bac97858d917e99e6030402fa35406a5da8 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Wed, 30 Nov 2016 01:06:21 +0100 Subject: [PATCH 5/7] check: rebuild manifest if it's corrupted --- borg/archive.py | 10 ++++++++-- borg/testsuite/archiver.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index f69834ce4..3c27c45d5 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -19,7 +19,7 @@ from . import xattr from .helpers import Error, uid2user, user2uid, gid2group, group2gid, bin_to_hex, \ parse_timestamp, to_localtime, format_time, format_timedelta, remove_surrogates, \ Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, \ - ProgressIndicatorPercent + ProgressIndicatorPercent, IntegrityError from .platform import acl_get, acl_set from .chunker import Chunker from .hashindex import ChunkIndex @@ -849,7 +849,13 @@ class ArchiveChecker: self.error_found = True self.manifest = self.rebuild_manifest() else: - self.manifest, _ = Manifest.load(repository, key=self.key) + try: + self.manifest, _ = Manifest.load(repository, key=self.key) + except IntegrityError as exc: + logger.error('Repository manifest is corrupted: %s', exc) + self.error_found = True + del self.chunks[Manifest.MANIFEST_ID] + self.manifest = self.rebuild_manifest() self.rebuild_refcounts(archive=archive, last=last, prefix=prefix) self.orphan_chunks_check() self.finish(save_space=save_space) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index a7d8ff3e6..b493be534 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1433,6 +1433,19 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): self.assert_in('archive2', output) self.cmd('check', self.repository_location, exit_code=0) + def test_corrupted_manifest(self): + archive, repository = self.open_archive('archive1') + with repository: + manifest = repository.get(Manifest.MANIFEST_ID) + corrupted_manifest = manifest + b'corrupted!' + repository.put(Manifest.MANIFEST_ID, corrupted_manifest) + repository.commit() + self.cmd('check', self.repository_location, exit_code=1) + output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0) + self.assert_in('archive1', output) + self.assert_in('archive2', output) + self.cmd('check', self.repository_location, exit_code=0) + def test_extra_chunks(self): self.cmd('check', self.repository_location, exit_code=0) with Repository(self.repository_location, exclusive=True) as repository: From 146d586b3b31fba57bb5c1987606c362f345dc10 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Wed, 30 Nov 2016 01:43:01 +0100 Subject: [PATCH 6/7] check: skip corrupted chunks during manifest rebuild --- borg/archive.py | 7 ++++++- borg/testsuite/archiver.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/borg/archive.py b/borg/archive.py index 3c27c45d5..ebfa091f9 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -912,7 +912,12 @@ class ArchiveChecker: archive_keys_serialized = [msgpack.packb(name) for name in ARCHIVE_KEYS] for chunk_id, _ in self.chunks.iteritems(): cdata = self.repository.get(chunk_id) - data = self.key.decrypt(chunk_id, cdata) + try: + data = self.key.decrypt(chunk_id, cdata) + except IntegrityError as exc: + logger.error('Skipping corrupted chunk: %s', exc) + self.error_found = True + continue if not valid_msgpacked_dict(data, archive_keys_serialized): continue if b'cmdline' not in data or b'\xa7version\x01' not in data: diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index b493be534..67b5e581b 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -1446,6 +1446,22 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase): self.assert_in('archive2', output) self.cmd('check', self.repository_location, exit_code=0) + def test_manifest_rebuild_corrupted_chunk(self): + archive, repository = self.open_archive('archive1') + with repository: + manifest = repository.get(Manifest.MANIFEST_ID) + corrupted_manifest = manifest + b'corrupted!' + repository.put(Manifest.MANIFEST_ID, corrupted_manifest) + + chunk = repository.get(archive.id) + corrupted_chunk = chunk + b'corrupted!' + repository.put(archive.id, corrupted_chunk) + repository.commit() + self.cmd('check', self.repository_location, exit_code=1) + output = self.cmd('check', '-v', '--repair', self.repository_location, exit_code=0) + self.assert_in('archive2', output) + self.cmd('check', self.repository_location, exit_code=0) + def test_extra_chunks(self): self.cmd('check', self.repository_location, exit_code=0) with Repository(self.repository_location, exclusive=True) as repository: From d6d3f275df646ec03f9f9bcf457aec9490af6430 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 30 Nov 2016 02:50:20 +0100 Subject: [PATCH 7/7] docs: add python3-devel as a dependency for cygwin-based installation --- docs/installation.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index ff5cf7d1a..d3bc73007 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -266,7 +266,7 @@ Cygwin Use the Cygwin installer to install the dependencies:: - python3 python3-setuptools + python3 python3-devel python3-setuptools binutils gcc-g++ libopenssl openssl-devel liblz4_1 liblz4-devel