From 39051ac5f148eb2792ddfb4b69cb79dbd8ab7928 Mon Sep 17 00:00:00 2001
From: Marian Beermann <public@enkore.de>
Date: Thu, 25 May 2017 13:22:24 +0200
Subject: [PATCH 01/12] file_integrity: split in IntegrityCheckedFile +
 Detached variant

---
 src/borg/crypto/file_integrity.py    | 79 ++++++++++++++++++----------
 src/borg/testsuite/file_integrity.py | 38 ++++++-------
 2 files changed, 70 insertions(+), 47 deletions(-)

diff --git a/src/borg/crypto/file_integrity.py b/src/borg/crypto/file_integrity.py
index 5c1fa4e1c..53cbcea3e 100644
--- a/src/borg/crypto/file_integrity.py
+++ b/src/borg/crypto/file_integrity.py
@@ -104,7 +104,7 @@ class FileIntegrityError(IntegrityError):
 
 
 class IntegrityCheckedFile(FileLikeWrapper):
-    def __init__(self, path, write, filename=None, override_fd=None):
+    def __init__(self, path, write, filename=None, override_fd=None, integrity_data=None):
         self.path = path
         self.writing = write
         mode = 'wb' if write else 'rb'
@@ -114,10 +114,10 @@ class IntegrityCheckedFile(FileLikeWrapper):
 
         self.hash_filename(filename)
 
-        if write:
+        if write or not integrity_data:
             self.digests = {}
         else:
-            self.digests = self.read_integrity_file(path, self.hasher)
+            self.digests = self.parse_integrity_data(path, integrity_data, self.hasher)
             # TODO: When we're reading but don't have any digests, i.e. no integrity file existed,
             # TODO: then we could just short-circuit.
 
@@ -126,32 +126,27 @@ class IntegrityCheckedFile(FileLikeWrapper):
         # In Borg the name itself encodes the context (eg. index.N, cache, files),
         # while the path doesn't matter, and moving e.g. a repository or cache directory is supported.
         # Changing the name however imbues a change of context that is not permissible.
+        # While Borg does not use anything except ASCII in these file names, it's important to use
+        # the same encoding everywhere for portability. Using os.fsencode() would be wrong.
         filename = os.path.basename(filename or self.path)
         self.hasher.update(('%10d' % len(filename)).encode())
         self.hasher.update(filename.encode())
 
-    @staticmethod
-    def integrity_file_path(path):
-        return path + '.integrity'
-
     @classmethod
-    def read_integrity_file(cls, path, hasher):
+    def parse_integrity_data(cls, path: str, data: str, hasher: SHA512FileHashingWrapper):
         try:
-            with open(cls.integrity_file_path(path), 'r') as fd:
-                integrity_file = json.load(fd)
-                # Provisions for agility now, implementation later, but make sure the on-disk joint is oiled.
-                algorithm = integrity_file['algorithm']
-                if algorithm != hasher.ALGORITHM:
-                    logger.warning('Cannot verify integrity of %s: Unknown algorithm %r', path, algorithm)
-                    return
-                digests = integrity_file['digests']
-                # Require at least presence of the final digest
-                digests['final']
-                return digests
-        except FileNotFoundError:
-            logger.info('No integrity file found for %s', path)
-        except (OSError, ValueError, TypeError, KeyError) as e:
-            logger.warning('Could not read integrity file for %s: %s', path, e)
+            integrity_file = json.loads(data)
+            # Provisions for agility now, implementation later, but make sure the on-disk joint is oiled.
+            algorithm = integrity_file['algorithm']
+            if algorithm != hasher.ALGORITHM:
+                logger.warning('Cannot verify integrity of %s: Unknown algorithm %r', path, algorithm)
+                return
+            digests = integrity_file['digests']
+            # Require at least presence of the final digest
+            digests['final']
+            return digests
+        except (ValueError, TypeError, KeyError) as e:
+            logger.warning('Could not parse integrity data for %s: %s', path, e)
             raise FileIntegrityError(path)
 
     def hash_part(self, partname, is_final=False):
@@ -173,10 +168,38 @@ class IntegrityCheckedFile(FileLikeWrapper):
         if exception:
             return
         if self.writing:
-            with open(self.integrity_file_path(self.path), 'w') as fd:
-                json.dump({
-                    'algorithm': self.hasher.ALGORITHM,
-                    'digests': self.digests,
-                }, fd)
+            self.store_integrity_data(json.dumps({
+                'algorithm': self.hasher.ALGORITHM,
+                'digests': self.digests,
+            }))
         elif self.digests:
             logger.debug('Verified integrity of %s', self.path)
+
+    def store_integrity_data(self, data: str):
+        self.integrity_data = data
+
+
+class DetachedIntegrityCheckedFile(IntegrityCheckedFile):
+    def __init__(self, path, write, filename=None, override_fd=None):
+        super().__init__(path, write, filename, override_fd)
+        if not write:
+            self.digests = self.read_integrity_file(self.path, self.hasher)
+
+    @staticmethod
+    def integrity_file_path(path):
+        return path + '.integrity'
+
+    @classmethod
+    def read_integrity_file(cls, path, hasher):
+        try:
+            with open(cls.integrity_file_path(path), 'r') as fd:
+                return cls.parse_integrity_data(path, fd.read(), hasher)
+        except FileNotFoundError:
+            logger.info('No integrity file found for %s', path)
+        except OSError as e:
+            logger.warning('Could not read integrity file for %s: %s', path, e)
+            raise FileIntegrityError(path)
+
+    def store_integrity_data(self, data: str):
+        with open(self.integrity_file_path(self.path), 'w') as fd:
+            fd.write(data)
diff --git a/src/borg/testsuite/file_integrity.py b/src/borg/testsuite/file_integrity.py
index a8ef95f74..0dd323d61 100644
--- a/src/borg/testsuite/file_integrity.py
+++ b/src/borg/testsuite/file_integrity.py
@@ -1,21 +1,21 @@
 
 import pytest
 
-from ..crypto.file_integrity import IntegrityCheckedFile, FileIntegrityError
+from ..crypto.file_integrity import IntegrityCheckedFile, DetachedIntegrityCheckedFile, FileIntegrityError
 
 
 class TestReadIntegrityFile:
     def test_no_integrity(self, tmpdir):
         protected_file = tmpdir.join('file')
         protected_file.write('1234')
-        assert IntegrityCheckedFile.read_integrity_file(str(protected_file), None) is None
+        assert DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file), None) is None
 
     def test_truncated_integrity(self, tmpdir):
         protected_file = tmpdir.join('file')
         protected_file.write('1234')
         tmpdir.join('file.integrity').write('')
         with pytest.raises(FileIntegrityError):
-            IntegrityCheckedFile.read_integrity_file(str(protected_file), None)
+            DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file), None)
 
     def test_unknown_algorithm(self, tmpdir):
         class SomeHasher:
@@ -24,7 +24,7 @@ class TestReadIntegrityFile:
         protected_file = tmpdir.join('file')
         protected_file.write('1234')
         tmpdir.join('file.integrity').write('{"algorithm": "HMAC_SERIOUSHASH", "digests": "1234"}')
-        assert IntegrityCheckedFile.read_integrity_file(str(protected_file), SomeHasher()) is None
+        assert DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file), SomeHasher()) is None
 
     @pytest.mark.parametrize('json', (
         '{"ALGORITHM": "HMAC_SERIOUSHASH", "digests": "1234"}',
@@ -38,7 +38,7 @@ class TestReadIntegrityFile:
         protected_file.write('1234')
         tmpdir.join('file.integrity').write(json)
         with pytest.raises(FileIntegrityError):
-            IntegrityCheckedFile.read_integrity_file(str(protected_file), None)
+            DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file), None)
 
     def test_valid(self, tmpdir):
         class SomeHasher:
@@ -47,35 +47,35 @@ class TestReadIntegrityFile:
         protected_file = tmpdir.join('file')
         protected_file.write('1234')
         tmpdir.join('file.integrity').write('{"algorithm": "HMAC_FOO1", "digests": {"final": "1234"}}')
-        assert IntegrityCheckedFile.read_integrity_file(str(protected_file), SomeHasher()) == {'final': '1234'}
+        assert DetachedIntegrityCheckedFile.read_integrity_file(str(protected_file), SomeHasher()) == {'final': '1234'}
 
 
-class TestIntegrityCheckedFile:
+class TestDetachedIntegrityCheckedFile:
     @pytest.fixture
     def integrity_protected_file(self, tmpdir):
         path = str(tmpdir.join('file'))
-        with IntegrityCheckedFile(path, write=True) as fd:
+        with DetachedIntegrityCheckedFile(path, write=True) as fd:
             fd.write(b'foo and bar')
         return path
 
     def test_simple(self, tmpdir, integrity_protected_file):
         assert tmpdir.join('file').check(file=True)
         assert tmpdir.join('file.integrity').check(file=True)
-        with IntegrityCheckedFile(integrity_protected_file, write=False) as fd:
+        with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
             assert fd.read() == b'foo and bar'
 
     def test_corrupted_file(self, integrity_protected_file):
         with open(integrity_protected_file, 'ab') as fd:
             fd.write(b' extra data')
         with pytest.raises(FileIntegrityError):
-            with IntegrityCheckedFile(integrity_protected_file, write=False) as fd:
+            with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
                 assert fd.read() == b'foo and bar extra data'
 
     def test_corrupted_file_partial_read(self, integrity_protected_file):
         with open(integrity_protected_file, 'ab') as fd:
             fd.write(b' extra data')
         with pytest.raises(FileIntegrityError):
-            with IntegrityCheckedFile(integrity_protected_file, write=False) as fd:
+            with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
                 data = b'foo and bar'
                 assert fd.read(len(data)) == data
 
@@ -88,7 +88,7 @@ class TestIntegrityCheckedFile:
         tmpdir.join('file').move(new_path)
         tmpdir.join('file.integrity').move(new_path + '.integrity')
         with pytest.raises(FileIntegrityError):
-            with IntegrityCheckedFile(str(new_path), write=False) as fd:
+            with DetachedIntegrityCheckedFile(str(new_path), write=False) as fd:
                 assert fd.read() == b'foo and bar'
 
     def test_moved_file(self, tmpdir, integrity_protected_file):
@@ -96,27 +96,27 @@ class TestIntegrityCheckedFile:
         tmpdir.join('file').move(new_dir.join('file'))
         tmpdir.join('file.integrity').move(new_dir.join('file.integrity'))
         new_path = str(new_dir.join('file'))
-        with IntegrityCheckedFile(new_path, write=False) as fd:
+        with DetachedIntegrityCheckedFile(new_path, write=False) as fd:
             assert fd.read() == b'foo and bar'
 
     def test_no_integrity(self, tmpdir, integrity_protected_file):
         tmpdir.join('file.integrity').remove()
-        with IntegrityCheckedFile(integrity_protected_file, write=False) as fd:
+        with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
             assert fd.read() == b'foo and bar'
 
 
-class TestIntegrityCheckedFileParts:
+class TestDetachedIntegrityCheckedFileParts:
     @pytest.fixture
     def integrity_protected_file(self, tmpdir):
         path = str(tmpdir.join('file'))
-        with IntegrityCheckedFile(path, write=True) as fd:
+        with DetachedIntegrityCheckedFile(path, write=True) as fd:
             fd.write(b'foo and bar')
             fd.hash_part('foopart')
             fd.write(b' other data')
         return path
 
     def test_simple(self, integrity_protected_file):
-        with IntegrityCheckedFile(integrity_protected_file, write=False) as fd:
+        with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
             data1 = b'foo and bar'
             assert fd.read(len(data1)) == data1
             fd.hash_part('foopart')
@@ -127,7 +127,7 @@ class TestIntegrityCheckedFileParts:
             # Because some hash_part failed, the final digest will fail as well - again - even if we catch
             # the failing hash_part. This is intentional: (1) it makes the code simpler (2) it's a good fail-safe
             # against overly broad exception handling.
-            with IntegrityCheckedFile(integrity_protected_file, write=False) as fd:
+            with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
                 data1 = b'foo and bar'
                 assert fd.read(len(data1)) == data1
                 with pytest.raises(FileIntegrityError):
@@ -140,7 +140,7 @@ class TestIntegrityCheckedFileParts:
         with open(integrity_protected_file, 'ab') as fd:
             fd.write(b'some extra stuff that does not belong')
         with pytest.raises(FileIntegrityError):
-            with IntegrityCheckedFile(integrity_protected_file, write=False) as fd:
+            with DetachedIntegrityCheckedFile(integrity_protected_file, write=False) as fd:
                 data1 = b'foo and bar'
                 try:
                     assert fd.read(len(data1)) == data1

From 06cf15cc6dcee0e490755e5d17a8eccd9bc0ea42 Mon Sep 17 00:00:00 2001
From: Marian Beermann <public@enkore.de>
Date: Thu, 25 May 2017 13:43:04 +0200
Subject: [PATCH 02/12] hashindex: read/write: accept file-like objects for
 path

---
 src/borg/hashindex.pyx | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/borg/hashindex.pyx b/src/borg/hashindex.pyx
index fba8c7a38..2409836fe 100644
--- a/src/borg/hashindex.pyx
+++ b/src/borg/hashindex.pyx
@@ -67,8 +67,11 @@ cdef class IndexBase:
     def __cinit__(self, capacity=0, path=None, key_size=32):
         self.key_size = key_size
         if path:
-            with open(path, 'rb') as fd:
-                self.index = hashindex_read(fd)
+            if isinstance(path, (str, bytes)):
+                with open(path, 'rb') as fd:
+                    self.index = hashindex_read(fd)
+            else:
+                self.index = hashindex_read(path)
             assert self.index, 'hashindex_read() returned NULL with no exception set'
         else:
             self.index = hashindex_init(capacity, self.key_size, self.value_size)
@@ -84,8 +87,11 @@ cdef class IndexBase:
         return cls(path=path)
 
     def write(self, path):
-        with open(path, 'wb') as fd:
-            hashindex_write(self.index, fd)
+        if isinstance(path, (str, bytes)):
+            with open(path, 'wb') as fd:
+                hashindex_write(self.index, fd)
+        else:
+            hashindex_write(self.index, path)
 
     def clear(self):
         hashindex_free(self.index)

From 2b518b71884c612a00f7951c5516f412db9ca5c0 Mon Sep 17 00:00:00 2001
From: Marian Beermann <public@enkore.de>
Date: Thu, 25 May 2017 13:43:15 +0200
Subject: [PATCH 03/12] cache: add integrity checking of chunks and files
 caches

---
 src/borg/cache.py | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/src/borg/cache.py b/src/borg/cache.py
index 882d98631..fa2e6fe40 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -24,6 +24,7 @@ from .helpers import remove_surrogates
 from .helpers import ProgressIndicatorPercent, ProgressIndicatorMessage
 from .item import ArchiveItem, ChunkListEntry
 from .crypto.key import PlaintextKey
+from .crypto.file_integrity import IntegrityCheckedFile, DetachedIntegrityCheckedFile
 from .locking import Lock
 from .platform import SaveFile
 from .remote import cache_if_remote
@@ -237,6 +238,7 @@ class CacheConfig:
         config.set('cache', 'version', '1')
         config.set('cache', 'repository', self.repository.id_str)
         config.set('cache', 'manifest', '')
+        config.add_section('integrity')
         with SaveFile(self.config_path) as fd:
             config.write(fd)
 
@@ -253,6 +255,12 @@ class CacheConfig:
         self.manifest_id = unhexlify(self._config.get('cache', 'manifest'))
         self.timestamp = self._config.get('cache', 'timestamp', fallback=None)
         self.key_type = self._config.get('cache', 'key_type', fallback=None)
+        if not self._config.has_section('integrity'):
+            self._config.add_section('integrity')
+        try:
+            self.integrity = dict(self._config.items('integrity'))
+        except configparser.NoSectionError:
+            self.integrity = {}
         previous_location = self._config.get('cache', 'previous_location', fallback=None)
         if previous_location:
             self.previous_location = recanonicalize_relative_location(previous_location, self.repository)
@@ -266,6 +274,8 @@ class CacheConfig:
         if key:
             self._config.set('cache', 'key_type', str(key.TYPE))
         self._config.set('cache', 'previous_location', self.repository._location.canonical_path())
+        for file, integrity_data in self.integrity.items():
+            self._config.set('integrity', file, integrity_data)
         with SaveFile(self.config_path) as fd:
             self._config.write(fd)
 
@@ -392,14 +402,16 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
         with open(os.path.join(self.path, 'README'), 'w') as fd:
             fd.write(CACHE_README)
         self.cache_config.create()
-        ChunkIndex().write(os.path.join(self.path, 'chunks').encode('utf-8'))
+        ChunkIndex().write(os.path.join(self.path, 'chunks'))
         os.makedirs(os.path.join(self.path, 'chunks.archive.d'))
         with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd:
             pass  # empty file
 
     def _do_open(self):
         self.cache_config.load()
-        self.chunks = ChunkIndex.read(os.path.join(self.path, 'chunks').encode('utf-8'))
+        with IntegrityCheckedFile(path=os.path.join(self.path, 'chunks'), write=False,
+                                  integrity_data=self.cache_config.integrity.get('chunks')) as fd:
+            self.chunks = ChunkIndex.read(fd)
         self.files = None
 
     def open(self):
@@ -417,7 +429,9 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
         self.files = {}
         self._newest_mtime = None
         logger.debug('Reading files cache ...')
-        with open(os.path.join(self.path, 'files'), 'rb') as fd:
+
+        with IntegrityCheckedFile(path=os.path.join(self.path, 'files'), write=False,
+                                  integrity_data=self.cache_config.integrity.get('files')) as fd:
             u = msgpack.Unpacker(use_list=True)
             while True:
                 data = fd.read(64 * 1024)
@@ -458,7 +472,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
                 self._newest_mtime = 2 ** 63 - 1  # nanoseconds, good until y2262
             ttl = int(os.environ.get('BORG_FILES_CACHE_TTL', 20))
             pi.output('Saving files cache')
-            with SaveFile(os.path.join(self.path, 'files'), binary=True) as fd:
+            with IntegrityCheckedFile(path=os.path.join(self.path, 'files'), write=True) as fd:
                 for path_hash, item in self.files.items():
                     # Only keep files seen in this backup that are older than newest mtime seen in this backup -
                     # this is to avoid issues with filesystem snapshots and mtime granularity.
@@ -467,10 +481,13 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
                     if entry.age == 0 and bigint_to_int(entry.mtime) < self._newest_mtime or \
                        entry.age > 0 and entry.age < ttl:
                         msgpack.pack((path_hash, entry), fd)
+            self.cache_config.integrity['files'] = fd.integrity_data
+        pi.output('Saving chunks cache')
+        with IntegrityCheckedFile(path=os.path.join(self.path, 'chunks'), write=True) as fd:
+            self.chunks.write(fd)
+        self.cache_config.integrity['chunks'] = fd.integrity_data
         pi.output('Saving cache config')
         self.cache_config.save(self.manifest, self.key)
-        pi.output('Saving chunks cache')
-        self.chunks.write(os.path.join(self.path, 'chunks').encode('utf-8'))
         os.rename(os.path.join(self.path, 'txn.active'),
                   os.path.join(self.path, 'txn.tmp'))
         shutil.rmtree(os.path.join(self.path, 'txn.tmp'))

From 1dfe6930032540e03e57160c4ce80d5822e4cd24 Mon Sep 17 00:00:00 2001
From: Marian Beermann <public@enkore.de>
Date: Thu, 25 May 2017 13:49:03 +0200
Subject: [PATCH 04/12] cache: integrity checking in archive.chunks.d

---
 src/borg/cache.py                 | 22 ++++++++++++++--------
 src/borg/crypto/file_integrity.py |  5 ++++-
 2 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/borg/cache.py b/src/borg/cache.py
index fa2e6fe40..9a920a706 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -527,7 +527,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
         def mkpath(id, suffix=''):
             id_hex = bin_to_hex(id)
             path = os.path.join(archive_path, id_hex + suffix)
-            return path.encode('utf-8')
+            return path
 
         def cached_archives():
             if self.do_cache:
@@ -543,6 +543,10 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
         def cleanup_outdated(ids):
             for id in ids:
                 os.unlink(mkpath(id))
+                try:
+                    os.unlink(mkpath(id) + '.integrity')
+                except FileNotFoundError:
+                    pass
 
         def fetch_and_build_idx(archive_id, repository, key, chunk_idx):
             cdata = repository.get(archive_id)
@@ -565,12 +569,13 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
             if self.do_cache:
                 fn = mkpath(archive_id)
                 fn_tmp = mkpath(archive_id, suffix='.tmp')
-                try:
-                    chunk_idx.write(fn_tmp)
-                except Exception:
-                    os.unlink(fn_tmp)
-                else:
-                    os.rename(fn_tmp, fn)
+                with DetachedIntegrityCheckedFile(path=fn_tmp, write=True, filename=bin_to_hex(archive_id)) as fd:
+                    try:
+                        chunk_idx.write(fd)
+                    except Exception:
+                        os.unlink(fn_tmp)
+                    else:
+                        os.rename(fn_tmp, fn)
 
         def lookup_name(archive_id):
             for info in self.manifest.archives.list():
@@ -601,7 +606,8 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
                         if archive_id in cached_ids:
                             archive_chunk_idx_path = mkpath(archive_id)
                             logger.info("Reading cached archive chunk index for %s ..." % archive_name)
-                            archive_chunk_idx = ChunkIndex.read(archive_chunk_idx_path)
+                            with DetachedIntegrityCheckedFile(path=archive_chunk_idx_path, write=False) as fd:
+                                archive_chunk_idx = ChunkIndex.read(fd)
                         else:
                             logger.info('Fetching and building archive index for %s ...' % archive_name)
                             archive_chunk_idx = ChunkIndex()
diff --git a/src/borg/crypto/file_integrity.py b/src/borg/crypto/file_integrity.py
index 53cbcea3e..de4dcba3d 100644
--- a/src/borg/crypto/file_integrity.py
+++ b/src/borg/crypto/file_integrity.py
@@ -182,6 +182,9 @@ class IntegrityCheckedFile(FileLikeWrapper):
 class DetachedIntegrityCheckedFile(IntegrityCheckedFile):
     def __init__(self, path, write, filename=None, override_fd=None):
         super().__init__(path, write, filename, override_fd)
+        filename = filename or os.path.basename(path)
+        output_dir = os.path.dirname(path)
+        self.output_integrity_file = self.integrity_file_path(os.path.join(output_dir, filename))
         if not write:
             self.digests = self.read_integrity_file(self.path, self.hasher)
 
@@ -201,5 +204,5 @@ class DetachedIntegrityCheckedFile(IntegrityCheckedFile):
             raise FileIntegrityError(path)
 
     def store_integrity_data(self, data: str):
-        with open(self.integrity_file_path(self.path), 'w') as fd:
+        with open(self.output_integrity_file, 'w') as fd:
             fd.write(data)

From addd7addfec2fbc8690210796494bd5e608068d5 Mon Sep 17 00:00:00 2001
From: Marian Beermann <public@enkore.de>
Date: Thu, 25 May 2017 14:00:03 +0200
Subject: [PATCH 05/12] cache: chunks.archive.d: autofix corruption

---
 src/borg/cache.py | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/src/borg/cache.py b/src/borg/cache.py
index 9a920a706..369735f25 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -24,7 +24,7 @@ from .helpers import remove_surrogates
 from .helpers import ProgressIndicatorPercent, ProgressIndicatorMessage
 from .item import ArchiveItem, ChunkListEntry
 from .crypto.key import PlaintextKey
-from .crypto.file_integrity import IntegrityCheckedFile, DetachedIntegrityCheckedFile
+from .crypto.file_integrity import IntegrityCheckedFile, DetachedIntegrityCheckedFile, FileIntegrityError
 from .locking import Lock
 from .platform import SaveFile
 from .remote import cache_if_remote
@@ -542,11 +542,14 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
 
         def cleanup_outdated(ids):
             for id in ids:
-                os.unlink(mkpath(id))
-                try:
-                    os.unlink(mkpath(id) + '.integrity')
-                except FileNotFoundError:
-                    pass
+                cleanup_cached_archive(id)
+
+        def cleanup_cached_archive(id):
+            os.unlink(mkpath(id))
+            try:
+                os.unlink(mkpath(id) + '.integrity')
+            except FileNotFoundError:
+                pass
 
         def fetch_and_build_idx(archive_id, repository, key, chunk_idx):
             cdata = repository.get(archive_id)
@@ -606,9 +609,17 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
                         if archive_id in cached_ids:
                             archive_chunk_idx_path = mkpath(archive_id)
                             logger.info("Reading cached archive chunk index for %s ..." % archive_name)
-                            with DetachedIntegrityCheckedFile(path=archive_chunk_idx_path, write=False) as fd:
-                                archive_chunk_idx = ChunkIndex.read(fd)
-                        else:
+                            try:
+                                with DetachedIntegrityCheckedFile(path=archive_chunk_idx_path, write=False) as fd:
+                                    archive_chunk_idx = ChunkIndex.read(fd)
+                            except FileIntegrityError as fie:
+                                logger.error('Cached archive chunk index of %s is corrupted: %s', archive_name, fie)
+                                # Delete it and fetch a new index
+                                cleanup_cached_archive(archive_id)
+                                cached_ids.remove(archive_id)
+                        if archive_id not in cached_ids:
+                            # Do not make this an else branch; the FileIntegrityError exception handler
+                            # above can remove *archive_id* from *cached_ids*.
                             logger.info('Fetching and building archive index for %s ...' % archive_name)
                             archive_chunk_idx = ChunkIndex()
                             fetch_and_build_idx(archive_id, repository, self.key, archive_chunk_idx)

From f59affe5858d127680c06b108c5c3454bc29ac49 Mon Sep 17 00:00:00 2001
From: Marian Beermann <public@enkore.de>
Date: Thu, 25 May 2017 14:01:21 +0200
Subject: [PATCH 06/12] cache: fix possible printf issue with archive names in
 sync

---
 src/borg/cache.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/borg/cache.py b/src/borg/cache.py
index 369735f25..ce8cef251 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -566,7 +566,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
                 for item in unpacker:
                     if not isinstance(item, dict):
                         logger.error('Error: Did not get expected metadata dict - archive corrupted!')
-                        continue
+                        continue   # XXX: continue?!
                     for chunk_id, size, csize in item.get(b'chunks', []):
                         chunk_idx.add(chunk_id, 1, size, csize)
             if self.do_cache:
@@ -589,9 +589,9 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
             logger.info('Synchronizing chunks cache...')
             cached_ids = cached_archives()
             archive_ids = repo_archives()
-            logger.info('Archives: %d, w/ cached Idx: %d, w/ outdated Idx: %d, w/o cached Idx: %d.' % (
+            logger.info('Archives: %d, w/ cached Idx: %d, w/ outdated Idx: %d, w/o cached Idx: %d.',
                 len(archive_ids), len(cached_ids),
-                len(cached_ids - archive_ids), len(archive_ids - cached_ids), ))
+                len(cached_ids - archive_ids), len(archive_ids - cached_ids))
             # deallocates old hashindex, creates empty hashindex:
             chunk_idx.clear()
             cleanup_outdated(cached_ids - archive_ids)
@@ -608,7 +608,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
                     if self.do_cache:
                         if archive_id in cached_ids:
                             archive_chunk_idx_path = mkpath(archive_id)
-                            logger.info("Reading cached archive chunk index for %s ..." % archive_name)
+                            logger.info("Reading cached archive chunk index for %s ...", archive_name)
                             try:
                                 with DetachedIntegrityCheckedFile(path=archive_chunk_idx_path, write=False) as fd:
                                     archive_chunk_idx = ChunkIndex.read(fd)
@@ -620,7 +620,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
                         if archive_id not in cached_ids:
                             # Do not make this an else branch; the FileIntegrityError exception handler
                             # above can remove *archive_id* from *cached_ids*.
-                            logger.info('Fetching and building archive index for %s ...' % archive_name)
+                            logger.info('Fetching and building archive index for %s ...', archive_name)
                             archive_chunk_idx = ChunkIndex()
                             fetch_and_build_idx(archive_id, repository, self.key, archive_chunk_idx)
                         logger.info("Merging into master chunks index ...")
@@ -633,7 +633,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
                             chunk_idx.merge(archive_chunk_idx)
                     else:
                         chunk_idx = chunk_idx or ChunkIndex()
-                        logger.info('Fetching archive index for %s ...' % archive_name)
+                        logger.info('Fetching archive index for %s ...', archive_name)
                         fetch_and_build_idx(archive_id, repository, self.key, chunk_idx)
                 if self.progress:
                     pi.finish()

From d463dd89aa8c93ed5dee3a51456ef9a5a398d0ca Mon Sep 17 00:00:00 2001
From: Marian Beermann <public@enkore.de>
Date: Thu, 25 May 2017 15:18:45 +0200
Subject: [PATCH 07/12] hashindex: read/write: use hash_part for HashHeader

---
 src/borg/_hashindex.c           | 27 +++++++++++++++++++++++++++
 src/borg/testsuite/hashindex.py | 22 ++++++++++++++++++++++
 2 files changed, 49 insertions(+)

diff --git a/src/borg/_hashindex.c b/src/borg/_hashindex.c
index 289457fe5..b41d57b70 100644
--- a/src/borg/_hashindex.c
+++ b/src/borg/_hashindex.c
@@ -291,6 +291,20 @@ hashindex_read(PyObject *file_py)
         goto fail_decref_header;
     }
 
+    /*
+     * Hash the header
+     * If the header is corrupted this bails before doing something stupid (like allocating 3.8 TB of memory)
+     */
+    Py_XDECREF(PyObject_CallMethod(file_py, "hash_part", "s", "HashHeader"));
+    if(PyErr_Occurred()) {
+        if(PyErr_ExceptionMatches(PyExc_AttributeError)) {
+            /* Be able to work with regular file objects which do not have a hash_part method. */
+            PyErr_Clear();
+        } else {
+            goto fail_decref_header;
+        }
+    }
+
     /* Find length of file */
     length_object = PyObject_CallMethod(file_py, "seek", "ni", (Py_ssize_t)0, SEEK_END);
     if(PyErr_Occurred()) {
@@ -473,6 +487,19 @@ hashindex_write(HashIndex *index, PyObject *file_py)
         return;
     }
 
+    /*
+     * Hash the header
+     */
+    Py_XDECREF(PyObject_CallMethod(file_py, "hash_part", "s", "HashHeader"));
+    if(PyErr_Occurred()) {
+        if(PyErr_ExceptionMatches(PyExc_AttributeError)) {
+            /* Be able to work with regular file objects which do not have a hash_part method. */
+            PyErr_Clear();
+        } else {
+            return;
+        }
+    }
+
     /* Note: explicitly construct view; BuildValue can convert (pointer, length) to Python objects, but copies them for doing so */
     buckets_view = PyMemoryView_FromMemory((char*)index->buckets, buckets_length, PyBUF_READ);
     if(!buckets_view) {
diff --git a/src/borg/testsuite/hashindex.py b/src/borg/testsuite/hashindex.py
index 116399071..120c01b44 100644
--- a/src/borg/testsuite/hashindex.py
+++ b/src/borg/testsuite/hashindex.py
@@ -6,6 +6,7 @@ import zlib
 
 from ..hashindex import NSIndex, ChunkIndex
 from .. import hashindex
+from ..crypto.file_integrity import IntegrityCheckedFile, FileIntegrityError
 from . import BaseTestCase
 
 # Note: these tests are part of the self test, do not use or import py.test functionality here.
@@ -319,6 +320,27 @@ class HashIndexDataTestCase(BaseTestCase):
         assert idx1[H(3)] == (ChunkIndex.MAX_VALUE, 6, 7)
 
 
+class HashIndexIntegrityTestCase(HashIndexDataTestCase):
+    def write_integrity_checked_index(self, tempdir):
+        idx = self._deserialize_hashindex(self.HASHINDEX)
+        file = os.path.join(tempdir, 'idx')
+        with IntegrityCheckedFile(path=file, write=True) as fd:
+            idx.write(fd)
+        integrity_data = fd.integrity_data
+        assert 'final' in integrity_data
+        assert 'HashHeader' in integrity_data
+        return file, integrity_data
+
+    def test_integrity_checked_file(self):
+        with tempfile.TemporaryDirectory() as tempdir:
+            file, integrity_data = self.write_integrity_checked_index(tempdir)
+            with open(file, 'r+b') as fd:
+                fd.write(b'Foo')
+            with self.assert_raises(FileIntegrityError):
+                with IntegrityCheckedFile(path=file, write=False, integrity_data=integrity_data) as fd:
+                    ChunkIndex.read(fd)
+
+
 class NSIndexTestCase(BaseTestCase):
     def test_nsindex_segment_limit(self):
         idx = NSIndex()

From 83bca02a4e283afedd7e04dbe47064d1890d6497 Mon Sep 17 00:00:00 2001
From: Marian Beermann <public@enkore.de>
Date: Thu, 25 May 2017 15:31:30 +0200
Subject: [PATCH 08/12] file_integrity: hash_part: mix length into state

---
 src/borg/crypto/file_integrity.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/borg/crypto/file_integrity.py b/src/borg/crypto/file_integrity.py
index de4dcba3d..eb796c67b 100644
--- a/src/borg/crypto/file_integrity.py
+++ b/src/borg/crypto/file_integrity.py
@@ -152,6 +152,7 @@ class IntegrityCheckedFile(FileLikeWrapper):
     def hash_part(self, partname, is_final=False):
         if not self.writing and not self.digests:
             return
+        self.hasher.update(('%10d' % len(partname)).encode())
         self.hasher.update(partname.encode())
         self.hasher.hash_length(seek_to_end=is_final)
         digest = self.hasher.hexdigest()

From 50ac9d914dd54cde5a9c69d8be0f5abcfc8a59c4 Mon Sep 17 00:00:00 2001
From: Marian Beermann <public@enkore.de>
Date: Thu, 25 May 2017 15:54:38 +0200
Subject: [PATCH 09/12] testsuite: add ArchiverCorruptionTestCase

---
 src/borg/cache.py              |  2 ++
 src/borg/testsuite/archiver.py | 64 ++++++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+)

diff --git a/src/borg/cache.py b/src/borg/cache.py
index ce8cef251..aad48ca5f 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -22,6 +22,7 @@ from .helpers import safe_ns
 from .helpers import yes, hostname_is_unique
 from .helpers import remove_surrogates
 from .helpers import ProgressIndicatorPercent, ProgressIndicatorMessage
+from .helpers import set_ec, EXIT_WARNING
 from .item import ArchiveItem, ChunkListEntry
 from .crypto.key import PlaintextKey
 from .crypto.file_integrity import IntegrityCheckedFile, DetachedIntegrityCheckedFile, FileIntegrityError
@@ -617,6 +618,7 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
                                 # Delete it and fetch a new index
                                 cleanup_cached_archive(archive_id)
                                 cached_ids.remove(archive_id)
+                                set_ec(EXIT_WARNING)
                         if archive_id not in cached_ids:
                             # Do not make this an else branch; the FileIntegrityError exception handler
                             # above can remove *archive_id* from *cached_ids*.
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
index 508b1586c..13c24106e 100644
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -1,5 +1,6 @@
 import argparse
 import errno
+import io
 import json
 import logging
 import os
@@ -37,6 +38,7 @@ from ..constants import *  # NOQA
 from ..crypto.low_level import bytes_to_long, num_aes_blocks
 from ..crypto.key import KeyfileKeyBase, RepoKey, KeyfileKey, Passphrase, TAMRequiredError
 from ..crypto.keymanager import RepoIdMismatch, NotABorgKeyFile
+from ..crypto.file_integrity import FileIntegrityError
 from ..helpers import Location, get_security_dir
 from ..helpers import Manifest
 from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
@@ -2886,6 +2888,68 @@ class RemoteArchiverTestCase(ArchiverTestCase):
                 self.assert_true(marker not in res)
 
 
+class ArchiverCorruptionTestCase(ArchiverTestCaseBase):
+    def corrupt(self, file):
+        with open(file, 'r+b') as fd:
+            fd.seek(-1, io.SEEK_END)
+            fd.write(b'1')
+
+    def test_cache_chunks(self):
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        cache_path = json.loads(self.cmd('info', self.repository_location, '--json'))['cache']['path']
+        self.corrupt(os.path.join(cache_path, 'chunks'))
+
+        if self.FORK_DEFAULT:
+            out = self.cmd('info', self.repository_location, exit_code=2)
+            assert 'failed integrity check' in out
+        else:
+            with pytest.raises(FileIntegrityError):
+                self.cmd('info', self.repository_location)
+
+    def test_cache_files(self):
+        self.create_test_files()
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        self.cmd('create', self.repository_location + '::test', 'input')
+        cache_path = json.loads(self.cmd('info', self.repository_location, '--json'))['cache']['path']
+        self.corrupt(os.path.join(cache_path, 'files'))
+
+        if self.FORK_DEFAULT:
+            out = self.cmd('create', self.repository_location + '::test1', 'input', exit_code=2)
+            assert 'failed integrity check' in out
+        else:
+            with pytest.raises(FileIntegrityError):
+                self.cmd('create', self.repository_location + '::test1', 'input')
+
+    def test_chunks_archive(self):
+        self.create_test_files()
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        self.cmd('create', self.repository_location + '::test1', 'input')
+        # Find ID of test1 so we can corrupt it later :)
+        target_id = self.cmd('list', self.repository_location, '--format={id}{LF}').strip()
+        self.cmd('create', self.repository_location + '::test2', 'input')
+        self.cmd('delete', '--cache-only', self.repository_location)
+
+        cache_path = json.loads(self.cmd('info', self.repository_location, '--json'))['cache']['path']
+        chunks_archive = os.path.join(cache_path, 'chunks.archive.d')
+        assert len(os.listdir(chunks_archive)) == 4  # two archives, one chunks cache and one .integrity file each
+
+        self.corrupt(os.path.join(chunks_archive, target_id))
+
+        # Trigger cache sync by changing the manifest ID in the cache config
+        config_path = os.path.join(cache_path, 'config')
+        config = ConfigParser(interpolation=None)
+        config.read(config_path)
+        config.set('cache', 'manifest', bin_to_hex(bytes(32)))
+        with open(config_path, 'w') as fd:
+            config.write(fd)
+
+        # Cache sync will notice corrupted archive chunks, but automatically recover.
+        out = self.cmd('create', '-v', self.repository_location + '::test3', 'input', exit_code=1)
+        assert 'Reading cached archive chunk index for test1' in out
+        assert 'Cached archive chunk index of test1 is corrupted' in out
+        assert 'Fetching and building archive index for test1' in out
+
+
 class DiffArchiverTestCase(ArchiverTestCaseBase):
     def test_basic_functionality(self):
         # Initialize test folder

From d35d388d9ce9cd183da3ee8551228c70bbf88d36 Mon Sep 17 00:00:00 2001
From: Marian Beermann <public@enkore.de>
Date: Thu, 25 May 2017 16:13:40 +0200
Subject: [PATCH 10/12] cache integrity: handle interference from old versions

---
 src/borg/cache.py              | 20 ++++++++++++++++----
 src/borg/testsuite/archiver.py | 17 +++++++++++++++++
 2 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/src/borg/cache.py b/src/borg/cache.py
index aad48ca5f..9af9dbf97 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -240,6 +240,7 @@ class CacheConfig:
         config.set('cache', 'repository', self.repository.id_str)
         config.set('cache', 'manifest', '')
         config.add_section('integrity')
+        config.set('integrity', 'manifest', '')
         with SaveFile(self.config_path) as fd:
             config.write(fd)
 
@@ -256,11 +257,19 @@ class CacheConfig:
         self.manifest_id = unhexlify(self._config.get('cache', 'manifest'))
         self.timestamp = self._config.get('cache', 'timestamp', fallback=None)
         self.key_type = self._config.get('cache', 'key_type', fallback=None)
-        if not self._config.has_section('integrity'):
-            self._config.add_section('integrity')
         try:
             self.integrity = dict(self._config.items('integrity'))
+            if self._config.get('cache', 'manifest') != self.integrity.pop('manifest'):
+                # The cache config file is updated (parsed with ConfigParser, the state of the ConfigParser
+                # is modified and then written out.), not re-created.
+                # Thus, older versions will leave our [integrity] section alone, making the section's data invalid.
+                # Therefore, we also add the manifest ID to this section and
+                # can discern whether an older version interfere by comparing the manifest IDs of this section
+                # and the main [cache] section.
+                self.integrity = {}
+                logger.warning('Cache integrity data lost: old Borg version modified the cache.')
         except configparser.NoSectionError:
+            logger.debug('Cache integrity: No integrity data found (files, chunks). Cache is from old version.')
             self.integrity = {}
         previous_location = self._config.get('cache', 'previous_location', fallback=None)
         if previous_location:
@@ -272,11 +281,14 @@ class CacheConfig:
         if manifest:
             self._config.set('cache', 'manifest', manifest.id_str)
             self._config.set('cache', 'timestamp', manifest.timestamp)
+            if not self._config.has_section('integrity'):
+                self._config.add_section('integrity')
+            for file, integrity_data in self.integrity.items():
+                self._config.set('integrity', file, integrity_data)
+            self._config.set('integrity', 'manifest', manifest.id_str)
         if key:
             self._config.set('cache', 'key_type', str(key.TYPE))
         self._config.set('cache', 'previous_location', self.repository._location.canonical_path())
-        for file, integrity_data in self.integrity.items():
-            self._config.set('integrity', file, integrity_data)
         with SaveFile(self.config_path) as fd:
             self._config.write(fd)
 
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
index 13c24106e..a2d0ff806 100644
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -2949,6 +2949,23 @@ class ArchiverCorruptionTestCase(ArchiverTestCaseBase):
         assert 'Cached archive chunk index of test1 is corrupted' in out
         assert 'Fetching and building archive index for test1' in out
 
+    def test_old_version_intefered(self):
+        self.create_test_files()
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        cache_path = json.loads(self.cmd('info', self.repository_location, '--json'))['cache']['path']
+
+        # Modify the main manifest ID without touching the manifest ID in the integrity section.
+        # This happens if a version without integrity checking modifies the cache.
+        config_path = os.path.join(cache_path, 'config')
+        config = ConfigParser(interpolation=None)
+        config.read(config_path)
+        config.set('cache', 'manifest', bin_to_hex(bytes(32)))
+        with open(config_path, 'w') as fd:
+            config.write(fd)
+
+        out = self.cmd('info', self.repository_location)
+        assert 'Cache integrity data lost: old Borg version modified the cache.' in out
+
 
 class DiffArchiverTestCase(ArchiverTestCaseBase):
     def test_basic_functionality(self):

From 0a5d9b6f7cae7d75f183d9b3a959558393dee4f5 Mon Sep 17 00:00:00 2001
From: Marian Beermann <public@enkore.de>
Date: Wed, 31 May 2017 18:06:28 +0200
Subject: [PATCH 11/12] cache sync: close archive chunks file before renaming

---
 src/borg/cache.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/borg/cache.py b/src/borg/cache.py
index 9af9dbf97..4b59dfbd8 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -585,13 +585,14 @@ Chunk index:    {0.total_unique_chunks:20d} {0.total_chunks:20d}"""
             if self.do_cache:
                 fn = mkpath(archive_id)
                 fn_tmp = mkpath(archive_id, suffix='.tmp')
-                with DetachedIntegrityCheckedFile(path=fn_tmp, write=True, filename=bin_to_hex(archive_id)) as fd:
-                    try:
+                try:
+                    with DetachedIntegrityCheckedFile(path=fn_tmp, write=True,
+                                                      filename=bin_to_hex(archive_id)) as fd:
                         chunk_idx.write(fd)
-                    except Exception:
-                        os.unlink(fn_tmp)
-                    else:
-                        os.rename(fn_tmp, fn)
+                except Exception:
+                    os.unlink(fn_tmp)
+                else:
+                    os.rename(fn_tmp, fn)
 
         def lookup_name(archive_id):
             for info in self.manifest.archives.list():

From 9032aa062b9487f8737cfb6ab5efb67924a72127 Mon Sep 17 00:00:00 2001
From: Marian Beermann <public@enkore.de>
Date: Wed, 31 May 2017 18:08:02 +0200
Subject: [PATCH 12/12] testsuite: simplify ArchiverCorruptionTestCase

---
 src/borg/cache.py                 |  4 ++--
 src/borg/crypto/file_integrity.py |  6 ++---
 src/borg/testsuite/archiver.py    | 39 ++++++++++++++-----------------
 3 files changed, 23 insertions(+), 26 deletions(-)

diff --git a/src/borg/cache.py b/src/borg/cache.py
index 4b59dfbd8..13045f0e9 100644
--- a/src/borg/cache.py
+++ b/src/borg/cache.py
@@ -264,10 +264,10 @@ class CacheConfig:
                 # is modified and then written out.), not re-created.
                 # Thus, older versions will leave our [integrity] section alone, making the section's data invalid.
                 # Therefore, we also add the manifest ID to this section and
-                # can discern whether an older version interfere by comparing the manifest IDs of this section
+                # can discern whether an older version interfered by comparing the manifest IDs of this section
                 # and the main [cache] section.
                 self.integrity = {}
-                logger.warning('Cache integrity data lost: old Borg version modified the cache.')
+                logger.warning('Cache integrity data not available: old Borg version modified the cache.')
         except configparser.NoSectionError:
             logger.debug('Cache integrity: No integrity data found (files, chunks). Cache is from old version.')
             self.integrity = {}
diff --git a/src/borg/crypto/file_integrity.py b/src/borg/crypto/file_integrity.py
index eb796c67b..032b8672d 100644
--- a/src/borg/crypto/file_integrity.py
+++ b/src/borg/crypto/file_integrity.py
@@ -135,13 +135,13 @@ class IntegrityCheckedFile(FileLikeWrapper):
     @classmethod
     def parse_integrity_data(cls, path: str, data: str, hasher: SHA512FileHashingWrapper):
         try:
-            integrity_file = json.loads(data)
+            integrity_data = json.loads(data)
             # Provisions for agility now, implementation later, but make sure the on-disk joint is oiled.
-            algorithm = integrity_file['algorithm']
+            algorithm = integrity_data['algorithm']
             if algorithm != hasher.ALGORITHM:
                 logger.warning('Cannot verify integrity of %s: Unknown algorithm %r', path, algorithm)
                 return
-            digests = integrity_file['digests']
+            digests = integrity_data['digests']
             # Require at least presence of the final digest
             digests['final']
             return digests
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
index a2d0ff806..36d32ba82 100644
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -2889,15 +2889,19 @@ class RemoteArchiverTestCase(ArchiverTestCase):
 
 
 class ArchiverCorruptionTestCase(ArchiverTestCaseBase):
+    def setUp(self):
+        super().setUp()
+        self.create_test_files()
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        self.cache_path = json.loads(self.cmd('info', self.repository_location, '--json'))['cache']['path']
+
     def corrupt(self, file):
         with open(file, 'r+b') as fd:
             fd.seek(-1, io.SEEK_END)
             fd.write(b'1')
 
     def test_cache_chunks(self):
-        self.cmd('init', '--encryption=repokey', self.repository_location)
-        cache_path = json.loads(self.cmd('info', self.repository_location, '--json'))['cache']['path']
-        self.corrupt(os.path.join(cache_path, 'chunks'))
+        self.corrupt(os.path.join(self.cache_path, 'chunks'))
 
         if self.FORK_DEFAULT:
             out = self.cmd('info', self.repository_location, exit_code=2)
@@ -2907,11 +2911,8 @@ class ArchiverCorruptionTestCase(ArchiverTestCaseBase):
                 self.cmd('info', self.repository_location)
 
     def test_cache_files(self):
-        self.create_test_files()
-        self.cmd('init', '--encryption=repokey', self.repository_location)
         self.cmd('create', self.repository_location + '::test', 'input')
-        cache_path = json.loads(self.cmd('info', self.repository_location, '--json'))['cache']['path']
-        self.corrupt(os.path.join(cache_path, 'files'))
+        self.corrupt(os.path.join(self.cache_path, 'files'))
 
         if self.FORK_DEFAULT:
             out = self.cmd('create', self.repository_location + '::test1', 'input', exit_code=2)
@@ -2921,42 +2922,38 @@ class ArchiverCorruptionTestCase(ArchiverTestCaseBase):
                 self.cmd('create', self.repository_location + '::test1', 'input')
 
     def test_chunks_archive(self):
-        self.create_test_files()
-        self.cmd('init', '--encryption=repokey', self.repository_location)
         self.cmd('create', self.repository_location + '::test1', 'input')
         # Find ID of test1 so we can corrupt it later :)
         target_id = self.cmd('list', self.repository_location, '--format={id}{LF}').strip()
         self.cmd('create', self.repository_location + '::test2', 'input')
-        self.cmd('delete', '--cache-only', self.repository_location)
 
-        cache_path = json.loads(self.cmd('info', self.repository_location, '--json'))['cache']['path']
-        chunks_archive = os.path.join(cache_path, 'chunks.archive.d')
+        # Force cache sync, creating archive chunks of test1 and test2 in chunks.archive.d
+        self.cmd('delete', '--cache-only', self.repository_location)
+        self.cmd('info', self.repository_location, '--json')
+
+        chunks_archive = os.path.join(self.cache_path, 'chunks.archive.d')
         assert len(os.listdir(chunks_archive)) == 4  # two archives, one chunks cache and one .integrity file each
 
         self.corrupt(os.path.join(chunks_archive, target_id))
 
         # Trigger cache sync by changing the manifest ID in the cache config
-        config_path = os.path.join(cache_path, 'config')
+        config_path = os.path.join(self.cache_path, 'config')
         config = ConfigParser(interpolation=None)
         config.read(config_path)
         config.set('cache', 'manifest', bin_to_hex(bytes(32)))
         with open(config_path, 'w') as fd:
             config.write(fd)
 
-        # Cache sync will notice corrupted archive chunks, but automatically recover.
+        # Cache sync notices corrupted archive chunks, but automatically recovers.
         out = self.cmd('create', '-v', self.repository_location + '::test3', 'input', exit_code=1)
         assert 'Reading cached archive chunk index for test1' in out
         assert 'Cached archive chunk index of test1 is corrupted' in out
         assert 'Fetching and building archive index for test1' in out
 
-    def test_old_version_intefered(self):
-        self.create_test_files()
-        self.cmd('init', '--encryption=repokey', self.repository_location)
-        cache_path = json.loads(self.cmd('info', self.repository_location, '--json'))['cache']['path']
-
+    def test_old_version_interfered(self):
         # Modify the main manifest ID without touching the manifest ID in the integrity section.
         # This happens if a version without integrity checking modifies the cache.
-        config_path = os.path.join(cache_path, 'config')
+        config_path = os.path.join(self.cache_path, 'config')
         config = ConfigParser(interpolation=None)
         config.read(config_path)
         config.set('cache', 'manifest', bin_to_hex(bytes(32)))
@@ -2964,7 +2961,7 @@ class ArchiverCorruptionTestCase(ArchiverTestCaseBase):
             config.write(fd)
 
         out = self.cmd('info', self.repository_location)
-        assert 'Cache integrity data lost: old Borg version modified the cache.' in out
+        assert 'Cache integrity data not available: old Borg version modified the cache.' in out
 
 
 class DiffArchiverTestCase(ArchiverTestCaseBase):