From 0575f87580ec09978ee28645fec3adc91d4a9069 Mon Sep 17 00:00:00 2001 From: William Bonnaventure Date: Sun, 14 Jul 2024 07:23:06 +0200 Subject: [PATCH 1/2] Add BORG_USE_CHUNKS_ARCHIVE option --- src/borg/cache.py | 4 +--- src/borg/testsuite/archiver.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/borg/cache.py b/src/borg/cache.py index f6d5baeb0..9a50238c6 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -478,6 +478,7 @@ class LocalCache(CacheStatsMixin): self.consider_part_files = consider_part_files self.timestamp = None self.txn_active = False + self.do_cache = os.environ.get("BORG_USE_CHUNKS_ARCHIVE", "yes").lower() in ["yes", "1", "true"] self.path = cache_dir(repository, path) self.security_manager = SecurityManager(repository) @@ -907,9 +908,6 @@ class LocalCache(CacheStatsMixin): self.begin_txn() with cache_if_remote(self.repository, decrypted_cache=self.key) as decrypted_repository: legacy_cleanup() - # TEMPORARY HACK: to avoid archive index caching, create a FILE named ~/.cache/borg/REPOID/chunks.archive.d - - # this is only recommended if you have a fast, low latency connection to your repo (e.g. if repo is local disk) - self.do_cache = os.path.isdir(archive_path) self.chunks = create_master_idx(self.chunks) def check_cache_compatibility(self): diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 45645711a..896bd28e7 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -3075,6 +3075,21 @@ class ArchiverTestCase(ArchiverTestCaseBase): with pytest.raises(AssertionError): self.check_cache() + def test_env_use_chunks_archive(self): + self.create_test_files() + with environment_variable(BORG_USE_CHUNKS_ARCHIVE="no"): + self.cmd("init", "--encryption=repokey", self.repository_location) + self.cmd("create", self.repository_location + "::test", "input") + repository_id = bin_to_hex(self._extract_repository_id(self.repository_path)) + cache_path = os.path.join(self.cache_path, repository_id) + assert os.path.exists(cache_path) + assert os.path.exists(os.path.join(cache_path, "chunks.archive.d")) + assert len(os.listdir(os.path.join(cache_path, "chunks.archive.d"))) == 0 + self.cmd("delete", self.repository_location, "--cache-only") + with environment_variable(BORG_USE_CHUNKS_ARCHIVE="yes"): + self.cmd("create", self.repository_location + "::test2", "input") + assert len(os.listdir(os.path.join(cache_path, "chunks.archive.d"))) > 0 + def test_recreate_target_rc(self): self.cmd('init', '--encryption=repokey', self.repository_location) if self.FORK_DEFAULT: From f3c1444a41248d8a01cb2db7bc24ee043a1f5b4f Mon Sep 17 00:00:00 2001 From: William Bonnaventure Date: Sun, 14 Jul 2024 07:24:37 +0200 Subject: [PATCH 2/2] Update docs for BORG_USE_CHUNKS_ARCHIVE --- docs/faq.rst | 14 ++------------ docs/usage/general/environment.rst.inc | 3 +++ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index f782629f8..8e8d4b229 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -840,18 +840,8 @@ will make the subsequent rebuilds faster (because it needs to transfer less data from the repository). While being faster, the cache needs quite some disk space, which might be unwanted. -There is a temporary (but maybe long lived) hack to avoid using lots of disk -space for chunks.archive.d (see :issue:`235` for details): - -:: - - # this assumes you are working with the same user as the backup. - cd ~/.cache/borg/$(borg config /path/to/repo id) - rm -rf chunks.archive.d ; touch chunks.archive.d - -This deletes all the cached archive chunk indexes and replaces the directory -that kept them with a file, so borg won't be able to store anything "in" there -in future. +You can disable the cached archive chunk indexes by setting the environment +variable ``BORG_USE_CHUNKS_ARCHIVE`` to ``no``. This has some pros and cons, though: diff --git a/docs/usage/general/environment.rst.inc b/docs/usage/general/environment.rst.inc index 991281e6a..89c1437de 100644 --- a/docs/usage/general/environment.rst.inc +++ b/docs/usage/general/environment.rst.inc @@ -63,6 +63,9 @@ General: When set to a numeric value, this determines the maximum "time to live" for the files cache entries (default: 20). The files cache is used to quickly determine whether a file is unchanged. The FAQ explains this more detailed in: :ref:`always_chunking` + BORG_USE_CHUNKS_ARCHIVE + When set to no (default: yes), the ``chunks.archive.d`` folder will not be used. This reduces + disk space usage but slows down cache resyncs. BORG_SHOW_SYSINFO When set to no (default: yes), system information (like OS, Python version, ...) in exceptions is not shown.