Merge pull request #9645 from ThomasWaldmann/related-repos-1.4

Minimal implementation of "related repositories" for Borg 1.4.x.

docs/faq.rst

@@ -99,6 +99,18 @@ Also, you must not run borg against multiple instances of the same repo
 
 See also: :ref:`faq_corrupt_repo`
 
+Prepare for borg2 "Related repositories" and borg transfer
+----------------------------------------------------------
+
+A related repository is a repository that shares the same deduplication
+secrets (``id_key`` and ``chunk_seed``) as another repository, but uses
+its own independent encryption keys.
+
+This will allow archives to be transferred between related repositories (e.g.
+using ``borg transfer`` in Borg 2.0) without breaking deduplication.
+
+For more information and detailed instructions, see :ref:`borg_key_export-related-secrets`.
+
 "this is either an attack or unsafe" warning
 --------------------------------------------
 

docs/man/borg-init.1

@@ -29,7 +29,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
 .in \\n[rst2man-indent\\n[rst2man-indent-level]]u
 ..
-.TH "borg-init" "1" "2026-03-18" "" "borg backup tool"
+.TH "borg-init" "1" "2026-05-15" "" "borg backup tool"
 .SH Name
 borg-init \- Initialize an empty repository
 .SH SYNOPSIS
@@ -276,11 +276,11 @@ BLAKE2b\-256 hash from the other BLAKE2b modes. This mode is only
 compatible with Borg 1.1 and later.
 .sp
 \fBnone\fP mode uses no encryption and no authentication. It uses SHA256
-as chunk ID hash. This mode is not recommended. You should instead
-consider using an authenticated or authenticated/encrypted mode. This
-mode has possible denial\-of\-service issues when running \fBborg create\fP
-on contents controlled by an attacker. See above for alternatives.
-This mode is compatible with all Borg versions.
+as chunk ID hash. This mode is not recommended
+as it is vulnerable to DoS attacks by an attacker (for example,
+crafting content that causes hash index collisions). Do not use it if
+untrusted clients use the repository. See \fIinternals_hashindex\fP for
+details. This mode is compatible with all Borg versions.
 .SH OPTIONS
 .sp
 See \fIborg\-common(1)\fP for common options of Borg commands.
@@ -304,6 +304,9 @@ Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota.
 .TP
 .B  \-\-make\-parent\-dirs
 create the parent directories of the repository directory, if they are missing.
+.TP
+.BI \-\-import\-related\-secrets \ PATH
+import related secrets from PATH
 .UNINDENT
 .SH EXAMPLES
 .INDENT 0.0

docs/man/borg-key-export-related-secrets.1

@@ -0,0 +1,109 @@
+.\" Man page generated from reStructuredText
+.\" by the Docutils 0.22.4 manpage writer.
+.
+.
+.nr rst2man-indent-level 0
+.
+.de1 rstReportMargin
+\\$1 \\n[an-margin]
+level \\n[rst2man-indent-level]
+level margin: \\n[rst2man-indent\\n[rst2man-indent-level]]
+-
+\\n[rst2man-indent0]
+\\n[rst2man-indent1]
+\\n[rst2man-indent2]
+..
+.de1 INDENT
+.\" .rstReportMargin pre:
+. RS \\$1
+. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin]
+. nr rst2man-indent-level +1
+.\" .rstReportMargin post:
+..
+.de UNINDENT
+. RE
+.\" indent \\n[an-margin]
+.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.nr rst2man-indent-level -1
+.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]]
+.in \\n[rst2man-indent\\n[rst2man-indent-level]]u
+..
+.TH "borg-key-export-related-secrets" "1" "2026-05-15" "" "borg backup tool"
+.SH Name
+borg-key-export-related-secrets \- Export secrets for creating related repositories
+.SH SYNOPSIS
+.sp
+borg [common options] key export\-related\-secrets [options] [REPOSITORY] [PATH]
+.SH DESCRIPTION
+.sp
+This command exports the deduplication secrets (\fBid_key\fP and \fBchunk_seed\fP)
+of a repository. These secrets can be used to initialize a \fBrelated repository\fP\&.
+.sp
+Related repositories share the same deduplication metadata but have their own
+independent encryption keys. This is useful for:
+.INDENT 0.0
+.IP 1. 3
+Creating independent backup targets that still benefit from being
+\(dqcompatible\(dq for future archive transfers.
+.IP 2. 3
+Preparing for a migration to Borg 2.0, where archives can be transferred
+between related repositories using \fBborg transfer\fP\&.
+.UNINDENT
+.sp
+The exported secrets are stored in a JSON file. This file contains sensitive
+information and should be deleted immediately after usage.
+.sp
+Examples:
+.INDENT 0.0
+.INDENT 3.5
+.sp
+.EX
+# Export secrets from an existing repository
+$ borg key export\-related\-secrets /path/to/repo1 secrets.json
+
+# Initialize a new related repository using these secrets
+$ borg init \-\-import\-related\-secrets=secrets.json \-\-encryption=repokey /path/to/repo2
+$ rm secrets.json
+.EE
+.UNINDENT
+.UNINDENT
+.sp
+\fBImportant:\fP
+.INDENT 0.0
+.INDENT 3.5
+When initializing a related repository using \fBborg init \-\-import\-related\-secrets\fP,
+the new repository must use the same ID hash algorithm (either both HMAC\-SHA256
+or both BLAKE2) as the original repository.
+.INDENT 0.0
+.IP \(bu 2
+HMAC\-SHA256: \fBrepokey\fP, \fBkeyfile\fP, \fBauthenticated\fP
+.IP \(bu 2
+BLAKE2: \fBrepokey\-blake2\fP, \fBkeyfile\-blake2\fP, \fBauthenticated\-blake2\fP
+.UNINDENT
+.UNINDENT
+.UNINDENT
+.sp
+\fBWarning:\fP
+.INDENT 0.0
+.INDENT 3.5
+Please note that future Borg 2.0 versions might remove support for BLAKE2
+in new repositories (see #8867).
+.UNINDENT
+.UNINDENT
+.SH OPTIONS
+.sp
+See \fIborg\-common(1)\fP for common options of Borg commands.
+.SS arguments
+.sp
+REPOSITORY
+.INDENT 0.0
+.TP
+.B PATH
+where to store the secrets
+.UNINDENT
+.SH SEE ALSO
+.sp
+\fIborg\-common(1)\fP
+.SH Author
+The Borg Collective
+.\" End of generated man page.

docs/usage/init.rst.inc

@@ -27,6 +27,8 @@ borg init
     +-------------------------------------------------------+------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
     |                                                       | ``--make-parent-dirs``             | create the parent directories of the repository directory, if they are missing.                                                                                                                                                        |
     +-------------------------------------------------------+------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
+    |                                                       | ``--import-related-secrets PATH``  | import related secrets from PATH                                                                                                                                                                                                       |
+    +-------------------------------------------------------+------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
     | .. class:: borg-common-opt-ref                                                                                                                                                                                                                                                                                                      |
     |                                                                                                                                                                                                                                                                                                                                     |
     | :ref:`common_options`                                                                                                                                                                                                                                                                                                               |
@@ -51,6 +53,7 @@ borg init
         --append-only                  create an append-only mode repository. Note that this only affects the low level structure of the repository, and running `delete` or `prune` will still be allowed. See :ref:`append_only_mode` in Additional Notes for more details.
         --storage-quota QUOTA          Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota.
         --make-parent-dirs             create the parent directories of the repository directory, if they are missing.
+        --import-related-secrets PATH    import related secrets from PATH
 
 
     :ref:`common_options`
@@ -265,8 +268,8 @@ BLAKE2b-256 hash from the other BLAKE2b modes. This mode is only
 compatible with Borg 1.1 and later.
 
 ``none`` mode uses no encryption and no authentication. It uses SHA256
-as chunk ID hash. This mode is not recommended. You should instead
-consider using an authenticated or authenticated/encrypted mode. This
-mode has possible denial-of-service issues when running ``borg create``
-on contents controlled by an attacker. See above for alternatives.
-This mode is compatible with all Borg versions.
+as chunk ID hash. This mode is not recommended
+as it is vulnerable to DoS attacks by an attacker (for example,
+crafting content that causes hash index collisions). Do not use it if
+untrusted clients use the repository. See :ref:`internals_hashindex` for
+details. This mode is compatible with all Borg versions.

docs/usage/key.rst

@@ -44,3 +44,7 @@ Fully automated using environment variables:
 .. include:: key_export.rst.inc
 
 .. include:: key_import.rst.inc
+
+This command can be used to create a related repository:
+
+.. include:: key_export-related-secrets.rst.inc

docs/usage/key_export-related-secrets.rst

@@ -0,0 +1 @@
+.. include:: key_export-related-secrets.rst.inc

docs/usage/key_export-related-secrets.rst.inc

@@ -0,0 +1,82 @@
+.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit!
+
+.. _borg_key_export-related-secrets:
+
+borg key export-related-secrets
+-------------------------------
+.. code-block:: none
+
+    borg [common options] key export-related-secrets [options] [REPOSITORY] [PATH]
+
+.. only:: html
+
+    .. class:: borg-options-table
+
+    +-------------------------------------------------------+----------------+----------------------------+
+    | **positional arguments**                                                                            |
+    +-------------------------------------------------------+----------------+----------------------------+
+    |                                                       | ``REPOSITORY`` |                            |
+    +-------------------------------------------------------+----------------+----------------------------+
+    |                                                       | ``PATH``       | where to store the secrets |
+    +-------------------------------------------------------+----------------+----------------------------+
+    | .. class:: borg-common-opt-ref                                                                      |
+    |                                                                                                     |
+    | :ref:`common_options`                                                                               |
+    +-------------------------------------------------------+----------------+----------------------------+
+
+    .. raw:: html
+
+        <script type='text/javascript'>
+        $(document).ready(function () {
+            $('.borg-options-table colgroup').remove();
+        })
+        </script>
+
+.. only:: latex
+
+    REPOSITORY
+
+    PATH
+        where to store the secrets
+
+
+    :ref:`common_options`
+        |
+
+Description
+~~~~~~~~~~~
+
+This command exports the deduplication secrets (``id_key`` and ``chunk_seed``)
+of a repository. These secrets can be used to initialize a **related repository**.
+
+Related repositories share the same deduplication metadata but have their own
+independent encryption keys. This is useful for:
+
+1. Creating independent backup targets that still benefit from being
+   "compatible" for future archive transfers.
+2. Preparing for a migration to Borg 2.0, where archives can be transferred
+   between related repositories using ``borg transfer``.
+
+The exported secrets are stored in a JSON file. This file contains sensitive
+information and should be deleted immediately after usage.
+
+Examples::
+
+    # Export secrets from an existing repository
+    $ borg key export-related-secrets /path/to/repo1 secrets.json
+
+    # Initialize a new related repository using these secrets
+    $ borg init --import-related-secrets=secrets.json --encryption=repokey /path/to/repo2
+    $ rm secrets.json
+
+.. IMPORTANT::
+   When initializing a related repository using ``borg init --import-related-secrets``,
+   the new repository must use the same ID hash algorithm (either both HMAC-SHA256
+   or both BLAKE2) as the original repository.
+
+   - HMAC-SHA256: ``repokey``, ``keyfile``, ``authenticated``
+   - BLAKE2: ``repokey-blake2``, ``keyfile-blake2``, ``authenticated-blake2``
+
+.. WARNING::
+   Please note that future Borg 2.0 versions might remove support for BLAKE2
+   in new repositories (see :issue:`8867`).

src/borg/archiver.py

@@ -333,8 +333,21 @@ class Archiver:
         """Initialize an empty repository"""
         path = args.location.canonical_path()
         logger.info('Initializing repository at "%s"' % path)
+        related_secrets = None
+        if args.import_related_secrets:
+            with dash_open(args.import_related_secrets, 'r') as fd:
+                try:
+                    related_secrets = json.load(fd)
+                except ValueError:
+                    raise CommandError(f"Invalid JSON in related secrets file: {args.import_related_secrets}")
+            if related_secrets.get('version') != 1:
+                raise CommandError(f"Unsupported related secrets version: {related_secrets.get('version')}")
+            try:
+                related_secrets['id_key'] = hex_to_bin(related_secrets['id_key'])
+            except (KeyError, ValueError):
+                raise CommandError(f"Invalid id_key in related secrets file: {args.import_related_secrets}")
         try:
-            key = key_creator(repository, args)
+            key = key_creator(repository, args, related_secrets=related_secrets)
         except (EOFError, KeyboardInterrupt):
             repository.destroy()
             raise CancelledByUser()
@@ -413,6 +426,19 @@ class Archiver:
             # print key location to make backing it up easier
             logger.info('Key location: %s', key.find_key())
 
+    @with_repository(manifest=True, compatibility=(Manifest.Operation.READ,))
+    def do_key_export_related_secrets(self, args, repository, manifest, key):
+        """Export secrets for creating related repositories"""
+        secrets = {
+            'version': 1,
+            'id_key': bin_to_hex(key.id_key),
+            'chunk_seed': key.chunk_seed,
+            'key_name': key.NAME,
+        }
+        with dash_open(args.path, 'w') as fd:
+            json.dump(secrets, fd, indent=4)
+            fd.write('\n')
+
     @with_repository(lock=False, exclusive=False, manifest=False, cache=False)
     def do_key_export(self, args, repository):
         """Export the repository key for backup"""
@@ -4784,6 +4810,8 @@ class Archiver:
                                help='Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota.')
         subparser.add_argument('--make-parent-dirs', dest='make_parent_dirs', action='store_true',
                                help='create the parent directories of the repository directory, if they are missing.')
+        subparser.add_argument('--import-related-secrets', metavar='PATH', dest='import_related_secrets',
+                               type=PathSpec, help='import related secrets from PATH')
 
         # borg key
         subparser = subparsers.add_parser('key', parents=[mid_common_parser], add_help=False,
@@ -4851,6 +4879,54 @@ class Archiver:
         subparser.add_argument('--qr-html', dest='qr', action='store_true',
                                help='Create an html file suitable for printing and later type-in or qr scan')
 
+        export_related_secrets_epilog = process_epilog("""
+        This command exports the deduplication secrets (``id_key`` and ``chunk_seed``)
+        of a repository. These secrets can be used to initialize a **related repository**.
+
+        Related repositories share the same deduplication metadata but have their own
+        independent encryption keys. This is useful for:
+
+        1. Creating independent backup targets that still benefit from being
+           "compatible" for future archive transfers.
+        2. Preparing for a migration to Borg 2.0, where archives can be transferred
+           between related repositories using ``borg transfer``.
+
+        The exported secrets are stored in a JSON file. This file contains sensitive
+        information and should be deleted immediately after usage.
+
+        Examples::
+
+            # Export secrets from an existing repository
+            $ borg key export-related-secrets /path/to/repo1 secrets.json
+
+            # Initialize a new related repository using these secrets
+            $ borg init --import-related-secrets=secrets.json --encryption=repokey /path/to/repo2
+            $ rm secrets.json
+
+        .. IMPORTANT::
+           When initializing a related repository using ``borg init --import-related-secrets``,
+           the new repository must use the same ID hash algorithm (either both HMAC-SHA256
+           or both BLAKE2) as the original repository.
+
+           - HMAC-SHA256: ``repokey``, ``keyfile``, ``authenticated``
+           - BLAKE2: ``repokey-blake2``, ``keyfile-blake2``, ``authenticated-blake2``
+
+        .. WARNING::
+           Please note that future Borg 2.0 versions might remove support for BLAKE2
+           in new repositories (see :issue:`8867`).
+        """)
+
+        subparser = key_parsers.add_parser('export-related-secrets', parents=[common_parser], add_help=False,
+                                          description=self.do_key_export_related_secrets.__doc__,
+                                          epilog=export_related_secrets_epilog,
+                                          formatter_class=argparse.RawDescriptionHelpFormatter,
+                                          help='export related secrets for related repositories')
+        subparser.set_defaults(func=self.do_key_export_related_secrets)
+        subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
+                               type=location_validator(archive=False))
+        subparser.add_argument('path', metavar='PATH', nargs='?', type=PathSpec,
+                               help='where to store the secrets')
+
         key_import_epilog = process_epilog("""
         This command restores a key previously backed up with the export command.
 

src/borg/crypto/key.py

@@ -134,15 +134,24 @@ class KeyBlobStorage:
     REPO = 'repository'
 
 
-def key_creator(repository, args):
+def key_creator(repository, args, related_secrets=None):
     for key in AVAILABLE_KEY_TYPES:
         if key.ARG_NAME == args.encryption:
             assert key.ARG_NAME is not None
-            return key.create(repository, args)
+            return key.create(repository, args, related_secrets=related_secrets)
     else:
         raise ValueError('Invalid encryption mode "%s"' % args.encryption)
 
 
+def uses_same_id_hash(other_key_name, key):
+    """is the id hash the same?"""
+    # avoid breaking the deduplication by changing the id hash
+    hmac_sha256_names = ('repokey', 'key file', 'authenticated')
+    blake2_names = ('repokey BLAKE2b', 'key file BLAKE2b', 'authenticated BLAKE2b')
+    return (other_key_name in hmac_sha256_names and key.NAME in hmac_sha256_names or
+            other_key_name in blake2_names and key.NAME in blake2_names)
+
+
 def key_argument_names():
     return [key.ARG_NAME for key in AVAILABLE_KEY_TYPES if key.ARG_NAME]
 
@@ -355,7 +364,7 @@ class PlaintextKey(KeyBase):
         self.tam_required = False
 
     @classmethod
-    def create(cls, repository, args):
+    def create(cls, repository, args, related_secrets=None):
         logger.info('Encryption NOT enabled.\nUse the "--encryption=repokey|keyfile" to enable encryption.')
         return cls(repository)
 
@@ -622,11 +631,13 @@ class PassphraseKey(ID_HMAC_SHA_256, AESKeyBase):
     iterations = 100000  # must not be changed ever!
 
     @classmethod
-    def create(cls, repository, args):
+    def create(cls, repository, args, related_secrets=None):
         key = cls(repository)
         logger.warning('WARNING: "passphrase" mode is unsupported since borg 1.0.')
         passphrase = Passphrase.new(allow_empty=False)
         key.init(repository, passphrase)
+        if related_secrets:
+             raise Error('Importing related secrets is not supported for "passphrase" mode.')
         return key
 
     @classmethod
@@ -762,11 +773,16 @@ class KeyfileKeyBase(AESKeyBase):
         self.save(self.target, passphrase)
 
     @classmethod
-    def create(cls, repository, args):
+    def create(cls, repository, args, related_secrets=None):
         passphrase = Passphrase.new(allow_empty=True)
         key = cls(repository)
         key.repository_id = repository.id
         key.init_from_random_data()
+        if related_secrets:
+            if not uses_same_id_hash(related_secrets['key_name'], key):
+                raise Error('You must keep the same ID hash (HMAC-SHA256 or BLAKE2b) or deduplication will break.')
+            key.id_key = related_secrets['id_key']
+            key.chunk_seed = related_secrets['chunk_seed']
         key.init_ciphers()
         target = key.get_new_target(args)
         key.save(target, passphrase, create=True)

src/borg/testsuite/archiver.py

@@ -3209,7 +3209,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         assert "is invalid" in output
 
     def test_init_interrupt(self):
-        def raise_eof(*args):
+        def raise_eof(*args, **kwargs):
             raise EOFError
 
         with patch.object(KeyfileKeyBase, 'create', raise_eof):
@@ -4217,6 +4217,76 @@ id: 2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02
         self.cmd('recreate', '--chunker-params=10,12,11,63', self.repository_location + '::archive')
         assert original_size('archive') == sum(sizes)
 
+    def test_related_repos_deduplication(self):
+        CHUNKER_PARAMS = 'buzhash,10,18,14,4095'  # ~16kiB chunks
+        # 1. Create repo1
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        self.create_regular_file('file1', contents=os.urandom(128 * 1024))
+        self.cmd('create', f'--chunker-params={CHUNKER_PARAMS}', self.repository_location + '::archive1', 'input')
+
+        # 2. Export related secrets
+        secrets_path = os.path.join(self.tmpdir, 'secrets.json')
+        self.cmd('key', 'export-related-secrets', self.repository_location, secrets_path)
+
+        with open(secrets_path, 'r') as f:
+            secrets = json.load(f)
+        assert secrets['version'] == 1
+        assert 'id_key' in secrets
+        assert 'chunk_seed' in secrets
+        assert 'key_name' in secrets
+
+        # 3. Create repo2 using imported secrets
+        repo2_path = os.path.join(self.tmpdir, 'repo2')
+        repo2_location = self.prefix + repo2_path
+        self.cmd('init', '--encryption=repokey', '--import-related-secrets', secrets_path, repo2_location)
+
+        # 4. Create backup in repo2 with same data
+        self.cmd('create', f'--chunker-params={CHUNKER_PARAMS}', repo2_location + '::archive2', 'input')
+
+        # 5. Verify chunk IDs are identical
+        def get_chunk_ids(repo_path, archive_name):
+            with Repository(repo_path) as repository:
+                manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
+                archive = Archive(repository, key, manifest, archive_name)
+                ids = []
+                for item in archive.iter_items():
+                    chunks = getattr(item, 'chunks', None)
+                    if chunks:
+                        ids.extend(c.id for c in chunks)
+                return ids
+
+        ids1 = get_chunk_ids(self.repository_path, 'archive1')
+        ids2 = get_chunk_ids(repo2_path, 'archive2')
+
+        assert ids1 == ids2
+        assert len(ids1) > 3
+
+        # 6. Verify encryption keys are different, but id_key and chunk_seed are same
+        def get_keys(repo_path):
+            with Repository(repo_path) as repository:
+                manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
+                return key.enc_key, key.enc_hmac_key, key.id_key, key.chunk_seed
+
+        enc_key1, hmac_key1, id_key1, chunk_seed1 = get_keys(self.repository_path)
+        enc_key2, hmac_key2, id_key2, chunk_seed2 = get_keys(repo2_path)
+
+        assert enc_key1 != enc_key2
+        assert hmac_key1 != hmac_key2
+        assert id_key1 == id_key2
+        assert chunk_seed1 == chunk_seed2
+
+    def test_related_repos_incompatible_key_name(self):
+        # Create repo1 with default (HMAC-SHA256)
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        secrets_path = os.path.join(self.tmpdir, 'secrets.json')
+        self.cmd('key', 'export-related-secrets', self.repository_location, secrets_path)
+ 
+        # Try to create repo2 with BLAKE2b (incompatible)
+        repo2_path = os.path.join(self.tmpdir, 'repo2')
+        repo2_location = self.prefix + repo2_path
+        # This should fail
+        out = self.cmd('init', '--encryption=repokey-blake2', '--import-related-secrets', secrets_path, repo2_location, exit_code=2, fork=True)
+        assert 'deduplication will break' in out
 
 @unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available')
 class ArchiverTestCaseBinary(ArchiverTestCase):