From 94aec46a01b50e436506bef3489f24df87b19ee8 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 9 Jul 2022 00:40:51 +0200 Subject: [PATCH] move rcreate command to archiver.rcreate --- src/borg/archiver/__init__.py | 198 +------------------------------ src/borg/archiver/rcreate.py | 208 +++++++++++++++++++++++++++++++++ src/borg/testsuite/archiver.py | 3 +- 3 files changed, 215 insertions(+), 194 deletions(-) create mode 100644 src/borg/archiver/rcreate.py diff --git a/src/borg/archiver/__init__.py b/src/borg/archiver/__init__.py index 9e3748334..8f3ca5267 100644 --- a/src/borg/archiver/__init__.py +++ b/src/borg/archiver/__init__.py @@ -25,7 +25,7 @@ try: logger = create_logger() - from .common import with_repository, with_other_repository, with_archive, Highlander + from .common import with_repository, with_archive, Highlander from .. import __version__ from .. import helpers from ..archive import Archive, ArchiveRecreater, Statistics, is_special @@ -34,13 +34,12 @@ try: from ..cache import Cache, SecurityManager from ..constants import * # NOQA from ..compress import CompressionSpec - from ..crypto.key import key_creator, key_argument_names, tam_required_file from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, EXIT_SIGNAL_BASE from ..helpers import Error, NoManifestError, set_ec from ..helpers import location_validator, archivename_validator, ChunkerParams, Location from ..helpers import NameSpec, CommentSpec, FilesCacheMode from ..helpers import BaseFormatter, ItemFormatter, ArchiveFormatter - from ..helpers import format_timedelta, format_file_size, format_archive, parse_storage_quota + from ..helpers import format_timedelta, format_file_size, format_archive from ..helpers import remove_surrogates, bin_to_hex, eval_escapes from ..helpers import timestamp from ..helpers import get_cache_dir, os_stat @@ -100,6 +99,7 @@ from .keys import KeysMixIn from .locks import LocksMixIn from .mount import MountMixIn from .prune import PruneMixIn +from .rcreate import RCreateMixIn from .serve import ServeMixIn from .tar import TarMixIn from .transfer import TransferMixIn @@ -118,6 +118,7 @@ class Archiver( MountMixIn, PruneMixIn, HelpMixIn, + RCreateMixIn, ServeMixIn, TransferMixIn, ): @@ -154,37 +155,6 @@ class Archiver( matcher.add_includepaths(include_paths) return matcher - @with_repository(create=True, exclusive=True, manifest=False) - @with_other_repository(key=True, compatibility=(Manifest.Operation.READ,)) - def do_rcreate(self, args, repository, *, other_repository=None, other_key=None): - """Create a new, empty repository""" - path = args.location.canonical_path() - logger.info('Initializing repository at "%s"' % path) - try: - key = key_creator(repository, args, other_key=other_key) - except (EOFError, KeyboardInterrupt): - repository.destroy() - return EXIT_WARNING - manifest = Manifest(key, repository) - manifest.key = key - manifest.write() - repository.commit(compact=False) - with Cache(repository, key, manifest, warn_if_unencrypted=False): - pass - if key.tam_required: - tam_file = tam_required_file(repository) - open(tam_file, "w").close() - - if key.NAME != "plaintext": - logger.warning( - "\n" - "IMPORTANT: you will need both KEY AND PASSPHRASE to access this repo!\n" - "If you used a repokey mode, the key is stored in the repo, but you should back it up separately.\n" - 'Use "borg key export" to export the key, optionally in printable format.\n' - "Write down the passphrase. Store both at safe place(s).\n" - ) - return self.exit_code - @with_repository(fake="dry_run", exclusive=True, compatibility=(Manifest.Operation.WRITE,)) def do_create(self, args, repository, manifest=None, key=None): """Create new archive""" @@ -2029,166 +1999,8 @@ class Archiver( subparser.add_argument("--json", action="store_true", help="format output as JSON") define_archive_filters_group(subparser) - # borg rcreate - rcreate_epilog = process_epilog( - """ - This command creates a new, empty repository. A repository is a filesystem - directory containing the deduplicated data from zero or more archives. - - Encryption mode TLDR - ++++++++++++++++++++ - - The encryption mode can only be configured when creating a new repository - you can - neither configure it on a per-archive basis nor change the mode of an existing repository. - This example will likely NOT give optimum performance on your machine (performance - tips will come below): - - :: - - borg rcreate --encryption repokey-aes-ocb - - Borg will: - - 1. Ask you to come up with a passphrase. - 2. Create a borg key (which contains some random secrets. See :ref:`key_files`). - 3. Derive a "key encryption key" from your passphrase - 4. Encrypt and sign the key with the key encryption key - 5. Store the encrypted borg key inside the repository directory (in the repo config). - This is why it is essential to use a secure passphrase. - 6. Encrypt and sign your backups to prevent anyone from reading or forging them unless they - have the key and know the passphrase. Make sure to keep a backup of - your key **outside** the repository - do not lock yourself out by - "leaving your keys inside your car" (see :ref:`borg_key_export`). - For remote backups the encryption is done locally - the remote machine - never sees your passphrase, your unencrypted key or your unencrypted files. - Chunking and id generation are also based on your key to improve - your privacy. - 7. Use the key when extracting files to decrypt them and to verify that the contents of - the backups have not been accidentally or maliciously altered. - - Picking a passphrase - ++++++++++++++++++++ - - Make sure you use a good passphrase. Not too short, not too simple. The real - encryption / decryption key is encrypted with / locked by your passphrase. - If an attacker gets your key, he can't unlock and use it without knowing the - passphrase. - - Be careful with special or non-ascii characters in your passphrase: - - - Borg processes the passphrase as unicode (and encodes it as utf-8), - so it does not have problems dealing with even the strangest characters. - - BUT: that does not necessarily apply to your OS / VM / keyboard configuration. - - So better use a long passphrase made from simple ascii chars than one that - includes non-ascii stuff or characters that are hard/impossible to enter on - a different keyboard layout. - - You can change your passphrase for existing repos at any time, it won't affect - the encryption/decryption key or other secrets. - - Choosing an encryption mode - +++++++++++++++++++++++++++ - - Depending on your hardware, hashing and crypto performance may vary widely. - The easiest way to find out about what's fastest is to run ``borg benchmark cpu``. - - `repokey` modes: if you want ease-of-use and "passphrase" security is good enough - - the key will be stored in the repository (in ``repo_dir/config``). - - `keyfile` modes: if you rather want "passphrase and having-the-key" security - - the key will be stored in your home directory (in ``~/.config/borg/keys``). - - The following table is roughly sorted in order of preference, the better ones are - in the upper part of the table, in the lower part is the old and/or unsafe(r) stuff: - - .. nanorst: inline-fill - - +-----------------------------------+--------------+----------------+--------------------+---------+ - | Mode (K = keyfile or repokey) | ID-Hash | Encryption | Authentication | V >= | - +-----------------------------------+--------------+----------------+--------------------+---------+ - | K-blake2-chacha20-poly1305 | BLAKE2b | CHACHA20 | POLY1305 | 2.0 | - +-----------------------------------+--------------+----------------+--------------------+---------+ - | K-chacha20-poly1305 | HMAC-SHA-256 | CHACHA20 | POLY1305 | 2.0 | - +-----------------------------------+--------------+----------------+--------------------+---------+ - | K-blake2-aes-ocb | BLAKE2b | AES256-OCB | AES256-OCB | 2.0 | - +-----------------------------------+--------------+----------------+--------------------+---------+ - | K-aes-ocb | HMAC-SHA-256 | AES256-OCB | AES256-OCB | 2.0 | - +-----------------------------------+--------------+----------------+--------------------+---------+ - | K-blake2 | BLAKE2b | AES256-CTR | BLAKE2b | 1.1 | - +-----------------------------------+--------------+----------------+--------------------+---------+ - | K | HMAC-SHA-256 | AES256-CTR | HMAC-SHA256 | any | - +-----------------------------------+--------------+----------------+--------------------+---------+ - | authenticated-blake2 | BLAKE2b | none | BLAKE2b | 1.1 | - +-----------------------------------+--------------+----------------+--------------------+---------+ - | authenticated | HMAC-SHA-256 | none | HMAC-SHA256 | 1.1 | - +-----------------------------------+--------------+----------------+--------------------+---------+ - | none | SHA-256 | none | none | any | - +-----------------------------------+--------------+----------------+--------------------+---------+ - - .. nanorst: inline-replace - - `none` mode uses no encryption and no authentication. You're advised to NOT use this mode - as it would expose you to all sorts of issues (DoS, confidentiality, tampering, ...) in - case of malicious activity in the repository. - - If you do **not** want to encrypt the contents of your backups, but still want to detect - malicious tampering use an `authenticated` mode. It's like `repokey` minus encryption. - - """ - ) - subparser = subparsers.add_parser( - "rcreate", - parents=[common_parser], - add_help=False, - description=self.do_rcreate.__doc__, - epilog=rcreate_epilog, - formatter_class=argparse.RawDescriptionHelpFormatter, - help="create a new, empty repository", - ) - subparser.set_defaults(func=self.do_rcreate) - subparser.add_argument( - "--other-repo", - metavar="SRC_REPOSITORY", - dest="other_location", - type=location_validator(other=True), - default=Location(other=True), - help="reuse the key material from the other repository", - ) - subparser.add_argument( - "-e", - "--encryption", - metavar="MODE", - dest="encryption", - required=True, - choices=key_argument_names(), - help="select encryption key mode **(required)**", - ) - subparser.add_argument( - "--append-only", - dest="append_only", - action="store_true", - help="create an append-only mode repository. Note that this only affects " - "the low level structure of the repository, and running `delete` " - "or `prune` will still be allowed. See :ref:`append_only_mode` in " - "Additional Notes for more details.", - ) - subparser.add_argument( - "--storage-quota", - metavar="QUOTA", - dest="storage_quota", - default=None, - type=parse_storage_quota, - help="Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota.", - ) - subparser.add_argument( - "--make-parent-dirs", - dest="make_parent_dirs", - action="store_true", - help="create the parent directories of the repository directory, if they are missing.", - ) - self.build_parser_keys(subparsers, common_parser, mid_common_parser) + self.build_parser_rcreate(subparsers, common_parser, mid_common_parser) # borg list list_epilog = ( diff --git a/src/borg/archiver/rcreate.py b/src/borg/archiver/rcreate.py new file mode 100644 index 000000000..c30fbc7b1 --- /dev/null +++ b/src/borg/archiver/rcreate.py @@ -0,0 +1,208 @@ +import argparse + +from .common import with_repository, with_other_repository +from ..cache import Cache +from ..constants import * # NOQA +from ..crypto.key import key_creator, key_argument_names, tam_required_file +from ..helpers import EXIT_WARNING +from ..helpers import location_validator, Location +from ..helpers import parse_storage_quota +from ..helpers import Manifest + +from ..logger import create_logger + +logger = create_logger() + + +class RCreateMixIn: + @with_repository(create=True, exclusive=True, manifest=False) + @with_other_repository(key=True, compatibility=(Manifest.Operation.READ,)) + def do_rcreate(self, args, repository, *, other_repository=None, other_key=None): + """Create a new, empty repository""" + path = args.location.canonical_path() + logger.info('Initializing repository at "%s"' % path) + try: + key = key_creator(repository, args, other_key=other_key) + except (EOFError, KeyboardInterrupt): + repository.destroy() + return EXIT_WARNING + manifest = Manifest(key, repository) + manifest.key = key + manifest.write() + repository.commit(compact=False) + with Cache(repository, key, manifest, warn_if_unencrypted=False): + pass + if key.tam_required: + tam_file = tam_required_file(repository) + open(tam_file, "w").close() + + if key.NAME != "plaintext": + logger.warning( + "\n" + "IMPORTANT: you will need both KEY AND PASSPHRASE to access this repo!\n" + "If you used a repokey mode, the key is stored in the repo, but you should back it up separately.\n" + 'Use "borg key export" to export the key, optionally in printable format.\n' + "Write down the passphrase. Store both at safe place(s).\n" + ) + return self.exit_code + + def build_parser_rcreate(self, subparsers, common_parser, mid_common_parser): + from .common import process_epilog + + rcreate_epilog = process_epilog( + """ + This command creates a new, empty repository. A repository is a filesystem + directory containing the deduplicated data from zero or more archives. + + Encryption mode TLDR + ++++++++++++++++++++ + + The encryption mode can only be configured when creating a new repository - you can + neither configure it on a per-archive basis nor change the mode of an existing repository. + This example will likely NOT give optimum performance on your machine (performance + tips will come below): + + :: + + borg rcreate --encryption repokey-aes-ocb + + Borg will: + + 1. Ask you to come up with a passphrase. + 2. Create a borg key (which contains some random secrets. See :ref:`key_files`). + 3. Derive a "key encryption key" from your passphrase + 4. Encrypt and sign the key with the key encryption key + 5. Store the encrypted borg key inside the repository directory (in the repo config). + This is why it is essential to use a secure passphrase. + 6. Encrypt and sign your backups to prevent anyone from reading or forging them unless they + have the key and know the passphrase. Make sure to keep a backup of + your key **outside** the repository - do not lock yourself out by + "leaving your keys inside your car" (see :ref:`borg_key_export`). + For remote backups the encryption is done locally - the remote machine + never sees your passphrase, your unencrypted key or your unencrypted files. + Chunking and id generation are also based on your key to improve + your privacy. + 7. Use the key when extracting files to decrypt them and to verify that the contents of + the backups have not been accidentally or maliciously altered. + + Picking a passphrase + ++++++++++++++++++++ + + Make sure you use a good passphrase. Not too short, not too simple. The real + encryption / decryption key is encrypted with / locked by your passphrase. + If an attacker gets your key, he can't unlock and use it without knowing the + passphrase. + + Be careful with special or non-ascii characters in your passphrase: + + - Borg processes the passphrase as unicode (and encodes it as utf-8), + so it does not have problems dealing with even the strangest characters. + - BUT: that does not necessarily apply to your OS / VM / keyboard configuration. + + So better use a long passphrase made from simple ascii chars than one that + includes non-ascii stuff or characters that are hard/impossible to enter on + a different keyboard layout. + + You can change your passphrase for existing repos at any time, it won't affect + the encryption/decryption key or other secrets. + + Choosing an encryption mode + +++++++++++++++++++++++++++ + + Depending on your hardware, hashing and crypto performance may vary widely. + The easiest way to find out about what's fastest is to run ``borg benchmark cpu``. + + `repokey` modes: if you want ease-of-use and "passphrase" security is good enough - + the key will be stored in the repository (in ``repo_dir/config``). + + `keyfile` modes: if you rather want "passphrase and having-the-key" security - + the key will be stored in your home directory (in ``~/.config/borg/keys``). + + The following table is roughly sorted in order of preference, the better ones are + in the upper part of the table, in the lower part is the old and/or unsafe(r) stuff: + + .. nanorst: inline-fill + + +-----------------------------------+--------------+----------------+--------------------+---------+ + | Mode (K = keyfile or repokey) | ID-Hash | Encryption | Authentication | V >= | + +-----------------------------------+--------------+----------------+--------------------+---------+ + | K-blake2-chacha20-poly1305 | BLAKE2b | CHACHA20 | POLY1305 | 2.0 | + +-----------------------------------+--------------+----------------+--------------------+---------+ + | K-chacha20-poly1305 | HMAC-SHA-256 | CHACHA20 | POLY1305 | 2.0 | + +-----------------------------------+--------------+----------------+--------------------+---------+ + | K-blake2-aes-ocb | BLAKE2b | AES256-OCB | AES256-OCB | 2.0 | + +-----------------------------------+--------------+----------------+--------------------+---------+ + | K-aes-ocb | HMAC-SHA-256 | AES256-OCB | AES256-OCB | 2.0 | + +-----------------------------------+--------------+----------------+--------------------+---------+ + | K-blake2 | BLAKE2b | AES256-CTR | BLAKE2b | 1.1 | + +-----------------------------------+--------------+----------------+--------------------+---------+ + | K | HMAC-SHA-256 | AES256-CTR | HMAC-SHA256 | any | + +-----------------------------------+--------------+----------------+--------------------+---------+ + | authenticated-blake2 | BLAKE2b | none | BLAKE2b | 1.1 | + +-----------------------------------+--------------+----------------+--------------------+---------+ + | authenticated | HMAC-SHA-256 | none | HMAC-SHA256 | 1.1 | + +-----------------------------------+--------------+----------------+--------------------+---------+ + | none | SHA-256 | none | none | any | + +-----------------------------------+--------------+----------------+--------------------+---------+ + + .. nanorst: inline-replace + + `none` mode uses no encryption and no authentication. You're advised to NOT use this mode + as it would expose you to all sorts of issues (DoS, confidentiality, tampering, ...) in + case of malicious activity in the repository. + + If you do **not** want to encrypt the contents of your backups, but still want to detect + malicious tampering use an `authenticated` mode. It's like `repokey` minus encryption. + + """ + ) + subparser = subparsers.add_parser( + "rcreate", + parents=[common_parser], + add_help=False, + description=self.do_rcreate.__doc__, + epilog=rcreate_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help="create a new, empty repository", + ) + subparser.set_defaults(func=self.do_rcreate) + subparser.add_argument( + "--other-repo", + metavar="SRC_REPOSITORY", + dest="other_location", + type=location_validator(other=True), + default=Location(other=True), + help="reuse the key material from the other repository", + ) + subparser.add_argument( + "-e", + "--encryption", + metavar="MODE", + dest="encryption", + required=True, + choices=key_argument_names(), + help="select encryption key mode **(required)**", + ) + subparser.add_argument( + "--append-only", + dest="append_only", + action="store_true", + help="create an append-only mode repository. Note that this only affects " + "the low level structure of the repository, and running `delete` " + "or `prune` will still be allowed. See :ref:`append_only_mode` in " + "Additional Notes for more details.", + ) + subparser.add_argument( + "--storage-quota", + metavar="QUOTA", + dest="storage_quota", + default=None, + type=parse_storage_quota, + help="Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota.", + ) + subparser.add_argument( + "--make-parent-dirs", + dest="make_parent_dirs", + action="store_true", + help="create the parent directories of the repository directory, if they are missing.", + ) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 5b371bab1..0561dabe7 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -31,7 +31,7 @@ import borg import borg.helpers.errors from .. import xattr, helpers, platform from ..archive import Archive, ChunkBuffer -from ..archiver import Archiver, parse_storage_quota, PURE_PYTHON_MSGPACK_WARNING +from ..archiver import Archiver, PURE_PYTHON_MSGPACK_WARNING from ..cache import Cache, LocalCache from ..chunker import has_seek_hole from ..constants import * # NOQA @@ -43,6 +43,7 @@ from ..helpers import Manifest, MandatoryFeatureUnsupported from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from ..helpers import bin_to_hex from ..helpers import msgpack +from ..helpers import parse_storage_quota from ..helpers import flags_noatime, flags_normal from ..nanorst import RstToTextLazy, rst_to_terminal from ..patterns import IECommand, PatternMatcher, parse_pattern