From 5b47cf6fa54cb96d5903c5febf3f91f6ece99adf Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Fri, 10 Nov 2017 20:30:02 -0800 Subject: [PATCH 1/2] Add borg config command (fixes #3304) This command works similarly to "git config" - it parses repo and cache configs to get, set, and delete values. It only works on local repos so a malicious client can't e.g. override their storage quota or reset the append_only flag. Add tests for borg config Add documentation for borg config Change manual config edits -> borg config There were a couple places in the documentation where it was advised to edit the repository or cache config file, a process that is stream- lined by borg config. --- docs/faq.rst | 5 +-- docs/quickstart.rst | 9 +++-- docs/usage.rst | 1 + docs/usage/config.rst | 22 +++++++++++ docs/usage/notes.rst | 5 ++- src/borg/archiver.py | 67 +++++++++++++++++++++++++++++++++ src/borg/helpers/parseformat.py | 9 ++++- src/borg/testsuite/archiver.py | 17 +++++++++ 8 files changed, 124 insertions(+), 11 deletions(-) create mode 100644 docs/usage/config.rst diff --git a/docs/faq.rst b/docs/faq.rst index f4d7fe5c2..d45cc74ce 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -502,8 +502,7 @@ space for chunks.archive.d (see :issue:`235` for details): :: # this assumes you are working with the same user as the backup. - # you can get the REPOID from the "config" file inside the repository. - cd ~/.cache/borg/ + cd ~/.cache/borg/$(borg config /path/to/repo id) rm -rf chunks.archive.d ; touch chunks.archive.d This deletes all the cached archive chunk indexes and replaces the directory @@ -808,7 +807,7 @@ There are some caveats: - If the repository is in "keyfile" encryption mode, the keyfile must exist locally or it must be manually moved after performing the upgrade: - 1. Locate the repository ID, contained in the ``config`` file in the repository. + 1. Get the repository ID with ``borg config /path/to/repo id``. 2. Locate the attic key file at ``~/.attic/keys/``. The correct key for the repository starts with the line ``ATTIC_KEY ``. 3. Copy the attic key file to ``~/.config/borg/keys/`` diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 4ea265b11..3c197aee3 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -22,10 +22,11 @@ a good amount of free space on the filesystem that has your backup repository repositories. See also :ref:`cache-memory-usage`. Borg doesn't use space reserved for root on repository disks (even when run as root), -on file systems which do not support this mechanism (e.g. XFS) we recommend to -reserve some space in Borg itself just to be safe by adjusting the -``additional_free_space`` setting in the ``[repository]`` section of a repositories -``config`` file. A good starting point is ``2G``. +on file systems which do not support this mechanism (e.g. XFS) we recommend to reserve +some space in Borg itself just to be safe by adjusting the ``additional_free_space`` +setting (a good starting point is ``2G``):: + + borg config /path/to/repo additional_free_space 2G If Borg runs out of disk space, it tries to free as much space as it can while aborting the current operation safely, which allows the user to free more space diff --git a/docs/usage.rst b/docs/usage.rst index de335c1c0..f8e2b48bf 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -51,6 +51,7 @@ Usage usage/serve usage/lock usage/benchmark + usage/config usage/help usage/debug diff --git a/docs/usage/config.rst b/docs/usage/config.rst new file mode 100644 index 000000000..dc21eb39d --- /dev/null +++ b/docs/usage/config.rst @@ -0,0 +1,22 @@ +.. include:: config.rst.inc + +.. note:: + + The repository & cache config files are some of the only directly manipulable + parts of a repository that aren't versioned or backed up, so be careful when + making changes\! + +Examples +~~~~~~~~ +:: + + # find cache directory + $ cd ~/.cache/borg/$(borg config /path/to/repo id) + + # reserve some space + $ borg config /path/to/repo additional_free_space 2G + + # make a repo append-only + $ borg config /path/to/repo append_only 1 + + diff --git a/docs/usage/notes.rst b/docs/usage/notes.rst index c45ef3f8c..4342a8c12 100644 --- a/docs/usage/notes.rst +++ b/docs/usage/notes.rst @@ -149,8 +149,9 @@ reject to delete the repository completely). This is useful for scenarios where backup client machine backups remotely to a backup server using ``borg serve``, since a hacked client machine cannot delete backups on the server permanently. -To activate append-only mode, edit the repository ``config`` file and add a line -``append_only=1`` to the ``[repository]`` section (or edit the line if it exists). +To activate append-only mode, set ``append_only`` to 1 in the repository config:: + + borg config /path/to/repo append_only 1 In append-only mode Borg will create a transaction log in the ``transactions`` file, where each line is a transaction and a UTC timestamp. diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 32dec1bdd..bb664d5ce 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1,5 +1,6 @@ import argparse import collections +import configparser import faulthandler import functools import hashlib @@ -1517,6 +1518,41 @@ class Archiver: # see issue #1867. repository.commit() + @with_repository(exclusive=True, cache=True, compatibility=(Manifest.Operation.WRITE,)) + def do_config(self, args, repository, manifest, key, cache): + """get, set, and delete values in a repository or cache config file""" + try: + section, name = args.name.split('.') + except ValueError: + section = args.cache and "cache" or "repository" + name = args.name + + if args.cache: + cache.cache_config.load() + config = cache.cache_config._config + save = cache.cache_config.save + else: + config = repository.config + save = lambda: repository.save_config(repository.path, repository.config) + + if args.delete: + config.remove_option(section, name) + if len(config.options(section)) == 0: + config.remove_section(section) + save() + elif args.value: + if section not in config.sections(): + config.add_section(section) + config.set(section, name, args.value) + save() + else: + try: + print(config.get(section, name)) + except (configparser.NoOptionError, configparser.NoSectionError) as e: + print(e, file=sys.stderr) + return EXIT_WARNING + return EXIT_SUCCESS + def do_debug_info(self, args): """display system information for debugging / bug reports""" print(sysinfo()) @@ -3469,6 +3505,37 @@ class Archiver: subparser.add_argument('args', metavar='ARGS', nargs=argparse.REMAINDER, help='command arguments') + config_epilog = process_epilog(""" + This command gets and sets options in a local repository or cache config file. + For security reasons, this command only works on local repositories. + + To delete a config value entirely, use ``--delete``. To get an existing key, pass + only the key name. To set a key, pass both the key name and the new value. Keys + can be specified in the format "section.name" or simply "name"; the section will + default to "repository" and "cache" for the repo and cache configs, respectively. + + By default, borg config manipulates the repository config file. Using ``--cache`` + edits the repository cache's config file instead. + """) + subparser = subparsers.add_parser('config', parents=[common_parser], add_help=False, + description=self.do_config.__doc__, + epilog=config_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='get and set repository config options') + subparser.set_defaults(func=self.do_config) + subparser.add_argument('-c', '--cache', dest='cache', action='store_true', + help='get and set values from the repo cache') + subparser.add_argument('-d', '--delete', dest='delete', action='store_true', + help='delete the key from the config file') + + subparser.add_argument('location', metavar='REPOSITORY', + type=location_validator(archive=False, proto='file'), + help='repository to configure') + subparser.add_argument('name', metavar='NAME', + help='name of config key') + subparser.add_argument('value', metavar='VALUE', nargs='?', + help='new value for key') + subparser = subparsers.add_parser('help', parents=[common_parser], add_help=False, description='Extra help') subparser.add_argument('--epilog-only', dest='epilog_only', action='store_true') diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 0889eea98..948fe8d72 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -447,7 +447,7 @@ class Location: path) -def location_validator(archive=None): +def location_validator(archive=None, proto=None): def validator(text): try: loc = Location(text) @@ -456,7 +456,12 @@ def location_validator(archive=None): if archive is True and not loc.archive: raise argparse.ArgumentTypeError('"%s": No archive specified' % text) elif archive is False and loc.archive: - raise argparse.ArgumentTypeError('"%s" No archive can be specified' % text) + raise argparse.ArgumentTypeError('"%s": No archive can be specified' % text) + if proto is not None and loc.proto != proto: + if proto == 'file': + raise argparse.ArgumentTypeError('"%s": Repository must be local' % text) + else: + raise argparse.ArgumentTypeError('"%s": Repository must be remote' % text) return loc return validator diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 29fc50f73..48013080d 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -2709,6 +2709,19 @@ id: 2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02 with environment_variable(_BORG_BENCHMARK_CRUD_TEST='YES'): self.cmd('benchmark', 'crud', self.repository_location, self.input_path) + def test_config(self): + self.create_test_files() + os.unlink('input/flagfile') + self.cmd('init', '--encryption=repokey', self.repository_location) + for flags in [[], ['--cache']]: + for key in {'testkey', 'testsection.testkey'}: + self.cmd('config', self.repository_location, *flags, key, exit_code=1) + self.cmd('config', self.repository_location, *flags, key, 'testcontents') + output = self.cmd('config', self.repository_location, *flags, key) + assert output == 'testcontents\n' + self.cmd('config', self.repository_location, *flags, '--delete', key) + self.cmd('config', self.repository_location, *flags, key, exit_code=1) + requires_gnutar = pytest.mark.skipif(not have_gnutar(), reason='GNU tar must be installed for this test.') requires_gzip = pytest.mark.skipif(not shutil.which('gzip'), reason='gzip must be installed for this test.') @@ -3191,6 +3204,10 @@ class RemoteArchiverTestCase(ArchiverTestCase): def test_debug_put_get_delete_obj(self): pass + @unittest.skip('only works locally') + def test_config(self): + pass + def test_strip_components_doesnt_leak(self): self.cmd('init', '--encryption=repokey', self.repository_location) self.create_regular_file('dir/file', contents=b"test file contents 1") From 1e520203cb3cec7d7d387df161f829b0ee6240ab Mon Sep 17 00:00:00 2001 From: Milkey Mouse Date: Tue, 21 Nov 2017 11:41:36 -0800 Subject: [PATCH 2/2] Fix borg config flake8 failures Suppressed E731 so lambdas can be assigned to vars --- setup.cfg | 2 +- src/borg/testsuite/archiver.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.cfg b/setup.cfg index 8e9414ac6..40ca52dba 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,7 +5,7 @@ python_files = testsuite/*.py # please note that the values are adjusted so that they do not cause failures # with existing code. if you want to change them, you should first fix all # flake8 failures that appear with your change. -ignore = E122,E123,E125,E126,E127,E128,E226,E402,E722,E741,F401,F405,F811 +ignore = E122,E123,E125,E126,E127,E128,E226,E402,E722,E731,E741,F401,F405,F811 # line length long term target: 120 max-line-length = 255 exclude = build,dist,.git,.idea,.cache,.tox,docs/conf.py diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 48013080d..9014829a5 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -2714,13 +2714,13 @@ id: 2 / e29442 3506da 4e1ea7 / 25f62a 5a3d41 - 02 os.unlink('input/flagfile') self.cmd('init', '--encryption=repokey', self.repository_location) for flags in [[], ['--cache']]: - for key in {'testkey', 'testsection.testkey'}: - self.cmd('config', self.repository_location, *flags, key, exit_code=1) - self.cmd('config', self.repository_location, *flags, key, 'testcontents') - output = self.cmd('config', self.repository_location, *flags, key) + for cfg_key in {'testkey', 'testsection.testkey'}: + self.cmd('config', self.repository_location, *flags, cfg_key, exit_code=1) + self.cmd('config', self.repository_location, *flags, cfg_key, 'testcontents') + output = self.cmd('config', self.repository_location, *flags, cfg_key) assert output == 'testcontents\n' - self.cmd('config', self.repository_location, *flags, '--delete', key) - self.cmd('config', self.repository_location, *flags, key, exit_code=1) + self.cmd('config', self.repository_location, *flags, '--delete', cfg_key) + self.cmd('config', self.repository_location, *flags, cfg_key, exit_code=1) requires_gnutar = pytest.mark.skipif(not have_gnutar(), reason='GNU tar must be installed for this test.') requires_gzip = pytest.mark.skipif(not shutil.which('gzip'), reason='gzip must be installed for this test.')