From 4c1a920ed46b4a17c465fe43b9dccfa1bb68a230 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Wed, 5 Oct 2016 12:27:26 +0200 Subject: [PATCH 01/17] debug delete/get obj: fix wrong reference to exception iow RemoteRepository doesn't define this type, only Repository does. --- borg/archiver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index ce7655fc5..0b57e14c4 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -767,7 +767,7 @@ class Archiver: else: try: data = repository.get(id) - except repository.ObjectNotFound: + except Repository.ObjectNotFound: print("object %s not found." % hex_id) else: with open(args.path, "wb") as f: @@ -801,7 +801,7 @@ class Archiver: repository.delete(id) modified = True print("object %s deleted." % hex_id) - except repository.ObjectNotFound: + except Repository.ObjectNotFound: print("object %s not found." % hex_id) if modified: repository.commit() From e829e8372d0a9f862c4f048ac7fdc5dc8ad60366 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 13 Oct 2016 00:38:04 +0200 Subject: [PATCH 02/17] implement /./relpath hack, fixes #1655 --- borg/helpers.py | 16 +++++++++++----- borg/remote.py | 4 +++- borg/testsuite/helpers.py | 10 +++++++++- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index 14caeec3c..2cceb2af1 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -835,26 +835,32 @@ class Location: return True def _parse(self, text): + def normpath_special(p): + # avoid that normpath strips away our relative path hack and even makes p absolute + relative = p.startswith('/./') + p = os.path.normpath(p) + return ('/.' + p) if relative else p + m = self.ssh_re.match(text) if m: self.proto = m.group('proto') self.user = m.group('user') self.host = m.group('host') self.port = m.group('port') and int(m.group('port')) or None - self.path = os.path.normpath(m.group('path')) + self.path = normpath_special(m.group('path')) self.archive = m.group('archive') return True m = self.file_re.match(text) if m: self.proto = m.group('proto') - self.path = os.path.normpath(m.group('path')) + self.path = normpath_special(m.group('path')) self.archive = m.group('archive') return True m = self.scp_re.match(text) if m: self.user = m.group('user') self.host = m.group('host') - self.path = os.path.normpath(m.group('path')) + self.path = normpath_special(m.group('path')) self.archive = m.group('archive') self.proto = self.host and 'ssh' or 'file' return True @@ -885,9 +891,9 @@ class Location: return self.path else: if self.path and self.path.startswith('~'): - path = '/' + self.path + path = '/' + self.path # /~/x = path x relative to home dir elif self.path and not self.path.startswith('/'): - path = '/~/' + self.path + path = '/./' + self.path # /./x = path x relative to cwd else: path = self.path return 'ssh://{}{}{}{}'.format('{}@'.format(self.user) if self.user else '', diff --git a/borg/remote.py b/borg/remote.py index 20d0c1857..e04802314 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -129,8 +129,10 @@ class RepositoryServer: # pragma: no cover def open(self, path, create=False, lock_wait=None, lock=True, exclusive=None, append_only=False): path = os.fsdecode(path) - if path.startswith('/~'): + if path.startswith('/~'): # /~/x = path x relative to home dir, /~username/x = relative to "user" home dir path = path[1:] + elif path.startswith('/./'): # /./x = path x relative to cwd + path = path[3:] path = os.path.realpath(os.path.expanduser(path)) if self.restrict_to_paths: # if --restrict-to-path P is given, we make sure that we only operate in/below path P. diff --git a/borg/testsuite/helpers.py b/borg/testsuite/helpers.py index 10574f5a1..1087b552d 100644 --- a/borg/testsuite/helpers.py +++ b/borg/testsuite/helpers.py @@ -69,6 +69,8 @@ class TestLocationWithoutEnv: "Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive='archive')" assert repr(Location('/some/absolute/path')) == \ "Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive=None)" + assert repr(Location('ssh://user@host/some/path')) == \ + "Location(proto='ssh', user='user', host='host', port=None, path='/some/path', archive=None)" def test_relpath(self, monkeypatch): monkeypatch.delenv('BORG_REPO', raising=False) @@ -76,6 +78,12 @@ class TestLocationWithoutEnv: "Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive='archive')" assert repr(Location('some/relative/path')) == \ "Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive=None)" + assert repr(Location('ssh://user@host/./some/path')) == \ + "Location(proto='ssh', user='user', host='host', port=None, path='/./some/path', archive=None)" + assert repr(Location('ssh://user@host/~/some/path')) == \ + "Location(proto='ssh', user='user', host='host', port=None, path='/~/some/path', archive=None)" + assert repr(Location('ssh://user@host/~user/some/path')) == \ + "Location(proto='ssh', user='user', host='host', port=None, path='/~user/some/path', archive=None)" def test_with_colons(self, monkeypatch): monkeypatch.delenv('BORG_REPO', raising=False) @@ -107,7 +115,7 @@ class TestLocationWithoutEnv: 'ssh://user@host:1234/some/path::archive'] for location in locations: assert Location(location).canonical_path() == \ - Location(Location(location).canonical_path()).canonical_path() + Location(Location(location).canonical_path()).canonical_path(), "failed: %s" % location def test_format_path(self, monkeypatch): monkeypatch.delenv('BORG_REPO', raising=False) From 4ca0ed150b628a164c524e8df5fd58ed09d1c41f Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 15 Oct 2016 15:21:38 +0200 Subject: [PATCH 03/17] vagrant / openbsd: use own box, fixes #1695 self-made basic openbsd box includes pre-installed rsync as vagrant is unable to install rsync. as the self-made box now has enough space on /, remove workaround for low space. delete line from tox.ini with sed so it does not try to install llfuse (which is broken on openbsd). --- Vagrantfile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 7b09c375e..5d81b398a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -106,10 +106,6 @@ end def packages_openbsd return <<-EOF . ~/.profile - mkdir -p /home/vagrant/borg - rsync -aH /vagrant/borg/ /home/vagrant/borg/ - rm -rf /vagrant/borg - ln -sf /home/vagrant/borg /vagrant/ pkg_add bash chsh -s /usr/local/bin/bash vagrant pkg_add openssl @@ -121,6 +117,8 @@ def packages_openbsd easy_install-3.4 pip pip3 install virtualenv touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile + # avoid that breaking llfuse install breaks borgbackup install under tox: + sed -i.bak '/fuse.txt/d' /vagrant/borg/borg/tox.ini EOF end @@ -271,12 +269,12 @@ def run_tests(boxname) . ~/.bash_profile cd /vagrant/borg/borg . ../borg-env/bin/activate - if which pyenv > /dev/null; then + if which pyenv 2> /dev/null; then # for testing, use the earliest point releases of the supported python versions: pyenv global 3.4.0 3.5.0 fi # otherwise: just use the system python - if which fakeroot > /dev/null; then + if which fakeroot 2> /dev/null; then echo "Running tox WITH fakeroot -u" fakeroot -u tox --skip-missing-interpreters else @@ -440,7 +438,7 @@ Vagrant.configure(2) do |config| end config.vm.define "openbsd64" do |b| - b.vm.box = "kaorimatz/openbsd-5.9-amd64" + b.vm.box = "openbsd60-64" # note: basic openbsd install for vagrant WITH sudo and rsync pre-installed b.vm.provider :virtualbox do |v| v.memory = 768 end From 63ff002b19fd71697a6341db3ebe92a20387406b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 17 Oct 2016 02:03:52 +0200 Subject: [PATCH 04/17] vagrant / netbsd: use own box, fixes #1671, fixes #1728 self-made basic netbsd box includes pre-installed rsync as vagrant is unable to install rsync. delete line from tox.ini with sed so it does not try to install llfuse (which is broken on netbsd). --- Vagrantfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index 5d81b398a..d5e3bb57d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -144,6 +144,8 @@ def packages_netbsd easy_install-3.4 pip pip install virtualenv touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile + # fuse does not work good enough (see above), do not install llfuse: + sed -i.bak '/fuse.txt/d' /vagrant/borg/borg/tox.ini EOF end @@ -449,7 +451,7 @@ Vagrant.configure(2) do |config| end config.vm.define "netbsd64" do |b| - b.vm.box = "alex-skimlinks/netbsd-6.1.5-amd64" + b.vm.box = "netbsd70-64" b.vm.provider :virtualbox do |v| v.memory = 768 end From 498e71846c92bc938cd49fcbd5d03e95e9f6bafb Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 17 Oct 2016 03:16:13 +0200 Subject: [PATCH 05/17] add borg debug refcount-obj ID subcommand --- borg/archiver.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/borg/archiver.py b/borg/archiver.py index fb75f8dd9..ad822452f 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -808,6 +808,22 @@ class Archiver: print('Done.') return EXIT_SUCCESS + @with_repository(manifest=False, exclusive=True, cache=True) + def do_debug_refcount_obj(self, args, repository, manifest, key, cache): + """display refcounts for the objects with the given IDs""" + for hex_id in args.ids: + try: + id = unhexlify(hex_id) + except ValueError: + print("object id %s is invalid." % hex_id) + else: + try: + refcount = cache.chunks[id][0] + print("object %s has %d referrers [info from chunks cache]." % (hex_id, refcount)) + except KeyError: + print("object %s not found [info from chunks cache]." % hex_id) + return EXIT_SUCCESS + @with_repository(lock=False, manifest=False) def do_break_lock(self, args, repository): """Break the repository lock (e.g. in case it was left by a dead borg.""" @@ -1782,6 +1798,33 @@ class Archiver: subparser.add_argument('ids', metavar='IDs', nargs='+', type=str, help='hex object ID(s) to delete from the repo') + debug_refcount_obj_epilog = textwrap.dedent(""" + This command displays the reference count for objects from the repository. + """) + subparser = subparsers.add_parser('debug-refcount-obj', parents=[common_parser], + description=self.do_debug_refcount_obj.__doc__, + epilog=debug_refcount_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='show refcount for object from repository (debug)') + subparser.set_defaults(func=self.do_debug_refcount_obj) + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False), + help='repository to use') + subparser.add_argument('ids', metavar='IDs', nargs='+', type=str, + help='hex object ID(s) to show refcounts for') + + subparser = debug_parsers.add_parser('refcount-obj', parents=[common_parser], + description=self.do_debug_refcount_obj.__doc__, + epilog=debug_refcount_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='show refcount for object from repository (debug)') + subparser.set_defaults(func=self.do_debug_refcount_obj) + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False), + help='repository to use') + subparser.add_argument('ids', metavar='IDs', nargs='+', type=str, + help='hex object ID(s) to show refcounts for') + return parser def get_args(self, argv, cmd): From 4d7af95ad23db3be40b36a856e57172b1e0f1743 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 17 Oct 2016 02:14:19 +0200 Subject: [PATCH 06/17] update 1.0 CHANGES --- docs/changes.rst | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index b1c8a1ba5..8f8dc9a87 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -50,8 +50,8 @@ The best check that everything is ok is to run a dry-run extraction:: borg extract -v --dry-run REPO::ARCHIVE -Version 1.0.8rc1 (not released yet) ------------------------------------ +Version 1.0.8rc1 (2016-10-17) +----------------------------- Bug fixes: @@ -63,15 +63,22 @@ Bug fixes: also correctly processes broken symlinks. before this regressed to a crash (5b45385) a broken symlink would've been skipped. - process_symlink: fix missing backup_io() - Fixes a chmod/chown/chgrp/unlink/rename/... crash race between getting dirents - and dispatching to process_symlink. -- yes(): abort on wrong answers, saying so + Fixes a chmod/chown/chgrp/unlink/rename/... crash race between getting + dirents and dispatching to process_symlink. +- yes(): abort on wrong answers, saying so, #1622 - fixed exception borg serve raised when connection was closed before reposiory was openend. add an error message for this. - fix read-from-closed-FD issue, #1551 (this seems not to get triggered in 1.0.x, but was discovered in master) - hashindex: fix iterators (always raise StopIteration when exhausted) (this seems not to get triggered in 1.0.x, but was discovered in master) +- enable relative pathes in ssh:// repo URLs, via /./relpath hack, fixes #1655 +- allow repo pathes with colons, fixes #1705 +- update changed repo location immediately after acceptance, #1524 +- fix debug get-obj / delete-obj crash if object not found and remote repo, + #1684 +- pyinstaller: use a spec file to build borg.exe binary, exclude osxfuse dylib + on Mac OS X (avoids mismatch lib <-> driver), #1619 New features: @@ -82,6 +89,8 @@ New features: special "paper" format with by line checksums for printed backups. For the paper format, the import is an interactive process which checks each line as soon as it is input. +- add "borg debug-refcount-obj" to determine a repo objects' referrer counts, + #1352 Other changes: @@ -90,10 +99,19 @@ Other changes: - setup.py: Add subcommand support to build_usage. - remote: change exception message for unexpected RPC data format to indicate dataflow direction. -- vagrant: +- improved messages / error reporting: + + - IntegrityError: add placeholder for message, so that the message we give + appears not only in the traceback, but also in the (short) error message, + #1572 + - borg.key: include chunk id in exception msgs, #1571 + - better messages for cache newer than repo, fixes #1700 +- vagrant (testing/build VMs): - upgrade OSXfuse / FUSE for macOS to 3.5.2 - - update Debian Wheezy boxes to 7.11 + - update Debian Wheezy boxes, #1686 + - openbsd / netbsd: use own boxes, fixes misc rsync installation and + fuse/llfuse related testing issues, #1695 #1696 #1670 #1671 #1728 - docs: - add docs for "key export" and "key import" commands, #1641 @@ -109,12 +127,17 @@ Other changes: - add debug-info usage help file - internals.rst: fix typos - setup.py: fix build_usage to always process all commands + - added docs explaining multiple --restrict-to-path flags, #1602 + - add more specific warning about write-access debug commands, #1587 + - clarify FAQ regarding backup of virtual machines, #1672 - tests: - work around fuse xattr test issue with recent fakeroot - simplify repo/hashindex tests - travis: test fuse-enabled borg, use trusty to have a recent FUSE - re-enable fuse tests for RemoteArchiver (no deadlocks any more) + - clean env for pytest based tests, #1714 + - fuse_mount contextmanager: accept any options Version 1.0.7 (2016-08-19) From b3133f6394230662f5cffd55c294908dc750777a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 17 Oct 2016 03:36:49 +0200 Subject: [PATCH 07/17] update api.rst --- docs/api.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/api.rst b/docs/api.rst index 8dfc8cce0..89a95304a 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -27,6 +27,10 @@ API Documentation :members: :undoc-members: +.. automodule:: borg.keymanager + :members: + :undoc-members: + .. automodule:: borg.logger :members: :undoc-members: From 3550b24ed99807d5dbad2ec28d2f2e8f72933461 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 17 Oct 2016 03:39:38 +0200 Subject: [PATCH 08/17] ran build_usage --- docs/usage/info.rst.inc | 9 +++++++++ docs/usage/key_export.rst.inc | 10 +++++----- docs/usage/serve.rst.inc | 6 +++++- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/docs/usage/info.rst.inc b/docs/usage/info.rst.inc index 68c9194ec..cd745b531 100644 --- a/docs/usage/info.rst.inc +++ b/docs/usage/info.rst.inc @@ -35,3 +35,12 @@ Description ~~~~~~~~~~~ This command displays some detailed information about the specified archive. + +Please note that the deduplicated sizes of the individual archives do not add +up to the deduplicated size of the repository ("all archives"), because the two +are meaning different things: + +This archive / deduplicated size = amount of data stored ONLY for this archive + = unique chunks of this archive. +All archives / deduplicated size = amount of data stored in the repo + = all chunks in the repository. diff --git a/docs/usage/key_export.rst.inc b/docs/usage/key_export.rst.inc index 47e9e119e..a87c8192d 100644 --- a/docs/usage/key_export.rst.inc +++ b/docs/usage/key_export.rst.inc @@ -42,14 +42,14 @@ If repository encryption is used, the repository is inaccessible without the key. This command allows to backup this essential key. There are two backup formats. The normal backup format is suitable for -digital storage as a file. The ``--paper`` backup format is optimized for -print out and later type-in, with per line checks to reduce problems -with manual input. +digital storage as a file. The ``--paper`` backup format is optimized +for printing and typing in while importing, with per line checks to +reduce problems with manual input. For repositories using keyfile encryption the key is saved locally on the system that is capable of doing backups. To guard against loss -of this key the key needs to be backed up independent of the main data -backup. +of this key, the key needs to be backed up independently of the main +data backup. For repositories using the repokey encryption the key is saved in the repository in the config file. A backup is thus not strictly needed, diff --git a/docs/usage/serve.rst.inc b/docs/usage/serve.rst.inc index eb57a975b..48f97074c 100644 --- a/docs/usage/serve.rst.inc +++ b/docs/usage/serve.rst.inc @@ -29,7 +29,11 @@ borg serve --umask M set umask to M (local and remote, default: 0077) --remote-path PATH set remote path to executable (default: "borg") --restrict-to-path PATH - restrict repository access to PATH + restrict repository access to PATH. Can be specified + multiple times to allow the client access to several + directories. Access to all sub-directories is granted + implicitly; PATH doesn't need to directly point to a + repository. --append-only only allow appending to repository segment files Description From 57402b120bdc51e8b574010a01aaa6d72abf8c16 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 17 Oct 2016 17:37:33 +0200 Subject: [PATCH 09/17] vagrant: no chown when rsyncing it tries to do chown -R vagrant.vagrant, but some boxes do not have a vagrant group and break there. also, we do our own chown in the provisioning scripts. --- Vagrantfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index d5e3bb57d..53ea02c3b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -301,7 +301,7 @@ end Vagrant.configure(2) do |config| # use rsync to copy content to the folder - config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"] + config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"], :rsync__chown => false # do not let the VM access . on the host machine via the default shared folder! config.vm.synced_folder ".", "/vagrant", disabled: true From f10700841faaea976300dfc48cfef0e985935ad2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 19 Oct 2016 00:56:22 +0200 Subject: [PATCH 10/17] add comment about requiring llfuse, fixes #1726 --- requirements.d/fuse.txt | 2 +- setup.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/requirements.d/fuse.txt b/requirements.d/fuse.txt index be35d2ae9..0df0f3381 100644 --- a/requirements.d/fuse.txt +++ b/requirements.d/fuse.txt @@ -1,4 +1,4 @@ # low-level FUSE support library for "borg mount" -# see comments setup.py about this version requirement. +# please see the comments in setup.py about llfuse. llfuse<2.0 diff --git a/setup.py b/setup.py index 8ba33e73b..ffb9aae9e 100644 --- a/setup.py +++ b/setup.py @@ -21,12 +21,17 @@ on_rtd = os.environ.get('READTHEDOCS') # Also, we might use some rather recent API features. install_requires = ['msgpack-python>=0.4.6', ] +# note for package maintainers: if you package borgbackup for distribution, +# please add llfuse as a *requirement* on all platforms that have a working +# llfuse package. "borg mount" needs llfuse to work. +# if you do not have llfuse, do not require it, most of borgbackup will work. extras_require = { # llfuse 0.40 (tested, proven, ok), needs FUSE version >= 2.8.0 # llfuse 0.41 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 0.41.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0 + # llfuse 1.1.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 2.0 will break API 'fuse': ['llfuse<2.0', ], } From c000eb8083f07c1143e1dcae2ef1514c725fac96 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 19 Oct 2016 01:13:44 +0200 Subject: [PATCH 11/17] add clarification about append-only mode, fixes #1689 --- docs/usage.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/usage.rst b/docs/usage.rst index 0452c164a..d9fcc3a22 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -800,9 +800,14 @@ That's all to it. Drawbacks +++++++++ -As data is only appended, and nothing deleted, commands like ``prune`` or ``delete`` +As data is only appended, and nothing removed, commands like ``prune`` or ``delete`` won't free disk space, they merely tag data as deleted in a new transaction. +Be aware that as soon as you write to the repo in non-append-only mode (e.g. prune, +delete or create archives from an admin machine), it will remove the deleted objects +permanently (including the ones that were already marked as deleted, but not removed, +in append-only mode). + Note that you can go back-and-forth between normal and append-only operation by editing the configuration file, it's not a "one way trip". From ade405eae012629b397cdc0c88f2044ee64f9c31 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 19 Oct 2016 01:51:25 +0200 Subject: [PATCH 12/17] ignore security.selinux xattrs, fixes #1735 they fail the FUSE tests on centos7. --- borg/testsuite/__init__.py | 13 +++++++++++-- borg/testsuite/archiver.py | 4 ++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/borg/testsuite/__init__.py b/borg/testsuite/__init__.py index 6b85538b4..8e09771d7 100644 --- a/borg/testsuite/__init__.py +++ b/borg/testsuite/__init__.py @@ -35,6 +35,15 @@ if sys.platform.startswith('netbsd'): setup_logging() +def no_selinux(x): + # selinux fails our FUSE tests, thus ignore selinux xattrs + SELINUX_KEY = 'security.selinux' + if isinstance(x, dict): + return {k: v for k, v in x.items() if k != SELINUX_KEY} + if isinstance(x, list): + return [k for k in x if k != SELINUX_KEY] + + class BaseTestCase(unittest.TestCase): """ """ @@ -87,8 +96,8 @@ class BaseTestCase(unittest.TestCase): else: d1.append(round(s1.st_mtime_ns, st_mtime_ns_round)) d2.append(round(s2.st_mtime_ns, st_mtime_ns_round)) - d1.append(get_all(path1, follow_symlinks=False)) - d2.append(get_all(path2, follow_symlinks=False)) + d1.append(no_selinux(get_all(path1, follow_symlinks=False))) + d2.append(no_selinux(get_all(path2, follow_symlinks=False))) self.assert_equal(d1, d2) for sub_diff in diff.subdirs.values(): self._assert_dirs_equal_cmp(sub_diff) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 8508639e0..4a470c2bf 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -26,7 +26,7 @@ from ..key import RepoKey, KeyfileKey, Passphrase from ..keymanager import RepoIdMismatch, NotABorgKeyFile from ..remote import RemoteRepository, PathNotAllowed from ..repository import Repository -from . import BaseTestCase, changedir, environment_variable +from . import BaseTestCase, changedir, environment_variable, no_selinux try: import llfuse @@ -1067,7 +1067,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): in_fn = 'input/fusexattr' out_fn = os.path.join(mountpoint, 'input', 'fusexattr') if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): - assert xattr.listxattr(out_fn) == ['user.foo', ] + assert no_selinux(xattr.listxattr(out_fn)) == ['user.foo', ] assert xattr.getxattr(out_fn, 'user.foo') == b'bar' else: assert xattr.listxattr(out_fn) == [] From 2b90e45dd18d204e57c7973138e1384a5f1bcde3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 4 Sep 2016 03:01:29 +0200 Subject: [PATCH 13/17] vagrant: fix fuse permission issues on linux/freebsd, fixes #1544 --- Vagrantfile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Vagrantfile b/Vagrantfile index 53ea02c3b..cf440fb5a 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -25,6 +25,8 @@ def packages_debianoid # for building borgbackup and dependencies: apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config usermod -a -G fuse $username + chgrp fuse /dev/fuse + chmod 666 /dev/fuse apt-get install -y fakeroot build-essential git apt-get install -y python3-dev python3-setuptools # for building python: @@ -45,6 +47,8 @@ def packages_redhatted # for building borgbackup and dependencies: yum install -y openssl-devel openssl libacl-devel libacl lz4-devel fuse-devel fuse pkgconfig usermod -a -G fuse vagrant + chgrp fuse /dev/fuse + chmod 666 /dev/fuse yum install -y fakeroot gcc git patch # needed to compile msgpack-python (otherwise it will use slow fallback code): yum install -y gcc-c++ @@ -96,6 +100,8 @@ def packages_freebsd kldload fuse sysctl vfs.usermount=1 pw groupmod operator -M vagrant + # /dev/fuse has group operator + chmod 666 /dev/fuse touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile # install all the (security and other) updates, packages pkg update From f9aa74e7e1a5baebc105a35323dd0c446dd56032 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 4 Sep 2016 04:32:16 +0200 Subject: [PATCH 14/17] skip fuse test for borg binary + fakeroot strange: works on wheezy, blows up on xenial --- borg/testsuite/archiver.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 4a470c2bf..ee68f61b1 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -27,6 +27,7 @@ from ..keymanager import RepoIdMismatch, NotABorgKeyFile from ..remote import RemoteRepository, PathNotAllowed from ..repository import Repository from . import BaseTestCase, changedir, environment_variable, no_selinux +from .platform import fakeroot_detected try: import llfuse @@ -1316,6 +1317,12 @@ class ArchiverTestCaseBinary(ArchiverTestCase): def test_overwrite(self): pass + def test_fuse(self): + if fakeroot_detected(): + unittest.skip('test_fuse with the binary is not compatible with fakeroot') + else: + super().test_fuse() + class ArchiverCheckTestCase(ArchiverTestCaseBase): From d49a782796413e4b1492f73b3dd213fd696fefe6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 21 Oct 2016 03:43:38 +0200 Subject: [PATCH 15/17] implement borgmajor/minor/patch placeholders, fixes #1694 --- borg/__init__.py | 10 +++++++++- borg/archiver.py | 18 +++++++++++++++--- borg/helpers.py | 4 ++++ 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/borg/__init__.py b/borg/__init__.py index e292841a6..2dcf95316 100644 --- a/borg/__init__.py +++ b/borg/__init__.py @@ -1,3 +1,11 @@ -# This is a python package +import re from ._version import version as __version__ + +version_re = r'(\d+)\.(\d+)\.(\d+)' + +m = re.match(version_re, __version__) +if m: + __version_tuple__ = tuple(map(int, m.groups())) +else: + raise RuntimeError("Can't parse __version__: %r" % __version__) diff --git a/borg/archiver.py b/borg/archiver.py index ad822452f..f417c245f 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -949,7 +949,19 @@ class Archiver: {borgversion} - The version of borg. + The version of borg, e.g.: 1.0.8rc1 + + {borgmajor} + + The version of borg, only the major version, e.g.: 1 + + {borgminor} + + The version of borg, only major and minor version, e.g.: 1.0 + + {borgpatch} + + The version of borg, only major, minor and patch version, e.g.: 1.0.8 Examples:: @@ -1229,8 +1241,8 @@ class Archiver: '.checkpoint.N' (with N being a number), because these names are used for checkpoints and treated in special ways. - In the archive name, you may use the following format tags: - {now}, {utcnow}, {fqdn}, {hostname}, {user}, {pid}, {borgversion} + In the archive name, you may use the following placeholders: + {now}, {utcnow}, {fqdn}, {hostname}, {user} and some others. To speed up pulling backups over sshfs and similar network file systems which do not provide correct inode information the --ignore-inode flag can be used. This diff --git a/borg/helpers.py b/borg/helpers.py index 2cceb2af1..a4469c990 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -28,6 +28,7 @@ from fnmatch import translate from operator import attrgetter from . import __version__ as borg_version +from . import __version_tuple__ as borg_version_tuple from . import hashindex from . import chunker from . import crypto @@ -585,6 +586,9 @@ def replace_placeholders(text): 'utcnow': current_time.utcnow(), 'user': uid2user(os.getuid(), os.getuid()), 'borgversion': borg_version, + 'borgmajor': '%d' % borg_version_tuple[:1], + 'borgminor': '%d.%d' % borg_version_tuple[:2], + 'borgpatch': '%d.%d.%d' % borg_version_tuple[:3], } return format_line(text, data) From 22f77b908f31cd4d0de77a16a77f868843322f34 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 22 Oct 2016 00:20:50 +0200 Subject: [PATCH 16/17] implement borgmajor/minor/patch placeholders, fixes #1694 --- borg/__init__.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/borg/__init__.py b/borg/__init__.py index 2dcf95316..9ac0e0f90 100644 --- a/borg/__init__.py +++ b/borg/__init__.py @@ -1,11 +1,6 @@ -import re +from distutils.version import LooseVersion from ._version import version as __version__ -version_re = r'(\d+)\.(\d+)\.(\d+)' -m = re.match(version_re, __version__) -if m: - __version_tuple__ = tuple(map(int, m.groups())) -else: - raise RuntimeError("Can't parse __version__: %r" % __version__) +__version_tuple__ = tuple(LooseVersion(__version__).version[:3]) From 88f7f8673dacc5d00ffdd34f8e17f5b19a7e938b Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Mon, 24 Oct 2016 01:09:09 +0200 Subject: [PATCH 17/17] docs/deployment.rst: do not use bare variables in ansible snippet The current snippet throws deprecation warnings: ``` [DEPRECATION WARNING]: Using bare variables is deprecated. Update your playbooks so that the environment value uses the full variable syntax ('{{auth_users}}'). This feature will be removed in a future release. Deprecation warnings can be disabled by setting deprecation_warnings=False in ansible.cfg. ``` --- docs/deployment.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/deployment.rst b/docs/deployment.rst index c73c6ddb2..9586c72a2 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -149,10 +149,10 @@ package manager to install and keep borg up-to-date. - authorized_key: user="{{ user }}" key="{{ item.key }}" key_options='command="cd {{ pool }}/{{ item.host }};borg serve --restrict-to-path {{ pool }}/{{ item.host }}",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc' - with_items: auth_users + with_items: "{{ auth_users }}" - file: path="{{ home }}/.ssh/authorized_keys" owner="{{ user }}" group="{{ group }}" mode=0600 state=file - file: path="{{ pool }}/{{ item.host }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory - with_items: auth_users + with_items: "{{ auth_users }}" Enhancements ------------