diff --git a/Vagrantfile b/Vagrantfile index 84e7a91d7..4c4303f0d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -25,6 +25,8 @@ def packages_debianoid # for building borgbackup and dependencies: apt-get install -y libssl-dev libacl1-dev liblz4-dev libfuse-dev fuse pkg-config usermod -a -G fuse $username + chgrp fuse /dev/fuse + chmod 666 /dev/fuse apt-get install -y fakeroot build-essential git apt-get install -y python3-dev python3-setuptools # for building python: @@ -45,6 +47,8 @@ def packages_redhatted # for building borgbackup and dependencies: yum install -y openssl-devel openssl libacl-devel libacl lz4-devel fuse-devel fuse pkgconfig usermod -a -G fuse vagrant + chgrp fuse /dev/fuse + chmod 666 /dev/fuse yum install -y fakeroot gcc git patch # needed to compile msgpack-python (otherwise it will use slow fallback code): yum install -y gcc-c++ @@ -96,6 +100,8 @@ def packages_freebsd kldload fuse sysctl vfs.usermount=1 pw groupmod operator -M vagrant + # /dev/fuse has group operator + chmod 666 /dev/fuse touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile # install all the (security and other) updates, packages pkg update @@ -106,10 +112,6 @@ end def packages_openbsd return <<-EOF . ~/.profile - mkdir -p /home/vagrant/borg - rsync -aH /vagrant/borg/ /home/vagrant/borg/ - rm -rf /vagrant/borg - ln -sf /home/vagrant/borg /vagrant/ pkg_add bash chsh -s /usr/local/bin/bash vagrant pkg_add openssl @@ -121,6 +123,8 @@ def packages_openbsd easy_install-3.4 pip pip3 install virtualenv touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile + # avoid that breaking llfuse install breaks borgbackup install under tox: + sed -i.bak '/fuse.txt/d' /vagrant/borg/borg/tox.ini EOF end @@ -146,6 +150,8 @@ def packages_netbsd easy_install-3.4 pip pip install virtualenv touch ~vagrant/.bash_profile ; chown vagrant ~vagrant/.bash_profile + # fuse does not work good enough (see above), do not install llfuse: + sed -i.bak '/fuse.txt/d' /vagrant/borg/borg/tox.ini EOF end @@ -273,13 +279,13 @@ def run_tests(boxname) . ~/.bash_profile cd /vagrant/borg/borg . ../borg-env/bin/activate - if which pyenv > /dev/null; then + if which pyenv 2> /dev/null; then # for testing, use the earliest point releases of the supported python versions: pyenv global 3.4.0 3.5.0 pyenv local 3.4.0 3.5.0 fi # otherwise: just use the system python - if which fakeroot > /dev/null; then + if which fakeroot 2> /dev/null; then echo "Running tox WITH fakeroot -u" fakeroot -u tox --skip-missing-interpreters else @@ -304,7 +310,7 @@ end Vagrant.configure(2) do |config| # use rsync to copy content to the folder - config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"] + config.vm.synced_folder ".", "/vagrant/borg/borg", :type => "rsync", :rsync__args => ["--verbose", "--archive", "--delete", "-z"], :rsync__chown => false # do not let the VM access . on the host machine via the default shared folder! config.vm.synced_folder ".", "/vagrant", disabled: true @@ -443,7 +449,7 @@ Vagrant.configure(2) do |config| end config.vm.define "openbsd64" do |b| - b.vm.box = "kaorimatz/openbsd-5.9-amd64" + b.vm.box = "openbsd60-64" # note: basic openbsd install for vagrant WITH sudo and rsync pre-installed b.vm.provider :virtualbox do |v| v.memory = 768 end @@ -454,7 +460,7 @@ Vagrant.configure(2) do |config| end config.vm.define "netbsd64" do |b| - b.vm.box = "alex-skimlinks/netbsd-6.1.5-amd64" + b.vm.box = "netbsd70-64" b.vm.provider :virtualbox do |v| v.memory = 768 end diff --git a/docs/api.rst b/docs/api.rst index 7ce030192..9fd1a4928 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -26,6 +26,10 @@ API Documentation :members: :undoc-members: +.. automodule:: borg.keymanager + :members: + :undoc-members: + .. automodule:: borg.nonces :members: :undoc-members: diff --git a/docs/changes.rst b/docs/changes.rst index dbeb110b8..edcdcfd5e 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -218,8 +218,8 @@ Other changes: - ChunkBuffer: add test for leaving partial chunk in buffer, fixes #945 -Version 1.0.8rc1 (not released yet) ------------------------------------ +Version 1.0.8rc1 (2016-10-17) +----------------------------- Bug fixes: @@ -231,15 +231,22 @@ Bug fixes: also correctly processes broken symlinks. before this regressed to a crash (5b45385) a broken symlink would've been skipped. - process_symlink: fix missing backup_io() - Fixes a chmod/chown/chgrp/unlink/rename/... crash race between getting dirents - and dispatching to process_symlink. -- yes(): abort on wrong answers, saying so + Fixes a chmod/chown/chgrp/unlink/rename/... crash race between getting + dirents and dispatching to process_symlink. +- yes(): abort on wrong answers, saying so, #1622 - fixed exception borg serve raised when connection was closed before reposiory was openend. add an error message for this. - fix read-from-closed-FD issue, #1551 (this seems not to get triggered in 1.0.x, but was discovered in master) - hashindex: fix iterators (always raise StopIteration when exhausted) (this seems not to get triggered in 1.0.x, but was discovered in master) +- enable relative pathes in ssh:// repo URLs, via /./relpath hack, fixes #1655 +- allow repo pathes with colons, fixes #1705 +- update changed repo location immediately after acceptance, #1524 +- fix debug get-obj / delete-obj crash if object not found and remote repo, + #1684 +- pyinstaller: use a spec file to build borg.exe binary, exclude osxfuse dylib + on Mac OS X (avoids mismatch lib <-> driver), #1619 New features: @@ -250,6 +257,8 @@ New features: special "paper" format with by line checksums for printed backups. For the paper format, the import is an interactive process which checks each line as soon as it is input. +- add "borg debug-refcount-obj" to determine a repo objects' referrer counts, + #1352 Other changes: @@ -258,10 +267,19 @@ Other changes: - setup.py: Add subcommand support to build_usage. - remote: change exception message for unexpected RPC data format to indicate dataflow direction. -- vagrant: +- improved messages / error reporting: + + - IntegrityError: add placeholder for message, so that the message we give + appears not only in the traceback, but also in the (short) error message, + #1572 + - borg.key: include chunk id in exception msgs, #1571 + - better messages for cache newer than repo, fixes #1700 +- vagrant (testing/build VMs): - upgrade OSXfuse / FUSE for macOS to 3.5.2 - - update Debian Wheezy boxes to 7.11 + - update Debian Wheezy boxes, #1686 + - openbsd / netbsd: use own boxes, fixes misc rsync installation and + fuse/llfuse related testing issues, #1695 #1696 #1670 #1671 #1728 - docs: - add docs for "key export" and "key import" commands, #1641 @@ -277,12 +295,17 @@ Other changes: - add debug-info usage help file - internals.rst: fix typos - setup.py: fix build_usage to always process all commands + - added docs explaining multiple --restrict-to-path flags, #1602 + - add more specific warning about write-access debug commands, #1587 + - clarify FAQ regarding backup of virtual machines, #1672 - tests: - work around fuse xattr test issue with recent fakeroot - simplify repo/hashindex tests - travis: test fuse-enabled borg, use trusty to have a recent FUSE - re-enable fuse tests for RemoteArchiver (no deadlocks any more) + - clean env for pytest based tests, #1714 + - fuse_mount contextmanager: accept any options Version 1.0.7 (2016-08-19) diff --git a/docs/deployment.rst b/docs/deployment.rst index 010820e8e..d57132fd8 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -149,10 +149,10 @@ package manager to install and keep borg up-to-date. - authorized_key: user="{{ user }}" key="{{ item.key }}" key_options='command="cd {{ pool }}/{{ item.host }};borg serve --restrict-to-path {{ pool }}/{{ item.host }}",no-port-forwarding,no-X11-forwarding,no-pty,no-agent-forwarding,no-user-rc' - with_items: auth_users + with_items: "{{ auth_users }}" - file: path="{{ home }}/.ssh/authorized_keys" owner="{{ user }}" group="{{ group }}" mode=0600 state=file - file: path="{{ pool }}/{{ item.host }}" owner="{{ user }}" group="{{ group }}" mode=0700 state=directory - with_items: auth_users + with_items: "{{ auth_users }}" Salt ---- diff --git a/docs/usage.rst b/docs/usage.rst index 54bda11d7..a2442b9ab 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -897,9 +897,14 @@ That's all to it. Drawbacks +++++++++ -As data is only appended, and nothing deleted, commands like ``prune`` or ``delete`` +As data is only appended, and nothing removed, commands like ``prune`` or ``delete`` won't free disk space, they merely tag data as deleted in a new transaction. +Be aware that as soon as you write to the repo in non-append-only mode (e.g. prune, +delete or create archives from an admin machine), it will remove the deleted objects +permanently (including the ones that were already marked as deleted, but not removed, +in append-only mode). + Note that you can go back-and-forth between normal and append-only operation by editing the configuration file, it's not a "one way trip". diff --git a/requirements.d/fuse.txt b/requirements.d/fuse.txt index be35d2ae9..0df0f3381 100644 --- a/requirements.d/fuse.txt +++ b/requirements.d/fuse.txt @@ -1,4 +1,4 @@ # low-level FUSE support library for "borg mount" -# see comments setup.py about this version requirement. +# please see the comments in setup.py about llfuse. llfuse<2.0 diff --git a/setup.py b/setup.py index a1b4b1b2c..02dd18863 100644 --- a/setup.py +++ b/setup.py @@ -23,12 +23,17 @@ on_rtd = os.environ.get('READTHEDOCS') # Also, we might use some rather recent API features. install_requires = ['msgpack-python>=0.4.6', ] +# note for package maintainers: if you package borgbackup for distribution, +# please add llfuse as a *requirement* on all platforms that have a working +# llfuse package. "borg mount" needs llfuse to work. +# if you do not have llfuse, do not require it, most of borgbackup will work. extras_require = { # llfuse 0.40 (tested, proven, ok), needs FUSE version >= 2.8.0 # llfuse 0.41 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 0.41.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 0.42 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 1.0 (tested shortly, looks ok), needs FUSE version >= 2.8.0 + # llfuse 1.1.1 (tested shortly, looks ok), needs FUSE version >= 2.8.0 # llfuse 2.0 will break API 'fuse': ['llfuse<2.0', ], } diff --git a/src/borg/__init__.py b/src/borg/__init__.py index e292841a6..9ac0e0f90 100644 --- a/src/borg/__init__.py +++ b/src/borg/__init__.py @@ -1,3 +1,6 @@ -# This is a python package +from distutils.version import LooseVersion from ._version import version as __version__ + + +__version_tuple__ = tuple(LooseVersion(__version__).version[:3]) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index bc0f5ae57..785ef668f 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1176,7 +1176,7 @@ class Archiver: else: try: data = repository.get(id) - except repository.ObjectNotFound: + except Repository.ObjectNotFound: print("object %s not found." % hex_id) else: with open(args.path, "wb") as f: @@ -1210,13 +1210,29 @@ class Archiver: repository.delete(id) modified = True print("object %s deleted." % hex_id) - except repository.ObjectNotFound: + except Repository.ObjectNotFound: print("object %s not found." % hex_id) if modified: repository.commit() print('Done.') return EXIT_SUCCESS + @with_repository(manifest=False, exclusive=True, cache=True) + def do_debug_refcount_obj(self, args, repository, manifest, key, cache): + """display refcounts for the objects with the given IDs""" + for hex_id in args.ids: + try: + id = unhexlify(hex_id) + except ValueError: + print("object id %s is invalid." % hex_id) + else: + try: + refcount = cache.chunks[id][0] + print("object %s has %d referrers [info from chunks cache]." % (hex_id, refcount)) + except KeyError: + print("object %s not found [info from chunks cache]." % hex_id) + return EXIT_SUCCESS + @with_repository(lock=False, manifest=False) def do_break_lock(self, args, repository): """Break the repository lock (e.g. in case it was left by a dead borg.""" @@ -1344,7 +1360,19 @@ class Archiver: {borgversion} - The version of borg. + The version of borg, e.g.: 1.0.8rc1 + + {borgmajor} + + The version of borg, only the major version, e.g.: 1 + + {borgminor} + + The version of borg, only major and minor version, e.g.: 1.0 + + {borgpatch} + + The version of borg, only major, minor and patch version, e.g.: 1.0.8 Examples:: @@ -1777,8 +1805,8 @@ class Archiver: '.checkpoint.N' (with N being a number), because these names are used for checkpoints and treated in special ways. - In the archive name, you may use the following format tags: - {now}, {utcnow}, {fqdn}, {hostname}, {user}, {pid}, {uuid4}, {borgversion} + In the archive name, you may use the following placeholders: + {now}, {utcnow}, {fqdn}, {hostname}, {user} and some others. To speed up pulling backups over sshfs and similar network file systems which do not provide correct inode information the --ignore-inode flag can be used. This @@ -2541,6 +2569,21 @@ class Archiver: subparser.add_argument('ids', metavar='IDs', nargs='+', type=str, help='hex object ID(s) to delete from the repo') + debug_refcount_obj_epilog = textwrap.dedent(""" + This command displays the reference count for objects from the repository. + """) + subparser = debug_parsers.add_parser('refcount-obj', parents=[common_parser], add_help=False, + description=self.do_debug_refcount_obj.__doc__, + epilog=debug_refcount_obj_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='show refcount for object from repository (debug)') + subparser.set_defaults(func=self.do_debug_refcount_obj) + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False), + help='repository to use') + subparser.add_argument('ids', metavar='IDs', nargs='+', type=str, + help='hex object ID(s) to show refcounts for') + return parser @staticmethod diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 42d96b166..875d708fd 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -34,6 +34,7 @@ from .logger import create_logger logger = create_logger() from . import __version__ as borg_version +from . import __version_tuple__ as borg_version_tuple from . import chunker from . import crypto from . import hashindex @@ -664,6 +665,9 @@ def replace_placeholders(text): 'user': uid2user(os.getuid(), os.getuid()), 'uuid4': str(uuid.uuid4()), 'borgversion': borg_version, + 'borgmajor': '%d' % borg_version_tuple[:1], + 'borgminor': '%d.%d' % borg_version_tuple[:2], + 'borgpatch': '%d.%d.%d' % borg_version_tuple[:3], } return format_line(text, data) @@ -945,26 +949,32 @@ class Location: return True def _parse(self, text): + def normpath_special(p): + # avoid that normpath strips away our relative path hack and even makes p absolute + relative = p.startswith('/./') + p = os.path.normpath(p) + return ('/.' + p) if relative else p + m = self.ssh_re.match(text) if m: self.proto = m.group('proto') self.user = m.group('user') self.host = m.group('host') self.port = m.group('port') and int(m.group('port')) or None - self.path = os.path.normpath(m.group('path')) + self.path = normpath_special(m.group('path')) self.archive = m.group('archive') return True m = self.file_re.match(text) if m: self.proto = m.group('proto') - self.path = os.path.normpath(m.group('path')) + self.path = normpath_special(m.group('path')) self.archive = m.group('archive') return True m = self.scp_re.match(text) if m: self.user = m.group('user') self.host = m.group('host') - self.path = os.path.normpath(m.group('path')) + self.path = normpath_special(m.group('path')) self.archive = m.group('archive') self.proto = self.host and 'ssh' or 'file' return True @@ -995,9 +1005,9 @@ class Location: return self.path else: if self.path and self.path.startswith('~'): - path = '/' + self.path + path = '/' + self.path # /~/x = path x relative to home dir elif self.path and not self.path.startswith('/'): - path = '/~/' + self.path + path = '/./' + self.path # /./x = path x relative to cwd else: path = self.path return 'ssh://{}{}{}{}'.format('{}@'.format(self.user) if self.user else '', diff --git a/src/borg/remote.py b/src/borg/remote.py index 824161f04..a4988eb91 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -149,8 +149,10 @@ class RepositoryServer: # pragma: no cover def open(self, path, create=False, lock_wait=None, lock=True, exclusive=None, append_only=False): path = os.fsdecode(path) - if path.startswith('/~'): - path = os.path.join(get_home_dir(), path[2:]) + if path.startswith('/~'): # /~/x = path x relative to home dir, /~username/x = relative to "user" home dir + path = os.path.join(get_home_dir(), path[2:]) # XXX check this (see also 1.0-maint), is it correct for ~u? + elif path.startswith('/./'): # /./x = path x relative to cwd + path = path[3:] path = os.path.realpath(path) if self.restrict_to_paths: # if --restrict-to-path P is given, we make sure that we only operate in/below path P. diff --git a/src/borg/testsuite/__init__.py b/src/borg/testsuite/__init__.py index a8e2a3b3f..ddcb443a0 100644 --- a/src/borg/testsuite/__init__.py +++ b/src/borg/testsuite/__init__.py @@ -117,6 +117,15 @@ def is_utime_fully_supported(): return False +def no_selinux(x): + # selinux fails our FUSE tests, thus ignore selinux xattrs + SELINUX_KEY = 'security.selinux' + if isinstance(x, dict): + return {k: v for k, v in x.items() if k != SELINUX_KEY} + if isinstance(x, list): + return [k for k in x if k != SELINUX_KEY] + + class BaseTestCase(unittest.TestCase): """ """ @@ -176,8 +185,8 @@ class BaseTestCase(unittest.TestCase): else: d1.append(round(s1.st_mtime_ns, st_mtime_ns_round)) d2.append(round(s2.st_mtime_ns, st_mtime_ns_round)) - d1.append(get_all(path1, follow_symlinks=False)) - d2.append(get_all(path2, follow_symlinks=False)) + d1.append(no_selinux(get_all(path1, follow_symlinks=False))) + d2.append(no_selinux(get_all(path2, follow_symlinks=False))) self.assert_equal(d1, d2) for sub_diff in diff.subdirs.values(): self._assert_dirs_equal_cmp(sub_diff) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index ecaa6be6d..d1db84ce9 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -39,8 +39,10 @@ from ..keymanager import RepoIdMismatch, NotABorgKeyFile from ..remote import RemoteRepository, PathNotAllowed from ..repository import Repository from . import has_lchflags, has_llfuse -from . import BaseTestCase, changedir, environment_variable +from . import BaseTestCase, changedir, environment_variable, no_selinux from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported +from .platform import fakeroot_detected + src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) @@ -1428,7 +1430,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): in_fn = 'input/fusexattr' out_fn = os.path.join(mountpoint, 'input', 'fusexattr') if not xattr.XATTR_FAKEROOT and xattr.is_enabled(self.input_path): - assert xattr.listxattr(out_fn) == ['user.foo', ] + assert no_selinux(xattr.listxattr(out_fn)) == ['user.foo', ] assert xattr.getxattr(out_fn, 'user.foo') == b'bar' else: assert xattr.listxattr(out_fn) == [] @@ -1988,6 +1990,12 @@ class ArchiverTestCaseBinary(ArchiverTestCase): def test_overwrite(self): pass + def test_fuse(self): + if fakeroot_detected(): + unittest.skip('test_fuse with the binary is not compatible with fakeroot') + else: + super().test_fuse() + class ArchiverCheckTestCase(ArchiverTestCaseBase): diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index e9d2f1865..bd4865f0d 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -84,6 +84,8 @@ class TestLocationWithoutEnv: "Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive='archive')" assert repr(Location('/some/absolute/path')) == \ "Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path', archive=None)" + assert repr(Location('ssh://user@host/some/path')) == \ + "Location(proto='ssh', user='user', host='host', port=None, path='/some/path', archive=None)" def test_relpath(self, monkeypatch): monkeypatch.delenv('BORG_REPO', raising=False) @@ -91,6 +93,12 @@ class TestLocationWithoutEnv: "Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive='archive')" assert repr(Location('some/relative/path')) == \ "Location(proto='file', user=None, host=None, port=None, path='some/relative/path', archive=None)" + assert repr(Location('ssh://user@host/./some/path')) == \ + "Location(proto='ssh', user='user', host='host', port=None, path='/./some/path', archive=None)" + assert repr(Location('ssh://user@host/~/some/path')) == \ + "Location(proto='ssh', user='user', host='host', port=None, path='/~/some/path', archive=None)" + assert repr(Location('ssh://user@host/~user/some/path')) == \ + "Location(proto='ssh', user='user', host='host', port=None, path='/~user/some/path', archive=None)" def test_with_colons(self, monkeypatch): monkeypatch.delenv('BORG_REPO', raising=False) @@ -122,7 +130,7 @@ class TestLocationWithoutEnv: 'ssh://user@host:1234/some/path::archive'] for location in locations: assert Location(location).canonical_path() == \ - Location(Location(location).canonical_path()).canonical_path() + Location(Location(location).canonical_path()).canonical_path(), "failed: %s" % location def test_format_path(self, monkeypatch): monkeypatch.delenv('BORG_REPO', raising=False)