diff --git a/Vagrantfile b/Vagrantfile index f6670bbb5..f0ea5584f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -109,7 +109,6 @@ def packages_openbsd pkg_add lz4 # pkg_add fuse # does not install, sdl dependency missing pkg_add git # no fakeroot - pkg_add python-3.4.2 pkg_add py3-setuptools ln -sf /usr/local/bin/python3.4 /usr/local/bin/python3 ln -sf /usr/local/bin/python3.4 /usr/local/bin/python @@ -166,7 +165,7 @@ def install_pythons(boxname) . ~/.bash_profile pyenv install 3.4.0 # tests pyenv install 3.5.0 # tests - pyenv install 3.5.1 # binary build, use latest 3.5.x release + pyenv install 3.5.2 # binary build, use latest 3.5.x release pyenv rehash EOF end @@ -184,8 +183,8 @@ def build_pyenv_venv(boxname) . ~/.bash_profile cd /vagrant/borg # use the latest 3.5 release - pyenv global 3.5.1 - pyenv virtualenv 3.5.1 borg-env + pyenv global 3.5.2 + pyenv virtualenv 3.5.2 borg-env ln -s ~/.pyenv/versions/borg-env . EOF end @@ -207,6 +206,22 @@ def install_borg(boxname) EOF end +def install_borg_no_fuse(boxname) + return <<-EOF + . ~/.bash_profile + cd /vagrant/borg + . borg-env/bin/activate + pip install -U wheel # upgrade wheel, too old for 3.5 + cd borg + # clean up (wrong/outdated) stuff we likely got via rsync: + rm -f borg/*.so borg/*.cpy* + rm -f borg/{chunker,crypto,compress,hashindex,platform_linux}.c + rm -rf borg/__pycache__ borg/support/__pycache__ borg/testsuite/__pycache__ + pip install -r requirements.d/development.txt + pip install -e . + EOF +end + def install_pyinstaller(boxname) return <<-EOF . ~/.bash_profile @@ -417,13 +432,13 @@ Vagrant.configure(2) do |config| end config.vm.define "openbsd64" do |b| - b.vm.box = "bodgit/openbsd-5.7-amd64" + b.vm.box = "kaorimatz/openbsd-5.9-amd64" b.vm.provider :virtualbox do |v| v.memory = 768 end b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("openbsd64") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg_no_fuse("openbsd64") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd64") end diff --git a/docs/misc/prune-example.txt b/docs/misc/prune-example.txt new file mode 100644 index 000000000..6c8f8e553 --- /dev/null +++ b/docs/misc/prune-example.txt @@ -0,0 +1,93 @@ +borg prune visualized +===================== + +Assume it is 2016-01-01, today's backup has not yet been made and you have +created at least one backup on each day in 2015 except on 2015-12-20 (no +backup made on that day). + +This is what borg prune --keep-daily 14 --keep-monthly 6 would keep. + +Backups kept by the --keep-daily rule are marked by a "d" to the right, +backups kept by the --keep-monthly rule are marked by a "m" to the right. + +Calendar view +------------- + + 2015 + January February March +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 1 1 + 5 6 7 8 9 10 11 2 3 4 5 6 7 8 2 3 4 5 6 7 8 +12 13 14 15 16 17 18 9 10 11 12 13 14 15 9 10 11 12 13 14 15 +19 20 21 22 23 24 25 16 17 18 19 20 21 22 16 17 18 19 20 21 22 +26 27 28 29 30 31 23 24 25 26 27 28 23 24 25 26 27 28 29 + 30 31 + + April May June +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 5 1 2 3 1 2 3 4 5 6 7 + 6 7 8 9 10 11 12 4 5 6 7 8 9 10 8 9 10 11 12 13 14 +13 14 15 16 17 18 19 11 12 13 14 15 16 17 15 16 17 18 19 20 21 +20 21 22 23 24 25 26 18 19 20 21 22 23 24 22 23 24 25 26 27 28 +27 28 29 30 25 26 27 28 29 30 31 29 30m + + + July August September +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 5 1 2 1 2 3 4 5 6 + 6 7 8 9 10 11 12 3 4 5 6 7 8 9 7 8 9 10 11 12 13 +13 14 15 16 17 18 19 10 11 12 13 14 15 16 14 15 16 17 18 19 20 +20 21 22 23 24 25 26 17 18 19 20 21 22 23 21 22 23 24 25 26 27 +27 28 29 30 31m 24 25 26 27 28 29 30 28 29 30m + 31m + + October November December +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 1 1 2 3 4 5 6 + 5 6 7 8 9 10 11 2 3 4 5 6 7 8 7 8 9 10 11 12 13 +12 13 14 15 16 17 18 9 10 11 12 13 14 15 14 15 16 17d18d19d20 +19 20 21 22 23 24 25 16 17 18 19 20 21 22 21d22d23d24d25d26d27d +26 27 28 29 30 31m 23 24 25 26 27 28 29 28d29d30d31d + 30m + +List view +--------- + +--keep-daily 14 --keep-monthly 6 +------------------------------------------------- + 1. 2015-12-31 (2015-12-31 kept by daily rule) + 2. 2015-12-30 1. 2015-11-30 + 3. 2015-12-29 2. 2015-10-31 + 4. 2015-12-28 3. 2015-09-30 + 5. 2015-12-27 4. 2015-08-31 + 6. 2015-12-26 5. 2015-07-31 + 7. 2015-12-25 6. 2015-06-30 + 8. 2015-12-24 + 9. 2015-12-23 +10. 2015-12-22 +11. 2015-12-21 + (no backup made on 2015-12-20) +12. 2015-12-19 +13. 2015-12-18 +14. 2015-12-17 + + +Notes +----- + +2015-12-31 is kept due to the --keep-daily 14 rule (because it is applied +first), not due to the --keep-monthly rule. + +Because of that, the --keep-monthly 6 rule keeps Nov, Oct, Sep, Aug, Jul and +Jun. December is not considered for this rule, because that backup was already +kept because of the daily rule. + +2015-12-17 is kept to satisfy the --keep-daily 14 rule - because no backup was +made on 2015-12-20. If a backup had been made on that day, it would not keep +the one from 2015-12-17. + +We did not include yearly, weekly, hourly, minutely or secondly rules to keep +this example simple. They all work in basically the same way. + +The weekly rule is easy to understand roughly, but hard to understand in all +details. If interested, read "ISO 8601:2000 standard week-based year". diff --git a/docs/usage.rst b/docs/usage.rst index 49e90c68b..8b06430fd 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -446,6 +446,8 @@ prefix "foo" if you do not also want to match "foobar". It is strongly recommended to always run ``prune --dry-run ...`` first so you will see what it would do without it actually doing anything. +There is also a visualized prune example in ``docs/misc/prune-example.txt``. + :: # Keep 7 end of day and 4 additional end of week archives. diff --git a/src/borg/archive.py b/src/borg/archive.py index a362f9542..8f02bcfc6 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -4,6 +4,7 @@ import socket import stat import sys import time +from contextlib import contextmanager from datetime import datetime, timezone from getpass import getuser from io import BytesIO @@ -97,6 +98,37 @@ class Statistics: print(msg, file=stream or sys.stderr, end="\r", flush=True) +class InputOSError(Exception): + """Wrapper for OSError raised while accessing input files.""" + def __init__(self, os_error): + self.os_error = os_error + self.errno = os_error.errno + self.strerror = os_error.strerror + self.filename = os_error.filename + + def __str__(self): + return str(self.os_error) + + +@contextmanager +def input_io(): + """Context manager changing OSError to InputOSError.""" + try: + yield + except OSError as os_error: + raise InputOSError(os_error) from os_error + + +def input_io_iter(iterator): + while True: + try: + with input_io(): + item = next(iterator) + except StopIteration: + return + yield item + + class DownloadPipeline: def __init__(self, repository, key): @@ -560,13 +592,15 @@ Number of files: {0.stats.nfiles}'''.format( ) if self.numeric_owner: attrs['user'] = attrs['group'] = None - xattrs = xattr.get_all(path, follow_symlinks=False) + with input_io(): + xattrs = xattr.get_all(path, follow_symlinks=False) if xattrs: attrs['xattrs'] = StableDict(xattrs) bsdflags = get_flags(path, st) if bsdflags: attrs['bsdflags'] = bsdflags - acl_get(path, attrs, st, self.numeric_owner) + with input_io(): + acl_get(path, attrs, st, self.numeric_owner) return attrs def process_dir(self, path, st): @@ -601,7 +635,7 @@ Number of files: {0.stats.nfiles}'''.format( uid, gid = 0, 0 fd = sys.stdin.buffer # binary chunks = [] - for data in self.chunker.chunkify(fd): + for data in input_io_iter(self.chunker.chunkify(fd)): chunks.append(cache.add_chunk(self.key.id_hash(data), Chunk(data), self.stats)) self.stats.nfiles += 1 t = int(time.time()) * 1000000000 @@ -654,10 +688,11 @@ Number of files: {0.stats.nfiles}'''.format( if chunks is None: compress = self.compression_decider1.decide(path) logger.debug('%s -> compression %s', path, compress['name']) - fh = Archive._open_rb(path) + with input_io(): + fh = Archive._open_rb(path) with os.fdopen(fh, 'rb') as fd: chunks = [] - for data in self.chunker.chunkify(fd, fh): + for data in input_io_iter(self.chunker.chunkify(fd, fh)): chunks.append(cache.add_chunk(self.key.id_hash(data), Chunk(data, compress=compress), self.stats)) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index f44d28983..eab535c36 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -24,6 +24,7 @@ logger = create_logger() from . import __version__ from . import helpers from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics +from .archive import InputOSError, CHUNKER_PARAMS from .cache import Cache from .constants import * # NOQA from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR @@ -254,7 +255,7 @@ class Archiver: if not dry_run: try: status = archive.process_stdin(path, cache) - except OSError as e: + except InputOSError as e: status = 'E' self.print_warning('%s: %s', path, e) else: @@ -312,7 +313,15 @@ class Archiver: return if st is None: try: - st = os.lstat(path) + # usually, do not follow symlinks (if we have a symlink, we want to + # backup it as such). + # but if we are in --read-special mode, we later process as + # a regular file (we open and read the symlink target file's content). + # thus, in read_special mode, we also want to stat the symlink target + # file, for consistency. if we did not, we also have issues extracting + # this file, as it would be in the archive as a symlink, not as the + # target's file type (which could be e.g. a block device). + st = os.stat(path, follow_symlinks=read_special) except OSError as e: self.print_warning('%s: %s', path, e) return @@ -330,7 +339,7 @@ class Archiver: if not dry_run: try: status = archive.process_file(path, st, cache, self.ignore_inode) - except OSError as e: + except InputOSError as e: status = 'E' self.print_warning('%s: %s', path, e) elif stat.S_ISDIR(st.st_mode): diff --git a/src/borg/remote.py b/src/borg/remote.py index 170a9f2ef..fb67e1d57 100644 --- a/src/borg/remote.py +++ b/src/borg/remote.py @@ -249,6 +249,24 @@ class RemoteRepository: del self.cache[args] return msgid + def handle_error(error, res): + if error == b'DoesNotExist': + raise Repository.DoesNotExist(self.location.orig) + elif error == b'AlreadyExists': + raise Repository.AlreadyExists(self.location.orig) + elif error == b'CheckNeeded': + raise Repository.CheckNeeded(self.location.orig) + elif error == b'IntegrityError': + raise IntegrityError(res) + elif error == b'PathNotAllowed': + raise PathNotAllowed(*res) + elif error == b'ObjectNotFound': + raise Repository.ObjectNotFound(res[0], self.location.orig) + elif error == b'InvalidRPCMethod': + raise InvalidRPCMethod(*res) + else: + raise self.RPCError(res.decode('utf-8')) + calls = list(calls) waiting_for = [] while wait or calls: @@ -257,22 +275,7 @@ class RemoteRepository: error, res = self.responses.pop(waiting_for[0]) waiting_for.pop(0) if error: - if error == b'DoesNotExist': - raise Repository.DoesNotExist(self.location.orig) - elif error == b'AlreadyExists': - raise Repository.AlreadyExists(self.location.orig) - elif error == b'CheckNeeded': - raise Repository.CheckNeeded(self.location.orig) - elif error == b'IntegrityError': - raise IntegrityError(res) - elif error == b'PathNotAllowed': - raise PathNotAllowed(*res) - elif error == b'ObjectNotFound': - raise Repository.ObjectNotFound(res[0], self.location.orig) - elif error == b'InvalidRPCMethod': - raise InvalidRPCMethod(*res) - else: - raise self.RPCError(res.decode('utf-8')) + handle_error(error, res) else: yield res if not waiting_for and not calls: @@ -298,6 +301,8 @@ class RemoteRepository: type, msgid, error, res = unpacked if msgid in self.ignore_responses: self.ignore_responses.remove(msgid) + if error: + handle_error(error, res) else: self.responses[msgid] = error, res elif fd is self.stderr_fd: diff --git a/src/borg/testsuite/archive.py b/src/borg/testsuite/archive.py index 70a03eb6b..ee4a86c32 100644 --- a/src/borg/testsuite/archive.py +++ b/src/borg/testsuite/archive.py @@ -7,6 +7,7 @@ import pytest import msgpack from ..archive import Archive, CacheChunkBuffer, RobustUnpacker, valid_msgpacked_dict, ITEM_KEYS, Statistics +from ..archive import InputOSError, input_io, input_io_iter from ..item import Item from ..key import PlaintextKey from ..helpers import Manifest @@ -216,3 +217,27 @@ def test_key_length_msgpacked_items(): data = {key: b''} item_keys_serialized = [msgpack.packb(key), ] assert valid_msgpacked_dict(msgpack.packb(data), item_keys_serialized) + + +def test_input_io(): + with pytest.raises(InputOSError): + with input_io(): + raise OSError(123) + + +def test_input_io_iter(): + class Iterator: + def __init__(self, exc): + self.exc = exc + + def __next__(self): + raise self.exc() + + oserror_iterator = Iterator(OSError) + with pytest.raises(InputOSError): + for _ in input_io_iter(oserror_iterator): + pass + + normal_iterator = Iterator(StopIteration) + for _ in input_io_iter(normal_iterator): + assert False, 'StopIteration handled incorrectly'