From c6a72bb16f94d9f14c659535f9445b068a117272 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 27 Jun 2016 18:42:12 +0200 Subject: [PATCH 1/7] use python 3.5.2 to build the binaries --- Vagrantfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index c9306b54d..184959181 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -166,7 +166,7 @@ def install_pythons(boxname) . ~/.bash_profile pyenv install 3.4.0 # tests pyenv install 3.5.0 # tests - pyenv install 3.5.1 # binary build, use latest 3.5.x release + pyenv install 3.5.2 # binary build, use latest 3.5.x release pyenv rehash EOF end @@ -184,8 +184,8 @@ def build_pyenv_venv(boxname) . ~/.bash_profile cd /vagrant/borg # use the latest 3.5 release - pyenv global 3.5.1 - pyenv virtualenv 3.5.1 borg-env + pyenv global 3.5.2 + pyenv virtualenv 3.5.2 borg-env ln -s ~/.pyenv/versions/borg-env . EOF end From 5b453856ec239bdc18b14e6f1f79ec002a5ed92d Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Mon, 27 Jun 2016 20:56:41 +0200 Subject: [PATCH 2/7] Fix incorrect propagation of OSErrors in create code --- borg/archive.py | 45 ++++++++++++++++++++++++++++++++++++++++----- borg/archiver.py | 6 +++--- 2 files changed, 43 insertions(+), 8 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 1894ad7a7..870069953 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -1,4 +1,5 @@ from binascii import hexlify +from contextlib import contextmanager from datetime import datetime, timezone from getpass import getuser from itertools import groupby @@ -45,6 +46,37 @@ flags_normal = os.O_RDONLY | getattr(os, 'O_BINARY', 0) flags_noatime = flags_normal | getattr(os, 'O_NOATIME', 0) +class InputOSError(Exception): + """Wrapper for OSError raised while accessing input files.""" + def __init__(self, os_error): + self.os_error = os_error + self.errno = os_error.errno + self.strerror = os_error.strerror + self.filename = os_error.filename + + def __str__(self): + return str(self.os_error) + + +@contextmanager +def input_io(): + """Context manager changing OSError to InputOSError.""" + try: + yield + except OSError as os_error: + raise InputOSError(os_error) from os_error + + +def input_io_iter(iterator): + while True: + try: + with input_io(): + item = next(iterator) + except StopIteration: + return + yield item + + class DownloadPipeline: def __init__(self, repository, key): @@ -464,12 +496,14 @@ Number of files: {0.stats.nfiles}'''.format( } if self.numeric_owner: item[b'user'] = item[b'group'] = None - xattrs = xattr.get_all(path, follow_symlinks=False) + with input_io(): + xattrs = xattr.get_all(path, follow_symlinks=False) if xattrs: item[b'xattrs'] = StableDict(xattrs) if has_lchflags and st.st_flags: item[b'bsdflags'] = st.st_flags - acl_get(path, item, st, self.numeric_owner) + with input_io(): + acl_get(path, item, st, self.numeric_owner) return item def process_dir(self, path, st): @@ -504,7 +538,7 @@ Number of files: {0.stats.nfiles}'''.format( uid, gid = 0, 0 fd = sys.stdin.buffer # binary chunks = [] - for chunk in self.chunker.chunkify(fd): + for chunk in input_io_iter(self.chunker.chunkify(fd)): chunks.append(cache.add_chunk(self.key.id_hash(chunk), chunk, self.stats)) self.stats.nfiles += 1 t = int_to_bigint(int(time.time()) * 1000000000) @@ -552,10 +586,11 @@ Number of files: {0.stats.nfiles}'''.format( item = {b'path': safe_path} # Only chunkify the file if needed if chunks is None: - fh = Archive._open_rb(path) + with input_io(): + fh = Archive._open_rb(path) with os.fdopen(fh, 'rb') as fd: chunks = [] - for chunk in self.chunker.chunkify(fd, fh): + for chunk in input_io_iter(self.chunker.chunkify(fd, fh)): chunks.append(cache.add_chunk(self.key.id_hash(chunk), chunk, self.stats)) if self.show_progress: self.stats.show_progress(item=item, dt=0.2) diff --git a/borg/archiver.py b/borg/archiver.py index 5cac525ce..0f07907c6 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -29,7 +29,7 @@ from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader from .repository import Repository from .cache import Cache from .key import key_creator, RepoKey, PassphraseKey -from .archive import Archive, ArchiveChecker, CHUNKER_PARAMS +from .archive import input_io, InputOSError, Archive, ArchiveChecker, CHUNKER_PARAMS from .remote import RepositoryServer, RemoteRepository, cache_if_remote has_lchflags = hasattr(os, 'lchflags') @@ -198,7 +198,7 @@ class Archiver: if not dry_run: try: status = archive.process_stdin(path, cache) - except OSError as e: + except InputOSError as e: status = 'E' self.print_warning('%s: %s', path, e) else: @@ -273,7 +273,7 @@ class Archiver: if not dry_run: try: status = archive.process_file(path, st, cache, self.ignore_inode) - except OSError as e: + except InputOSError as e: status = 'E' self.print_warning('%s: %s', path, e) elif stat.S_ISDIR(st.st_mode): From c773c882b9a94cfa43693e5d98a184fe4e4824da Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 27 Jun 2016 21:20:12 +0200 Subject: [PATCH 3/7] add prune visualization / example, fixes #723 --- docs/misc/prune-example.txt | 93 +++++++++++++++++++++++++++++++++++++ docs/usage.rst | 2 + 2 files changed, 95 insertions(+) create mode 100644 docs/misc/prune-example.txt diff --git a/docs/misc/prune-example.txt b/docs/misc/prune-example.txt new file mode 100644 index 000000000..6c8f8e553 --- /dev/null +++ b/docs/misc/prune-example.txt @@ -0,0 +1,93 @@ +borg prune visualized +===================== + +Assume it is 2016-01-01, today's backup has not yet been made and you have +created at least one backup on each day in 2015 except on 2015-12-20 (no +backup made on that day). + +This is what borg prune --keep-daily 14 --keep-monthly 6 would keep. + +Backups kept by the --keep-daily rule are marked by a "d" to the right, +backups kept by the --keep-monthly rule are marked by a "m" to the right. + +Calendar view +------------- + + 2015 + January February March +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 1 1 + 5 6 7 8 9 10 11 2 3 4 5 6 7 8 2 3 4 5 6 7 8 +12 13 14 15 16 17 18 9 10 11 12 13 14 15 9 10 11 12 13 14 15 +19 20 21 22 23 24 25 16 17 18 19 20 21 22 16 17 18 19 20 21 22 +26 27 28 29 30 31 23 24 25 26 27 28 23 24 25 26 27 28 29 + 30 31 + + April May June +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 5 1 2 3 1 2 3 4 5 6 7 + 6 7 8 9 10 11 12 4 5 6 7 8 9 10 8 9 10 11 12 13 14 +13 14 15 16 17 18 19 11 12 13 14 15 16 17 15 16 17 18 19 20 21 +20 21 22 23 24 25 26 18 19 20 21 22 23 24 22 23 24 25 26 27 28 +27 28 29 30 25 26 27 28 29 30 31 29 30m + + + July August September +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 5 1 2 1 2 3 4 5 6 + 6 7 8 9 10 11 12 3 4 5 6 7 8 9 7 8 9 10 11 12 13 +13 14 15 16 17 18 19 10 11 12 13 14 15 16 14 15 16 17 18 19 20 +20 21 22 23 24 25 26 17 18 19 20 21 22 23 21 22 23 24 25 26 27 +27 28 29 30 31m 24 25 26 27 28 29 30 28 29 30m + 31m + + October November December +Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su + 1 2 3 4 1 1 2 3 4 5 6 + 5 6 7 8 9 10 11 2 3 4 5 6 7 8 7 8 9 10 11 12 13 +12 13 14 15 16 17 18 9 10 11 12 13 14 15 14 15 16 17d18d19d20 +19 20 21 22 23 24 25 16 17 18 19 20 21 22 21d22d23d24d25d26d27d +26 27 28 29 30 31m 23 24 25 26 27 28 29 28d29d30d31d + 30m + +List view +--------- + +--keep-daily 14 --keep-monthly 6 +------------------------------------------------- + 1. 2015-12-31 (2015-12-31 kept by daily rule) + 2. 2015-12-30 1. 2015-11-30 + 3. 2015-12-29 2. 2015-10-31 + 4. 2015-12-28 3. 2015-09-30 + 5. 2015-12-27 4. 2015-08-31 + 6. 2015-12-26 5. 2015-07-31 + 7. 2015-12-25 6. 2015-06-30 + 8. 2015-12-24 + 9. 2015-12-23 +10. 2015-12-22 +11. 2015-12-21 + (no backup made on 2015-12-20) +12. 2015-12-19 +13. 2015-12-18 +14. 2015-12-17 + + +Notes +----- + +2015-12-31 is kept due to the --keep-daily 14 rule (because it is applied +first), not due to the --keep-monthly rule. + +Because of that, the --keep-monthly 6 rule keeps Nov, Oct, Sep, Aug, Jul and +Jun. December is not considered for this rule, because that backup was already +kept because of the daily rule. + +2015-12-17 is kept to satisfy the --keep-daily 14 rule - because no backup was +made on 2015-12-20. If a backup had been made on that day, it would not keep +the one from 2015-12-17. + +We did not include yearly, weekly, hourly, minutely or secondly rules to keep +this example simple. They all work in basically the same way. + +The weekly rule is easy to understand roughly, but hard to understand in all +details. If interested, read "ISO 8601:2000 standard week-based year". diff --git a/docs/usage.rst b/docs/usage.rst index acca302ac..b7863d9a9 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -419,6 +419,8 @@ prefix "foo" if you do not also want to match "foobar". It is strongly recommended to always run ``prune --dry-run ...`` first so you will see what it would do without it actually doing anything. +There is also a visualized prune example in ``docs/misc/prune-example.txt``. + :: # Keep 7 end of day and 4 additional end of week archives. From 431441f0d6134c02a7cd3d9931d52f5ffc7a89d8 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Mon, 27 Jun 2016 22:00:24 +0200 Subject: [PATCH 4/7] input_io_* tests --- borg/testsuite/archive.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/borg/testsuite/archive.py b/borg/testsuite/archive.py index 919d57a0c..229ff8bda 100644 --- a/borg/testsuite/archive.py +++ b/borg/testsuite/archive.py @@ -5,6 +5,7 @@ import msgpack import pytest from ..archive import Archive, CacheChunkBuffer, RobustUnpacker, valid_msgpacked_dict, ITEM_KEYS +from ..archive import InputOSError, input_io, input_io_iter from ..key import PlaintextKey from ..helpers import Manifest from . import BaseTestCase @@ -145,3 +146,27 @@ def test_key_length_msgpacked_items(): data = {key: b''} item_keys_serialized = [msgpack.packb(key), ] assert valid_msgpacked_dict(msgpack.packb(data), item_keys_serialized) + + +def test_input_io(): + with pytest.raises(InputOSError): + with input_io(): + raise OSError(123) + + +def test_input_io_iter(): + class Iterator: + def __init__(self, exc): + self.exc = exc + + def __next__(self): + raise self.exc() + + oserror_iterator = Iterator(OSError) + with pytest.raises(InputOSError): + for _ in input_io_iter(oserror_iterator): + pass + + normal_iterator = Iterator(StopIteration) + for _ in input_io_iter(normal_iterator): + assert False, 'StopIteration handled incorrectly' From e96905c6b150e9b486500baf7fc83d120b11843a Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Mon, 27 Jun 2016 22:44:41 +0200 Subject: [PATCH 5/7] Fix error swallowing of ignored responses in RPC code --- borg/remote.py | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/borg/remote.py b/borg/remote.py index 5768efad6..e95c38978 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -241,6 +241,24 @@ class RemoteRepository: del self.cache[args] return msgid + def handle_error(error, res): + if error == b'DoesNotExist': + raise Repository.DoesNotExist(self.location.orig) + elif error == b'AlreadyExists': + raise Repository.AlreadyExists(self.location.orig) + elif error == b'CheckNeeded': + raise Repository.CheckNeeded(self.location.orig) + elif error == b'IntegrityError': + raise IntegrityError(res) + elif error == b'PathNotAllowed': + raise PathNotAllowed(*res) + elif error == b'ObjectNotFound': + raise Repository.ObjectNotFound(res[0], self.location.orig) + elif error == b'InvalidRPCMethod': + raise InvalidRPCMethod(*res) + else: + raise self.RPCError(res.decode('utf-8')) + calls = list(calls) waiting_for = [] w_fds = [self.stdin_fd] @@ -250,22 +268,7 @@ class RemoteRepository: error, res = self.responses.pop(waiting_for[0]) waiting_for.pop(0) if error: - if error == b'DoesNotExist': - raise Repository.DoesNotExist(self.location.orig) - elif error == b'AlreadyExists': - raise Repository.AlreadyExists(self.location.orig) - elif error == b'CheckNeeded': - raise Repository.CheckNeeded(self.location.orig) - elif error == b'IntegrityError': - raise IntegrityError(res) - elif error == b'PathNotAllowed': - raise PathNotAllowed(*res) - elif error == b'ObjectNotFound': - raise Repository.ObjectNotFound(res[0], self.location.orig) - elif error == b'InvalidRPCMethod': - raise InvalidRPCMethod(*res) - else: - raise self.RPCError(res.decode('utf-8')) + handle_error(error, res) else: yield res if not waiting_for and not calls: @@ -287,6 +290,8 @@ class RemoteRepository: type, msgid, error, res = unpacked if msgid in self.ignore_responses: self.ignore_responses.remove(msgid) + if error: + handle_error(error, res) else: self.responses[msgid] = error, res elif fd is self.stderr_fd: From c3073bacbb4850cf3a526ac08db42b05f3e33684 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 28 Jun 2016 18:49:51 +0200 Subject: [PATCH 6/7] in --read-special mode, follow symlinks, fixes #1215 --- borg/archiver.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/borg/archiver.py b/borg/archiver.py index 5cac525ce..f98db7fd0 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -256,7 +256,15 @@ class Archiver: return try: - st = os.lstat(path) + # usually, do not follow symlinks (if we have a symlink, we want to + # backup it as such). + # but if we are in --read-special mode, we later process as + # a regular file (we open and read the symlink target file's content). + # thus, in read_special mode, we also want to stat the symlink target + # file, for consistency. if we did not, we also have issues extracting + # this file, as it would be in the archive as a symlink, not as the + # target's file type (which could be e.g. a block device). + st = os.stat(path, follow_symlinks=read_special) except OSError as e: self.print_warning('%s: %s', path, e) return From e81fc4381cd555c6f4b36d7212921a07a314165c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 29 Jun 2016 01:04:24 +0200 Subject: [PATCH 7/7] Vagrantfile: use openbsd 5.9, fixes #716 machine already has python3.4. rsync works now. fuse does not work on openbsd (and breaks borg install, if tried). --- Vagrantfile | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 184959181..5b5b071f0 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -109,7 +109,6 @@ def packages_openbsd pkg_add lz4 # pkg_add fuse # does not install, sdl dependency missing pkg_add git # no fakeroot - pkg_add python-3.4.2 pkg_add py3-setuptools ln -sf /usr/local/bin/python3.4 /usr/local/bin/python3 ln -sf /usr/local/bin/python3.4 /usr/local/bin/python @@ -207,6 +206,22 @@ def install_borg(boxname) EOF end +def install_borg_no_fuse(boxname) + return <<-EOF + . ~/.bash_profile + cd /vagrant/borg + . borg-env/bin/activate + pip install -U wheel # upgrade wheel, too old for 3.5 + cd borg + # clean up (wrong/outdated) stuff we likely got via rsync: + rm -f borg/*.so borg/*.cpy* + rm -f borg/{chunker,crypto,compress,hashindex,platform_linux}.c + rm -rf borg/__pycache__ borg/support/__pycache__ borg/testsuite/__pycache__ + pip install -r requirements.d/development.txt + pip install -e . + EOF +end + def install_pyinstaller(boxname) return <<-EOF . ~/.bash_profile @@ -406,13 +421,13 @@ Vagrant.configure(2) do |config| end config.vm.define "openbsd64" do |b| - b.vm.box = "bodgit/openbsd-5.7-amd64" + b.vm.box = "kaorimatz/openbsd-5.9-amd64" b.vm.provider :virtualbox do |v| v.memory = 768 end b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64") - b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("openbsd64") + b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg_no_fuse("openbsd64") b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd64") end