upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-06-08 16:23:42 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge branch '1.0-maint'

Browse source
This commit is contained in:
Thomas Waldmann 2016-06-29 18:28:33 +02:00
commit 87d6755108
7 changed files with 214 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
Vagrantfile vendored
View file

@ -109,7 +109,6 @@ def packages_openbsd
pkg_add lz4
# pkg_add fuse # does not install, sdl dependency missing
pkg_add git # no fakeroot
pkg_add python-3.4.2
pkg_add py3-setuptools
ln -sf /usr/local/bin/python3.4 /usr/local/bin/python3
ln -sf /usr/local/bin/python3.4 /usr/local/bin/python
@ -166,7 +165,7 @@ def install_pythons(boxname)
. ~/.bash_profile
pyenv install 3.4.0 # tests
pyenv install 3.5.0 # tests
pyenv install 3.5.1 # binary build, use latest 3.5.x release
pyenv install 3.5.2 # binary build, use latest 3.5.x release
pyenv rehash
EOF
end
@ -184,8 +183,8 @@ def build_pyenv_venv(boxname)
. ~/.bash_profile
cd /vagrant/borg
# use the latest 3.5 release
pyenv global 3.5.1
pyenv virtualenv 3.5.1 borg-env
pyenv global 3.5.2
pyenv virtualenv 3.5.2 borg-env
ln -s ~/.pyenv/versions/borg-env .
EOF
end
@ -207,6 +206,22 @@ def install_borg(boxname)
EOF
end
def install_borg_no_fuse(boxname)
return <<-EOF
. ~/.bash_profile
cd /vagrant/borg
. borg-env/bin/activate
pip install -U wheel # upgrade wheel, too old for 3.5
cd borg
# clean up (wrong/outdated) stuff we likely got via rsync:
rm -f borg/*.so borg/*.cpy*
rm -f borg/{chunker,crypto,compress,hashindex,platform_linux}.c
rm -rf borg/__pycache__ borg/support/__pycache__ borg/testsuite/__pycache__
pip install -r requirements.d/development.txt
pip install -e .
EOF
end
def install_pyinstaller(boxname)
return <<-EOF
. ~/.bash_profile
@ -417,13 +432,13 @@ Vagrant.configure(2) do |config|
end
config.vm.define "openbsd64" do |b|
b.vm.box = "bodgit/openbsd-5.7-amd64"
b.vm.box = "kaorimatz/openbsd-5.9-amd64"
b.vm.provider :virtualbox do |v|
v.memory = 768
end
b.vm.provision "packages openbsd", :type => :shell, :inline => packages_openbsd
b.vm.provision "build env", :type => :shell, :privileged => false, :inline => build_sys_venv("openbsd64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg("openbsd64")
b.vm.provision "install borg", :type => :shell, :privileged => false, :inline => install_borg_no_fuse("openbsd64")
b.vm.provision "run tests", :type => :shell, :privileged => false, :inline => run_tests("openbsd64")
end

93
docs/misc/prune-example.txt Normal file
View file

@ -0,0 +1,93 @@
borg prune visualized
=====================
Assume it is 2016-01-01, today's backup has not yet been made and you have
created at least one backup on each day in 2015 except on 2015-12-20 (no
backup made on that day).
This is what borg prune --keep-daily 14 --keep-monthly 6 would keep.
Backups kept by the --keep-daily rule are marked by a "d" to the right,
backups kept by the --keep-monthly rule are marked by a "m" to the right.
Calendar view
-------------
2015
January February March
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 1 1
5 6 7 8 9 10 11 2 3 4 5 6 7 8 2 3 4 5 6 7 8
12 13 14 15 16 17 18 9 10 11 12 13 14 15 9 10 11 12 13 14 15
19 20 21 22 23 24 25 16 17 18 19 20 21 22 16 17 18 19 20 21 22
26 27 28 29 30 31 23 24 25 26 27 28 23 24 25 26 27 28 29
30 31
April May June
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 5 1 2 3 1 2 3 4 5 6 7
6 7 8 9 10 11 12 4 5 6 7 8 9 10 8 9 10 11 12 13 14
13 14 15 16 17 18 19 11 12 13 14 15 16 17 15 16 17 18 19 20 21
20 21 22 23 24 25 26 18 19 20 21 22 23 24 22 23 24 25 26 27 28
27 28 29 30 25 26 27 28 29 30 31 29 30m
July August September
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 5 1 2 1 2 3 4 5 6
6 7 8 9 10 11 12 3 4 5 6 7 8 9 7 8 9 10 11 12 13
13 14 15 16 17 18 19 10 11 12 13 14 15 16 14 15 16 17 18 19 20
20 21 22 23 24 25 26 17 18 19 20 21 22 23 21 22 23 24 25 26 27
27 28 29 30 31m 24 25 26 27 28 29 30 28 29 30m
31m
October November December
Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su
1 2 3 4 1 1 2 3 4 5 6
5 6 7 8 9 10 11 2 3 4 5 6 7 8 7 8 9 10 11 12 13
12 13 14 15 16 17 18 9 10 11 12 13 14 15 14 15 16 17d18d19d20
19 20 21 22 23 24 25 16 17 18 19 20 21 22 21d22d23d24d25d26d27d
26 27 28 29 30 31m 23 24 25 26 27 28 29 28d29d30d31d
30m
List view
---------
--keep-daily 14 --keep-monthly 6
-------------------------------------------------
1. 2015-12-31 (2015-12-31 kept by daily rule)
2. 2015-12-30 1. 2015-11-30
3. 2015-12-29 2. 2015-10-31
4. 2015-12-28 3. 2015-09-30
5. 2015-12-27 4. 2015-08-31
6. 2015-12-26 5. 2015-07-31
7. 2015-12-25 6. 2015-06-30
8. 2015-12-24
9. 2015-12-23
10. 2015-12-22
11. 2015-12-21
(no backup made on 2015-12-20)
12. 2015-12-19
13. 2015-12-18
14. 2015-12-17
Notes
-----
2015-12-31 is kept due to the --keep-daily 14 rule (because it is applied
first), not due to the --keep-monthly rule.
Because of that, the --keep-monthly 6 rule keeps Nov, Oct, Sep, Aug, Jul and
Jun. December is not considered for this rule, because that backup was already
kept because of the daily rule.
2015-12-17 is kept to satisfy the --keep-daily 14 rule - because no backup was
made on 2015-12-20. If a backup had been made on that day, it would not keep
the one from 2015-12-17.
We did not include yearly, weekly, hourly, minutely or secondly rules to keep
this example simple. They all work in basically the same way.
The weekly rule is easy to understand roughly, but hard to understand in all
details. If interested, read "ISO 8601:2000 standard week-based year".

2
docs/usage.rst
View file

@ -446,6 +446,8 @@ prefix "foo" if you do not also want to match "foobar".
It is strongly recommended to always run ``prune --dry-run ...`` first so you
will see what it would do without it actually doing anything.
There is also a visualized prune example in ``docs/misc/prune-example.txt``.
::
# Keep 7 end of day and 4 additional end of week archives.

45
src/borg/archive.py
View file

@ -4,6 +4,7 @@ import socket
import stat
import sys
import time
from contextlib import contextmanager
from datetime import datetime, timezone
from getpass import getuser
from io import BytesIO
@ -97,6 +98,37 @@ class Statistics:
print(msg, file=stream or sys.stderr, end="\r", flush=True)
class InputOSError(Exception):
"""Wrapper for OSError raised while accessing input files."""
def __init__(self, os_error):
self.os_error = os_error
self.errno = os_error.errno
self.strerror = os_error.strerror
self.filename = os_error.filename
def __str__(self):
return str(self.os_error)
@contextmanager
def input_io():
"""Context manager changing OSError to InputOSError."""
try:
yield
except OSError as os_error:
raise InputOSError(os_error) from os_error
def input_io_iter(iterator):
while True:
try:
with input_io():
item = next(iterator)
except StopIteration:
return
yield item
class DownloadPipeline:
def __init__(self, repository, key):
@ -560,13 +592,15 @@ Number of files: {0.stats.nfiles}'''.format(
)
if self.numeric_owner:
attrs['user'] = attrs['group'] = None
xattrs = xattr.get_all(path, follow_symlinks=False)
with input_io():
xattrs = xattr.get_all(path, follow_symlinks=False)
if xattrs:
attrs['xattrs'] = StableDict(xattrs)
bsdflags = get_flags(path, st)
if bsdflags:
attrs['bsdflags'] = bsdflags
acl_get(path, attrs, st, self.numeric_owner)
with input_io():
acl_get(path, attrs, st, self.numeric_owner)
return attrs
def process_dir(self, path, st):
@ -601,7 +635,7 @@ Number of files: {0.stats.nfiles}'''.format(
uid, gid = 0, 0
fd = sys.stdin.buffer # binary
chunks = []
for data in self.chunker.chunkify(fd):
for data in input_io_iter(self.chunker.chunkify(fd)):
chunks.append(cache.add_chunk(self.key.id_hash(data), Chunk(data), self.stats))
self.stats.nfiles += 1
t = int(time.time()) * 1000000000
@ -654,10 +688,11 @@ Number of files: {0.stats.nfiles}'''.format(
if chunks is None:
compress = self.compression_decider1.decide(path)
logger.debug('%s -> compression %s', path, compress['name'])
fh = Archive._open_rb(path)
with input_io():
fh = Archive._open_rb(path)
with os.fdopen(fh, 'rb') as fd:
chunks = []
for data in self.chunker.chunkify(fd, fh):
for data in input_io_iter(self.chunker.chunkify(fd, fh)):
chunks.append(cache.add_chunk(self.key.id_hash(data),
Chunk(data, compress=compress),
self.stats))

15
src/borg/archiver.py
View file

@ -24,6 +24,7 @@ logger = create_logger()
from . import __version__
from . import helpers
from .archive import Archive, ArchiveChecker, ArchiveRecreater, Statistics
from .archive import InputOSError, CHUNKER_PARAMS
from .cache import Cache
from .constants import * # NOQA
from .helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR
@ -254,7 +255,7 @@ class Archiver:
if not dry_run:
try:
status = archive.process_stdin(path, cache)
except OSError as e:
except InputOSError as e:
status = 'E'
self.print_warning('%s: %s', path, e)
else:
@ -312,7 +313,15 @@ class Archiver:
return
if st is None:
try:
st = os.lstat(path)
# usually, do not follow symlinks (if we have a symlink, we want to
# backup it as such).
# but if we are in --read-special mode, we later process <path> as
# a regular file (we open and read the symlink target file's content).
# thus, in read_special mode, we also want to stat the symlink target
# file, for consistency. if we did not, we also have issues extracting
# this file, as it would be in the archive as a symlink, not as the
# target's file type (which could be e.g. a block device).
st = os.stat(path, follow_symlinks=read_special)
except OSError as e:
self.print_warning('%s: %s', path, e)
return
@ -330,7 +339,7 @@ class Archiver:
if not dry_run:
try:
status = archive.process_file(path, st, cache, self.ignore_inode)
except OSError as e:
except InputOSError as e:
status = 'E'
self.print_warning('%s: %s', path, e)
elif stat.S_ISDIR(st.st_mode):

37
src/borg/remote.py
View file

@ -249,6 +249,24 @@ class RemoteRepository:
del self.cache[args]
return msgid
def handle_error(error, res):
if error == b'DoesNotExist':
raise Repository.DoesNotExist(self.location.orig)
elif error == b'AlreadyExists':
raise Repository.AlreadyExists(self.location.orig)
elif error == b'CheckNeeded':
raise Repository.CheckNeeded(self.location.orig)
elif error == b'IntegrityError':
raise IntegrityError(res)
elif error == b'PathNotAllowed':
raise PathNotAllowed(*res)
elif error == b'ObjectNotFound':
raise Repository.ObjectNotFound(res[0], self.location.orig)
elif error == b'InvalidRPCMethod':
raise InvalidRPCMethod(*res)
else:
raise self.RPCError(res.decode('utf-8'))
calls = list(calls)
waiting_for = []
while wait or calls:
@ -257,22 +275,7 @@ class RemoteRepository:
error, res = self.responses.pop(waiting_for[0])
waiting_for.pop(0)
if error:
if error == b'DoesNotExist':
raise Repository.DoesNotExist(self.location.orig)
elif error == b'AlreadyExists':
raise Repository.AlreadyExists(self.location.orig)
elif error == b'CheckNeeded':
raise Repository.CheckNeeded(self.location.orig)
elif error == b'IntegrityError':
raise IntegrityError(res)
elif error == b'PathNotAllowed':
raise PathNotAllowed(*res)
elif error == b'ObjectNotFound':
raise Repository.ObjectNotFound(res[0], self.location.orig)
elif error == b'InvalidRPCMethod':
raise InvalidRPCMethod(*res)
else:
raise self.RPCError(res.decode('utf-8'))
handle_error(error, res)
else:
yield res
if not waiting_for and not calls:
@ -298,6 +301,8 @@ class RemoteRepository:
type, msgid, error, res = unpacked
if msgid in self.ignore_responses:
self.ignore_responses.remove(msgid)
if error:
handle_error(error, res)
else:
self.responses[msgid] = error, res
elif fd is self.stderr_fd:

25
src/borg/testsuite/archive.py
View file

@ -7,6 +7,7 @@ import pytest
import msgpack
from ..archive import Archive, CacheChunkBuffer, RobustUnpacker, valid_msgpacked_dict, ITEM_KEYS, Statistics
from ..archive import InputOSError, input_io, input_io_iter
from ..item import Item
from ..key import PlaintextKey
from ..helpers import Manifest
@ -216,3 +217,27 @@ def test_key_length_msgpacked_items():
data = {key: b''}
item_keys_serialized = [msgpack.packb(key), ]
assert valid_msgpacked_dict(msgpack.packb(data), item_keys_serialized)
def test_input_io():
with pytest.raises(InputOSError):
with input_io():
raise OSError(123)
def test_input_io_iter():
class Iterator:
def __init__(self, exc):
self.exc = exc
def __next__(self):
raise self.exc()
oserror_iterator = Iterator(OSError)
with pytest.raises(InputOSError):
for _ in input_io_iter(oserror_iterator):
pass
normal_iterator = Iterator(StopIteration)
for _ in input_io_iter(normal_iterator):
assert False, 'StopIteration handled incorrectly'