From 8c4802312d3d33acdc02c6b45b7b3e74f8a6a907 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 Jul 2016 02:47:43 +0200 Subject: [PATCH 01/11] add picture made by anarcat --- docs/misc/internals-picture.txt | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/misc/internals-picture.txt diff --git a/docs/misc/internals-picture.txt b/docs/misc/internals-picture.txt new file mode 100644 index 000000000..ae76f0c19 --- /dev/null +++ b/docs/misc/internals-picture.txt @@ -0,0 +1,41 @@ +BorgBackup from 10.000m +======================= + ++--------+ +--------+ +--------+ +|archive0| |archive1| ... |archiveN| ++--------+ +--------+ +--+-----+ + | | | + | | | + | +---+ | + | | | + | | | + +------+-------+ | + | | | | + /chunk\/chunk\/chunk\... /maybe different chunks lists\ ++-----------------------------------------------------------------+ +|item list | ++-----------------------------------------------------------------+ + | + +-------------------------------------+--------------+ + | | | + | | | ++-------------+ +-------------+ | +|item0 | |item1 | | +| - owner | | - owner | | +| - size | | - size | ... +| - ... | | - ... | +| - chunks | | - chunks | ++----+--------+ +-----+-------+ + | | + | +-----+----------------------------+-----------------+ + | | | | + +-o-----o------------+ | + | | | | | + /chunk0\/chunk1\ ... /chunkN\ /chunk0\/chunk1\ ... /chunkN'\ + +-----------------------------+ +------------------------------+ + |file0 | |file0' | + +-----------------------------+ +------------------------------+ + + +Thanks to anarcat for drawing the picture! + From 0fb6cb8417e3f84afc03fbfaef3cb7ca6212e61b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 Jul 2016 20:20:10 +0200 Subject: [PATCH 02/11] more compatible sparse file testing, fixes #1310 removed the pointless platform check. just first test the input file with the same checks we expect succeeding on the extracted file. skip sparse archiving / extraction testing if the input file checks fail - likely we have a problem with the OS or the FS then. --- borg/testsuite/archiver.py | 71 ++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index dbe21b0f3..ab69770e0 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -393,11 +393,30 @@ class ArchiverTestCase(ArchiverTestCaseBase): return repository.id def test_sparse_file(self): - # no sparse file support on Mac OS X - sparse_support = sys.platform != 'darwin' + def is_sparse(fn, total_size, hole_size): + st = os.stat(fn) + assert st.st_size == total_size + sparse = True + if sparse and hasattr(st, 'st_blocks') and st.st_blocks * 512 >= st.st_size: + sparse = False + if sparse and hasattr(os, 'SEEK_HOLE') and hasattr(os, 'SEEK_DATA'): + with open(fn, 'rb') as fd: + # only check if the first hole is as expected, because the 2nd hole check + # is problematic on xfs due to its "dynamic speculative EOF preallocation + try: + if fd.seek(0, os.SEEK_HOLE) != 0: + sparse = False + if fd.seek(0, os.SEEK_DATA) != hole_size: + sparse = False + except OSError: + # OS/FS does not really support SEEK_HOLE/SEEK_DATA + sparse = False + return sparse + filename = os.path.join(self.input_path, 'sparse') content = b'foobar' hole_size = 5 * (1 << CHUNK_MAX_EXP) # 5 full chunker buffers + total_size = hole_size + len(content) + hole_size with open(filename, 'wb') as fd: # create a file that has a hole at the beginning and end (if the # OS and filesystem supports sparse files) @@ -406,39 +425,23 @@ class ArchiverTestCase(ArchiverTestCaseBase): fd.seek(hole_size, 1) pos = fd.tell() fd.truncate(pos) - total_len = hole_size + len(content) + hole_size - st = os.stat(filename) - self.assert_equal(st.st_size, total_len) - if sparse_support and hasattr(st, 'st_blocks'): - self.assert_true(st.st_blocks * 512 < total_len) # is input sparse? - self.cmd('init', self.repository_location) - self.cmd('create', self.repository_location + '::test', 'input') - with changedir('output'): - self.cmd('extract', '--sparse', self.repository_location + '::test') - self.assert_dirs_equal('input', 'output/input') - filename = os.path.join(self.output_path, 'input', 'sparse') - with open(filename, 'rb') as fd: - # check if file contents are as expected - self.assert_equal(fd.read(hole_size), b'\0' * hole_size) - self.assert_equal(fd.read(len(content)), content) - self.assert_equal(fd.read(hole_size), b'\0' * hole_size) - st = os.stat(filename) - self.assert_equal(st.st_size, total_len) + # we first check if we could create a sparse input file: + sparse_support = is_sparse(filename, total_size, hole_size) if sparse_support: - if hasattr(st, 'st_blocks'): - # do only check if it is less, do NOT check if it is much less - # as that causes troubles on xfs, zfs, ntfs: - self.assert_true(st.st_blocks * 512 < total_len) - if hasattr(os, 'SEEK_HOLE') and hasattr(os, 'SEEK_DATA'): - with open(filename, 'rb') as fd: - # only check if the first hole is as expected, because the 2nd hole check - # is problematic on xfs due to its "dynamic speculative EOF preallocation - try: - self.assert_equal(fd.seek(0, os.SEEK_HOLE), 0) - self.assert_equal(fd.seek(0, os.SEEK_DATA), hole_size) - except OSError: - # does not really support SEEK_HOLE/SEEK_DATA - pass + # we could create a sparse input file, so creating a backup of it and + # extracting it again (as sparse) should also work: + self.cmd('init', self.repository_location) + self.cmd('create', self.repository_location + '::test', 'input') + with changedir(self.output_path): + self.cmd('extract', '--sparse', self.repository_location + '::test') + self.assert_dirs_equal('input', 'output/input') + filename = os.path.join(self.output_path, 'input', 'sparse') + with open(filename, 'rb') as fd: + # check if file contents are as expected + self.assert_equal(fd.read(hole_size), b'\0' * hole_size) + self.assert_equal(fd.read(len(content)), content) + self.assert_equal(fd.read(hole_size), b'\0' * hole_size) + self.assert_true(is_sparse(filename, total_size, hole_size)) def test_unusual_filenames(self): filenames = ['normal', 'with some blanks', '(with_parens)', ] From cf4d76104254078633c9d94bc3972fc2f9c11e2d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 11 Jul 2016 23:16:53 +0200 Subject: [PATCH 03/11] docs: fix the highlighting default is "python", that's why some help fragments and bash scripts looked strange. --- README.rst | 7 ++++++- docs/api.rst | 1 + docs/deployment.rst | 1 + docs/development.rst | 1 + docs/faq.rst | 3 ++- docs/installation.rst | 1 + docs/internals.rst | 1 + docs/quickstart.rst | 1 + docs/usage.rst | 1 + 9 files changed, 15 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index a9d338151..035a38d91 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,7 @@ |screencast| +.. highlight:: bash + What is BorgBackup? =================== @@ -87,7 +89,10 @@ Initialize a new backup repository and create a backup archive:: $ borg init /path/to/repo $ borg create /path/to/repo::Saturday1 ~/Documents -Now doing another backup, just to show off the great deduplication:: +Now doing another backup, just to show off the great deduplication: + +.. code-block:: none + :emphasize-lines: 11 $ borg create -v --stats /path/to/repo::Saturday2 ~/Documents ----------------------------------------------------------------------------- diff --git a/docs/api.rst b/docs/api.rst index 31a22fbb3..8dfc8cce0 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,3 +1,4 @@ +.. highlight:: python API Documentation ================= diff --git a/docs/deployment.rst b/docs/deployment.rst index 1620aace9..3c76500fe 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -1,4 +1,5 @@ .. include:: global.rst.inc +.. highlight:: none .. _deployment: Deployment diff --git a/docs/development.rst b/docs/development.rst index 524957e01..2dd47e70c 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -1,4 +1,5 @@ .. include:: global.rst.inc +.. highlight:: bash .. _development: Development diff --git a/docs/faq.rst b/docs/faq.rst index 3effa0002..877944930 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -1,5 +1,6 @@ -.. _faq: .. include:: global.rst.inc +.. highlight:: none +.. _faq: Frequently asked questions ========================== diff --git a/docs/installation.rst b/docs/installation.rst index 84b7b8feb..b39bb0afa 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -1,4 +1,5 @@ .. include:: global.rst.inc +.. highlight:: bash .. _installation: Installation diff --git a/docs/internals.rst b/docs/internals.rst index 1bd14bb1e..61d845893 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1,4 +1,5 @@ .. include:: global.rst.inc +.. highlight:: none .. _internals: Internals diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 31d2e216b..e3186f353 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -1,4 +1,5 @@ .. include:: global.rst.inc +.. highlight:: bash .. _quickstart: Quick Start diff --git a/docs/usage.rst b/docs/usage.rst index 443eda1c1..fd3027526 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,4 +1,5 @@ .. include:: global.rst.inc +.. highlight:: none .. _detailed_usage: Usage From 98530fffd3f81977a03c838167921d0d58f697c4 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Tue, 12 Jul 2016 11:57:58 +0200 Subject: [PATCH 04/11] xattr: don't log before logging is set up --- borg/xattr.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/borg/xattr.py b/borg/xattr.py index f2f764737..2058f61ca 100644 --- a/borg/xattr.py +++ b/borg/xattr.py @@ -63,12 +63,10 @@ if sys.platform.startswith('linux') and 'fakeroot' in LD_PRELOAD: libc_name = LD_PRELOAD XATTR_FAKEROOT = True - try: libc = CDLL(libc_name, use_errno=True) except OSError as e: msg = "Can't find C library [%s]. Try installing ldconfig, gcc/cc or objdump." % e - logger.error(msg) raise Exception(msg) From 0e0f487b95d4668660a29aea8012c4b30a1fcc5a Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Tue, 12 Jul 2016 13:17:54 +0200 Subject: [PATCH 05/11] test_atime: try to open with O_NOATIME to determine support --- borg/testsuite/archiver.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index ab69770e0..11bbe6f79 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -17,7 +17,7 @@ from hashlib import sha256 import pytest from .. import xattr -from ..archive import Archive, ChunkBuffer, CHUNK_MAX_EXP +from ..archive import Archive, ChunkBuffer, CHUNK_MAX_EXP, flags_noatime, flags_normal from ..archiver import Archiver from ..cache import Cache from ..crypto import bytes_to_long, num_aes_blocks @@ -365,6 +365,12 @@ class ArchiverTestCase(ArchiverTestCaseBase): def test_atime(self): self.create_test_files() atime, mtime = 123456780, 234567890 + try: + os.close(os.open('input/file1', flags_noatime)) + except PermissionError: + have_noatime = False + else: + have_noatime = flags_noatime != flags_normal os.utime('input/file1', (atime, mtime)) self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') @@ -373,7 +379,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): sti = os.stat('input/file1') sto = os.stat('output/input/file1') assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9 - if hasattr(os, 'O_NOATIME'): + if have_noatime: assert sti.st_atime_ns == sto.st_atime_ns == atime * 1e9 else: # it touched the input file's atime while backing it up From c07d91e9caf5c77f1570607371091c0d6d3f81b0 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Tue, 12 Jul 2016 14:42:00 +0200 Subject: [PATCH 06/11] test_atime: exclude GNU Hurd from this test It has O_NOATIME, opening doesn't fail with EPERM, but it still updates the atime of the file. --- borg/testsuite/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 11bbe6f79..0ec78587f 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -370,7 +370,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): except PermissionError: have_noatime = False else: - have_noatime = flags_noatime != flags_normal + have_noatime = (flags_noatime != flags_normal and sys.platform != 'gnu0') os.utime('input/file1', (atime, mtime)) self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') From 2c616527cfd6a893a6b214af336ab5fc1416aa4a Mon Sep 17 00:00:00 2001 From: James Clarke Date: Sun, 19 Jun 2016 22:04:03 +0100 Subject: [PATCH 07/11] Correctly handle multiple LD_PRELOAD entries; fixes #1111 --- borg/xattr.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/borg/xattr.py b/borg/xattr.py index 2058f61ca..9f4ab34d0 100644 --- a/borg/xattr.py +++ b/borg/xattr.py @@ -2,6 +2,7 @@ """ import errno import os +import re import subprocess import sys import tempfile @@ -52,16 +53,20 @@ if libc_name is None: # the 'test_extract_capabilities' test, but also allows xattrs to work with fakeroot on Linux in normal use. # TODO: Check whether fakeroot supports xattrs on all platforms supported below. # TODO: If that's the case then we can make Borg fakeroot-xattr-compatible on these as well. -LD_PRELOAD = os.environ.get('LD_PRELOAD', '') XATTR_FAKEROOT = False -if sys.platform.startswith('linux') and 'fakeroot' in LD_PRELOAD: - fakeroot_version = LooseVersion(subprocess.check_output(['fakeroot', '-v']).decode('ascii').split()[-1]) - if fakeroot_version >= LooseVersion("1.20.2"): - # 1.20.2 has been confirmed to have xattr support - # 1.18.2 has been confirmed not to have xattr support - # Versions in-between are unknown - libc_name = LD_PRELOAD - XATTR_FAKEROOT = True +if sys.platform.startswith('linux'): + LD_PRELOAD = os.environ.get('LD_PRELOAD', '') + preloads = re.split("[ :]", LD_PRELOAD) + for preload in preloads: + if preload.startswith("libfakeroot"): + fakeroot_version = LooseVersion(subprocess.check_output(['fakeroot', '-v']).decode('ascii').split()[-1]) + if fakeroot_version >= LooseVersion("1.20.2"): + # 1.20.2 has been confirmed to have xattr support + # 1.18.2 has been confirmed not to have xattr support + # Versions in-between are unknown + libc_name = preload + XATTR_FAKEROOT = True + break try: libc = CDLL(libc_name, use_errno=True) From 70c2d69799112cd6762282d5a1a472f4c2f308bb Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Tue, 12 Jul 2016 16:07:22 +0200 Subject: [PATCH 08/11] xattr LD_PRELOAD: diaper --- borg/xattr.py | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/borg/xattr.py b/borg/xattr.py index 9f4ab34d0..e88d7ce8c 100644 --- a/borg/xattr.py +++ b/borg/xattr.py @@ -53,20 +53,23 @@ if libc_name is None: # the 'test_extract_capabilities' test, but also allows xattrs to work with fakeroot on Linux in normal use. # TODO: Check whether fakeroot supports xattrs on all platforms supported below. # TODO: If that's the case then we can make Borg fakeroot-xattr-compatible on these as well. -XATTR_FAKEROOT = False -if sys.platform.startswith('linux'): - LD_PRELOAD = os.environ.get('LD_PRELOAD', '') - preloads = re.split("[ :]", LD_PRELOAD) - for preload in preloads: - if preload.startswith("libfakeroot"): - fakeroot_version = LooseVersion(subprocess.check_output(['fakeroot', '-v']).decode('ascii').split()[-1]) - if fakeroot_version >= LooseVersion("1.20.2"): - # 1.20.2 has been confirmed to have xattr support - # 1.18.2 has been confirmed not to have xattr support - # Versions in-between are unknown - libc_name = preload - XATTR_FAKEROOT = True - break +try: + XATTR_FAKEROOT = False + if sys.platform.startswith('linux'): + LD_PRELOAD = os.environ.get('LD_PRELOAD', '') + preloads = re.split("[ :]", LD_PRELOAD) + for preload in preloads: + if preload.startswith("libfakeroot"): + fakeroot_version = LooseVersion(subprocess.check_output(['fakeroot', '-v']).decode('ascii').split()[-1]) + if fakeroot_version >= LooseVersion("1.20.2"): + # 1.20.2 has been confirmed to have xattr support + # 1.18.2 has been confirmed not to have xattr support + # Versions in-between are unknown + libc_name = preload + XATTR_FAKEROOT = True + break +except: + pass try: libc = CDLL(libc_name, use_errno=True) From 4891d33e2a6dda7b91d3585f8fa36397ec5286a4 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Tue, 12 Jul 2016 16:45:20 +0200 Subject: [PATCH 09/11] test_atime: detect O_NOATIME support by checking atime and flags --- borg/testsuite/archiver.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 0ec78587f..e523e7233 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -363,14 +363,20 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.assert_equal(filter(info_output), filter(info_output2)) def test_atime(self): + def has_noatime(some_file): + atime_before = os.stat(some_file).st_atime_ns + try: + os.close(os.open(some_file, flags_noatime)) + except PermissionError: + return False + else: + atime_after = os.stat(some_file).st_atime_ns + noatime_used = flags_noatime != flags_normal + return noatime_used and atime_before == atime_after + self.create_test_files() atime, mtime = 123456780, 234567890 - try: - os.close(os.open('input/file1', flags_noatime)) - except PermissionError: - have_noatime = False - else: - have_noatime = (flags_noatime != flags_normal and sys.platform != 'gnu0') + have_noatime = has_noatime('input/file1') os.utime('input/file1', (atime, mtime)) self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') From 06a1a899ee1fa0b9c4fecda36e52de7be983828b Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Tue, 12 Jul 2016 20:55:26 +0200 Subject: [PATCH 10/11] update CHANGES --- docs/changes.rst | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 4d5a74d18..0696da51f 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -50,6 +50,25 @@ The best check that everything is ok is to run a dry-run extraction:: borg extract -v --dry-run REPO::ARCHIVE +Version 1.0.6 (2016-07-12) +-------------------------- + +Bug fixes: + +- Linux: handle multiple LD_PRELOAD entries correctly, #1314, #1111 +- Fix crash with unclear message if the libc is not found, #1314, #1111 + +Other changes: + +- tests: + + - Fixed O_NOATIME tests for Solaris and GNU Hurd, #1315 + - Fixed sparse file tests for (file) systems not supporting it, #1310 +- docs: + + - Fixed syntax highlighting, #1313 + - misc docs: added data processing overview picture + Version 1.0.6rc1 (2016-07-10) ----------------------------- From bbb280c129a3fe7990e3f6e37976223e009e3641 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 12 Jul 2016 23:10:13 +0200 Subject: [PATCH 11/11] ran build_usage --- docs/usage/help.rst.inc | 76 ++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index f6618b677..4d7c776a7 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -1,5 +1,43 @@ .. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! +.. _borg_placeholders: + +borg help placeholders +~~~~~~~~~~~~~~~~~~~~~~ + + +Repository (or Archive) URLs and --prefix values support these placeholders: + +{hostname} + + The (short) hostname of the machine. + +{fqdn} + + The full name of the machine. + +{now} + + The current local date and time. + +{utcnow} + + The current UTC date and time. + +{user} + + The user name (or UID, if no name is available) of the user running borg. + +{pid} + + The current process ID. + +Examples:: + + borg create /path/to/repo::{hostname}-{user}-{utcnow} ... + borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ... + borg prune --prefix '{hostname}-' ... + .. _borg_patterns: borg help patterns @@ -93,41 +131,3 @@ Examples:: EOF $ borg create --exclude-from exclude.txt backup / -.. _borg_placeholders: - -borg help placeholders -~~~~~~~~~~~~~~~~~~~~~~ - - -Repository (or Archive) URLs and --prefix values support these placeholders: - -{hostname} - - The (short) hostname of the machine. - -{fqdn} - - The full name of the machine. - -{now} - - The current local date and time. - -{utcnow} - - The current UTC date and time. - -{user} - - The user name (or UID, if no name is available) of the user running borg. - -{pid} - - The current process ID. - -Examples:: - - borg create /path/to/repo::{hostname}-{user}-{utcnow} ... - borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ... - borg prune --prefix '{hostname}-' ... -