diff --git a/README.rst b/README.rst index a9d338151..035a38d91 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,7 @@ |screencast| +.. highlight:: bash + What is BorgBackup? =================== @@ -87,7 +89,10 @@ Initialize a new backup repository and create a backup archive:: $ borg init /path/to/repo $ borg create /path/to/repo::Saturday1 ~/Documents -Now doing another backup, just to show off the great deduplication:: +Now doing another backup, just to show off the great deduplication: + +.. code-block:: none + :emphasize-lines: 11 $ borg create -v --stats /path/to/repo::Saturday2 ~/Documents ----------------------------------------------------------------------------- diff --git a/docs/api.rst b/docs/api.rst index 31a22fbb3..8dfc8cce0 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -1,3 +1,4 @@ +.. highlight:: python API Documentation ================= diff --git a/docs/changes.rst b/docs/changes.rst index 15479a5ab..18d509764 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -123,6 +123,26 @@ Other changes: - ChunkBuffer: add test for leaving partial chunk in buffer, fixes #945 +Version 1.0.6 (2016-07-12) +-------------------------- + +Bug fixes: + +- Linux: handle multiple LD_PRELOAD entries correctly, #1314, #1111 +- Fix crash with unclear message if the libc is not found, #1314, #1111 + +Other changes: + +- tests: + + - Fixed O_NOATIME tests for Solaris and GNU Hurd, #1315 + - Fixed sparse file tests for (file) systems not supporting it, #1310 +- docs: + + - Fixed syntax highlighting, #1313 + - misc docs: added data processing overview picture + + Version 1.0.6rc1 (2016-07-10) ----------------------------- diff --git a/docs/deployment.rst b/docs/deployment.rst index c31f048f2..a29d09cd8 100644 --- a/docs/deployment.rst +++ b/docs/deployment.rst @@ -1,4 +1,5 @@ .. include:: global.rst.inc +.. highlight:: none .. _deployment: Deployment diff --git a/docs/development.rst b/docs/development.rst index b94554462..c477ee0b9 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -1,4 +1,5 @@ .. include:: global.rst.inc +.. highlight:: bash .. _development: Development diff --git a/docs/faq.rst b/docs/faq.rst index 1ae441ff9..3eee339a2 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -1,5 +1,6 @@ -.. _faq: .. include:: global.rst.inc +.. highlight:: none +.. _faq: Frequently asked questions ========================== diff --git a/docs/installation.rst b/docs/installation.rst index e57737440..a3ef9f01a 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -1,4 +1,5 @@ .. include:: global.rst.inc +.. highlight:: bash .. _installation: Installation diff --git a/docs/internals.rst b/docs/internals.rst index 1bd14bb1e..61d845893 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -1,4 +1,5 @@ .. include:: global.rst.inc +.. highlight:: none .. _internals: Internals diff --git a/docs/misc/internals-picture.txt b/docs/misc/internals-picture.txt new file mode 100644 index 000000000..ae76f0c19 --- /dev/null +++ b/docs/misc/internals-picture.txt @@ -0,0 +1,41 @@ +BorgBackup from 10.000m +======================= + ++--------+ +--------+ +--------+ +|archive0| |archive1| ... |archiveN| ++--------+ +--------+ +--+-----+ + | | | + | | | + | +---+ | + | | | + | | | + +------+-------+ | + | | | | + /chunk\/chunk\/chunk\... /maybe different chunks lists\ ++-----------------------------------------------------------------+ +|item list | ++-----------------------------------------------------------------+ + | + +-------------------------------------+--------------+ + | | | + | | | ++-------------+ +-------------+ | +|item0 | |item1 | | +| - owner | | - owner | | +| - size | | - size | ... +| - ... | | - ... | +| - chunks | | - chunks | ++----+--------+ +-----+-------+ + | | + | +-----+----------------------------+-----------------+ + | | | | + +-o-----o------------+ | + | | | | | + /chunk0\/chunk1\ ... /chunkN\ /chunk0\/chunk1\ ... /chunkN'\ + +-----------------------------+ +------------------------------+ + |file0 | |file0' | + +-----------------------------+ +------------------------------+ + + +Thanks to anarcat for drawing the picture! + diff --git a/docs/quickstart.rst b/docs/quickstart.rst index fbf98da00..76726d25b 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -1,4 +1,5 @@ .. include:: global.rst.inc +.. highlight:: bash .. _quickstart: Quick Start diff --git a/docs/usage.rst b/docs/usage.rst index 200e75046..354b950cf 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -1,4 +1,5 @@ .. include:: global.rst.inc +.. highlight:: none .. _detailed_usage: Usage diff --git a/docs/usage/help.rst.inc b/docs/usage/help.rst.inc index f6618b677..4d7c776a7 100644 --- a/docs/usage/help.rst.inc +++ b/docs/usage/help.rst.inc @@ -1,5 +1,43 @@ .. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! +.. _borg_placeholders: + +borg help placeholders +~~~~~~~~~~~~~~~~~~~~~~ + + +Repository (or Archive) URLs and --prefix values support these placeholders: + +{hostname} + + The (short) hostname of the machine. + +{fqdn} + + The full name of the machine. + +{now} + + The current local date and time. + +{utcnow} + + The current UTC date and time. + +{user} + + The user name (or UID, if no name is available) of the user running borg. + +{pid} + + The current process ID. + +Examples:: + + borg create /path/to/repo::{hostname}-{user}-{utcnow} ... + borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ... + borg prune --prefix '{hostname}-' ... + .. _borg_patterns: borg help patterns @@ -93,41 +131,3 @@ Examples:: EOF $ borg create --exclude-from exclude.txt backup / -.. _borg_placeholders: - -borg help placeholders -~~~~~~~~~~~~~~~~~~~~~~ - - -Repository (or Archive) URLs and --prefix values support these placeholders: - -{hostname} - - The (short) hostname of the machine. - -{fqdn} - - The full name of the machine. - -{now} - - The current local date and time. - -{utcnow} - - The current UTC date and time. - -{user} - - The user name (or UID, if no name is available) of the user running borg. - -{pid} - - The current process ID. - -Examples:: - - borg create /path/to/repo::{hostname}-{user}-{utcnow} ... - borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ... - borg prune --prefix '{hostname}-' ... - diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index f5a44bd9a..41e00c204 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -23,7 +23,7 @@ except ImportError: pass from .. import xattr, helpers, platform -from ..archive import Archive, ChunkBuffer, ArchiveRecreater +from ..archive import Archive, ChunkBuffer, ArchiveRecreater, flags_noatime, flags_normal from ..archiver import Archiver from ..cache import Cache from ..constants import * # NOQA @@ -390,8 +390,20 @@ class ArchiverTestCase(ArchiverTestCaseBase): assert os.readlink('input/link1') == 'somewhere' def test_atime(self): + def has_noatime(some_file): + atime_before = os.stat(some_file).st_atime_ns + try: + os.close(os.open(some_file, flags_noatime)) + except PermissionError: + return False + else: + atime_after = os.stat(some_file).st_atime_ns + noatime_used = flags_noatime != flags_normal + return noatime_used and atime_before == atime_after + self.create_test_files() atime, mtime = 123456780, 234567890 + have_noatime = has_noatime('input/file1') os.utime('input/file1', (atime, mtime)) self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') @@ -400,7 +412,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): sti = os.stat('input/file1') sto = os.stat('output/input/file1') assert sti.st_mtime_ns == sto.st_mtime_ns == mtime * 1e9 - if hasattr(os, 'O_NOATIME'): + if have_noatime: assert sti.st_atime_ns == sto.st_atime_ns == atime * 1e9 else: # it touched the input file's atime while backing it up @@ -420,12 +432,30 @@ class ArchiverTestCase(ArchiverTestCaseBase): return repository.id def test_sparse_file(self): - # Mac OS X has no sparse file support - # Solaris (ZFS) has sparse file support, but is less predictable about it - sparse_support = sys.platform not in ['darwin', 'sunos5'] + def is_sparse(fn, total_size, hole_size): + st = os.stat(fn) + assert st.st_size == total_size + sparse = True + if sparse and hasattr(st, 'st_blocks') and st.st_blocks * 512 >= st.st_size: + sparse = False + if sparse and hasattr(os, 'SEEK_HOLE') and hasattr(os, 'SEEK_DATA'): + with open(fn, 'rb') as fd: + # only check if the first hole is as expected, because the 2nd hole check + # is problematic on xfs due to its "dynamic speculative EOF preallocation + try: + if fd.seek(0, os.SEEK_HOLE) != 0: + sparse = False + if fd.seek(0, os.SEEK_DATA) != hole_size: + sparse = False + except OSError: + # OS/FS does not really support SEEK_HOLE/SEEK_DATA + sparse = False + return sparse + filename = os.path.join(self.input_path, 'sparse') content = b'foobar' hole_size = 5 * (1 << CHUNK_MAX_EXP) # 5 full chunker buffers + total_size = hole_size + len(content) + hole_size with open(filename, 'wb') as fd: # create a file that has a hole at the beginning and end (if the # OS and filesystem supports sparse files) @@ -434,39 +464,23 @@ class ArchiverTestCase(ArchiverTestCaseBase): fd.seek(hole_size, 1) pos = fd.tell() fd.truncate(pos) - total_len = hole_size + len(content) + hole_size - st = os.stat(filename) - self.assert_equal(st.st_size, total_len) - if sparse_support and hasattr(st, 'st_blocks'): - self.assert_true(st.st_blocks * 512 < total_len) # is input sparse? - self.cmd('init', self.repository_location) - self.cmd('create', self.repository_location + '::test', 'input') - with changedir('output'): - self.cmd('extract', '--sparse', self.repository_location + '::test') - self.assert_dirs_equal('input', 'output/input') - filename = os.path.join(self.output_path, 'input', 'sparse') - with open(filename, 'rb') as fd: - # check if file contents are as expected - self.assert_equal(fd.read(hole_size), b'\0' * hole_size) - self.assert_equal(fd.read(len(content)), content) - self.assert_equal(fd.read(hole_size), b'\0' * hole_size) - st = os.stat(filename) - self.assert_equal(st.st_size, total_len) + # we first check if we could create a sparse input file: + sparse_support = is_sparse(filename, total_size, hole_size) if sparse_support: - if hasattr(st, 'st_blocks'): - # do only check if it is less, do NOT check if it is much less - # as that causes troubles on xfs, zfs, ntfs: - self.assert_true(st.st_blocks * 512 < total_len) - if hasattr(os, 'SEEK_HOLE') and hasattr(os, 'SEEK_DATA'): - with open(filename, 'rb') as fd: - # only check if the first hole is as expected, because the 2nd hole check - # is problematic on xfs due to its "dynamic speculative EOF preallocation - try: - self.assert_equal(fd.seek(0, os.SEEK_HOLE), 0) - self.assert_equal(fd.seek(0, os.SEEK_DATA), hole_size) - except OSError: - # does not really support SEEK_HOLE/SEEK_DATA - pass + # we could create a sparse input file, so creating a backup of it and + # extracting it again (as sparse) should also work: + self.cmd('init', self.repository_location) + self.cmd('create', self.repository_location + '::test', 'input') + with changedir(self.output_path): + self.cmd('extract', '--sparse', self.repository_location + '::test') + self.assert_dirs_equal('input', 'output/input') + filename = os.path.join(self.output_path, 'input', 'sparse') + with open(filename, 'rb') as fd: + # check if file contents are as expected + self.assert_equal(fd.read(hole_size), b'\0' * hole_size) + self.assert_equal(fd.read(len(content)), content) + self.assert_equal(fd.read(hole_size), b'\0' * hole_size) + self.assert_true(is_sparse(filename, total_size, hole_size)) def test_unusual_filenames(self): filenames = ['normal', 'with some blanks', '(with_parens)', ] diff --git a/src/borg/xattr.py b/src/borg/xattr.py index 5110cd6d9..9f4ab34d0 100644 --- a/src/borg/xattr.py +++ b/src/borg/xattr.py @@ -72,7 +72,6 @@ try: libc = CDLL(libc_name, use_errno=True) except OSError as e: msg = "Can't find C library [%s]. Try installing ldconfig, gcc/cc or objdump." % e - logger.error(msg) raise Exception(msg)