diff --git a/MANIFEST.in b/MANIFEST.in index f8e9eda62..bbadcbcea 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,12 +1,8 @@ include README.rst AUTHORS LICENSE CHANGES.rst MANIFEST.in -graft src -recursive-exclude src *.pyc -recursive-exclude src *.pyo -recursive-exclude src *.so -recursive-include docs * -recursive-exclude docs *.pyc -recursive-exclude docs *.pyo -prune docs/_build +exclude .coveragerc .gitattributes .gitignore .travis.yml Vagrantfile prune .travis prune .github -exclude .coveragerc .gitattributes .gitignore .travis.yml Vagrantfile +graft src +graft docs +prune docs/_build +global-exclude *.py[co] *.orig *.so *.dll diff --git a/Vagrantfile b/Vagrantfile index 781312477..babbe432d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -63,11 +63,13 @@ end def packages_darwin return <<-EOF # install all the (security and other) updates + sudo softwareupdate --ignore iTunesX + sudo softwareupdate --ignore iTunes sudo softwareupdate --install --all # get osxfuse 3.x release code from github: - curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.5.3/osxfuse-3.5.3.dmg >osxfuse.dmg + curl -s -L https://github.com/osxfuse/osxfuse/releases/download/osxfuse-3.5.4/osxfuse-3.5.4.dmg >osxfuse.dmg MOUNTDIR=$(echo `hdiutil mount osxfuse.dmg | tail -1 | awk '{$1="" ; print $0}'` | xargs -0 echo) \ - && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.5.3.pkg" -target / + && sudo installer -pkg "${MOUNTDIR}/Extras/FUSE for macOS 3.5.4.pkg" -target / sudo chown -R vagrant /usr/local # brew must be able to create stuff here ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" brew update @@ -83,11 +85,13 @@ end def packages_freebsd return <<-EOF + # VM has no hostname set + hostname freebsd # install all the (security and other) updates, base system freebsd-update --not-running-from-cron fetch install # for building borgbackup and dependencies: pkg install -y openssl liblz4 fusefs-libs pkgconf - pkg install -y fakeroot git bash + pkg install -y git bash # for building python: pkg install -y sqlite3 # make bash default / work: @@ -227,7 +231,7 @@ def install_pythons(boxname) pyenv install 3.4.0 # tests pyenv install 3.5.0 # tests pyenv install 3.6.0 # tests - pyenv install 3.5.2 # binary build, use latest 3.5.x release + pyenv install 3.5.3 # binary build, use latest 3.5.x release pyenv rehash EOF end @@ -245,8 +249,8 @@ def build_pyenv_venv(boxname) . ~/.bash_profile cd /vagrant/borg # use the latest 3.5 release - pyenv global 3.5.2 - pyenv virtualenv 3.5.2 borg-env + pyenv global 3.5.3 + pyenv virtualenv 3.5.3 borg-env ln -s ~/.pyenv/versions/borg-env . EOF end @@ -446,6 +450,16 @@ Vagrant.configure(2) do |config| # OS X config.vm.define "darwin64" do |b| b.vm.box = "jhcook/yosemite-clitools" + b.vm.provider :virtualbox do |v| + v.customize ['modifyvm', :id, '--ostype', 'MacOS1010_64'] + v.customize ['modifyvm', :id, '--paravirtprovider', 'default'] + # Adjust CPU settings according to + # https://github.com/geerlingguy/macos-virtualbox-vm + v.customize ['modifyvm', :id, '--cpuidset', + '00000001', '000306a9', '00020800', '80000201', '178bfbff'] + # Disable USB variant requiring Virtualbox proprietary extension pack + v.customize ["modifyvm", :id, '--usbehci', 'off', '--usbxhci', 'off'] + end b.vm.provision "packages darwin", :type => :shell, :privileged => false, :inline => packages_darwin b.vm.provision "install pyenv", :type => :shell, :privileged => false, :inline => install_pyenv("darwin64") b.vm.provision "fix pyenv", :type => :shell, :privileged => false, :inline => fix_pyenv_darwin("darwin64") @@ -458,11 +472,11 @@ Vagrant.configure(2) do |config| end # BSD - # note: the FreeBSD-10.3-STABLE box needs "vagrant up" twice to start. + # note: the FreeBSD-10.3-RELEASE box needs "vagrant up" twice to start. config.vm.define "freebsd64" do |b| - b.vm.box = "freebsd/FreeBSD-10.3-STABLE" + b.vm.box = "freebsd/FreeBSD-10.3-RELEASE" b.vm.provider :virtualbox do |v| - v.memory = 768 + v.memory = 1536 end b.ssh.shell = "sh" b.vm.provision "install system packages", :type => :shell, :inline => packages_freebsd diff --git a/docs/changes.rst b/docs/changes.rst index 70bab92f8..134231030 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -5,8 +5,8 @@ This section is used for infos about security and corruption issues. .. _tam_vuln: -Pre-1.0.9 manifest spoofing vulnerability ------------------------------------------ +Pre-1.0.9 manifest spoofing vulnerability (CVE-2016-10099) +---------------------------------------------------------- A flaw in the cryptographic authentication scheme in Borg allowed an attacker to spoof the manifest. The attack requires an attacker to be able to @@ -54,7 +54,9 @@ Vulnerability time line: * 2016-11-14: Vulnerability and fix discovered during review of cryptography by Marian Beermann (@enkore) * 2016-11-20: First patch -* 2016-12-18: Released fixed versions: 1.0.9, 1.1.0b3 +* 2016-12-20: Released fixed version 1.0.9 +* 2017-01-02: CVE was assigned +* 2017-01-15: Released fixed version 1.1.0b3 (fix was previously only available from source) .. _attic013_check_corruption: @@ -219,8 +221,8 @@ Other changes: - remove all BORG_* env vars from the outer environment -Version 1.0.10rc1 (not released yet) ------------------------------------- +Version 1.0.10rc1 (2017-01-29) +------------------------------ Bug fixes: @@ -235,9 +237,16 @@ Bug fixes: - Fixed change-passphrase crashing with unencrypted repositories, #1978 - Fixed "borg check repo::archive" indicating success if "archive" does not exist, #1997 - borg check: print non-exit-code warning if --last or --prefix aren't fulfilled +- fix bad parsing of wrong repo location syntax +- create: don't create hard link refs to failed files, + mount: handle invalid hard link refs, #2092 +- detect mingw byte order, #2073 +- creating a new segment: use "xb" mode, #2099 +- mount: umount on SIGINT/^C when in foreground, #2082 Other changes: +- binary: use fixed AND freshly compiled pyinstaller bootloader, #2002 - xattr: ignore empty names returned by llistxattr(2) et al - Enable the fault handler: install handlers for the SIGSEGV, SIGFPE, SIGABRT, SIGBUS and SIGILL signals to dump the Python traceback. @@ -247,8 +256,11 @@ Other changes: - tests: - vagrant / travis / tox: add Python 3.6 based testing - - vagrant: fix openbsd repo, fixes #2042 - - vagrant: fix the freebsd64 machine, #2037 + - vagrant: fix openbsd repo, #2042 + - vagrant: fix the freebsd64 machine, #2037 #2067 + - vagrant: use python 3.5.3 to build binaries, #2078 + - vagrant: use osxfuse 3.5.4 for tests / to build binaries + vagrant: improve darwin64 VM settings - travis: fix osxfuse install (fixes OS X testing on Travis CI) - travis: require succeeding OS X tests, #2028 - travis: use latest pythons for OS X based testing @@ -260,12 +272,18 @@ Other changes: - language clarification - VM backup FAQ - borg create: document how to backup stdin, #2013 - borg upgrade: fix incorrect title levels + - add CVE numbers for issues fixed in 1.0.9, #2106 - fix typos (taken from Debian package patch) - remote: include data hexdump in "unexpected RPC data" error message - remote: log SSH command line at debug level - API_VERSION: use numberspaces, #2023 - remove .github from pypi package, #2051 - add pip and setuptools to requirements file, #2030 +- SyncFile: fix use of fd object after close (cosmetic) +- Manifest.in: simplify, exclude *.{so,dll,orig}, #2066 +- ignore posix_fadvise errors in repository.py, #2095 + (works around issues with docker on ARM) +- make LoggedIO.close_segment reentrant, avoid reentrance Version 1.0.9 (2016-12-20) @@ -276,10 +294,14 @@ Security fixes: - A flaw in the cryptographic authentication scheme in Borg allowed an attacker to spoof the manifest. See :ref:`tam_vuln` above for the steps you should take. + + CVE-2016-10099 was assigned to this vulnerability. - borg check: When rebuilding the manifest (which should only be needed very rarely) duplicate archive names would be handled on a "first come first serve" basis, allowing an attacker to apparently replace archives. + CVE-2016-10100 was assigned to this vulnerability. + Bug fixes: - borg check: diff --git a/docs/usage/mount.rst.inc b/docs/usage/mount.rst.inc index ee63cb42b..e15f25afa 100644 --- a/docs/usage/mount.rst.inc +++ b/docs/usage/mount.rst.inc @@ -61,3 +61,10 @@ The BORG_MOUNT_DATA_CACHE_ENTRIES environment variable is meant for advanced use to tweak the performance. It sets the number of cached data chunks; additional memory usage can be up to ~8 MiB times this number. The default is the number of CPU cores. + +When the daemonized process receives a signal or crashes, it does not unmount. +Unmounting in these cases could cause an active rsync or similar process +to unintentionally delete data. + +When running in the foreground ^C/SIGINT unmounts cleanly, but other +signals or crashes do not. diff --git a/src/borg/_hashindex.c b/src/borg/_hashindex.c index 5fea18f79..adcb90fd7 100644 --- a/src/borg/_hashindex.c +++ b/src/borg/_hashindex.c @@ -12,18 +12,26 @@ #include #endif -#if (defined(BYTE_ORDER)&&(BYTE_ORDER == BIG_ENDIAN)) || \ - (defined(_BIG_ENDIAN)&&defined(__SVR4)&&defined(__sun)) -#define _le32toh(x) __builtin_bswap32(x) -#define _htole32(x) __builtin_bswap32(x) -#elif (defined(BYTE_ORDER)&&(BYTE_ORDER == LITTLE_ENDIAN)) || \ - (defined(_LITTLE_ENDIAN)&&defined(__SVR4)&&defined(__sun)) -#define _le32toh(x) (x) -#define _htole32(x) (x) +#if (defined(BYTE_ORDER) && (BYTE_ORDER == BIG_ENDIAN)) || \ + (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) || \ + (defined(_BIG_ENDIAN) && defined(__SVR4)&&defined(__sun)) +#define BORG_BIG_ENDIAN 1 +#elif (defined(BYTE_ORDER) && (BYTE_ORDER == LITTLE_ENDIAN)) || \ + (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) || \ + (defined(_LITTLE_ENDIAN) && defined(__SVR4)&&defined(__sun)) +#define BORG_BIG_ENDIAN 0 #else #error Unknown byte order #endif +#if BORG_BIG_ENDIAN +#define _le32toh(x) __builtin_bswap32(x) +#define _htole32(x) __builtin_bswap32(x) +#else +#define _le32toh(x) (x) +#define _htole32(x) (x) +#endif + #define MAGIC "BORG_IDX" #define MAGIC_LEN 8 diff --git a/src/borg/archive.py b/src/borg/archive.py index 16424196b..f499f9233 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -838,8 +838,6 @@ Number of files: {0.stats.nfiles}'''.format( self.add_item(item) status = 'h' # regular file, hardlink (to already seen inodes) return status - else: - self.hard_links[st.st_ino, st.st_dev] = safe_path is_special_file = is_special(st.st_mode) if not is_special_file: path_hash = self.key.id_hash(safe_encode(os.path.join(self.cwd, path))) @@ -890,6 +888,9 @@ Number of files: {0.stats.nfiles}'''.format( item.mode = stat.S_IFREG | stat.S_IMODE(item.mode) self.stats.nfiles += 1 self.add_item(item) + if st.st_nlink > 1 and source is None: + # Add the hard link reference *after* the file has been added to the archive. + self.hard_links[st.st_ino, st.st_dev] = safe_path return status @staticmethod diff --git a/src/borg/archiver.py b/src/borg/archiver.py index a23a61fc6..9301b1a7b 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2247,6 +2247,13 @@ class Archiver: to tweak the performance. It sets the number of cached data chunks; additional memory usage can be up to ~8 MiB times this number. The default is the number of CPU cores. + + When the daemonized process receives a signal or crashes, it does not unmount. + Unmounting in these cases could cause an active rsync or similar process + to unintentionally delete data. + + When running in the foreground ^C/SIGINT unmounts cleanly, but other + signals or crashes do not. """) subparser = subparsers.add_parser('mount', parents=[common_parser], add_help=False, description=self.do_mount.__doc__, diff --git a/src/borg/fuse.py b/src/borg/fuse.py index 8de141fef..dbf34e1a6 100644 --- a/src/borg/fuse.py +++ b/src/borg/fuse.py @@ -5,6 +5,7 @@ import stat import tempfile import time from collections import defaultdict +from signal import SIGINT from distutils.version import LooseVersion from zlib import adler32 @@ -125,7 +126,8 @@ class FuseOperations(llfuse.Operations): umount = False try: signal = fuse_main() - umount = (signal is None) # no crash and no signal -> umount request + # no crash and no signal (or it's ^C and we're in the foreground) -> umount request + umount = (signal is None or (signal == SIGINT and foreground)) finally: llfuse.close(umount) @@ -190,6 +192,7 @@ class FuseOperations(llfuse.Operations): path = os.fsencode(os.path.normpath(item.path)) self.file_versions[path] = version + path = item.path del item.path # safe some space if 'source' in item and stat.S_ISREG(item.mode): # a hardlink, no contents, is the hardlink master @@ -199,7 +202,11 @@ class FuseOperations(llfuse.Operations): version = self.file_versions[source] source = make_versioned_name(source, version, add_dir=True) name = make_versioned_name(name, version) - inode = self._find_inode(source, prefix) + try: + inode = self._find_inode(source, prefix) + except KeyError: + logger.warning('Skipping broken hard link: %s -> %s', path, item.source) + return item = self.cache.get(inode) item.nlink = item.get('nlink', 1) + 1 self.items[inode] = item diff --git a/src/borg/helpers.py b/src/borg/helpers.py index e61a6d57c..19293f151 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -926,11 +926,17 @@ class Location: """ # path must not contain :: (it ends at :: or string end), but may contain single colons. - # to avoid ambiguities with other regexes, it must also not start with ":". + # to avoid ambiguities with other regexes, it must also not start with ":" nor with "//" nor with "ssh://". path_re = r""" - (?!:) # not starting with ":" + (?!(:|//|ssh://)) # not starting with ":" or // or ssh:// (?P([^:]|(:(?!:)))+) # any chars, but no "::" """ + # abs_path must not contain :: (it ends at :: or string end), but may contain single colons. + # it must start with a / and that slash is part of the path. + abs_path_re = r""" + (?P(/([^:]|(:(?!:)))+)) # start with /, then any chars, but no "::" + """ + # optional ::archive_name at the end, archive name must not contain "/". # borg mount's FUSE filesystem creates one level of directories from # the archive names and of course "/" is not valid in a directory name. @@ -945,7 +951,7 @@ class Location: (?Pssh):// # ssh:// """ + optional_user_re + r""" # user@ (optional) (?P[^:/]+)(?::(?P\d+))? # host or host:port - """ + path_re + optional_archive_re, re.VERBOSE) # path or path::archive + """ + abs_path_re + optional_archive_re, re.VERBOSE) # path or path::archive file_re = re.compile(r""" (?Pfile):// # file:// diff --git a/src/borg/platform/__init__.py b/src/borg/platform/__init__.py index 79fe949df..34202bf90 100644 --- a/src/borg/platform/__init__.py +++ b/src/borg/platform/__init__.py @@ -8,7 +8,7 @@ Public APIs are documented in platform.base. from .base import acl_get, acl_set from .base import set_flags, get_flags -from .base import SaveFile, SyncFile, sync_dir, fdatasync +from .base import SaveFile, SyncFile, sync_dir, fdatasync, safe_fadvise from .base import swidth, umount, API_VERSION from .base import process_alive, get_process_id, local_pid_alive diff --git a/src/borg/platform/base.py b/src/borg/platform/base.py index 1449a1f74..0d2fb51b8 100644 --- a/src/borg/platform/base.py +++ b/src/borg/platform/base.py @@ -63,6 +63,22 @@ def sync_dir(path): os.close(fd) +def safe_fadvise(fd, offset, len, advice): + if hasattr(os, 'posix_fadvise'): + advice = getattr(os, 'POSIX_FADV_' + advice) + try: + os.posix_fadvise(fd, offset, len, advice) + except OSError: + # usually, posix_fadvise can't fail for us, but there seem to + # be failures when running borg under docker on ARM, likely due + # to a bug outside of borg. + # also, there is a python wrapper bug, always giving errno = 0. + # https://github.com/borgbackup/borg/issues/2095 + # as this call is not critical for correct function (just to + # optimize cache usage), we ignore these errors. + pass + + class SyncFile: """ A file class that is supposed to enable write ordering (one way or another) and data durability after close(). @@ -103,15 +119,21 @@ class SyncFile: from .. import platform self.fd.flush() platform.fdatasync(self.fileno) - if hasattr(os, 'posix_fadvise'): - os.posix_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED) + # tell the OS that it does not need to cache what we just wrote, + # avoids spoiling the cache for the OS and other processes. + safe_fadvise(self.fileno, 0, 0, 'DONTNEED') def close(self): """sync() and close.""" from .. import platform - self.sync() - self.fd.close() - platform.sync_dir(os.path.dirname(self.fd.name)) + dirname = None + try: + dirname = os.path.dirname(self.fd.name) + self.sync() + finally: + self.fd.close() + if dirname: + platform.sync_dir(dirname) class SaveFile: diff --git a/src/borg/platform/linux.pyx b/src/borg/platform/linux.pyx index e0ee8992f..e87983c7d 100644 --- a/src/borg/platform/linux.pyx +++ b/src/borg/platform/linux.pyx @@ -8,6 +8,7 @@ from ..helpers import posix_acl_use_stored_uid_gid from ..helpers import user2uid, group2gid from ..helpers import safe_decode, safe_encode from .base import SyncFile as BaseSyncFile +from .base import safe_fadvise from .posix import swidth from libc cimport errno @@ -216,7 +217,7 @@ cdef _sync_file_range(fd, offset, length, flags): assert length & PAGE_MASK == 0, "length %d not page-aligned" % length if sync_file_range(fd, offset, length, flags) != 0: raise OSError(errno.errno, os.strerror(errno.errno)) - os.posix_fadvise(fd, offset, length, os.POSIX_FADV_DONTNEED) + safe_fadvise(fd, offset, length, 'DONTNEED') cdef unsigned PAGE_MASK = resource.getpagesize() - 1 @@ -251,7 +252,9 @@ class SyncFile(BaseSyncFile): def sync(self): self.fd.flush() os.fdatasync(self.fileno) - os.posix_fadvise(self.fileno, 0, 0, os.POSIX_FADV_DONTNEED) + # tell the OS that it does not need to cache what we just wrote, + # avoids spoiling the cache for the OS and other processes. + safe_fadvise(self.fileno, 0, 0, 'DONTNEED') def umount(mountpoint): diff --git a/src/borg/repository.py b/src/borg/repository.py index 824985da0..9d4d604dd 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -21,7 +21,7 @@ from .helpers import yes from .locking import Lock, LockError, LockErrorT from .logger import create_logger from .lrucache import LRUCache -from .platform import SaveFile, SyncFile, sync_dir +from .platform import SaveFile, SyncFile, sync_dir, safe_fadvise from .crc32 import crc32 logger = create_logger(__name__) @@ -909,8 +909,7 @@ class LoggedIO: self.fds = None # Just to make sure we're disabled def close_fd(self, fd): - if hasattr(os, 'posix_fadvise'): # only on UNIX - os.posix_fadvise(fd.fileno(), 0, 0, os.POSIX_FADV_DONTNEED) + safe_fadvise(fd.fileno(), 0, 0, 'DONTNEED') fd.close() def segment_iterator(self, segment=None, reverse=False): @@ -1017,11 +1016,12 @@ class LoggedIO: return fd def close_segment(self): - if self._write_fd: + # set self._write_fd to None early to guard against reentry from error handling code pathes: + fd, self._write_fd = self._write_fd, None + if fd is not None: self.segment += 1 self.offset = 0 - self._write_fd.close() - self._write_fd = None + fd.close() def delete_segment(self, segment): if segment in self.fds: diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index ed2b01576..49f32dfd4 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -137,6 +137,11 @@ class TestLocationWithoutEnv: location_time2 = Location('/some/path::archive{now:%s}') assert location_time1.archive != location_time2.archive + def test_bad_syntax(self): + with pytest.raises(ValueError): + # this is invalid due to the 2nd colon, correct: 'ssh://user@host/path' + Location('ssh://user@host:/path') + class TestLocationWithEnv: def test_ssh(self, monkeypatch):