diff --git a/borg/archive.py b/borg/archive.py index 0626acb04..710b52fb8 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -25,14 +25,13 @@ from .helpers import Chunk, Error, uid2user, user2uid, gid2group, group2gid, \ CompressionDecider1, CompressionDecider2, CompressionSpec, \ IntegrityError from .repository import Repository -from .platform import acl_get, acl_set +from .platform import acl_get, acl_set, set_flags, get_flags from .chunker import Chunker from .hashindex import ChunkIndex, ChunkIndexEntry from .cache import ChunkListEntry import msgpack has_lchmod = hasattr(os, 'lchmod') -has_lchflags = hasattr(os, 'lchflags') flags_normal = os.O_RDONLY | getattr(os, 'O_BINARY', 0) flags_noatime = flags_normal | getattr(os, 'O_NOATIME', 0) @@ -435,10 +434,9 @@ Number of files: {0.stats.nfiles}'''.format( else: os.utime(path, None, ns=(atime, mtime), follow_symlinks=False) acl_set(path, item, self.numeric_owner) - # Only available on OS X and FreeBSD - if has_lchflags and b'bsdflags' in item: + if b'bsdflags' in item: try: - os.lchflags(path, item[b'bsdflags']) + set_flags(path, item[b'bsdflags'], fd=fd) except OSError: pass # chown removes Linux capabilities, so set the extended attributes at the end, after chown, since they include @@ -506,8 +504,9 @@ Number of files: {0.stats.nfiles}'''.format( xattrs = xattr.get_all(path, follow_symlinks=False) if xattrs: item[b'xattrs'] = StableDict(xattrs) - if has_lchflags and st.st_flags: - item[b'bsdflags'] = st.st_flags + bsdflags = get_flags(path, st) + if bsdflags: + item[b'bsdflags'] = bsdflags acl_get(path, item, st, self.numeric_owner) return item diff --git a/borg/archiver.py b/borg/archiver.py index 8c9d018db..bd45da2ba 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -38,8 +38,7 @@ from .archive import Archive, ArchiveChecker, ArchiveRecreater from .remote import RepositoryServer, RemoteRepository, cache_if_remote from .selftest import selftest from .hashindex import ChunkIndexEntry - -has_lchflags = hasattr(os, 'lchflags') +from .platform import get_flags def argument(args, str_or_bool): @@ -316,7 +315,7 @@ class Archiver: return status = None # Ignore if nodump flag is set - if has_lchflags and (st.st_flags & stat.UF_NODUMP): + if get_flags(path, st) & stat.UF_NODUMP: return if stat.S_ISREG(st.st_mode) or read_special and not stat.S_ISDIR(st.st_mode): if not dry_run: diff --git a/borg/helpers.py b/borg/helpers.py index c73553e94..7e783f3eb 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -1166,7 +1166,7 @@ class ItemFormatter: 'NUL': 'NUL character for creating print0 / xargs -0 like ouput, see bpath', } KEY_GROUPS = ( - ('type', 'mode', 'uid', 'gid', 'user', 'group', 'path', 'bpath', 'source', 'linktarget'), + ('type', 'mode', 'uid', 'gid', 'user', 'group', 'path', 'bpath', 'source', 'linktarget', 'flags'), ('size', 'csize', 'num_chunks', 'unique_chunks'), ('mtime', 'ctime', 'atime', 'isomtime', 'isoctime', 'isoatime'), tuple(sorted(hashlib.algorithms_guaranteed)), @@ -1259,6 +1259,7 @@ class ItemFormatter: item_data['source'] = source item_data['linktarget'] = source item_data['extra'] = extra + item_data['flags'] = item.get(b'bsdflags') for key in self.used_call_keys: item_data[key] = self.call_keys[key](item) return item_data diff --git a/borg/platform.py b/borg/platform.py index 9c1c4ebed..e57c08707 100644 --- a/borg/platform.py +++ b/borg/platform.py @@ -1,9 +1,9 @@ import sys -from .platform_base import acl_get, acl_set, SyncFile, sync_dir, API_VERSION +from .platform_base import acl_get, acl_set, SyncFile, sync_dir, set_flags, get_flags, API_VERSION if sys.platform.startswith('linux'): # pragma: linux only - from .platform_linux import acl_get, acl_set, SyncFile, API_VERSION + from .platform_linux import acl_get, acl_set, SyncFile, set_flags, get_flags, API_VERSION elif sys.platform.startswith('freebsd'): # pragma: freebsd only from .platform_freebsd import acl_get, acl_set, API_VERSION elif sys.platform == 'darwin': # pragma: darwin only diff --git a/borg/platform_base.py b/borg/platform_base.py index a897f9f10..675b5af9e 100644 --- a/borg/platform_base.py +++ b/borg/platform_base.py @@ -21,6 +21,20 @@ def acl_set(path, item, numeric_owner=False): of the user/group names """ +try: + from os import lchflags + + def set_flags(path, bsd_flags, fd=None): + lchflags(path, bsd_flags) +except ImportError: + def set_flags(path, bsd_flags, fd=None): + pass + + +def get_flags(path, st): + """Return BSD-style file flags for path or stat without following symlinks.""" + return getattr(st, 'st_flags', 0) + def sync_dir(path): fd = os.open(path, os.O_RDONLY) diff --git a/borg/platform_linux.pyx b/borg/platform_linux.pyx index 854904f3f..8eb2b56d9 100644 --- a/borg/platform_linux.pyx +++ b/borg/platform_linux.pyx @@ -1,7 +1,8 @@ import os import re import resource -from stat import S_ISLNK +import stat + from .helpers import posix_acl_use_stored_uid_gid, user2uid, group2gid, safe_decode, safe_encode from .platform_base import SyncFile as BaseSyncFile from libc cimport errno @@ -33,10 +34,72 @@ cdef extern from "fcntl.h": unsigned int SYNC_FILE_RANGE_WAIT_BEFORE unsigned int SYNC_FILE_RANGE_WAIT_AFTER +cdef extern from "linux/fs.h": + # ioctls + int FS_IOC_SETFLAGS + int FS_IOC_GETFLAGS + + # inode flags + int FS_NODUMP_FL + int FS_IMMUTABLE_FL + int FS_APPEND_FL + int FS_COMPR_FL + +cdef extern from "stropts.h": + int ioctl(int fildes, int request, ...) + +cdef extern from "errno.h": + int errno + +cdef extern from "string.h": + char *strerror(int errnum) _comment_re = re.compile(' *#.*', re.M) +BSD_TO_LINUX_FLAGS = { + stat.UF_NODUMP: FS_NODUMP_FL, + stat.UF_IMMUTABLE: FS_IMMUTABLE_FL, + stat.UF_APPEND: FS_APPEND_FL, + stat.UF_COMPRESSED: FS_COMPR_FL, +} + + +def set_flags(path, bsd_flags, fd=None): + if fd is None and stat.S_ISLNK(os.lstat(path).st_mode): + return + cdef int flags = 0 + for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items(): + if bsd_flags & bsd_flag: + flags |= linux_flag + open_fd = fd is None + if open_fd: + fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW) + try: + if ioctl(fd, FS_IOC_SETFLAGS, &flags) == -1: + raise OSError(errno, strerror(errno).decode(), path) + finally: + if open_fd: + os.close(fd) + + +def get_flags(path, st): + if stat.S_ISLNK(st.st_mode): + return 0 + cdef int linux_flags + fd = os.open(path, os.O_RDONLY|os.O_NONBLOCK|os.O_NOFOLLOW) + try: + if ioctl(fd, FS_IOC_GETFLAGS, &linux_flags) == -1: + return 0 + finally: + os.close(fd) + bsd_flags = 0 + for bsd_flag, linux_flag in BSD_TO_LINUX_FLAGS.items(): + if linux_flags & linux_flag: + bsd_flags |= bsd_flag + return bsd_flags + + def acl_use_local_uid_gid(acl): """Replace the user/group field with the local uid/gid if possible """ @@ -93,7 +156,7 @@ def acl_get(path, item, st, numeric_owner=False): cdef char *access_text = NULL p = os.fsencode(path) - if S_ISLNK(st.st_mode) or acl_extended_file(p) <= 0: + if stat.S_ISLNK(st.st_mode) or acl_extended_file(p) <= 0: return if numeric_owner: converter = acl_numeric_ids diff --git a/borg/testsuite/__init__.py b/borg/testsuite/__init__.py index cccf97a82..50d005397 100644 --- a/borg/testsuite/__init__.py +++ b/borg/testsuite/__init__.py @@ -5,9 +5,13 @@ import posix import stat import sys import sysconfig +import tempfile import time import unittest + from ..xattr import get_all +from ..platform import get_flags +from .. import platform # Note: this is used by borg.selftest, do not use or import py.test functionality here. @@ -23,8 +27,20 @@ try: except ImportError: raises = None -has_lchflags = hasattr(os, 'lchflags') +has_lchflags = hasattr(os, 'lchflags') or sys.platform.startswith('linux') +no_lchlfags_because = '' if has_lchflags else '(not supported on this platform)' +try: + with tempfile.NamedTemporaryFile() as file: + platform.set_flags(file.name, stat.UF_NODUMP) +except OSError: + has_lchflags = False + no_lchlfags_because = '(the file system at %s does not support flags)' % tempfile.gettempdir() +try: + import llfuse + has_llfuse = True or llfuse # avoids "unused import" +except ImportError: + has_llfuse = False # The mtime get/set precision varies on different OS and Python versions if 'HAVE_FUTIMENS' in getattr(posix, '_have_functions', []): @@ -75,13 +91,13 @@ class BaseTestCase(unittest.TestCase): # Assume path2 is on FUSE if st_dev is different fuse = s1.st_dev != s2.st_dev attrs = ['st_mode', 'st_uid', 'st_gid', 'st_rdev'] - if has_lchflags: - attrs.append('st_flags') if not fuse or not os.path.isdir(path1): # dir nlink is always 1 on our fuse filesystem attrs.append('st_nlink') d1 = [filename] + [getattr(s1, a) for a in attrs] d2 = [filename] + [getattr(s2, a) for a in attrs] + d1.append(get_flags(path1, s1)) + d2.append(get_flags(path2, s2)) # ignore st_rdev if file is not a block/char device, fixes #203 if not stat.S_ISCHR(d1[1]) and not stat.S_ISBLK(d1[1]): d1[4] = None diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 363cc19e5..75ee0c04f 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -16,7 +16,7 @@ from hashlib import sha256 import pytest -from .. import xattr, helpers +from .. import xattr, helpers, platform from ..archive import Archive, ChunkBuffer, ArchiveRecreater from ..archiver import Archiver from ..cache import Cache @@ -26,15 +26,13 @@ from ..helpers import Chunk, Manifest, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, b from ..key import KeyfileKeyBase from ..remote import RemoteRepository, PathNotAllowed from ..repository import Repository +from . import has_lchflags, has_llfuse from . import BaseTestCase, changedir, environment_variable try: import llfuse - has_llfuse = True or llfuse # avoids "unused import" except ImportError: - has_llfuse = False - -has_lchflags = hasattr(os, 'lchflags') + pass src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) @@ -280,7 +278,7 @@ class ArchiverTestCaseBase(BaseTestCase): # FIFO node os.mkfifo(os.path.join(self.input_path, 'fifo1')) if has_lchflags: - os.lchflags(os.path.join(self.input_path, 'flagfile'), stat.UF_NODUMP) + platform.set_flags(os.path.join(self.input_path, 'flagfile'), stat.UF_NODUMP) try: # Block device os.mknod('input/bdev', 0o600 | stat.S_IFBLK, os.makedev(10, 20)) diff --git a/borg/testsuite/conftest.py b/borg/testsuite/conftest.py deleted file mode 100644 index 0c350fb7f..000000000 --- a/borg/testsuite/conftest.py +++ /dev/null @@ -1,4 +0,0 @@ -from ..logger import setup_logging - -# Ensure that the loggers exist for all tests -setup_logging() diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000..44ff5cde9 --- /dev/null +++ b/conftest.py @@ -0,0 +1,18 @@ +from borg.logger import setup_logging + +# Ensure that the loggers exist for all tests +setup_logging() + +from borg.testsuite import has_lchflags, no_lchlfags_because, has_llfuse +from borg.testsuite.platform import fakeroot_detected +from borg import xattr + + +def pytest_report_header(config, startdir): + yesno = ['no', 'yes'] + flags = 'Testing BSD-style flags: %s %s' % (yesno[has_lchflags], no_lchlfags_because) + fakeroot = 'fakeroot: %s (>=1.20.2: %s)' % ( + yesno[fakeroot_detected()], + yesno[xattr.XATTR_FAKEROOT]) + llfuse = 'Testing fuse: %s' % yesno[has_llfuse] + return '\n'.join((flags, llfuse, fakeroot))