upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-06-10 17:32:13 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Filesystem feature detection and test skipping

Browse source
This commit is contained in:
Lee Bousfield 2016-07-24 23:38:28 -04:00
parent 3c2d41898f
commit b86b5d952a
No known key found for this signature in database
GPG key ID: 380B74FB6AFFFB89
6 changed files with 230 additions and 83 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
conftest.py
View file

@ -1,10 +1,13 @@
import os
from borg.logger import setup_logging
# Ensure that the loggers exist for all tests
setup_logging()
from borg.testsuite import has_lchflags, no_lchlfags_because, has_llfuse
from borg.testsuite.platform import fakeroot_detected
from borg.testsuite import has_lchflags, has_llfuse
from borg.testsuite import are_symlinks_supported, are_hardlinks_supported, is_utime_fully_supported
from borg.testsuite.platform import fakeroot_detected, are_acls_working
from borg import xattr, constants
@ -14,10 +17,22 @@ def pytest_configure(config):
def pytest_report_header(config, startdir):
yesno = ['no', 'yes']
flags = 'Testing BSD-style flags: %s %s' % (yesno[has_lchflags], no_lchlfags_because)
fakeroot = 'fakeroot: %s (>=1.20.2: %s)' % (
yesno[fakeroot_detected()],
yesno[xattr.XATTR_FAKEROOT])
llfuse = 'Testing fuse: %s' % yesno[has_llfuse]
return '\n'.join((flags, llfuse, fakeroot))
tests = {
"BSD flags": has_lchflags,
"fuse": has_llfuse,
"root": not fakeroot_detected(),
"symlinks": are_symlinks_supported(),
"hardlinks": are_hardlinks_supported(),
"atime/mtime": is_utime_fully_supported(),
"modes": "BORG_TESTS_IGNORE_MODES" not in os.environ
}
enabled = []
disabled = []
for test in tests:
if tests[test]:
enabled.append(test)
else:
disabled.append(test)
output = "Tests enabled: " + ", ".join(enabled) + "\n"
output += "Tests disabled: " + ", ".join(disabled)
return output

12
src/borg/archive.py
View file

@ -553,10 +553,14 @@ Number of files: {0.stats.nfiles}'''.format(
else:
# old archives only had mtime in item metadata
atime = mtime
if fd:
os.utime(fd, None, ns=(atime, mtime))
else:
os.utime(path, None, ns=(atime, mtime), follow_symlinks=False)
try:
if fd:
os.utime(fd, None, ns=(atime, mtime))
else:
os.utime(path, None, ns=(atime, mtime), follow_symlinks=False)
except OSError:
# some systems don't support calling utime on a symlink
pass
acl_set(path, item, self.numeric_owner)
if 'bsdflags' in item:
try:

80
src/borg/testsuite/__init__.py
View file

@ -1,5 +1,6 @@
from contextlib import contextmanager
import filecmp
import functools
import os
import posix
import stat
@ -7,6 +8,7 @@ import sys
import sysconfig
import tempfile
import time
import uuid
import unittest
from ..xattr import get_all
@ -54,6 +56,67 @@ if sys.platform.startswith('netbsd'):
st_mtime_ns_round = -4 # only >1 microsecond resolution here?
@contextmanager
def unopened_tempfile():
with tempfile.TemporaryDirectory() as tempdir:
yield os.path.join(tempdir, "file")
@functools.lru_cache()
def are_symlinks_supported():
with unopened_tempfile() as filepath:
try:
os.symlink('somewhere', filepath)
if os.lstat(filepath) and os.readlink(filepath) == 'somewhere':
return True
except OSError:
pass
return False
@functools.lru_cache()
def are_hardlinks_supported():
with unopened_tempfile() as file1path, unopened_tempfile() as file2path:
open(file1path, 'w').close()
try:
os.link(file1path, file2path)
stat1 = os.stat(file1path)
stat2 = os.stat(file2path)
if stat1.st_nlink == stat2.st_nlink == 2 and stat1.st_ino == stat2.st_ino:
return True
except OSError:
pass
return False
@functools.lru_cache()
def are_fifos_supported():
with unopened_tempfile() as filepath:
try:
os.mkfifo(filepath)
return True
except OSError:
return False
@functools.lru_cache()
def is_utime_fully_supported():
with unopened_tempfile() as filepath:
# Some filesystems (such as SSHFS) don't support utime on symlinks
if are_symlinks_supported():
os.symlink('something', filepath)
else:
open(filepath, 'w').close()
try:
os.utime(filepath, (1000, 2000), follow_symlinks=False)
new_stats = os.lstat(filepath)
if new_stats.st_atime == 1000 and new_stats.st_mtime == 2000:
return True
except OSError as err:
pass
return False
class BaseTestCase(unittest.TestCase):
"""
"""
@ -103,13 +166,16 @@ class BaseTestCase(unittest.TestCase):
d1[4] = None
if not stat.S_ISCHR(d2[1]) and not stat.S_ISBLK(d2[1]):
d2[4] = None
# Older versions of llfuse do not support ns precision properly
if fuse and not have_fuse_mtime_ns:
d1.append(round(s1.st_mtime_ns, -4))
d2.append(round(s2.st_mtime_ns, -4))
else:
d1.append(round(s1.st_mtime_ns, st_mtime_ns_round))
d2.append(round(s2.st_mtime_ns, st_mtime_ns_round))
# If utime isn't fully supported, borg can't set mtime.
# Therefore, we shouldn't test it in that case.
if is_utime_fully_supported():
# Older versions of llfuse do not support ns precision properly
if fuse and not have_fuse_mtime_ns:
d1.append(round(s1.st_mtime_ns, -4))
d2.append(round(s2.st_mtime_ns, -4))
else:
d1.append(round(s1.st_mtime_ns, st_mtime_ns_round))
d2.append(round(s2.st_mtime_ns, st_mtime_ns_round))
d1.append(get_all(path1, follow_symlinks=False))
d2.append(get_all(path2, follow_symlinks=False))
self.assert_equal(d1, d2)

156
src/borg/testsuite/archiver.py
View file

@ -36,6 +36,7 @@ from ..remote import RemoteRepository, PathNotAllowed
from ..repository import Repository
from . import has_lchflags, has_llfuse
from . import BaseTestCase, changedir, environment_variable
from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported
src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
@ -274,10 +275,12 @@ class ArchiverTestCaseBase(BaseTestCase):
# File mode
os.chmod('input/file1', 0o4755)
# Hard link
os.link(os.path.join(self.input_path, 'file1'),
os.path.join(self.input_path, 'hardlink'))
if are_hardlinks_supported():
os.link(os.path.join(self.input_path, 'file1'),
os.path.join(self.input_path, 'hardlink'))
# Symlink
os.symlink('somewhere', os.path.join(self.input_path, 'link1'))
if are_symlinks_supported():
os.symlink('somewhere', os.path.join(self.input_path, 'link1'))
if xattr.is_enabled(self.input_path):
xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar')
# XXX this always fails for me
@ -287,7 +290,8 @@ class ArchiverTestCaseBase(BaseTestCase):
# so that the test setup for all tests using it does not fail here always for others.
# xattr.setxattr(os.path.join(self.input_path, 'link1'), 'user.foo_symlink', b'bar_symlink', follow_symlinks=False)
# FIFO node
os.mkfifo(os.path.join(self.input_path, 'fifo1'))
if are_fifos_supported():
os.mkfifo(os.path.join(self.input_path, 'fifo1'))
if has_lchflags:
platform.set_flags(os.path.join(self.input_path, 'flagfile'), stat.UF_NODUMP)
try:
@ -332,12 +336,15 @@ class ArchiverTestCase(ArchiverTestCaseBase):
'input/dir2',
'input/dir2/file2',
'input/empty',
'input/fifo1',
'input/file1',
'input/flagfile',
'input/hardlink',
'input/link1',
]
if are_fifos_supported():
expected.append('input/fifo1')
if are_symlinks_supported():
expected.append('input/link1')
if are_hardlinks_supported():
expected.append('input/hardlink')
if not have_root:
# we could not create these device files without (fake)root
expected.remove('input/bdev')
@ -373,14 +380,21 @@ class ArchiverTestCase(ArchiverTestCaseBase):
def test_unix_socket(self):
self.cmd('init', self.repository_location)
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(os.path.join(self.input_path, 'unix-socket'))
try:
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(os.path.join(self.input_path, 'unix-socket'))
except PermissionError as err:
if err.errno == errno.EPERM:
pytest.skip('unix sockets disabled or not supported')
elif err.errno == errno.EACCES:
pytest.skip('permission denied to create unix sockets')
self.cmd('create', self.repository_location + '::test', 'input')
sock.close()
with changedir('output'):
self.cmd('extract', self.repository_location + '::test')
assert not os.path.exists('input/unix-socket')
@pytest.mark.skipif(not are_symlinks_supported(), reason='symlinks not supported')
def test_symlink_extract(self):
self.create_test_files()
self.cmd('init', self.repository_location)
@ -389,6 +403,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.cmd('extract', self.repository_location + '::test')
assert os.readlink('input/link1') == 'somewhere'
@pytest.mark.skipif(not is_utime_fully_supported(), reason='cannot properly setup and execute test without utime')
def test_atime(self):
def has_noatime(some_file):
atime_before = os.stat(some_file).st_atime_ns
@ -557,6 +572,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.cmd('init', self.repository_location)
self.cmd('create', self.repository_location + '::test', 'input')
@pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported')
def test_strip_components_links(self):
self._extract_hardlinks_setup()
with changedir('output'):
@ -569,6 +585,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.cmd('extract', self.repository_location + '::test')
assert os.stat('input/dir1/hardlink').st_nlink == 4
@pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported')
def test_extract_hardlinks(self):
self._extract_hardlinks_setup()
with changedir('output'):
@ -988,6 +1005,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
# Restore permissions so shutil.rmtree is able to delete it
os.system('chmod -R u+w ' + self.repository_path)
@pytest.mark.skipif('BORG_TESTS_IGNORE_MODES' in os.environ, reason='modes unreliable')
def test_umask(self):
self.create_regular_file('file1', size=1024 * 80)
self.cmd('init', self.repository_location)
@ -1353,24 +1371,27 @@ class ArchiverTestCase(ArchiverTestCaseBase):
else:
assert False, "expected OSError(ENOATTR), but no error was raised"
# hardlink (to 'input/file1')
in_fn = 'input/hardlink'
out_fn = os.path.join(mountpoint, 'input', 'hardlink')
sti2 = os.stat(in_fn)
sto2 = os.stat(out_fn)
assert sti2.st_nlink == sto2.st_nlink == 2
assert sto1.st_ino == sto2.st_ino
if are_hardlinks_supported():
in_fn = 'input/hardlink'
out_fn = os.path.join(mountpoint, 'input', 'hardlink')
sti2 = os.stat(in_fn)
sto2 = os.stat(out_fn)
assert sti2.st_nlink == sto2.st_nlink == 2
assert sto1.st_ino == sto2.st_ino
# symlink
in_fn = 'input/link1'
out_fn = os.path.join(mountpoint, 'input', 'link1')
sti = os.stat(in_fn, follow_symlinks=False)
sto = os.stat(out_fn, follow_symlinks=False)
assert stat.S_ISLNK(sti.st_mode)
assert stat.S_ISLNK(sto.st_mode)
assert os.readlink(in_fn) == os.readlink(out_fn)
if are_symlinks_supported():
in_fn = 'input/link1'
out_fn = os.path.join(mountpoint, 'input', 'link1')
sti = os.stat(in_fn, follow_symlinks=False)
sto = os.stat(out_fn, follow_symlinks=False)
assert stat.S_ISLNK(sti.st_mode)
assert stat.S_ISLNK(sto.st_mode)
assert os.readlink(in_fn) == os.readlink(out_fn)
# FIFO
out_fn = os.path.join(mountpoint, 'input', 'fifo1')
sto = os.stat(out_fn)
assert stat.S_ISFIFO(sto.st_mode)
if are_fifos_supported():
out_fn = os.path.join(mountpoint, 'input', 'fifo1')
sto = os.stat(out_fn)
assert stat.S_ISFIFO(sto.st_mode)
@unittest.skipUnless(has_llfuse, 'llfuse not installed')
def test_fuse_allow_damaged_files(self):
@ -1481,6 +1502,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
assert 'dir2/file2' in listing
assert 'dir2/file3' not in listing
@pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported')
def test_recreate_subtree_hardlinks(self):
# This is essentially the same problem set as in test_extract_hardlinks
self._extract_hardlinks_setup()
@ -1884,17 +1906,19 @@ class DiffArchiverTestCase(ArchiverTestCaseBase):
self.create_regular_file('file_replaced', size=1024)
os.mkdir('input/dir_replaced_with_file')
os.chmod('input/dir_replaced_with_file', stat.S_IFDIR | 0o755)
os.mkdir('input/dir_replaced_with_link')
os.mkdir('input/dir_removed')
os.symlink('input/dir_replaced_with_file', 'input/link_changed')
os.symlink('input/file_unchanged', 'input/link_removed')
os.symlink('input/file_removed2', 'input/link_target_removed')
os.symlink('input/empty', 'input/link_target_contents_changed')
os.symlink('input/empty', 'input/link_replaced_by_file')
os.link('input/empty', 'input/hardlink_contents_changed')
os.link('input/file_removed', 'input/hardlink_removed')
os.link('input/file_removed2', 'input/hardlink_target_removed')
os.link('input/file_replaced', 'input/hardlink_target_replaced')
if are_symlinks_supported():
os.mkdir('input/dir_replaced_with_link')
os.symlink('input/dir_replaced_with_file', 'input/link_changed')
os.symlink('input/file_unchanged', 'input/link_removed')
os.symlink('input/file_removed2', 'input/link_target_removed')
os.symlink('input/empty', 'input/link_target_contents_changed')
os.symlink('input/empty', 'input/link_replaced_by_file')
if are_hardlinks_supported():
os.link('input/empty', 'input/hardlink_contents_changed')
os.link('input/file_removed', 'input/hardlink_removed')
os.link('input/file_removed2', 'input/hardlink_target_removed')
os.link('input/file_replaced', 'input/hardlink_target_replaced')
# Create the first snapshot
self.cmd('create', self.repository_location + '::test0', 'input')
@ -1910,16 +1934,18 @@ class DiffArchiverTestCase(ArchiverTestCaseBase):
os.chmod('input/dir_replaced_with_file', stat.S_IFREG | 0o755)
os.mkdir('input/dir_added')
os.rmdir('input/dir_removed')
os.rmdir('input/dir_replaced_with_link')
os.symlink('input/dir_added', 'input/dir_replaced_with_link')
os.unlink('input/link_changed')
os.symlink('input/dir_added', 'input/link_changed')
os.symlink('input/dir_added', 'input/link_added')
os.unlink('input/link_removed')
os.unlink('input/link_replaced_by_file')
self.create_regular_file('link_replaced_by_file', size=16384)
os.unlink('input/hardlink_removed')
os.link('input/file_added', 'input/hardlink_added')
if are_symlinks_supported():
os.rmdir('input/dir_replaced_with_link')
os.symlink('input/dir_added', 'input/dir_replaced_with_link')
os.unlink('input/link_changed')
os.symlink('input/dir_added', 'input/link_changed')
os.symlink('input/dir_added', 'input/link_added')
os.unlink('input/link_replaced_by_file')
self.create_regular_file('link_replaced_by_file', size=16384)
os.unlink('input/link_removed')
if are_hardlinks_supported():
os.unlink('input/hardlink_removed')
os.link('input/file_added', 'input/hardlink_added')
with open('input/empty', 'ab') as fd:
fd.write(b'appended_data')
@ -1936,49 +1962,57 @@ class DiffArchiverTestCase(ArchiverTestCaseBase):
assert 'input/file_unchanged' not in output
# Directory replaced with a regular file
assert '[drwxr-xr-x -> -rwxr-xr-x] input/dir_replaced_with_file' in output
if 'BORG_TESTS_IGNORE_MODES' not in os.environ:
assert '[drwxr-xr-x -> -rwxr-xr-x] input/dir_replaced_with_file' in output
# Basic directory cases
assert 'added directory input/dir_added' in output
assert 'removed directory input/dir_removed' in output
# Basic symlink cases
assert 'changed link input/link_changed' in output
assert 'added link input/link_added' in output
assert 'removed link input/link_removed' in output
if are_symlinks_supported():
# Basic symlink cases
assert 'changed link input/link_changed' in output
assert 'added link input/link_added' in output
assert 'removed link input/link_removed' in output
# Symlink replacing or being replaced
assert '] input/dir_replaced_with_link' in output
assert '] input/link_replaced_by_file' in output
# Symlink replacing or being replaced
assert '] input/dir_replaced_with_link' in output
assert '] input/link_replaced_by_file' in output
# Symlink target removed. Should not affect the symlink at all.
assert 'input/link_target_removed' not in output
# Symlink target removed. Should not affect the symlink at all.
assert 'input/link_target_removed' not in output
# The inode has two links and the file contents changed. Borg
# should notice the changes in both links. However, the symlink
# pointing to the file is not changed.
assert '0 B input/empty' in output
assert '0 B input/hardlink_contents_changed' in output
assert 'input/link_target_contents_changed' not in output
if are_hardlinks_supported():
assert '0 B input/hardlink_contents_changed' in output
if are_symlinks_supported():
assert 'input/link_target_contents_changed' not in output
# Added a new file and a hard link to it. Both links to the same
# inode should appear as separate files.
assert 'added 2.05 kB input/file_added' in output
assert 'added 2.05 kB input/hardlink_added' in output
if are_hardlinks_supported():
assert 'added 2.05 kB input/hardlink_added' in output
# The inode has two links and both of them are deleted. They should
# appear as two deleted files.
assert 'removed 256 B input/file_removed' in output
assert 'removed 256 B input/hardlink_removed' in output
if are_hardlinks_supported():
assert 'removed 256 B input/hardlink_removed' in output
# Another link (marked previously as the source in borg) to the
# same inode was removed. This should not change this link at all.
assert 'input/hardlink_target_removed' not in output
if are_hardlinks_supported():
assert 'input/hardlink_target_removed' not in output
# Another link (marked previously as the source in borg) to the
# same inode was replaced with a new regular file. This should not
# change this link at all.
assert 'input/hardlink_target_replaced' not in output
if are_hardlinks_supported():
assert 'input/hardlink_target_replaced' not in output
do_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1a'), '1a')
# We expect exit_code=1 due to the chunker params warning

27
src/borg/testsuite/platform.py
View file

@ -1,3 +1,4 @@
import functools
import os
import shutil
import sys
@ -6,7 +7,7 @@ import pwd
import unittest
from ..platform import acl_get, acl_set, swidth
from . import BaseTestCase
from . import BaseTestCase, unopened_tempfile
ACCESS_ACL = """
@ -31,6 +32,8 @@ mask::rw-
other::r--
""".strip().encode('ascii')
_acls_working = None
def fakeroot_detected():
return 'FAKEROOTKEY' in os.environ
@ -44,6 +47,24 @@ def user_exists(username):
return False
@functools.lru_cache()
def are_acls_working():
with unopened_tempfile() as filepath:
open(filepath, 'w').close()
try:
access = b'user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n'
acl = {'acl_access': access}
acl_set(filepath, acl)
read_acl = {}
acl_get(filepath, read_acl, os.stat(filepath))
read_acl_access = read_acl.get('acl_access', None)
if read_acl_access and b'user::rw-' in read_acl_access:
return True
except PermissionError:
pass
return False
@unittest.skipUnless(sys.platform.startswith('linux'), 'linux only test')
@unittest.skipIf(fakeroot_detected(), 'not compatible with fakeroot')
class PlatformLinuxTestCase(BaseTestCase):
@ -63,6 +84,7 @@ class PlatformLinuxTestCase(BaseTestCase):
item = {'acl_access': access, 'acl_default': default}
acl_set(path, item, numeric_owner=numeric_owner)
@unittest.skipIf(not are_acls_working(), 'ACLs do not work')
def test_access_acl(self):
file = tempfile.NamedTemporaryFile()
self.assert_equal(self.get_acl(file.name), {})
@ -75,6 +97,7 @@ class PlatformLinuxTestCase(BaseTestCase):
self.assert_in(b'user:9999:rw-:9999', self.get_acl(file2.name)['acl_access'])
self.assert_in(b'group:9999:rw-:9999', self.get_acl(file2.name)['acl_access'])
@unittest.skipIf(not are_acls_working(), 'ACLs do not work')
def test_default_acl(self):
self.assert_equal(self.get_acl(self.tmpdir), {})
self.set_acl(self.tmpdir, access=ACCESS_ACL, default=DEFAULT_ACL)
@ -82,6 +105,7 @@ class PlatformLinuxTestCase(BaseTestCase):
self.assert_equal(self.get_acl(self.tmpdir)['acl_default'], DEFAULT_ACL)
@unittest.skipIf(not user_exists('übel'), 'requires übel user')
@unittest.skipIf(not are_acls_working(), 'ACLs do not work')
def test_non_ascii_acl(self):
# Testing non-ascii ACL processing to see whether our code is robust.
# I have no idea whether non-ascii ACLs are allowed by the standard,
@ -138,6 +162,7 @@ class PlatformDarwinTestCase(BaseTestCase):
item = {'acl_extended': acl}
acl_set(path, item, numeric_owner=numeric_owner)
@unittest.skipIf(not are_acls_working(), 'ACLs do not work')
def test_access_acl(self):
file = tempfile.NamedTemporaryFile()
file2 = tempfile.NamedTemporaryFile()

5
src/borg/testsuite/upgrader.py
View file

@ -14,6 +14,7 @@ from ..upgrader import AtticRepositoryUpgrader, AtticKeyfileKey
from ..helpers import get_keys_dir
from ..key import KeyfileKey
from ..repository import Repository
from . import are_hardlinks_supported
def repo_valid(path):
@ -177,12 +178,14 @@ def test_convert_all(tmpdir, attic_repo, attic_key_file, inplace):
assert first_inode(repository.path) != first_inode(backup)
# i have seen cases where the copied tree has world-readable
# permissions, which is wrong
assert stat_segment(backup).st_mode & UMASK_DEFAULT == 0
if 'BORG_TESTS_IGNORE_MODES' not in os.environ:
assert stat_segment(backup).st_mode & UMASK_DEFAULT == 0
assert key_valid(attic_key_file.path)
assert repo_valid(tmpdir)
@pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported')
def test_hardlink(tmpdir, inplace):
"""test that we handle hard links properly