diff --git a/conftest.py b/conftest.py index b5f9d9822..5caf8a09a 100644 --- a/conftest.py +++ b/conftest.py @@ -1,10 +1,13 @@ +import os + from borg.logger import setup_logging # Ensure that the loggers exist for all tests setup_logging() -from borg.testsuite import has_lchflags, no_lchlfags_because, has_llfuse -from borg.testsuite.platform import fakeroot_detected +from borg.testsuite import has_lchflags, has_llfuse +from borg.testsuite import are_symlinks_supported, are_hardlinks_supported, is_utime_fully_supported +from borg.testsuite.platform import fakeroot_detected, are_acls_working from borg import xattr, constants @@ -14,10 +17,22 @@ def pytest_configure(config): def pytest_report_header(config, startdir): - yesno = ['no', 'yes'] - flags = 'Testing BSD-style flags: %s %s' % (yesno[has_lchflags], no_lchlfags_because) - fakeroot = 'fakeroot: %s (>=1.20.2: %s)' % ( - yesno[fakeroot_detected()], - yesno[xattr.XATTR_FAKEROOT]) - llfuse = 'Testing fuse: %s' % yesno[has_llfuse] - return '\n'.join((flags, llfuse, fakeroot)) + tests = { + "BSD flags": has_lchflags, + "fuse": has_llfuse, + "root": not fakeroot_detected(), + "symlinks": are_symlinks_supported(), + "hardlinks": are_hardlinks_supported(), + "atime/mtime": is_utime_fully_supported(), + "modes": "BORG_TESTS_IGNORE_MODES" not in os.environ + } + enabled = [] + disabled = [] + for test in tests: + if tests[test]: + enabled.append(test) + else: + disabled.append(test) + output = "Tests enabled: " + ", ".join(enabled) + "\n" + output += "Tests disabled: " + ", ".join(disabled) + return output diff --git a/src/borg/archive.py b/src/borg/archive.py index e45655153..6ba90797e 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -553,10 +553,14 @@ Number of files: {0.stats.nfiles}'''.format( else: # old archives only had mtime in item metadata atime = mtime - if fd: - os.utime(fd, None, ns=(atime, mtime)) - else: - os.utime(path, None, ns=(atime, mtime), follow_symlinks=False) + try: + if fd: + os.utime(fd, None, ns=(atime, mtime)) + else: + os.utime(path, None, ns=(atime, mtime), follow_symlinks=False) + except OSError: + # some systems don't support calling utime on a symlink + pass acl_set(path, item, self.numeric_owner) if 'bsdflags' in item: try: diff --git a/src/borg/testsuite/__init__.py b/src/borg/testsuite/__init__.py index 3f24c2478..4b8114ccc 100644 --- a/src/borg/testsuite/__init__.py +++ b/src/borg/testsuite/__init__.py @@ -1,5 +1,6 @@ from contextlib import contextmanager import filecmp +import functools import os import posix import stat @@ -7,6 +8,7 @@ import sys import sysconfig import tempfile import time +import uuid import unittest from ..xattr import get_all @@ -54,6 +56,67 @@ if sys.platform.startswith('netbsd'): st_mtime_ns_round = -4 # only >1 microsecond resolution here? +@contextmanager +def unopened_tempfile(): + with tempfile.TemporaryDirectory() as tempdir: + yield os.path.join(tempdir, "file") + + +@functools.lru_cache() +def are_symlinks_supported(): + with unopened_tempfile() as filepath: + try: + os.symlink('somewhere', filepath) + if os.lstat(filepath) and os.readlink(filepath) == 'somewhere': + return True + except OSError: + pass + return False + + +@functools.lru_cache() +def are_hardlinks_supported(): + with unopened_tempfile() as file1path, unopened_tempfile() as file2path: + open(file1path, 'w').close() + try: + os.link(file1path, file2path) + stat1 = os.stat(file1path) + stat2 = os.stat(file2path) + if stat1.st_nlink == stat2.st_nlink == 2 and stat1.st_ino == stat2.st_ino: + return True + except OSError: + pass + return False + + +@functools.lru_cache() +def are_fifos_supported(): + with unopened_tempfile() as filepath: + try: + os.mkfifo(filepath) + return True + except OSError: + return False + + +@functools.lru_cache() +def is_utime_fully_supported(): + with unopened_tempfile() as filepath: + # Some filesystems (such as SSHFS) don't support utime on symlinks + if are_symlinks_supported(): + os.symlink('something', filepath) + else: + open(filepath, 'w').close() + try: + os.utime(filepath, (1000, 2000), follow_symlinks=False) + new_stats = os.lstat(filepath) + if new_stats.st_atime == 1000 and new_stats.st_mtime == 2000: + return True + except OSError as err: + pass + return False + + class BaseTestCase(unittest.TestCase): """ """ @@ -103,13 +166,16 @@ class BaseTestCase(unittest.TestCase): d1[4] = None if not stat.S_ISCHR(d2[1]) and not stat.S_ISBLK(d2[1]): d2[4] = None - # Older versions of llfuse do not support ns precision properly - if fuse and not have_fuse_mtime_ns: - d1.append(round(s1.st_mtime_ns, -4)) - d2.append(round(s2.st_mtime_ns, -4)) - else: - d1.append(round(s1.st_mtime_ns, st_mtime_ns_round)) - d2.append(round(s2.st_mtime_ns, st_mtime_ns_round)) + # If utime isn't fully supported, borg can't set mtime. + # Therefore, we shouldn't test it in that case. + if is_utime_fully_supported(): + # Older versions of llfuse do not support ns precision properly + if fuse and not have_fuse_mtime_ns: + d1.append(round(s1.st_mtime_ns, -4)) + d2.append(round(s2.st_mtime_ns, -4)) + else: + d1.append(round(s1.st_mtime_ns, st_mtime_ns_round)) + d2.append(round(s2.st_mtime_ns, st_mtime_ns_round)) d1.append(get_all(path1, follow_symlinks=False)) d2.append(get_all(path2, follow_symlinks=False)) self.assert_equal(d1, d2) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 1c29fd407..fbb16dbbd 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -36,6 +36,7 @@ from ..remote import RemoteRepository, PathNotAllowed from ..repository import Repository from . import has_lchflags, has_llfuse from . import BaseTestCase, changedir, environment_variable +from . import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported, is_utime_fully_supported src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) @@ -274,10 +275,12 @@ class ArchiverTestCaseBase(BaseTestCase): # File mode os.chmod('input/file1', 0o4755) # Hard link - os.link(os.path.join(self.input_path, 'file1'), - os.path.join(self.input_path, 'hardlink')) + if are_hardlinks_supported(): + os.link(os.path.join(self.input_path, 'file1'), + os.path.join(self.input_path, 'hardlink')) # Symlink - os.symlink('somewhere', os.path.join(self.input_path, 'link1')) + if are_symlinks_supported(): + os.symlink('somewhere', os.path.join(self.input_path, 'link1')) if xattr.is_enabled(self.input_path): xattr.setxattr(os.path.join(self.input_path, 'file1'), 'user.foo', b'bar') # XXX this always fails for me @@ -287,7 +290,8 @@ class ArchiverTestCaseBase(BaseTestCase): # so that the test setup for all tests using it does not fail here always for others. # xattr.setxattr(os.path.join(self.input_path, 'link1'), 'user.foo_symlink', b'bar_symlink', follow_symlinks=False) # FIFO node - os.mkfifo(os.path.join(self.input_path, 'fifo1')) + if are_fifos_supported(): + os.mkfifo(os.path.join(self.input_path, 'fifo1')) if has_lchflags: platform.set_flags(os.path.join(self.input_path, 'flagfile'), stat.UF_NODUMP) try: @@ -332,12 +336,15 @@ class ArchiverTestCase(ArchiverTestCaseBase): 'input/dir2', 'input/dir2/file2', 'input/empty', - 'input/fifo1', 'input/file1', 'input/flagfile', - 'input/hardlink', - 'input/link1', ] + if are_fifos_supported(): + expected.append('input/fifo1') + if are_symlinks_supported(): + expected.append('input/link1') + if are_hardlinks_supported(): + expected.append('input/hardlink') if not have_root: # we could not create these device files without (fake)root expected.remove('input/bdev') @@ -373,14 +380,21 @@ class ArchiverTestCase(ArchiverTestCaseBase): def test_unix_socket(self): self.cmd('init', self.repository_location) - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - sock.bind(os.path.join(self.input_path, 'unix-socket')) + try: + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.bind(os.path.join(self.input_path, 'unix-socket')) + except PermissionError as err: + if err.errno == errno.EPERM: + pytest.skip('unix sockets disabled or not supported') + elif err.errno == errno.EACCES: + pytest.skip('permission denied to create unix sockets') self.cmd('create', self.repository_location + '::test', 'input') sock.close() with changedir('output'): self.cmd('extract', self.repository_location + '::test') assert not os.path.exists('input/unix-socket') + @pytest.mark.skipif(not are_symlinks_supported(), reason='symlinks not supported') def test_symlink_extract(self): self.create_test_files() self.cmd('init', self.repository_location) @@ -389,6 +403,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('extract', self.repository_location + '::test') assert os.readlink('input/link1') == 'somewhere' + @pytest.mark.skipif(not is_utime_fully_supported(), reason='cannot properly setup and execute test without utime') def test_atime(self): def has_noatime(some_file): atime_before = os.stat(some_file).st_atime_ns @@ -557,6 +572,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('init', self.repository_location) self.cmd('create', self.repository_location + '::test', 'input') + @pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported') def test_strip_components_links(self): self._extract_hardlinks_setup() with changedir('output'): @@ -569,6 +585,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd('extract', self.repository_location + '::test') assert os.stat('input/dir1/hardlink').st_nlink == 4 + @pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported') def test_extract_hardlinks(self): self._extract_hardlinks_setup() with changedir('output'): @@ -988,6 +1005,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): # Restore permissions so shutil.rmtree is able to delete it os.system('chmod -R u+w ' + self.repository_path) + @pytest.mark.skipif('BORG_TESTS_IGNORE_MODES' in os.environ, reason='modes unreliable') def test_umask(self): self.create_regular_file('file1', size=1024 * 80) self.cmd('init', self.repository_location) @@ -1353,24 +1371,27 @@ class ArchiverTestCase(ArchiverTestCaseBase): else: assert False, "expected OSError(ENOATTR), but no error was raised" # hardlink (to 'input/file1') - in_fn = 'input/hardlink' - out_fn = os.path.join(mountpoint, 'input', 'hardlink') - sti2 = os.stat(in_fn) - sto2 = os.stat(out_fn) - assert sti2.st_nlink == sto2.st_nlink == 2 - assert sto1.st_ino == sto2.st_ino + if are_hardlinks_supported(): + in_fn = 'input/hardlink' + out_fn = os.path.join(mountpoint, 'input', 'hardlink') + sti2 = os.stat(in_fn) + sto2 = os.stat(out_fn) + assert sti2.st_nlink == sto2.st_nlink == 2 + assert sto1.st_ino == sto2.st_ino # symlink - in_fn = 'input/link1' - out_fn = os.path.join(mountpoint, 'input', 'link1') - sti = os.stat(in_fn, follow_symlinks=False) - sto = os.stat(out_fn, follow_symlinks=False) - assert stat.S_ISLNK(sti.st_mode) - assert stat.S_ISLNK(sto.st_mode) - assert os.readlink(in_fn) == os.readlink(out_fn) + if are_symlinks_supported(): + in_fn = 'input/link1' + out_fn = os.path.join(mountpoint, 'input', 'link1') + sti = os.stat(in_fn, follow_symlinks=False) + sto = os.stat(out_fn, follow_symlinks=False) + assert stat.S_ISLNK(sti.st_mode) + assert stat.S_ISLNK(sto.st_mode) + assert os.readlink(in_fn) == os.readlink(out_fn) # FIFO - out_fn = os.path.join(mountpoint, 'input', 'fifo1') - sto = os.stat(out_fn) - assert stat.S_ISFIFO(sto.st_mode) + if are_fifos_supported(): + out_fn = os.path.join(mountpoint, 'input', 'fifo1') + sto = os.stat(out_fn) + assert stat.S_ISFIFO(sto.st_mode) @unittest.skipUnless(has_llfuse, 'llfuse not installed') def test_fuse_allow_damaged_files(self): @@ -1481,6 +1502,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): assert 'dir2/file2' in listing assert 'dir2/file3' not in listing + @pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported') def test_recreate_subtree_hardlinks(self): # This is essentially the same problem set as in test_extract_hardlinks self._extract_hardlinks_setup() @@ -1884,17 +1906,19 @@ class DiffArchiverTestCase(ArchiverTestCaseBase): self.create_regular_file('file_replaced', size=1024) os.mkdir('input/dir_replaced_with_file') os.chmod('input/dir_replaced_with_file', stat.S_IFDIR | 0o755) - os.mkdir('input/dir_replaced_with_link') os.mkdir('input/dir_removed') - os.symlink('input/dir_replaced_with_file', 'input/link_changed') - os.symlink('input/file_unchanged', 'input/link_removed') - os.symlink('input/file_removed2', 'input/link_target_removed') - os.symlink('input/empty', 'input/link_target_contents_changed') - os.symlink('input/empty', 'input/link_replaced_by_file') - os.link('input/empty', 'input/hardlink_contents_changed') - os.link('input/file_removed', 'input/hardlink_removed') - os.link('input/file_removed2', 'input/hardlink_target_removed') - os.link('input/file_replaced', 'input/hardlink_target_replaced') + if are_symlinks_supported(): + os.mkdir('input/dir_replaced_with_link') + os.symlink('input/dir_replaced_with_file', 'input/link_changed') + os.symlink('input/file_unchanged', 'input/link_removed') + os.symlink('input/file_removed2', 'input/link_target_removed') + os.symlink('input/empty', 'input/link_target_contents_changed') + os.symlink('input/empty', 'input/link_replaced_by_file') + if are_hardlinks_supported(): + os.link('input/empty', 'input/hardlink_contents_changed') + os.link('input/file_removed', 'input/hardlink_removed') + os.link('input/file_removed2', 'input/hardlink_target_removed') + os.link('input/file_replaced', 'input/hardlink_target_replaced') # Create the first snapshot self.cmd('create', self.repository_location + '::test0', 'input') @@ -1910,16 +1934,18 @@ class DiffArchiverTestCase(ArchiverTestCaseBase): os.chmod('input/dir_replaced_with_file', stat.S_IFREG | 0o755) os.mkdir('input/dir_added') os.rmdir('input/dir_removed') - os.rmdir('input/dir_replaced_with_link') - os.symlink('input/dir_added', 'input/dir_replaced_with_link') - os.unlink('input/link_changed') - os.symlink('input/dir_added', 'input/link_changed') - os.symlink('input/dir_added', 'input/link_added') - os.unlink('input/link_removed') - os.unlink('input/link_replaced_by_file') - self.create_regular_file('link_replaced_by_file', size=16384) - os.unlink('input/hardlink_removed') - os.link('input/file_added', 'input/hardlink_added') + if are_symlinks_supported(): + os.rmdir('input/dir_replaced_with_link') + os.symlink('input/dir_added', 'input/dir_replaced_with_link') + os.unlink('input/link_changed') + os.symlink('input/dir_added', 'input/link_changed') + os.symlink('input/dir_added', 'input/link_added') + os.unlink('input/link_replaced_by_file') + self.create_regular_file('link_replaced_by_file', size=16384) + os.unlink('input/link_removed') + if are_hardlinks_supported(): + os.unlink('input/hardlink_removed') + os.link('input/file_added', 'input/hardlink_added') with open('input/empty', 'ab') as fd: fd.write(b'appended_data') @@ -1936,49 +1962,57 @@ class DiffArchiverTestCase(ArchiverTestCaseBase): assert 'input/file_unchanged' not in output # Directory replaced with a regular file - assert '[drwxr-xr-x -> -rwxr-xr-x] input/dir_replaced_with_file' in output + if 'BORG_TESTS_IGNORE_MODES' not in os.environ: + assert '[drwxr-xr-x -> -rwxr-xr-x] input/dir_replaced_with_file' in output # Basic directory cases assert 'added directory input/dir_added' in output assert 'removed directory input/dir_removed' in output - # Basic symlink cases - assert 'changed link input/link_changed' in output - assert 'added link input/link_added' in output - assert 'removed link input/link_removed' in output + if are_symlinks_supported(): + # Basic symlink cases + assert 'changed link input/link_changed' in output + assert 'added link input/link_added' in output + assert 'removed link input/link_removed' in output - # Symlink replacing or being replaced - assert '] input/dir_replaced_with_link' in output - assert '] input/link_replaced_by_file' in output + # Symlink replacing or being replaced + assert '] input/dir_replaced_with_link' in output + assert '] input/link_replaced_by_file' in output - # Symlink target removed. Should not affect the symlink at all. - assert 'input/link_target_removed' not in output + # Symlink target removed. Should not affect the symlink at all. + assert 'input/link_target_removed' not in output # The inode has two links and the file contents changed. Borg # should notice the changes in both links. However, the symlink # pointing to the file is not changed. assert '0 B input/empty' in output - assert '0 B input/hardlink_contents_changed' in output - assert 'input/link_target_contents_changed' not in output + if are_hardlinks_supported(): + assert '0 B input/hardlink_contents_changed' in output + if are_symlinks_supported(): + assert 'input/link_target_contents_changed' not in output # Added a new file and a hard link to it. Both links to the same # inode should appear as separate files. assert 'added 2.05 kB input/file_added' in output - assert 'added 2.05 kB input/hardlink_added' in output + if are_hardlinks_supported(): + assert 'added 2.05 kB input/hardlink_added' in output # The inode has two links and both of them are deleted. They should # appear as two deleted files. assert 'removed 256 B input/file_removed' in output - assert 'removed 256 B input/hardlink_removed' in output + if are_hardlinks_supported(): + assert 'removed 256 B input/hardlink_removed' in output # Another link (marked previously as the source in borg) to the # same inode was removed. This should not change this link at all. - assert 'input/hardlink_target_removed' not in output + if are_hardlinks_supported(): + assert 'input/hardlink_target_removed' not in output # Another link (marked previously as the source in borg) to the # same inode was replaced with a new regular file. This should not # change this link at all. - assert 'input/hardlink_target_replaced' not in output + if are_hardlinks_supported(): + assert 'input/hardlink_target_replaced' not in output do_asserts(self.cmd('diff', self.repository_location + '::test0', 'test1a'), '1a') # We expect exit_code=1 due to the chunker params warning diff --git a/src/borg/testsuite/platform.py b/src/borg/testsuite/platform.py index 0001eeecd..9bd81d2b4 100644 --- a/src/borg/testsuite/platform.py +++ b/src/borg/testsuite/platform.py @@ -1,3 +1,4 @@ +import functools import os import shutil import sys @@ -6,7 +7,7 @@ import pwd import unittest from ..platform import acl_get, acl_set, swidth -from . import BaseTestCase +from . import BaseTestCase, unopened_tempfile ACCESS_ACL = """ @@ -31,6 +32,8 @@ mask::rw- other::r-- """.strip().encode('ascii') +_acls_working = None + def fakeroot_detected(): return 'FAKEROOTKEY' in os.environ @@ -44,6 +47,24 @@ def user_exists(username): return False +@functools.lru_cache() +def are_acls_working(): + with unopened_tempfile() as filepath: + open(filepath, 'w').close() + try: + access = b'user::rw-\ngroup::r--\nmask::rw-\nother::---\nuser:root:rw-:9999\ngroup:root:rw-:9999\n' + acl = {'acl_access': access} + acl_set(filepath, acl) + read_acl = {} + acl_get(filepath, read_acl, os.stat(filepath)) + read_acl_access = read_acl.get('acl_access', None) + if read_acl_access and b'user::rw-' in read_acl_access: + return True + except PermissionError: + pass + return False + + @unittest.skipUnless(sys.platform.startswith('linux'), 'linux only test') @unittest.skipIf(fakeroot_detected(), 'not compatible with fakeroot') class PlatformLinuxTestCase(BaseTestCase): @@ -63,6 +84,7 @@ class PlatformLinuxTestCase(BaseTestCase): item = {'acl_access': access, 'acl_default': default} acl_set(path, item, numeric_owner=numeric_owner) + @unittest.skipIf(not are_acls_working(), 'ACLs do not work') def test_access_acl(self): file = tempfile.NamedTemporaryFile() self.assert_equal(self.get_acl(file.name), {}) @@ -75,6 +97,7 @@ class PlatformLinuxTestCase(BaseTestCase): self.assert_in(b'user:9999:rw-:9999', self.get_acl(file2.name)['acl_access']) self.assert_in(b'group:9999:rw-:9999', self.get_acl(file2.name)['acl_access']) + @unittest.skipIf(not are_acls_working(), 'ACLs do not work') def test_default_acl(self): self.assert_equal(self.get_acl(self.tmpdir), {}) self.set_acl(self.tmpdir, access=ACCESS_ACL, default=DEFAULT_ACL) @@ -82,6 +105,7 @@ class PlatformLinuxTestCase(BaseTestCase): self.assert_equal(self.get_acl(self.tmpdir)['acl_default'], DEFAULT_ACL) @unittest.skipIf(not user_exists('übel'), 'requires übel user') + @unittest.skipIf(not are_acls_working(), 'ACLs do not work') def test_non_ascii_acl(self): # Testing non-ascii ACL processing to see whether our code is robust. # I have no idea whether non-ascii ACLs are allowed by the standard, @@ -138,6 +162,7 @@ class PlatformDarwinTestCase(BaseTestCase): item = {'acl_extended': acl} acl_set(path, item, numeric_owner=numeric_owner) + @unittest.skipIf(not are_acls_working(), 'ACLs do not work') def test_access_acl(self): file = tempfile.NamedTemporaryFile() file2 = tempfile.NamedTemporaryFile() diff --git a/src/borg/testsuite/upgrader.py b/src/borg/testsuite/upgrader.py index 088ee63b5..aba3ee9c1 100644 --- a/src/borg/testsuite/upgrader.py +++ b/src/borg/testsuite/upgrader.py @@ -14,6 +14,7 @@ from ..upgrader import AtticRepositoryUpgrader, AtticKeyfileKey from ..helpers import get_keys_dir from ..key import KeyfileKey from ..repository import Repository +from . import are_hardlinks_supported def repo_valid(path): @@ -177,12 +178,14 @@ def test_convert_all(tmpdir, attic_repo, attic_key_file, inplace): assert first_inode(repository.path) != first_inode(backup) # i have seen cases where the copied tree has world-readable # permissions, which is wrong - assert stat_segment(backup).st_mode & UMASK_DEFAULT == 0 + if 'BORG_TESTS_IGNORE_MODES' not in os.environ: + assert stat_segment(backup).st_mode & UMASK_DEFAULT == 0 assert key_valid(attic_key_file.path) assert repo_valid(tmpdir) +@pytest.mark.skipif(not are_hardlinks_supported(), reason='hardlinks not supported') def test_hardlink(tmpdir, inplace): """test that we handle hard links properly