upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-05-28 04:03:21 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #7171 from RayyanAnsari/archiver-tests-win

Browse source 
Fix archiver tests on Windows
This commit is contained in:
TW 2022-12-18 16:06:19 +01:00 committed by GitHub
commit edb28691d5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 105 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/usage/tar.rst
View file

@ -34,7 +34,7 @@ Outputs a script that copies all archives from repo1 to repo2:
::
for A T in `borg list --format='{archive} {time:%Y-%m-%dT%H:%M:%S}{LF}'`
for A T in `borg list --format='{archive} {time:%Y-%m-%dT%H:%M:%S}{NL}'`
do
echo "borg -r repo1 export-tar --tar-format=BORG $A - | borg -r repo2 import-tar --timestamp=$T $A -"
done

9
src/borg/archiver/create_cmd.py
View file

@ -28,6 +28,7 @@ from ..helpers import sig_int, ignore_sigint
from ..helpers import iter_separated
from ..manifest import Manifest
from ..patterns import PatternMatcher
from ..platform import is_win32
from ..platform import get_flags
from ..platform import uid2user, gid2group
@ -68,7 +69,9 @@ class CreateMixIn:
if not dry_run:
try:
try:
proc = subprocess.Popen(args.paths, stdout=subprocess.PIPE, preexec_fn=ignore_sigint)
proc = subprocess.Popen(
args.paths, stdout=subprocess.PIPE, preexec_fn=None if is_win32 else ignore_sigint
)
except (FileNotFoundError, PermissionError) as e:
self.print_error("Failed to execute command: %s", e)
return self.exit_code
@ -89,7 +92,9 @@ class CreateMixIn:
paths_sep = eval_escapes(args.paths_delimiter) if args.paths_delimiter is not None else "\n"
if args.paths_from_command:
try:
proc = subprocess.Popen(args.paths, stdout=subprocess.PIPE, preexec_fn=ignore_sigint)
proc = subprocess.Popen(
args.paths, stdout=subprocess.PIPE, preexec_fn=None if is_win32 else ignore_sigint
)
except (FileNotFoundError, PermissionError) as e:
self.print_error("Failed to execute command: %s", e)
return self.exit_code

2
src/borg/archiver/key_cmds.py
View file

@ -101,6 +101,8 @@ class KeysMixIn:
manager.export_paperkey(args.path)
else:
try:
if os.path.isdir(args.path):
raise IsADirectoryError
if args.qr:
manager.export_qr(args.path)
else:

4
src/borg/helpers/parseformat.py
View file

@ -593,8 +593,8 @@ class BaseFormatter:
"TAB": "\t",
"CR": "\r",
"NUL": "\0",
"NEWLINE": os.linesep,
"NL": os.linesep,
"NEWLINE": "\n",
"NL": "\n", # \n is automatically converted to os.linesep on write
}
def get_item_data(self, item):

4
src/borg/helpers/process.py
View file

@ -341,7 +341,7 @@ def create_filter_process(cmd, stream, stream_close, inbound=True):
stdin=filter_stream,
log_prefix="filter-process: ",
env=env,
preexec_fn=ignore_sigint,
preexec_fn=None if is_win32 else ignore_sigint,
)
else:
proc = popen_with_error_handling(
@ -350,7 +350,7 @@ def create_filter_process(cmd, stream, stream_close, inbound=True):
stdout=filter_stream,
log_prefix="filter-process: ",
env=env,
preexec_fn=ignore_sigint,
preexec_fn=None if is_win32 else ignore_sigint,
)
if not proc:
raise Error(f"filter {cmd}: process creation failed")

8
src/borg/platform/windows.pyx
View file

@ -16,22 +16,22 @@ cdef extern from 'windows.h':
@lru_cache(maxsize=None)
def uid2user(uid, default=None):
return default
return "root"
@lru_cache(maxsize=None)
def user2uid(user, default=None):
return default
return 0
@lru_cache(maxsize=None)
def gid2group(gid, default=None):
return default
return "root"
@lru_cache(maxsize=None)
def group2gid(group, default=None):
return default
return 0
def getosusername():

37
src/borg/testsuite/archiver/__init__.py
View file

@ -28,6 +28,7 @@ from ...repository import Repository
from .. import has_lchflags
from .. import BaseTestCase, changedir, environment_variable
from .. import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported
from ..platform import is_win32
RK_ENCRYPTION = "--encryption=repokey-aes-ocb"
KF_ENCRYPTION = "--encryption=keyfile-chacha20-poly1305"
@ -230,23 +231,27 @@ class ArchiverTestCaseBase(BaseTestCase):
os.mkfifo(os.path.join(self.input_path, "fifo1"))
if has_lchflags:
platform.set_flags(os.path.join(self.input_path, "flagfile"), stat.UF_NODUMP)
try:
# Block device
os.mknod("input/bdev", 0o600 | stat.S_IFBLK, os.makedev(10, 20))
# Char device
os.mknod("input/cdev", 0o600 | stat.S_IFCHR, os.makedev(30, 40))
# File owner
os.chown("input/file1", 100, 200) # raises OSError invalid argument on cygwin
# File mode
os.chmod("input/dir2", 0o555) # if we take away write perms, we need root to remove contents
have_root = True # we have (fake)root
except PermissionError:
have_root = False
except OSError as e:
# Note: ENOSYS "Function not implemented" happens as non-root on Win 10 Linux Subsystem.
if e.errno not in (errno.EINVAL, errno.ENOSYS):
raise
if is_win32:
have_root = False
else:
try:
# Block device
os.mknod("input/bdev", 0o600 | stat.S_IFBLK, os.makedev(10, 20))
# Char device
os.mknod("input/cdev", 0o600 | stat.S_IFCHR, os.makedev(30, 40))
# File owner
os.chown("input/file1", 100, 200) # raises OSError invalid argument on cygwin
# File mode
os.chmod("input/dir2", 0o555) # if we take away write perms, we need root to remove contents
have_root = True # we have (fake)root
except PermissionError:
have_root = False
except OSError as e:
# Note: ENOSYS "Function not implemented" happens as non-root on Win 10 Linux Subsystem.
if e.errno not in (errno.EINVAL, errno.ENOSYS):
raise
have_root = False
time.sleep(1) # "empty" must have newer timestamp than other files
self.create_regular_file("empty", size=0)
return have_root

4
src/borg/testsuite/archiver/check_cmd.py
View file

@ -71,7 +71,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
self.assert_in("New missing file chunk detected", output)
self.cmd(f"--repo={self.repository_location}", "check", exit_code=0)
output = self.cmd(
f"--repo={self.repository_location}", "list", "archive1", "--format={health}#{path}{LF}", exit_code=0
f"--repo={self.repository_location}", "list", "archive1", "--format={health}#{path}{NL}", exit_code=0
)
self.assert_in("broken#", output)
# check that the file in the old archives has now a different chunk list without the killed chunk
@ -104,7 +104,7 @@ class ArchiverCheckTestCase(ArchiverTestCaseBase):
self.fail("should not happen")
# list is also all-healthy again
output = self.cmd(
f"--repo={self.repository_location}", "list", "archive1", "--format={health}#{path}{LF}", exit_code=0
f"--repo={self.repository_location}", "list", "archive1", "--format={health}#{path}{NL}", exit_code=0
)
self.assert_not_in("broken#", output)

6
src/borg/testsuite/archiver/config_cmd.py
View file

@ -24,17 +24,17 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.assert_in("No option ", output)
self.cmd(f"--repo={self.repository_location}", "config", "last_segment_checked", "123")
output = self.cmd(f"--repo={self.repository_location}", "config", "last_segment_checked")
assert output == "123" + "\n"
assert output == "123" + os.linesep
output = self.cmd(f"--repo={self.repository_location}", "config", "--list")
self.assert_in("last_segment_checked", output)
self.cmd(f"--repo={self.repository_location}", "config", "--delete", "last_segment_checked")
for cfg_key, cfg_value in [("additional_free_space", "2G"), ("repository.append_only", "1")]:
output = self.cmd(f"--repo={self.repository_location}", "config", cfg_key)
assert output == "0" + "\n"
assert output == "0" + os.linesep
self.cmd(f"--repo={self.repository_location}", "config", cfg_key, cfg_value)
output = self.cmd(f"--repo={self.repository_location}", "config", cfg_key)
assert output == cfg_value + "\n"
assert output == cfg_value + os.linesep
self.cmd(f"--repo={self.repository_location}", "config", "--delete", cfg_key)
self.cmd(f"--repo={self.repository_location}", "config", cfg_key, exit_code=1)

2
src/borg/testsuite/archiver/corruption.py
View file

@ -60,7 +60,7 @@ class ArchiverCorruptionTestCase(ArchiverTestCaseBase):
def test_chunks_archive(self):
self.cmd(f"--repo={self.repository_location}", "create", "test1", "input")
# Find ID of test1 so we can corrupt it later :)
target_id = self.cmd(f"--repo={self.repository_location}", "rlist", "--format={id}{LF}").strip()
target_id = self.cmd(f"--repo={self.repository_location}", "rlist", "--format={id}{NL}").strip()
self.cmd(f"--repo={self.repository_location}", "create", "test2", "input")
# Force cache sync, creating archive chunks of test1 and test2 in chunks.archive.d

33
src/borg/testsuite/archiver/create_cmd.py
View file

@ -13,7 +13,7 @@ import pytest
from ... import platform
from ...constants import * # NOQA
from ...manifest import Manifest
from ...platform import is_cygwin
from ...platform import is_cygwin, is_win32
from ...repository import Repository
from .. import has_lchflags
from .. import changedir
@ -119,6 +119,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
# we have all fs items exactly once!
assert sorted(paths) == ["input", "input/a", "input/a/hardlink", "input/b", "input/b/hardlink"]
@pytest.mark.skipif(is_win32, reason="unix sockets not available on windows")
def test_unix_socket(self):
self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
try:
@ -204,14 +205,14 @@ class ArchiverTestCase(ArchiverTestCaseBase):
"exit 73;",
exit_code=2,
)
assert output.endswith("Command 'sh' exited with status 73\n")
assert output.endswith("Command 'sh' exited with status 73" + os.linesep)
archive_list = json.loads(self.cmd(f"--repo={self.repository_location}", "rlist", "--json"))
assert archive_list["archives"] == []
def test_create_content_from_command_missing_command(self):
self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
output = self.cmd(f"--repo={self.repository_location}", "create", "test", "--content-from-command", exit_code=2)
assert output.endswith("No command given.\n")
assert output.endswith("No command given." + os.linesep)
def test_create_paths_from_stdin(self):
self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
@ -242,9 +243,20 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.create_regular_file("file4", size=1024 * 80)
input_data = "input/file1\ninput/file2\ninput/file3"
if is_win32:
with open("filenames.cmd", "w") as script:
for filename in input_data.splitlines():
script.write(f"@echo {filename}\n")
self.cmd(
f"--repo={self.repository_location}", "create", "--paths-from-command", "test", "--", "echo", input_data
f"--repo={self.repository_location}",
"create",
"--paths-from-command",
"test",
"--",
"filenames.cmd" if is_win32 else "echo",
input_data,
)
archive_list = self.cmd(f"--repo={self.repository_location}", "list", "test", "--json-lines")
paths = [json.loads(line)["path"] for line in archive_list.split("\n") if line]
assert paths == ["input/file1", "input/file2", "input/file3"]
@ -262,14 +274,14 @@ class ArchiverTestCase(ArchiverTestCaseBase):
"exit 73;",
exit_code=2,
)
assert output.endswith("Command 'sh' exited with status 73\n")
assert output.endswith("Command 'sh' exited with status 73" + os.linesep)
archive_list = json.loads(self.cmd(f"--repo={self.repository_location}", "rlist", "--json"))
assert archive_list["archives"] == []
def test_create_paths_from_command_missing_command(self):
self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
output = self.cmd(f"--repo={self.repository_location}", "create", "test", "--paths-from-command", exit_code=2)
assert output.endswith("No command given.\n")
assert output.endswith("No command given." + os.linesep)
def test_create_without_root(self):
"""test create without a root"""
@ -500,6 +512,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.cmd(f"--repo={self.repository_location}", "create", "test", "input", "input")
@pytest.mark.skipif("BORG_TESTS_IGNORE_MODES" in os.environ, reason="modes unreliable")
@pytest.mark.skipif(is_win32, reason="modes unavailable on Windows")
def test_umask(self):
self.create_regular_file("file1", size=1024 * 80)
self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
@ -545,6 +558,9 @@ class ArchiverTestCase(ArchiverTestCaseBase):
# https://borgbackup.readthedocs.org/en/latest/faq.html#i-am-seeing-a-added-status-for-a-unchanged-file
self.assert_in("A input/file2", output)
@pytest.mark.skipif(
is_win32, reason="ctime attribute is file creation time on Windows"
) # see https://docs.python.org/3/library/os.html#os.stat_result.st_ctime
def test_file_status_cs_cache_mode(self):
"""test that a changed file with faked "previous" mtime still gets backed up in ctime,size cache_mode"""
self.create_regular_file("file1", contents=b"123")
@ -740,6 +756,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
extracted_data = f.read()
assert extracted_data == data
@pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported")
def test_create_read_special_broken_symlink(self):
os.symlink("somewhere does not exist", os.path.join(self.input_path, "link"))
self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
@ -784,7 +801,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
# Test case set up: create a repository and a file
self.cmd(f"--repo={self.repository_location}", "rcreate", "--encryption=none")
self.create_regular_file("testfile", contents=randbytes(15000000)) # more data might be needed for faster CPUs
self.create_regular_file("testfile", contents=randbytes(50000000))
# Archive
result = self.cmd(f"--repo={self.repository_location}", "create", "--stats", "test_archive", self.input_path)
hashing_time = extract_hashing_time(result)
@ -802,7 +819,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
# Test case set up: create a repository and a file
self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
self.create_regular_file("testfile", contents=randbytes(10000000))
self.create_regular_file("testfile", contents=randbytes(50000000))
# Archive
result = self.cmd(f"--repo={self.repository_location}", "create", "--stats", "test_archive", self.input_path)
chunking_time = extract_chunking_time(result)

2
src/borg/testsuite/archiver/debug_cmds.py
View file

@ -107,7 +107,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
# Invalid IDs do not abort or return an error
output = self.cmd(f"--repo={self.repository_location}", "debug", "refcount-obj", "124", "xyza").strip()
assert output == "object id 124 is invalid.\nobject id xyza is invalid."
assert output == "object id 124 is invalid." + os.linesep + "object id xyza is invalid."
def test_debug_info(self):
output = self.cmd("debug", "info")

5
src/borg/testsuite/archiver/diff_cmd.py
View file

@ -5,6 +5,7 @@ import unittest
from ...constants import * # NOQA
from .. import are_symlinks_supported, are_hardlinks_supported
from ..platform import is_win32
from . import ArchiverTestCaseBase, RemoteArchiverTestCaseBase, ArchiverTestCaseBinaryBase, RK_ENCRYPTION, BORG_EXES
@ -79,7 +80,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
assert "input/file_unchanged" not in output
# Directory replaced with a regular file
if "BORG_TESTS_IGNORE_MODES" not in os.environ:
if "BORG_TESTS_IGNORE_MODES" not in os.environ and not is_win32:
assert "[drwxr-xr-x -> -rwxr-xr-x] input/dir_replaced_with_file" in output
# Basic directory cases
@ -153,7 +154,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
assert not any(get_changes("input/file_unchanged", joutput))
# Directory replaced with a regular file
if "BORG_TESTS_IGNORE_MODES" not in os.environ:
if "BORG_TESTS_IGNORE_MODES" not in os.environ and not is_win32:
assert {"type": "mode", "old_mode": "drwxr-xr-x", "new_mode": "-rwxr-xr-x"} in get_changes(
"input/dir_replaced_with_file", joutput
)

7
src/borg/testsuite/archiver/extract_cmd.py
View file

@ -309,19 +309,18 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.create_regular_file("file3", size=1024 * 80)
self.create_regular_file("file4", size=1024 * 80)
self.create_regular_file("file333", size=1024 * 80)
self.create_regular_file("aa:something", size=1024 * 80)
# Create while excluding using mixed pattern styles
with open(self.exclude_file_path, "wb") as fd:
fd.write(b"re:input/file4$\n")
fd.write(b"fm:*aa:*thing\n")
fd.write(b"fm:*file3*\n")
self.cmd(
f"--repo={self.repository_location}", "create", "--exclude-from=" + self.exclude_file_path, "test", "input"
)
with changedir("output"):
self.cmd(f"--repo={self.repository_location}", "extract", "test")
self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2", "file3", "file333"])
self.assert_equal(sorted(os.listdir("output/input")), ["file1", "file2"])
shutil.rmtree("output/input")
# Exclude using regular expression
@ -346,7 +345,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.cmd(
f"--repo={self.repository_location}", "extract", "test", "--exclude-from=" + self.exclude_file_path
)
self.assert_equal(sorted(os.listdir("output/input")), ["file3"])
self.assert_equal(sorted(os.listdir("output/input")), [])
def test_extract_with_pattern(self):
self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)

5
src/borg/testsuite/archiver/info_cmd.py
View file

@ -1,4 +1,5 @@
import json
import os
import unittest
from ...constants import * # NOQA
@ -18,9 +19,9 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
self.cmd(f"--repo={self.repository_location}", "create", "test", "input")
info_archive = self.cmd(f"--repo={self.repository_location}", "info", "-a", "test")
assert "Archive name: test\n" in info_archive
assert "Archive name: test" + os.linesep in info_archive
info_archive = self.cmd(f"--repo={self.repository_location}", "info", "--first", "1")
assert "Archive name: test\n" in info_archive
assert "Archive name: test" + os.linesep in info_archive
def test_info_json(self):
self.create_regular_file("file1", size=1024 * 80)

10
src/borg/testsuite/archiver/key_cmds.py
View file

@ -163,6 +163,16 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.cmd(f"--repo={self.repository_location}", "key", "export", export_directory, exit_code=EXIT_ERROR)
def test_key_export_qr_directory(self):
export_directory = self.output_path + "/exported"
os.mkdir(export_directory)
self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)
self.cmd(
f"--repo={self.repository_location}", "key", "export", "--qr-html", export_directory, exit_code=EXIT_ERROR
)
def test_key_import_errors(self):
export_file = self.output_path + "/exported"
self.cmd(f"--repo={self.repository_location}", "rcreate", KF_ENCRYPTION)

4
src/borg/testsuite/archiver/recreate_cmd.py
View file

@ -265,7 +265,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.cmd(f"--repo={self.repository_location}", "create", "test2", "input", "--comment", "this is the comment")
self.cmd(f"--repo={self.repository_location}", "create", "test3", "input", "--comment", '"deleted" comment')
self.cmd(f"--repo={self.repository_location}", "create", "test4", "input", "--comment", "preserved comment")
assert "Comment: \n" in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test1")
assert "Comment: " + os.linesep in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test1")
assert "Comment: this is the comment" in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test2")
self.cmd(f"--repo={self.repository_location}", "recreate", "-a", "test1", "--comment", "added comment")
@ -274,7 +274,7 @@ class ArchiverTestCase(ArchiverTestCaseBase):
self.cmd(f"--repo={self.repository_location}", "recreate", "-a", "test4", "12345")
assert "Comment: added comment" in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test1")
assert "Comment: modified comment" in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test2")
assert "Comment: \n" in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test3")
assert "Comment: " + os.linesep in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test3")
assert "Comment: preserved comment" in self.cmd(f"--repo={self.repository_location}", "info", "-a", "test4")

7
src/borg/testsuite/archiver/rlist_cmd.py
View file

@ -1,4 +1,5 @@
import json
import os
import unittest
from ...constants import * # NOQA
@ -34,10 +35,10 @@ class ArchiverTestCase(ArchiverTestCaseBase):
)
self.assertEqual(output_1, output_2)
output_1 = self.cmd(f"--repo={self.repository_location}", "rlist", "--short")
self.assertEqual(output_1, "test-1\ntest-2\n")
self.assertEqual(output_1, "test-1" + os.linesep + "test-2" + os.linesep)
output_3 = self.cmd(f"--repo={self.repository_location}", "rlist", "--format", "{name} {comment}{NL}")
self.assert_in("test-1 comment 1\n", output_3)
self.assert_in("test-2 comment 2\n", output_3)
self.assert_in("test-1 comment 1" + os.linesep, output_3)
self.assert_in("test-2 comment 2" + os.linesep, output_3)
def test_rlist_consider_checkpoints(self):
self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION)

10
src/borg/testsuite/archiver/transfer_cmd.py
View file

@ -7,6 +7,7 @@ import unittest
from ...constants import * # NOQA
from ...helpers.time import parse_timestamp
from ..platform import is_win32
from . import ArchiverTestCaseBase, RemoteArchiverTestCaseBase, ArchiverTestCaseBinaryBase, RK_ENCRYPTION, BORG_EXES
@ -151,6 +152,11 @@ class ArchiverTestCase(ArchiverTestCaseBase):
# Note: size == 0 for all items without a size or chunks list (like e.g. directories)
# Note: healthy == True indicates the *absence* of the additional chunks_healthy list
del g["hlid"]
if e["type"] == "b" and is_win32:
# The S_IFBLK macro is broken on MINGW
del e["type"], g["type"]
del e["mode"], g["mode"]
assert g == e
if name == "archive1":
@ -250,7 +256,9 @@ class ArchiverTestCase(ArchiverTestCaseBase):
assert item.group in ("root", "wheel")
assert "hlid" not in item
elif item.path.endswith("bdev_12_34"):
assert stat.S_ISBLK(item.mode)
if not is_win32:
# The S_IFBLK macro is broken on MINGW
assert stat.S_ISBLK(item.mode)
# looks like we can't use os.major/minor with data coming from another platform,
# thus we only do a rather rough check here:
assert "rdev" in item and item.rdev != 0