From b0a680bb2a5bd54771b53dd9a9175ab967ff1d1d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 11 Sep 2025 21:14:04 +0200 Subject: [PATCH] tests: add sftp/rclone/s3 repo testing --- pyproject.toml | 21 ++-- .../testsuite/archiver/remote_repo_test.py | 105 ++++++++++++++++++ 2 files changed, 116 insertions(+), 10 deletions(-) create mode 100644 src/borg/testsuite/archiver/remote_repo_test.py diff --git a/pyproject.toml b/pyproject.toml index 5270db0ba..14ada71a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ llfuse = ["llfuse >= 1.3.8"] pyfuse3 = ["pyfuse3 >= 3.1.1"] nofuse = [] s3 = ["borgstore[s3] ~= 0.3.0"] +sftp = ["borgstore[sftp] ~= 0.3.0"] [project.urls] "Homepage" = "https://borgbackup.org/" @@ -179,51 +180,51 @@ pass_env = ["*"] # needed by tox4, so env vars are visible for building borg [tool.tox.env.py310-fuse2] set_env = {BORG_FUSE_IMPL = "llfuse"} -extras = ["llfuse"] +extras = ["llfuse", "sftp", "s3"] [tool.tox.env.py310-fuse3] set_env = {BORG_FUSE_IMPL = "pyfuse3"} -extras = ["pyfuse3"] +extras = ["pyfuse3", "sftp", "s3"] [tool.tox.env.py311-none] [tool.tox.env.py311-fuse2] set_env = {BORG_FUSE_IMPL = "llfuse"} -extras = ["llfuse"] +extras = ["llfuse", "sftp", "s3"] [tool.tox.env.py311-fuse3] set_env = {BORG_FUSE_IMPL = "pyfuse3"} -extras = ["pyfuse3"] +extras = ["pyfuse3", "sftp", "s3"] [tool.tox.env.py312-none] [tool.tox.env.py312-fuse2] set_env = {BORG_FUSE_IMPL = "llfuse"} -extras = ["llfuse"] +extras = ["llfuse", "sftp", "s3"] [tool.tox.env.py312-fuse3] set_env = {BORG_FUSE_IMPL = "pyfuse3"} -extras = ["pyfuse3"] +extras = ["pyfuse3", "sftp", "s3"] [tool.tox.env.py313-none] [tool.tox.env.py313-fuse2] set_env = {BORG_FUSE_IMPL = "llfuse"} -extras = ["llfuse"] +extras = ["llfuse", "sftp", "s3"] [tool.tox.env.py313-fuse3] set_env = {BORG_FUSE_IMPL = "pyfuse3"} -extras = ["pyfuse3"] +extras = ["pyfuse3", "sftp", "s3"] [tool.tox.env.py314-none] [tool.tox.env.py314-fuse2] set_env = {BORG_FUSE_IMPL = "llfuse"} -extras = ["llfuse"] +extras = ["llfuse", "sftp", "s3"] [tool.tox.env.py314-fuse3] set_env = {BORG_FUSE_IMPL = "pyfuse3"} -extras = ["pyfuse3"] +extras = ["pyfuse3", "sftp", "s3"] [tool.tox.env.ruff] skip_install = true diff --git a/src/borg/testsuite/archiver/remote_repo_test.py b/src/borg/testsuite/archiver/remote_repo_test.py new file mode 100644 index 000000000..294aa4b3a --- /dev/null +++ b/src/borg/testsuite/archiver/remote_repo_test.py @@ -0,0 +1,105 @@ +import json +import os +import shutil +import subprocess + +import pytest + +from .. import changedir +from . import cmd, create_regular_file, RK_ENCRYPTION, assert_dirs_equal + + +SFTP_URL = os.environ.get("BORG_TEST_SFTP_REPO") +S3_URL = os.environ.get("BORG_TEST_S3_REPO") + + +def have_rclone(): + rclone_path = shutil.which("rclone") + if not rclone_path: + return False # not installed + try: + # rclone returns JSON for core/version, e.g. {"decomposed": [1,59,2], "version": "v1.59.2"} + out = subprocess.check_output([rclone_path, "rc", "--loopback", "core/version"]) + info = json.loads(out.decode("utf-8")) + except Exception: + return False + try: + if info.get("decomposed", []) < [1, 57, 0]: + return False # too old + except Exception: + return False + return True # looks good + + +@pytest.mark.skipif(not have_rclone(), reason="rclone must be installed for this test.") +def test_rclone_repo_basics(archiver, tmp_path): + create_regular_file(archiver.input_path, "file1", size=100 * 1024) + create_regular_file(archiver.input_path, "file2", size=10 * 1024) + rclone_repo_dir = tmp_path / "rclone-repo" + os.makedirs(rclone_repo_dir, exist_ok=True) + archiver.repository_location = f"rclone:{os.fspath(rclone_repo_dir)}" + archive_name = "test-archive" + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", archive_name, "input") + list_output = cmd(archiver, "repo-list") + assert archive_name in list_output + archive_list_output = cmd(archiver, "list", archive_name) + assert "input/file1" in archive_list_output + assert "input/file2" in archive_list_output + with changedir("output"): + cmd(archiver, "extract", archive_name) + assert_dirs_equal( + archiver.input_path, os.path.join(archiver.output_path, "input"), ignore_flags=True, ignore_xattrs=True + ) + cmd(archiver, "delete", "-a", archive_name) + list_output = cmd(archiver, "repo-list") + assert archive_name not in list_output + cmd(archiver, "repo-delete") + + +@pytest.mark.skipif(not SFTP_URL, reason="BORG_TEST_SFTP_REPO not set.") +def test_sftp_repo_basics(archiver): + create_regular_file(archiver.input_path, "file1", size=100 * 1024) + create_regular_file(archiver.input_path, "file2", size=10 * 1024) + archiver.repository_location = SFTP_URL + archive_name = "test-archive" + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", archive_name, "input") + list_output = cmd(archiver, "repo-list") + assert archive_name in list_output + archive_list_output = cmd(archiver, "list", archive_name) + assert "input/file1" in archive_list_output + assert "input/file2" in archive_list_output + with changedir("output"): + cmd(archiver, "extract", archive_name) + assert_dirs_equal( + archiver.input_path, os.path.join(archiver.output_path, "input"), ignore_flags=True, ignore_xattrs=True + ) + cmd(archiver, "delete", "-a", archive_name) + list_output = cmd(archiver, "repo-list") + assert archive_name not in list_output + cmd(archiver, "repo-delete") + + +@pytest.mark.skipif(not S3_URL, reason="BORG_TEST_S3_REPO not set.") +def test_s3_repo_basics(archiver): + create_regular_file(archiver.input_path, "file1", size=100 * 1024) + create_regular_file(archiver.input_path, "file2", size=10 * 1024) + archiver.repository_location = S3_URL + archive_name = "test-archive" + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", archive_name, "input") + list_output = cmd(archiver, "repo-list") + assert archive_name in list_output + archive_list_output = cmd(archiver, "list", archive_name) + assert "input/file1" in archive_list_output + assert "input/file2" in archive_list_output + with changedir("output"): + cmd(archiver, "extract", archive_name) + assert_dirs_equal( + archiver.input_path, os.path.join(archiver.output_path, "input"), ignore_flags=True, ignore_xattrs=True + ) + cmd(archiver, "delete", "-a", archive_name) + list_output = cmd(archiver, "repo-list") + assert archive_name not in list_output + cmd(archiver, "repo-delete")