upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-06-13 10:50:21 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Merge pull request #9676 from ThomasWaldmann/support-rest-http-stdio

Browse source 
Support rest http stdio
This commit is contained in:
TW 2026-06-02 09:38:37 +02:00 committed by GitHub
commit 968f7dae00
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 161 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/canary.yml vendored
View file

@ -136,7 +136,7 @@ jobs:
run: |
# build borg.exe
. env/bin/activate
pip install -e ".[cockpit,s3,sftp,rest,rclone]"
pip install -e ".[cockpit,s3,sftp,rclone]"
mkdir -p dist/binary
pyinstaller -y --clean --distpath=dist/binary scripts/borg.exe.spec
# build sdist and wheel in dist/...

17
.github/workflows/ci.yml vendored
View file

@ -243,8 +243,13 @@ jobs:
# Start ssh-agent and add our key so paramiko can use the agent
eval "$(ssh-agent -s)"
ssh-add ~/.ssh/id_ed25519
sudo python3 -m venv /opt/borgstore-venv
sudo /opt/borgstore-venv/bin/pip install -U pip setuptools wheel
sudo /opt/borgstore-venv/bin/pip install "borgstore[rest]"
sudo ln -sf /opt/borgstore-venv/bin/borgstore-server-rest /usr/local/bin/borgstore-server-rest
# Export SFTP test URL for tox via GITHUB_ENV
echo "BORG_TEST_SFTP_REPO=sftp://sftpuser@localhost:22/borg/sftp-repo" >> $GITHUB_ENV
echo "BORG_TEST_REST_REPO=rest://sftpuser@localhost:22/borg/rest-repo" >> $GITHUB_ENV
- name: Install and configure MinIO S3 server (test only)
if: ${{ runner.os == 'Linux' && !contains(matrix.toxenv, 'mypy') && !contains(matrix.toxenv, 'docs') }}
@ -281,13 +286,13 @@ jobs:
- name: Install borgbackup
run: |
if [[ "$TOXENV" == *"llfuse"* ]]; then
pip install -ve ".[llfuse,cockpit,s3,sftp,rest,rclone]"
pip install -ve ".[llfuse,cockpit,s3,sftp,rclone]"
elif [[ "$TOXENV" == *"pyfuse3"* ]]; then
pip install -ve ".[pyfuse3,cockpit,s3,sftp,rest,rclone]"
pip install -ve ".[pyfuse3,cockpit,s3,sftp,rclone]"
elif [[ "$TOXENV" == *"mfusepy"* ]]; then
pip install -ve ".[mfusepy,cockpit,s3,sftp,rest,rclone]"
pip install -ve ".[mfusepy,cockpit,s3,sftp,rclone]"
else
pip install -ve ".[cockpit,s3,sftp,rest,rclone]"
pip install -ve ".[cockpit,s3,sftp,rclone]"
fi
- name: Build Borg fat binaries (${{ matrix.binary }})
@ -461,7 +466,7 @@ jobs:
pip -V
python -m pip install --upgrade pip wheel
pip install -r requirements.d/development.lock.txt
pip install -e ".[mfusepy,cockpit,s3,sftp,rest,rclone]"
pip install -e ".[mfusepy,cockpit,s3,sftp,rclone]"
tox -e py311-mfusepy
if [[ "${{ matrix.do_binaries }}" == "true" && "${{ startsWith(github.ref, 'refs/tags/') }}" == "true" ]]; then
@ -659,7 +664,7 @@ jobs:
run: |
# build borg.exe
. env/bin/activate
pip install -e ".[cockpit,s3,sftp,rest,rclone]"
pip install -e ".[cockpit,s3,sftp,rclone]"
mkdir -p dist/binary
pyinstaller -y --clean --distpath=dist/binary scripts/borg.exe.spec
# build sdist and wheel in dist/...

8
docs/usage/general/repository-urls.rst.inc
View file

@ -12,7 +12,13 @@ expanded by your shell).
Note: You may also prepend ``file://`` to a filesystem path to use URL style.
**Remote repositories** accessed via SSH user@host:
**Remote repositories** accessed via SSH user@host (REST http over stdio):
``rest://user@host:port//abs/path/to/repo`` — absolute path
``rest://user@host:port/rel/path/to/repo`` — path relative to the current directory
**Remote repositories** accessed via SSH user@host (legacy borg RPC protocol):
``ssh://user@host:port//abs/path/to/repo`` — absolute path

39
pyproject.toml
View file

@ -31,7 +31,7 @@ license = "BSD-3-Clause"
license-files = ["LICENSE", "AUTHORS"]
dependencies = [
"borghash ~= 0.1.0",
"borgstore ~= 0.5.0",
"borgstore[rest] ~= 0.5.1",
"msgpack >=1.0.3, <=1.1.2",
"packaging",
"platformdirs >=3.0.0, <5.0.0; sys_platform == 'darwin'", # for macOS: breaking changes in 3.0.0.
@ -51,10 +51,9 @@ mfusepy = ["mfusepy >= 3.1.0, <4.0.0"] # fuse 2+3, high-level
# a pypi release of borgbackup can't contain a dependency on github!
# mfusepym = ["mfusepy @ git+https://github.com/mxmlnkn/mfusepy.git@master"]
nofuse = []
s3 = ["borgstore[s3] ~= 0.5.0"]
sftp = ["borgstore[sftp] ~= 0.5.0"]
rclone = ["borgstore[rclone] ~= 0.5.0"]
rest = ["borgstore[rest] ~= 0.5.0"]
s3 = ["borgstore[rest,s3] ~= 0.5.1"]
sftp = ["borgstore[rest,sftp] ~= 0.5.1"]
rclone = ["borgstore[rest,rclone] ~= 0.5.1"]
cockpit = ["textual>=6.8.0"] # might also work with older versions, untested
[project.urls]
@ -192,71 +191,71 @@ pass_env = ["*"] # needed by tox4, so env vars are visible for building borg
[tool.tox.env.py310-llfuse]
set_env = {BORG_FUSE_IMPL = "llfuse"}
extras = ["llfuse", "sftp", "s3", "rest", "rclone"]
extras = ["llfuse", "sftp", "s3", "rclone"]
[tool.tox.env.py310-pyfuse3]
set_env = {BORG_FUSE_IMPL = "pyfuse3"}
extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"]
extras = ["pyfuse3", "sftp", "s3", "rclone"]
[tool.tox.env.py310-mfusepy]
set_env = {BORG_FUSE_IMPL = "mfusepy"}
extras = ["mfusepy", "sftp", "s3", "rest", "rclone"]
extras = ["mfusepy", "sftp", "s3", "rclone"]
[tool.tox.env.py311-none]
[tool.tox.env.py311-llfuse]
set_env = {BORG_FUSE_IMPL = "llfuse"}
extras = ["llfuse", "sftp", "s3", "rest", "rclone"]
extras = ["llfuse", "sftp", "s3", "rclone"]
[tool.tox.env.py311-pyfuse3]
set_env = {BORG_FUSE_IMPL = "pyfuse3"}
extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"]
extras = ["pyfuse3", "sftp", "s3", "rclone"]
[tool.tox.env.py311-mfusepy]
set_env = {BORG_FUSE_IMPL = "mfusepy"}
extras = ["mfusepy", "sftp", "s3", "rest", "rclone"]
extras = ["mfusepy", "sftp", "s3", "rclone"]
[tool.tox.env.py312-none]
[tool.tox.env.py312-llfuse]
set_env = {BORG_FUSE_IMPL = "llfuse"}
extras = ["llfuse", "sftp", "s3", "rest", "rclone"]
extras = ["llfuse", "sftp", "s3", "rclone"]
[tool.tox.env.py312-pyfuse3]
set_env = {BORG_FUSE_IMPL = "pyfuse3"}
extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"]
extras = ["pyfuse3", "sftp", "s3", "rclone"]
[tool.tox.env.py312-mfusepy]
set_env = {BORG_FUSE_IMPL = "mfusepy"}
extras = ["mfusepy", "sftp", "s3", "rest", "rclone"]
extras = ["mfusepy", "sftp", "s3", "rclone"]
[tool.tox.env.py313-none]
[tool.tox.env.py313-llfuse]
set_env = {BORG_FUSE_IMPL = "llfuse"}
extras = ["llfuse", "sftp", "s3", "rest", "rclone"]
extras = ["llfuse", "sftp", "s3", "rclone"]
[tool.tox.env.py313-pyfuse3]
set_env = {BORG_FUSE_IMPL = "pyfuse3"}
extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"]
extras = ["pyfuse3", "sftp", "s3", "rclone"]
[tool.tox.env.py313-mfusepy]
set_env = {BORG_FUSE_IMPL = "mfusepy"}
extras = ["mfusepy", "sftp", "s3", "rest", "rclone"]
extras = ["mfusepy", "sftp", "s3", "rclone"]
[tool.tox.env.py314-none]
[tool.tox.env.py314-llfuse]
set_env = {BORG_FUSE_IMPL = "llfuse"}
extras = ["llfuse", "sftp", "s3", "rest", "rclone"]
extras = ["llfuse", "sftp", "s3", "rclone"]
[tool.tox.env.py314-pyfuse3]
set_env = {BORG_FUSE_IMPL = "pyfuse3"}
extras = ["pyfuse3", "sftp", "s3", "rest", "rclone"]
extras = ["pyfuse3", "sftp", "s3", "rclone"]
[tool.tox.env.py314-mfusepy]
set_env = {BORG_FUSE_IMPL = "mfusepy"}
extras = ["mfusepy", "sftp", "s3", "rest", "rclone"]
extras = ["mfusepy", "sftp", "s3", "rclone"]
[tool.tox.env.ruff]
skip_install = true

2
src/borg/archiver/_common.py
View file

@ -42,7 +42,7 @@ def get_repository(location, *, create, exclusive, lock_wait, lock, args, v1_leg
)
elif (
location.proto in ("sftp", "file", "http", "https", "rclone", "s3", "b2") and not v1_legacy
location.proto in ("rest", "sftp", "file", "http", "https", "rclone", "s3", "b2") and not v1_legacy
): # stuff directly supported by borgstore
repository = Repository(location, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock)

4
src/borg/conftest.py
View file

@ -96,7 +96,7 @@ class ArchiverSetup:
self.patterns_file_path: str | None = None
def get_kind(self) -> str:
if self.repository_location.startswith("ssh://__testsuite__"):
if self.repository_location.startswith("ssh://__testsuite__") or self.repository_location.startswith("rest://"):
return "remote"
elif self.EXE == "borg.exe":
return "binary"
@ -152,7 +152,7 @@ def archiver(tmp_path, set_env_variables):
@pytest.fixture()
def remote_archiver(archiver):
archiver.repository_location = "ssh://__testsuite__/" + str(archiver.repository_path)
archiver.repository_location = "rest://" + "/" + str(archiver.repository_path)
yield archiver

23
src/borg/helpers/parseformat.py
View file

@ -552,6 +552,19 @@ class Location:
re.VERBOSE,
)
# REST http via stdio (via ssh, if host given):
rest_re = re.compile(
r"(?P<proto>(rest))://"
+ r"("
+ optional_user_re
+ host_re
+ optional_port_re
+ r")?"
+ r"/" # this is the separator, not part of the path!
+ abs_or_rel_path_re,
re.VERBOSE,
)
# BorgStore REST server
# (http|https)://user:pass@host:port/
http_re = re.compile(
@ -624,6 +637,14 @@ class Location:
def _parse(self, text):
m = self.ssh_or_sftp_re.match(text)
if m:
self.proto = m.group("proto")
self.user = m.group("user")
self._host = m.group("host")
self.port = m.group("port") and int(m.group("port")) or None
self.path = os.path.normpath(m.group("path"))
return True
m = self.rest_re.match(text)
if m:
self.proto = m.group("proto")
self.user = m.group("user")
@ -692,7 +713,7 @@ class Location:
return self.path
if self.proto == "rclone":
return f"{self.proto}:{self.path}"
if self.proto in ("sftp", "ssh", "s3", "b2", "http", "https"):
if self.proto in ("rest", "sftp", "ssh", "s3", "b2", "http", "https"):
return (
f"{self.proto}://"
f"{(self.user + '@') if self.user else ''}"

5
src/borg/testsuite/archiver/__init__.py
View file

@ -178,7 +178,10 @@ def open_archive(repo_path, name):
def open_repository(archiver):
if archiver.get_kind() == "remote":
return RemoteRepository(Location(archiver.repository_location))
location = Location(archiver.repository_location)
if location.proto == "rest":
return Repository(location, exclusive=True)
return RemoteRepository(location)
else:
return Repository(archiver.repository_path, exclusive=True)

4
src/borg/testsuite/archiver/checks_test.py
View file

@ -278,6 +278,8 @@ def test_unknown_mandatory_feature_in_cache(archivers, request):
# Begin Remote Tests
def test_remote_repo_restrict_to_path(remote_archiver):
if remote_archiver.repository_location.startswith("rest://"):
pytest.skip("Not applicable for rest:// protocol")
original_location, repo_path = remote_archiver.repository_location, remote_archiver.repository_path
# restricted to repo directory itself:
with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-path", repo_path]):
@ -306,6 +308,8 @@ def test_remote_repo_restrict_to_path(remote_archiver):
def test_remote_repo_restrict_to_repository(remote_archiver):
if remote_archiver.repository_location.startswith("rest://"):
pytest.skip("Not applicable for rest:// protocol")
repo_path = remote_archiver.repository_path
# restricted to repo directory itself:
with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-repository", repo_path]):

25
src/borg/testsuite/archiver/remote_repo_test.py
View file

@ -10,6 +10,7 @@ from . import cmd, create_regular_file, RK_ENCRYPTION, assert_dirs_equal
SFTP_URL = os.environ.get("BORG_TEST_SFTP_REPO")
REST_URL = os.environ.get("BORG_TEST_REST_REPO")
S3_URL = os.environ.get("BORG_TEST_S3_REPO")
@ -57,6 +58,30 @@ def test_rclone_repo_basics(archiver, tmp_path):
cmd(archiver, "repo-delete")
@pytest.mark.skipif(not REST_URL, reason="BORG_TEST_REST_REPO not set.")
def test_rest_repo_basics(archiver):
create_regular_file(archiver.input_path, "file1", size=100 * 1024)
create_regular_file(archiver.input_path, "file2", size=10 * 1024)
archiver.repository_location = REST_URL
archive_name = "test-archive"
cmd(archiver, "repo-create", RK_ENCRYPTION)
cmd(archiver, "create", archive_name, "input")
list_output = cmd(archiver, "repo-list")
assert archive_name in list_output
archive_list_output = cmd(archiver, "list", archive_name)
assert "input/file1" in archive_list_output
assert "input/file2" in archive_list_output
with changedir("output"):
cmd(archiver, "extract", archive_name)
assert_dirs_equal(
archiver.input_path, os.path.join(archiver.output_path, "input"), ignore_flags=True, ignore_xattrs=True
)
cmd(archiver, "delete", "-a", archive_name)
list_output = cmd(archiver, "repo-list")
assert archive_name not in list_output
cmd(archiver, "repo-delete")
@pytest.mark.skipif(not SFTP_URL, reason="BORG_TEST_SFTP_REPO not set.")
def test_sftp_repo_basics(archiver):
create_regular_file(archiver.input_path, "file1", size=100 * 1024)

65
src/borg/testsuite/helpers/parseformat_test.py
View file

@ -133,6 +133,71 @@ class TestLocationWithoutEnv:
"host='2a02:0001:0002:0003:0004:0005:0006:0007', port=1234, path='relative/path')"
)
def test_rest(self, monkeypatch):
monkeypatch.delenv("BORG_REPO", raising=False)
assert (
repr(Location("rest://user@host:1234//absolute/path"))
== "Location(proto='rest', user='user', pass=None, host='host', port=1234, path='/absolute/path')"
)
assert (
repr(Location("rest://user@host:1234/relative/path"))
== "Location(proto='rest', user='user', pass=None, host='host', port=1234, path='relative/path')"
)
assert (
repr(Location("rest://user@host/relative/path"))
== "Location(proto='rest', user='user', pass=None, host='host', port=None, path='relative/path')"
)
assert (
repr(Location("rest://user@[::]:1234/relative/path"))
== "Location(proto='rest', user='user', pass=None, host='::', port=1234, path='relative/path')"
)
assert (
repr(Location("rest://user@[::]/relative/path"))
== "Location(proto='rest', user='user', pass=None, host='::', port=None, path='relative/path')"
)
assert (
repr(Location("rest://user@[2001:db8::]:1234/relative/path"))
== "Location(proto='rest', user='user', pass=None, host='2001:db8::', port=1234, path='relative/path')"
)
assert (
repr(Location("rest://user@[2001:db8::]/relative/path"))
== "Location(proto='rest', user='user', pass=None, host='2001:db8::', port=None, path='relative/path')"
)
assert (
repr(Location("rest://user@[2001:db8::c0:ffee]:1234/relative/path"))
== "Location(proto='rest', user='user', pass=None, host='2001:db8::c0:ffee', port=1234, path='relative/path')" # noqa: E501
)
assert (
repr(Location("rest://user@[2001:db8::c0:ffee]/relative/path"))
== "Location(proto='rest', user='user', pass=None, host='2001:db8::c0:ffee', port=None, path='relative/path')" # noqa: E501
)
assert (
repr(Location("rest://user@[2001:db8::192.0.2.1]:1234/relative/path"))
== "Location(proto='rest', user='user', pass=None, host='2001:db8::192.0.2.1', port=1234, path='relative/path')" # noqa: E501
)
assert (
repr(Location("rest://user@[2001:db8::192.0.2.1]/relative/path"))
== "Location(proto='rest', user='user', pass=None, host='2001:db8::192.0.2.1', port=None, path='relative/path')" # noqa: E501
)
assert (
repr(Location("rest://user@[2a02:0001:0002:0003:0004:0005:0006:0007]/relative/path"))
== "Location(proto='rest', user='user', pass=None, "
"host='2a02:0001:0002:0003:0004:0005:0006:0007', port=None, path='relative/path')"
)
assert (
repr(Location("rest://user@[2a02:0001:0002:0003:0004:0005:0006:0007]:1234/relative/path"))
== "Location(proto='rest', user='user', pass=None, "
"host='2a02:0001:0002:0003:0004:0005:0006:0007', port=1234, path='relative/path')"
)
assert (
repr(Location("rest:///relative/path"))
== "Location(proto='rest', user=None, pass=None, host=None, port=None, path='relative/path')"
)
assert (
repr(Location("rest:////absolute/path"))
== "Location(proto='rest', user=None, pass=None, host=None, port=None, path='/absolute/path')"
)
def test_s3(self, monkeypatch):
monkeypatch.delenv("BORG_REPO", raising=False)
assert (