upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-06-11 01:41:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

feat(backend): enable s3 support using borgstore

Browse source 
Borg2 documentation mentions the support for the s3 backend however,
borg was missing the parsing bits for an s3 repo.

This updates the Location parser to parse the s3 url using the same
logic as borgstore.

Note: borgstore should be installed with the s3 dependencies in order
for the s3 backend to work.

Signed-off-by: Mike Mason <github@mikemrm.com>
This commit is contained in:
Mike Mason 2025-09-07 16:18:37 -05:00
parent 332ef28c7e
commit 0207c2176e
No known key found for this signature in database
4 changed files with 90 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
pyproject.toml
View file

@ -43,6 +43,7 @@ dependencies = [
llfuse = ["llfuse >= 1.3.8"]
pyfuse3 = ["pyfuse3 >= 3.1.1"]
nofuse = []
s3 = ["borgstore[s3] ~= 0.3.0"]
[project.urls]
"Homepage" = "https://borgbackup.org/"

4
src/borg/archiver/_common.py
View file

@ -37,7 +37,9 @@ def get_repository(location, *, create, exclusive, lock_wait, lock, args, v1_or_
location, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock, args=args
)
elif location.proto in ("sftp", "file", "rclone") and not v1_or_v2: # stuff directly supported by borgstore
elif (
location.proto in ("sftp", "file", "rclone", "s3", "b2") and not v1_or_v2
): # stuff directly supported by borgstore
repository = Repository(location, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock)
else:

34
src/borg/helpers/parseformat.py
View file

@ -472,6 +472,27 @@ class Location:
re.VERBOSE,
)
# (s3|b2):[profile|(access_key_id:access_key_secret)@][schema://hostname[:port]]/bucket/path
s3_re = re.compile(
r"""
(?P<s3type>(s3|b2)):
((
(?P<profile>[^@:]+) # profile (no colons allowed)
|
(?P<access_key_id>[^:@]+):(?P<access_key_secret>[^@]+) # access key and secret
)@)? # optional authentication
(
[^:/]+:// # scheme
(?P<hostname>[^:/]+)
(:(?P<port>\d+))?
)? # optional endpoint
/
(?P<bucket>[^/]+)/ # bucket name
(?P<path>.+) # path
""",
re.VERBOSE,
)
rclone_re = re.compile(r"(?P<proto>rclone):(?P<path>(.*))", re.VERBOSE)
file_or_socket_re = re.compile(r"(?P<proto>(file|socket))://" + abs_path_re, re.VERBOSE)
@ -483,6 +504,7 @@ class Location:
self.valid = False
self.proto = None
self.user = None
self._pass = None
self._host = None
self.port = None
self.path = None
@ -524,6 +546,15 @@ class Location:
self.proto = m.group("proto")
self.path = os.path.normpath(m.group("path"))
return True
m = self.s3_re.match(text)
if m:
self.proto = m.group("s3type")
self.user = m.group("profile") if m.group("profile") else m.group("access_key_id")
self._pass = True if m.group("access_key_secret") else False
self._host = m.group("hostname")
self.port = m.group("port") and int(m.group("port")) or None
self.path = m.group("bucket") + "/" + m.group("path")
return True
m = self.local_re.match(text)
if m:
self.proto = "file"
@ -535,6 +566,7 @@ class Location:
items = [
"proto=%r" % self.proto,
"user=%r" % self.user,
"pass=%r" % ("REDACTED" if self._pass else None),
"host=%r" % self.host,
"port=%r" % self.port,
"path=%r" % self.path,
@ -566,7 +598,7 @@ class Location:
return self.path
if self.proto == "rclone":
return f"{self.proto}:{self.path}"
if self.proto in ("sftp", "ssh"):
if self.proto in ("sftp", "ssh", "s3", "b2"):
return (
f"{self.proto}://"
f"{(self.user + '@') if self.user else ''}"

81
src/borg/testsuite/helpers/parseformat_test.py
View file

@ -86,30 +86,30 @@ class TestLocationWithoutEnv:
monkeypatch.delenv("BORG_REPO", raising=False)
assert (
repr(Location("ssh://user@host:1234//absolute/path"))
== "Location(proto='ssh', user='user', host='host', port=1234, path='/absolute/path')"
== "Location(proto='ssh', user='user', pass=None, host='host', port=1234, path='/absolute/path')"
)
assert Location("ssh://user@host:1234//absolute/path").to_key_filename() == keys_dir + "host___absolute_path"
assert (
repr(Location("ssh://user@host:1234/relative/path"))
== "Location(proto='ssh', user='user', host='host', port=1234, path='relative/path')"
== "Location(proto='ssh', user='user', pass=None, host='host', port=1234, path='relative/path')"
)
assert Location("ssh://user@host:1234/relative/path").to_key_filename() == keys_dir + "host__relative_path"
assert (
repr(Location("ssh://user@host/relative/path"))
== "Location(proto='ssh', user='user', host='host', port=None, path='relative/path')"
== "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='relative/path')"
)
assert (
repr(Location("ssh://user@[::]:1234/relative/path"))
== "Location(proto='ssh', user='user', host='::', port=1234, path='relative/path')"
== "Location(proto='ssh', user='user', pass=None, host='::', port=1234, path='relative/path')"
)
assert Location("ssh://user@[::]:1234/relative/path").to_key_filename() == keys_dir + "____relative_path"
assert (
repr(Location("ssh://user@[::]/relative/path"))
== "Location(proto='ssh', user='user', host='::', port=None, path='relative/path')"
== "Location(proto='ssh', user='user', pass=None, host='::', port=None, path='relative/path')"
)
assert (
repr(Location("ssh://user@[2001:db8::]:1234/relative/path"))
== "Location(proto='ssh', user='user', host='2001:db8::', port=1234, path='relative/path')"
== "Location(proto='ssh', user='user', pass=None, host='2001:db8::', port=1234, path='relative/path')"
)
assert (
Location("ssh://user@[2001:db8::]:1234/relative/path").to_key_filename()
@ -117,23 +117,23 @@ class TestLocationWithoutEnv:
)
assert (
repr(Location("ssh://user@[2001:db8::]/relative/path"))
== "Location(proto='ssh', user='user', host='2001:db8::', port=None, path='relative/path')"
== "Location(proto='ssh', user='user', pass=None, host='2001:db8::', port=None, path='relative/path')"
)
assert (
repr(Location("ssh://user@[2001:db8::c0:ffee]:1234/relative/path"))
== "Location(proto='ssh', user='user', host='2001:db8::c0:ffee', port=1234, path='relative/path')"
== "Location(proto='ssh', user='user', pass=None, host='2001:db8::c0:ffee', port=1234, path='relative/path')" # noqa: E501
)
assert (
repr(Location("ssh://user@[2001:db8::c0:ffee]/relative/path"))
== "Location(proto='ssh', user='user', host='2001:db8::c0:ffee', port=None, path='relative/path')"
== "Location(proto='ssh', user='user', pass=None, host='2001:db8::c0:ffee', port=None, path='relative/path')" # noqa: E501
)
assert (
repr(Location("ssh://user@[2001:db8::192.0.2.1]:1234/relative/path"))
== "Location(proto='ssh', user='user', host='2001:db8::192.0.2.1', port=1234, path='relative/path')"
== "Location(proto='ssh', user='user', pass=None, host='2001:db8::192.0.2.1', port=1234, path='relative/path')" # noqa: E501
)
assert (
repr(Location("ssh://user@[2001:db8::192.0.2.1]/relative/path"))
== "Location(proto='ssh', user='user', host='2001:db8::192.0.2.1', port=None, path='relative/path')"
== "Location(proto='ssh', user='user', pass=None, host='2001:db8::192.0.2.1', port=None, path='relative/path')" # noqa: E501
)
assert (
Location("ssh://user@[2001:db8::192.0.2.1]/relative/path").to_key_filename()
@ -141,20 +141,39 @@ class TestLocationWithoutEnv:
)
assert (
repr(Location("ssh://user@[2a02:0001:0002:0003:0004:0005:0006:0007]/relative/path"))
== "Location(proto='ssh', user='user', "
== "Location(proto='ssh', user='user', pass=None, "
"host='2a02:0001:0002:0003:0004:0005:0006:0007', port=None, path='relative/path')"
)
assert (
repr(Location("ssh://user@[2a02:0001:0002:0003:0004:0005:0006:0007]:1234/relative/path"))
== "Location(proto='ssh', user='user', "
== "Location(proto='ssh', user='user', pass=None, "
"host='2a02:0001:0002:0003:0004:0005:0006:0007', port=1234, path='relative/path')"
)
def test_s3(self, monkeypatch, keys_dir):
monkeypatch.delenv("BORG_REPO", raising=False)
assert (
repr(Location("s3:/test/path"))
== "Location(proto='s3', user=None, pass=None, host=None, port=None, path='test/path')"
)
assert (
repr(Location("s3:profile@http://172.28.52.116:9000/test/path"))
== "Location(proto='s3', user='profile', pass=None, host='172.28.52.116', port=9000, path='test/path')" # noqa: E501
)
assert (
repr(Location("s3:user:pass@http://172.28.52.116:9000/test/path"))
== "Location(proto='s3', user='user', pass='REDACTED', host='172.28.52.116', port=9000, path='test/path')" # noqa: E501
)
assert (
repr(Location("b2:user:pass@https://s3.us-east-005.backblazeb2.com/test/path"))
== "Location(proto='b2', user='user', pass='REDACTED', host='s3.us-east-005.backblazeb2.com', port=None, path='test/path')" # noqa: E501
)
def test_rclone(self, monkeypatch, keys_dir):
monkeypatch.delenv("BORG_REPO", raising=False)
assert (
repr(Location("rclone:remote:path"))
== "Location(proto='rclone', user=None, host=None, port=None, path='remote:path')"
== "Location(proto='rclone', user=None, pass=None, host=None, port=None, path='remote:path')"
)
assert Location("rclone:remote:path").to_key_filename() == keys_dir + "remote_path"
@ -163,13 +182,13 @@ class TestLocationWithoutEnv:
# relative path
assert (
repr(Location("sftp://user@host:1234/rel/path"))
== "Location(proto='sftp', user='user', host='host', port=1234, path='rel/path')"
== "Location(proto='sftp', user='user', pass=None, host='host', port=1234, path='rel/path')"
)
assert Location("sftp://user@host:1234/rel/path").to_key_filename() == keys_dir + "host__rel_path"
# absolute path
assert (
repr(Location("sftp://user@host:1234//abs/path"))
== "Location(proto='sftp', user='user', host='host', port=1234, path='/abs/path')"
== "Location(proto='sftp', user='user', pass=None, host='host', port=1234, path='/abs/path')"
)
assert Location("sftp://user@host:1234//abs/path").to_key_filename() == keys_dir + "host___abs_path"
@ -177,7 +196,7 @@ class TestLocationWithoutEnv:
monkeypatch.delenv("BORG_REPO", raising=False)
assert (
repr(Location("socket:///repo/path"))
== "Location(proto='socket', user=None, host=None, port=None, path='/repo/path')"
== "Location(proto='socket', user=None, pass=None, host=None, port=None, path='/repo/path')"
)
assert Location("socket:///some/path").to_key_filename() == keys_dir + "_some_path"
@ -185,11 +204,11 @@ class TestLocationWithoutEnv:
monkeypatch.delenv("BORG_REPO", raising=False)
assert (
repr(Location("file:///some/path"))
== "Location(proto='file', user=None, host=None, port=None, path='/some/path')"
== "Location(proto='file', user=None, pass=None, host=None, port=None, path='/some/path')"
)
assert (
repr(Location("file:///some/path"))
== "Location(proto='file', user=None, host=None, port=None, path='/some/path')"
== "Location(proto='file', user=None, pass=None, host=None, port=None, path='/some/path')"
)
assert Location("file:///some/path").to_key_filename() == keys_dir + "_some_path"
@ -197,7 +216,7 @@ class TestLocationWithoutEnv:
monkeypatch.delenv("BORG_REPO", raising=False)
assert (
repr(Location("file:////server/share/path"))
== "Location(proto='file', user=None, host=None, port=None, path='//server/share/path')"
== "Location(proto='file', user=None, pass=None, host=None, port=None, path='//server/share/path')"
)
assert Location("file:////server/share/path").to_key_filename() == keys_dir + "__server_share_path"
@ -205,19 +224,22 @@ class TestLocationWithoutEnv:
monkeypatch.delenv("BORG_REPO", raising=False)
rel_path = "path"
abs_path = os.path.abspath(rel_path)
assert repr(Location(rel_path)) == f"Location(proto='file', user=None, host=None, port=None, path='{abs_path}')"
assert (
repr(Location(rel_path))
== f"Location(proto='file', user=None, pass=None, host=None, port=None, path='{abs_path}')"
)
assert Location("path").to_key_filename().endswith(rel_path)
def test_abspath(self, monkeypatch, keys_dir):
monkeypatch.delenv("BORG_REPO", raising=False)
assert (
repr(Location("/some/absolute/path"))
== "Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path')"
== "Location(proto='file', user=None, pass=None, host=None, port=None, path='/some/absolute/path')"
)
assert Location("/some/absolute/path").to_key_filename() == keys_dir + "_some_absolute_path"
assert (
repr(Location("/some/../absolute/path"))
== "Location(proto='file', user=None, host=None, port=None, path='/absolute/path')"
== "Location(proto='file', user=None, pass=None, host=None, port=None, path='/absolute/path')"
)
assert Location("/some/../absolute/path").to_key_filename() == keys_dir + "_absolute_path"
@ -226,11 +248,14 @@ class TestLocationWithoutEnv:
# for a local path, borg creates a Location instance with an absolute path
rel_path = "relative/path"
abs_path = os.path.abspath(rel_path)
assert repr(Location(rel_path)) == f"Location(proto='file', user=None, host=None, port=None, path='{abs_path}')"
assert (
repr(Location(rel_path))
== f"Location(proto='file', user=None, pass=None, host=None, port=None, path='{abs_path}')"
)
assert Location(rel_path).to_key_filename().endswith("relative_path")
assert (
repr(Location("ssh://user@host/relative/path"))
== "Location(proto='ssh', user='user', host='host', port=None, path='relative/path')"
== "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='relative/path')"
)
assert Location("ssh://user@host/relative/path").to_key_filename() == keys_dir + "host__relative_path"
@ -238,17 +263,17 @@ class TestLocationWithoutEnv:
monkeypatch.delenv("BORG_REPO", raising=False)
assert (
repr(Location("/abs/path:w:cols"))
== "Location(proto='file', user=None, host=None, port=None, path='/abs/path:w:cols')"
== "Location(proto='file', user=None, pass=None, host=None, port=None, path='/abs/path:w:cols')"
)
assert Location("/abs/path:w:cols").to_key_filename() == keys_dir + "_abs_path_w_cols"
assert (
repr(Location("file:///abs/path:w:cols"))
== "Location(proto='file', user=None, host=None, port=None, path='/abs/path:w:cols')"
== "Location(proto='file', user=None, pass=None, host=None, port=None, path='/abs/path:w:cols')"
)
assert Location("file:///abs/path:w:cols").to_key_filename() == keys_dir + "_abs_path_w_cols"
assert (
repr(Location("ssh://user@host/abs/path:w:cols"))
== "Location(proto='ssh', user='user', host='host', port=None, path='abs/path:w:cols')"
== "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='abs/path:w:cols')"
)
assert Location("ssh://user@host/abs/path:w:cols").to_key_filename() == keys_dir + "host__abs_path_w_cols"