diff --git a/pyproject.toml b/pyproject.toml index 9aeb4b7c9..5270db0ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ dependencies = [ llfuse = ["llfuse >= 1.3.8"] pyfuse3 = ["pyfuse3 >= 3.1.1"] nofuse = [] +s3 = ["borgstore[s3] ~= 0.3.0"] [project.urls] "Homepage" = "https://borgbackup.org/" diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index 7900d5f1c..8e9d95a69 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -37,7 +37,9 @@ def get_repository(location, *, create, exclusive, lock_wait, lock, args, v1_or_ location, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock, args=args ) - elif location.proto in ("sftp", "file", "rclone") and not v1_or_v2: # stuff directly supported by borgstore + elif ( + location.proto in ("sftp", "file", "rclone", "s3", "b2") and not v1_or_v2 + ): # stuff directly supported by borgstore repository = Repository(location, create=create, exclusive=exclusive, lock_wait=lock_wait, lock=lock) else: diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 8f9385e27..61f52a4b0 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -472,6 +472,27 @@ class Location: re.VERBOSE, ) + # (s3|b2):[profile|(access_key_id:access_key_secret)@][schema://hostname[:port]]/bucket/path + s3_re = re.compile( + r""" + (?P(s3|b2)): + (( + (?P[^@:]+) # profile (no colons allowed) + | + (?P[^:@]+):(?P[^@]+) # access key and secret + )@)? # optional authentication + ( + [^:/]+:// # scheme + (?P[^:/]+) + (:(?P\d+))? + )? # optional endpoint + / + (?P[^/]+)/ # bucket name + (?P.+) # path + """, + re.VERBOSE, + ) + rclone_re = re.compile(r"(?Prclone):(?P(.*))", re.VERBOSE) file_or_socket_re = re.compile(r"(?P(file|socket))://" + abs_path_re, re.VERBOSE) @@ -483,6 +504,7 @@ class Location: self.valid = False self.proto = None self.user = None + self._pass = None self._host = None self.port = None self.path = None @@ -524,6 +546,15 @@ class Location: self.proto = m.group("proto") self.path = os.path.normpath(m.group("path")) return True + m = self.s3_re.match(text) + if m: + self.proto = m.group("s3type") + self.user = m.group("profile") if m.group("profile") else m.group("access_key_id") + self._pass = True if m.group("access_key_secret") else False + self._host = m.group("hostname") + self.port = m.group("port") and int(m.group("port")) or None + self.path = m.group("bucket") + "/" + m.group("path") + return True m = self.local_re.match(text) if m: self.proto = "file" @@ -535,6 +566,7 @@ class Location: items = [ "proto=%r" % self.proto, "user=%r" % self.user, + "pass=%r" % ("REDACTED" if self._pass else None), "host=%r" % self.host, "port=%r" % self.port, "path=%r" % self.path, @@ -566,7 +598,7 @@ class Location: return self.path if self.proto == "rclone": return f"{self.proto}:{self.path}" - if self.proto in ("sftp", "ssh"): + if self.proto in ("sftp", "ssh", "s3", "b2"): return ( f"{self.proto}://" f"{(self.user + '@') if self.user else ''}" diff --git a/src/borg/testsuite/helpers/parseformat_test.py b/src/borg/testsuite/helpers/parseformat_test.py index ef39e6714..61a710cea 100644 --- a/src/borg/testsuite/helpers/parseformat_test.py +++ b/src/borg/testsuite/helpers/parseformat_test.py @@ -86,30 +86,30 @@ class TestLocationWithoutEnv: monkeypatch.delenv("BORG_REPO", raising=False) assert ( repr(Location("ssh://user@host:1234//absolute/path")) - == "Location(proto='ssh', user='user', host='host', port=1234, path='/absolute/path')" + == "Location(proto='ssh', user='user', pass=None, host='host', port=1234, path='/absolute/path')" ) assert Location("ssh://user@host:1234//absolute/path").to_key_filename() == keys_dir + "host___absolute_path" assert ( repr(Location("ssh://user@host:1234/relative/path")) - == "Location(proto='ssh', user='user', host='host', port=1234, path='relative/path')" + == "Location(proto='ssh', user='user', pass=None, host='host', port=1234, path='relative/path')" ) assert Location("ssh://user@host:1234/relative/path").to_key_filename() == keys_dir + "host__relative_path" assert ( repr(Location("ssh://user@host/relative/path")) - == "Location(proto='ssh', user='user', host='host', port=None, path='relative/path')" + == "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='relative/path')" ) assert ( repr(Location("ssh://user@[::]:1234/relative/path")) - == "Location(proto='ssh', user='user', host='::', port=1234, path='relative/path')" + == "Location(proto='ssh', user='user', pass=None, host='::', port=1234, path='relative/path')" ) assert Location("ssh://user@[::]:1234/relative/path").to_key_filename() == keys_dir + "____relative_path" assert ( repr(Location("ssh://user@[::]/relative/path")) - == "Location(proto='ssh', user='user', host='::', port=None, path='relative/path')" + == "Location(proto='ssh', user='user', pass=None, host='::', port=None, path='relative/path')" ) assert ( repr(Location("ssh://user@[2001:db8::]:1234/relative/path")) - == "Location(proto='ssh', user='user', host='2001:db8::', port=1234, path='relative/path')" + == "Location(proto='ssh', user='user', pass=None, host='2001:db8::', port=1234, path='relative/path')" ) assert ( Location("ssh://user@[2001:db8::]:1234/relative/path").to_key_filename() @@ -117,23 +117,23 @@ class TestLocationWithoutEnv: ) assert ( repr(Location("ssh://user@[2001:db8::]/relative/path")) - == "Location(proto='ssh', user='user', host='2001:db8::', port=None, path='relative/path')" + == "Location(proto='ssh', user='user', pass=None, host='2001:db8::', port=None, path='relative/path')" ) assert ( repr(Location("ssh://user@[2001:db8::c0:ffee]:1234/relative/path")) - == "Location(proto='ssh', user='user', host='2001:db8::c0:ffee', port=1234, path='relative/path')" + == "Location(proto='ssh', user='user', pass=None, host='2001:db8::c0:ffee', port=1234, path='relative/path')" # noqa: E501 ) assert ( repr(Location("ssh://user@[2001:db8::c0:ffee]/relative/path")) - == "Location(proto='ssh', user='user', host='2001:db8::c0:ffee', port=None, path='relative/path')" + == "Location(proto='ssh', user='user', pass=None, host='2001:db8::c0:ffee', port=None, path='relative/path')" # noqa: E501 ) assert ( repr(Location("ssh://user@[2001:db8::192.0.2.1]:1234/relative/path")) - == "Location(proto='ssh', user='user', host='2001:db8::192.0.2.1', port=1234, path='relative/path')" + == "Location(proto='ssh', user='user', pass=None, host='2001:db8::192.0.2.1', port=1234, path='relative/path')" # noqa: E501 ) assert ( repr(Location("ssh://user@[2001:db8::192.0.2.1]/relative/path")) - == "Location(proto='ssh', user='user', host='2001:db8::192.0.2.1', port=None, path='relative/path')" + == "Location(proto='ssh', user='user', pass=None, host='2001:db8::192.0.2.1', port=None, path='relative/path')" # noqa: E501 ) assert ( Location("ssh://user@[2001:db8::192.0.2.1]/relative/path").to_key_filename() @@ -141,20 +141,39 @@ class TestLocationWithoutEnv: ) assert ( repr(Location("ssh://user@[2a02:0001:0002:0003:0004:0005:0006:0007]/relative/path")) - == "Location(proto='ssh', user='user', " + == "Location(proto='ssh', user='user', pass=None, " "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=None, path='relative/path')" ) assert ( repr(Location("ssh://user@[2a02:0001:0002:0003:0004:0005:0006:0007]:1234/relative/path")) - == "Location(proto='ssh', user='user', " + == "Location(proto='ssh', user='user', pass=None, " "host='2a02:0001:0002:0003:0004:0005:0006:0007', port=1234, path='relative/path')" ) + def test_s3(self, monkeypatch, keys_dir): + monkeypatch.delenv("BORG_REPO", raising=False) + assert ( + repr(Location("s3:/test/path")) + == "Location(proto='s3', user=None, pass=None, host=None, port=None, path='test/path')" + ) + assert ( + repr(Location("s3:profile@http://172.28.52.116:9000/test/path")) + == "Location(proto='s3', user='profile', pass=None, host='172.28.52.116', port=9000, path='test/path')" # noqa: E501 + ) + assert ( + repr(Location("s3:user:pass@http://172.28.52.116:9000/test/path")) + == "Location(proto='s3', user='user', pass='REDACTED', host='172.28.52.116', port=9000, path='test/path')" # noqa: E501 + ) + assert ( + repr(Location("b2:user:pass@https://s3.us-east-005.backblazeb2.com/test/path")) + == "Location(proto='b2', user='user', pass='REDACTED', host='s3.us-east-005.backblazeb2.com', port=None, path='test/path')" # noqa: E501 + ) + def test_rclone(self, monkeypatch, keys_dir): monkeypatch.delenv("BORG_REPO", raising=False) assert ( repr(Location("rclone:remote:path")) - == "Location(proto='rclone', user=None, host=None, port=None, path='remote:path')" + == "Location(proto='rclone', user=None, pass=None, host=None, port=None, path='remote:path')" ) assert Location("rclone:remote:path").to_key_filename() == keys_dir + "remote_path" @@ -163,13 +182,13 @@ class TestLocationWithoutEnv: # relative path assert ( repr(Location("sftp://user@host:1234/rel/path")) - == "Location(proto='sftp', user='user', host='host', port=1234, path='rel/path')" + == "Location(proto='sftp', user='user', pass=None, host='host', port=1234, path='rel/path')" ) assert Location("sftp://user@host:1234/rel/path").to_key_filename() == keys_dir + "host__rel_path" # absolute path assert ( repr(Location("sftp://user@host:1234//abs/path")) - == "Location(proto='sftp', user='user', host='host', port=1234, path='/abs/path')" + == "Location(proto='sftp', user='user', pass=None, host='host', port=1234, path='/abs/path')" ) assert Location("sftp://user@host:1234//abs/path").to_key_filename() == keys_dir + "host___abs_path" @@ -177,7 +196,7 @@ class TestLocationWithoutEnv: monkeypatch.delenv("BORG_REPO", raising=False) assert ( repr(Location("socket:///repo/path")) - == "Location(proto='socket', user=None, host=None, port=None, path='/repo/path')" + == "Location(proto='socket', user=None, pass=None, host=None, port=None, path='/repo/path')" ) assert Location("socket:///some/path").to_key_filename() == keys_dir + "_some_path" @@ -185,11 +204,11 @@ class TestLocationWithoutEnv: monkeypatch.delenv("BORG_REPO", raising=False) assert ( repr(Location("file:///some/path")) - == "Location(proto='file', user=None, host=None, port=None, path='/some/path')" + == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/some/path')" ) assert ( repr(Location("file:///some/path")) - == "Location(proto='file', user=None, host=None, port=None, path='/some/path')" + == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/some/path')" ) assert Location("file:///some/path").to_key_filename() == keys_dir + "_some_path" @@ -197,7 +216,7 @@ class TestLocationWithoutEnv: monkeypatch.delenv("BORG_REPO", raising=False) assert ( repr(Location("file:////server/share/path")) - == "Location(proto='file', user=None, host=None, port=None, path='//server/share/path')" + == "Location(proto='file', user=None, pass=None, host=None, port=None, path='//server/share/path')" ) assert Location("file:////server/share/path").to_key_filename() == keys_dir + "__server_share_path" @@ -205,19 +224,22 @@ class TestLocationWithoutEnv: monkeypatch.delenv("BORG_REPO", raising=False) rel_path = "path" abs_path = os.path.abspath(rel_path) - assert repr(Location(rel_path)) == f"Location(proto='file', user=None, host=None, port=None, path='{abs_path}')" + assert ( + repr(Location(rel_path)) + == f"Location(proto='file', user=None, pass=None, host=None, port=None, path='{abs_path}')" + ) assert Location("path").to_key_filename().endswith(rel_path) def test_abspath(self, monkeypatch, keys_dir): monkeypatch.delenv("BORG_REPO", raising=False) assert ( repr(Location("/some/absolute/path")) - == "Location(proto='file', user=None, host=None, port=None, path='/some/absolute/path')" + == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/some/absolute/path')" ) assert Location("/some/absolute/path").to_key_filename() == keys_dir + "_some_absolute_path" assert ( repr(Location("/some/../absolute/path")) - == "Location(proto='file', user=None, host=None, port=None, path='/absolute/path')" + == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/absolute/path')" ) assert Location("/some/../absolute/path").to_key_filename() == keys_dir + "_absolute_path" @@ -226,11 +248,14 @@ class TestLocationWithoutEnv: # for a local path, borg creates a Location instance with an absolute path rel_path = "relative/path" abs_path = os.path.abspath(rel_path) - assert repr(Location(rel_path)) == f"Location(proto='file', user=None, host=None, port=None, path='{abs_path}')" + assert ( + repr(Location(rel_path)) + == f"Location(proto='file', user=None, pass=None, host=None, port=None, path='{abs_path}')" + ) assert Location(rel_path).to_key_filename().endswith("relative_path") assert ( repr(Location("ssh://user@host/relative/path")) - == "Location(proto='ssh', user='user', host='host', port=None, path='relative/path')" + == "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='relative/path')" ) assert Location("ssh://user@host/relative/path").to_key_filename() == keys_dir + "host__relative_path" @@ -238,17 +263,17 @@ class TestLocationWithoutEnv: monkeypatch.delenv("BORG_REPO", raising=False) assert ( repr(Location("/abs/path:w:cols")) - == "Location(proto='file', user=None, host=None, port=None, path='/abs/path:w:cols')" + == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/abs/path:w:cols')" ) assert Location("/abs/path:w:cols").to_key_filename() == keys_dir + "_abs_path_w_cols" assert ( repr(Location("file:///abs/path:w:cols")) - == "Location(proto='file', user=None, host=None, port=None, path='/abs/path:w:cols')" + == "Location(proto='file', user=None, pass=None, host=None, port=None, path='/abs/path:w:cols')" ) assert Location("file:///abs/path:w:cols").to_key_filename() == keys_dir + "_abs_path_w_cols" assert ( repr(Location("ssh://user@host/abs/path:w:cols")) - == "Location(proto='ssh', user='user', host='host', port=None, path='abs/path:w:cols')" + == "Location(proto='ssh', user='user', pass=None, host='host', port=None, path='abs/path:w:cols')" ) assert Location("ssh://user@host/abs/path:w:cols").to_key_filename() == keys_dir + "host__abs_path_w_cols"