diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ec766998..9129072bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -243,10 +243,10 @@ jobs: # Start ssh-agent and add our key so paramiko can use the agent eval "$(ssh-agent -s)" ssh-add ~/.ssh/id_ed25519 - sudo python3 -m venv /opt/borgstore-venv - sudo /opt/borgstore-venv/bin/pip install -U pip setuptools wheel - sudo /opt/borgstore-venv/bin/pip install "borgstore[rest]" - sudo ln -sf /opt/borgstore-venv/bin/borgstore-server-rest /usr/local/bin/borgstore-server-rest + # The rest test starts "borg serve --rest" over ssh as sftpuser, which runs the borg + # under test from the tox venv under $HOME. Allow sftpuser to traverse into the runner + # home so it can reach that borg (the venv dirs/files are created world-r/x by tox/pip). + sudo chmod o+x "$HOME" # Export SFTP test URL for tox via GITHUB_ENV echo "BORG_TEST_SFTP_REPO=sftp://sftpuser@localhost:22/borg/sftp-repo" >> $GITHUB_ENV echo "BORG_TEST_REST_REPO=rest://sftpuser@localhost:22/borg/rest-repo" >> $GITHUB_ENV diff --git a/docs/changes.rst b/docs/changes.rst index 04100fde5..20533dd3f 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -171,6 +171,10 @@ New features: - WIP packs project, major repo format changes, you must create new repos! #8572 - rest:// repository URLs - connect via ssh to remote borgstore REST server, talking http via stdio, #9593 +- ``borg serve --rest`` serves a (non-legacy) repository as the remote-side + component of a rest:// repository (HTTP over stdio). A rest:// client then + starts ``borg serve --rest`` on the remote. + ``borg serve`` (without --rest) serves legacy borg 1.x repositories. - removed ssh:// and socket:// support for current repositories; use a rest:// repository instead (it can tunnel over ssh). ssh:// and ``borg serve`` remain available only for legacy (borg 1.x / v1) repositories, e.g. for diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 222e3eb4c..9ceede63e 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -65,7 +65,7 @@ If you only back up your own files, run it as your normal user (i.e. not root). For a local repository always use the same user to invoke borg. For a remote repository: always use e.g., rest://borg@remote_host (Borg connects -via ssh and runs a borgstore REST server on the remote). You can use this +via ssh and runs ``borg serve --rest`` on the remote). You can use this from different local users; the remote user running borg and accessing the repo will always be `borg`. @@ -363,8 +363,8 @@ Remote repositories Borg can initialize and access repositories on remote hosts if the host is accessible using SSH. This is fastest and easiest when Borg is installed on the remote host, in which case a ``rest://`` repository URL is -used. Borg connects via SSH and runs a borgstore REST server on the remote host -(talking HTTP over stdio):: +used. Borg connects via SSH and runs ``borg serve --rest`` on the remote host, +which serves the repository talking HTTP over stdio:: $ borg -r rest://user@hostname:port/path/to/repo repo-create ... @@ -533,8 +533,8 @@ Example with **borg extract**: Difference when using a **remote borg backup server**: It is basically all the same as with the local repository, but you need to -refer to the repo using a ``rest://`` URL (Borg connects via ssh and runs a -borgstore REST server on the remote host). +refer to the repo using a ``rest://`` URL (Borg connects via ssh and runs +``borg serve --rest`` on the remote host). In the given example, ``borg`` is the user name used to log into the machine ``backup.example.org`` which runs ssh on port ``2222`` and has the borg repo diff --git a/src/borg/archiver/__init__.py b/src/borg/archiver/__init__.py index db3a9bec0..16cf29366 100644 --- a/src/borg/archiver/__init__.py +++ b/src/borg/archiver/__init__.py @@ -339,7 +339,15 @@ class Archiver( # everything else comes from the forced "borg serve" command (or the defaults). # stuff from denylist must never be used from the client. denylist = {"restrict_to_paths", "restrict_to_repositories", "umask", "permissions"} - allowlist = {"debug_topics", "lock_wait", "log_level"} + # "backend" is given by the client to the REST server to contruct a posixfs backend + # that shall be used as a repository (borg serve --rest --backend FILE:). + # Like the legacy repository path (transmitted via the RPC protocol), + # the client chooses *which* repository it wants to use; the ssh forced command pins the + # restrictions, and do_serve_rest validates the client backend against restrict_to_paths + # and restrict_to_repositories. + # The --rest mode flag itself is intentionally NOT allowlisted, so the forced command + # keeps pinning the mode (legacy/rpc vs. non-legacy/rest). + allowlist = {"debug_topics", "lock_wait", "log_level", "backend"} not_present = object() for attr_name in allowlist: assert attr_name not in denylist, "allowlist has denylisted attribute name %s" % attr_name diff --git a/src/borg/archiver/serve_cmd.py b/src/borg/archiver/serve_cmd.py index af2418912..d8dd657f0 100644 --- a/src/borg/archiver/serve_cmd.py +++ b/src/borg/archiver/serve_cmd.py @@ -1,4 +1,7 @@ +import os + from ..constants import * # NOQA +from ..helpers import Error, PathNotAllowed from ..legacy.remote import RepositoryServer from ..logger import create_logger @@ -10,30 +13,72 @@ logger = create_logger() class ServeMixIn: def do_serve(self, args): """Starts in server mode. This command is usually not used manually.""" - RepositoryServer( - restrict_to_paths=args.restrict_to_paths, - restrict_to_repositories=args.restrict_to_repositories, - permissions=args.permissions, - ).serve() + if args.rest: + self.do_serve_rest(args) + else: + # note: legacy (borg 1.x) repositories have no permission system, so args.permissions + # is intentionally not forwarded here (it only applies to "borg serve --rest"). + RepositoryServer( + restrict_to_paths=args.restrict_to_paths, restrict_to_repositories=args.restrict_to_repositories + ).serve() + + def do_serve_rest(self, args): + """Serve a current (non-legacy) rest:// repository on stdio (borgstore REST server).""" + from borgstore.server.rest import serve as rest_serve + from ..repository import borg_permissions + + if not args.backend: + raise Error("borg serve --rest requires --backend FILE:.") + # enforce --restrict-to-path / --restrict-to-repository against the requested FILE: path + self.check_rest_restrictions(args.backend, args.restrict_to_paths, args.restrict_to_repositories) + permissions = ( + args.permissions if args.permissions is not None else os.environ.get("BORG_REPO_PERMISSIONS", "all") + ) + rest_serve(None, None, args.backend, permissions=borg_permissions(permissions), stdio=True) + + @staticmethod + def check_rest_restrictions(backend, restrict_to_paths, restrict_to_repositories): + if not (restrict_to_paths or restrict_to_repositories): + return + if not backend.startswith("FILE:"): + raise PathNotAllowed("only FILE: backends can be restricted") + path = os.path.realpath(os.path.expanduser(backend[len("FILE:") :])) + path_with_sep = os.path.join(path, "") # ensure trailing slash for prefix checks + if restrict_to_paths: + for p in restrict_to_paths: + if path_with_sep.startswith(os.path.join(os.path.realpath(p), "")): + break + else: + raise PathNotAllowed(path) + if restrict_to_repositories: + for p in restrict_to_repositories: + if os.path.join(os.path.realpath(p), "") == path_with_sep: + break + else: + raise PathNotAllowed(path) def build_parser_serve(self, subparsers, common_parser, mid_common_parser): from ._common import process_epilog serve_epilog = process_epilog( """ - This command starts a repository server process. + This command starts a repository server process. It is usually started automatically via + SSH by a borg client and runs until that SSH connection is terminated. - `borg serve` is only used to serve legacy (borg 1.x / v1) repositories over SSH, so that - such repositories can still be accessed remotely (e.g. for `borg transfer --from-borg1`). - Current repositories are accessed via a rest:// repository instead (which can itself tunnel - over SSH), so they do not use `borg serve`. + It operates in one of two modes: - It is automatically started via SSH when a borg client uses an ssh://... repository. - In this mode, `borg serve` will run until that SSH connection is terminated. + - default (no option): serve a **legacy** (borg 1.x / v1) repository using the legacy + RPC protocol. This is used e.g. for ``borg transfer --from-borg1`` and is command-line + compatible with borg 1.x ``borg serve``. - Please note that `borg serve` does not support providing a specific repository via the - `--repo` option or the `BORG_REPO` environment variable. It is always the borg client that - specifies the repository to use when communicating with `borg serve`. + - ``--rest``: serve a **current** (non-legacy) repository as the server-side component of + a ``rest://`` repository, talking HTTP over stdio. The repository to serve is given via + ``--backend FILE:``. A borg client using a ``rest://`` repository starts this + automatically (over SSH if a host is given). + + Please note that, in legacy mode, `borg serve` does not support providing a specific + repository via the `--repo` option or the `BORG_REPO` environment variable - it is the + borg client that specifies the repository to use. The --permissions option enforces repository permissions: @@ -50,6 +95,19 @@ class ServeMixIn: ) subparser = ArgumentParser(parents=[common_parser], description=self.do_serve.__doc__, epilog=serve_epilog) subparsers.add_subcommand("serve", subparser, help="start the repository server process") + subparser.add_argument( + "--rest", + dest="rest", + action="store_true", + help="serve a current (non-legacy) repository as a rest:// server (HTTP over stdio). " + "Requires --backend. Without this option, a legacy (borg 1.x) repository is served.", + ) + subparser.add_argument( + "--backend", + metavar="BACKEND_URL", + dest="backend", + help="(with --rest) backend URL of the repository to serve, e.g. FILE:/path/to/repo.", + ) subparser.add_argument( "--restrict-to-path", metavar="PATH", diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index 12db71b2a..2d7258e26 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -13,7 +13,7 @@ from ..constants import * # NOQA from .datastruct import StableDict, Buffer, EfficientCollectionQueue from .errors import Error, ErrorWithTraceback, IntegrityError, DecompressionError, CancelledByUser, CommandError -from .errors import RTError, modern_ec +from .errors import RTError, PathNotAllowed, modern_ec from .errors import BorgWarning, FileChangedWarning, BackupWarning, IncludePatternNeverMatchedWarning from .errors import BackupError, BackupOSError, BackupRaceConditionError, BackupItemExcluded from .errors import BackupPermissionError, BackupIOError, BackupFileNotFoundError diff --git a/src/borg/helpers/errors.py b/src/borg/helpers/errors.py index 7199177f7..4ea42607f 100644 --- a/src/borg/helpers/errors.py +++ b/src/borg/helpers/errors.py @@ -76,6 +76,12 @@ class CommandError(Error): exit_mcode = 4 +class PathNotAllowed(Error): + """Repository path not allowed: {}.""" + + exit_mcode = 83 + + class BorgWarning: """Warning: {}""" diff --git a/src/borg/legacy/remote.py b/src/borg/legacy/remote.py index abcc7fc85..091926d63 100644 --- a/src/borg/legacy/remote.py +++ b/src/borg/legacy/remote.py @@ -16,7 +16,7 @@ from subprocess import Popen, PIPE import borg.logger from .. import __version__ from ..constants import * # NOQA -from ..helpers import Error, ErrorWithTraceback, IntegrityError +from ..helpers import Error, ErrorWithTraceback, IntegrityError, PathNotAllowed from ..helpers import bin_to_hex from ..helpers import get_limited_unpacker from ..helpers import replace_placeholders @@ -55,12 +55,6 @@ class ConnectionClosedWithHint(ConnectionClosed): exit_mcode = 81 -class PathNotAllowed(Error): - """Repository path not allowed: {}.""" - - exit_mcode = 83 - - class InvalidRPCMethod(Error): """RPC method {} is not valid.""" @@ -758,13 +752,15 @@ class RepositoryServer: # pragma: no cover "get_manifest", # borg2 LegacyRepository has this ) - def __init__(self, restrict_to_paths, restrict_to_repositories, permissions=None): + def __init__(self, restrict_to_paths, restrict_to_repositories): self.repository = None self.RepoCls = None self.rpc_methods = ("open", "close", "negotiate") self.restrict_to_paths = restrict_to_paths self.restrict_to_repositories = restrict_to_repositories - self.permissions = permissions + # note: legacy (borg 1.x / v1) repositories have no permission system, so borg serve + # does not accept/forward permissions here (the --permissions option only applies to + # the non-legacy "borg serve --rest" path). self.client_version = None # we update this after client sends version information def filter_args(self, f, kwargs): diff --git a/src/borg/repository.py b/src/borg/repository.py index 04a6d9d51..879e5afde 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -1,9 +1,11 @@ import os +import sys import time from pathlib import Path from hashlib import sha256 from borgstore.store import Store +from borgstore.backends.rest import REST, ssh_cmd from borgstore.store import ObjectNotFound as StoreObjectNotFound from borgstore.backends.errors import BackendError as StoreBackendError from borgstore.backends.errors import BackendDoesNotExist as StoreBackendDoesNotExist @@ -34,6 +36,69 @@ def repo_lister(repository, *, limit=None): yield from result +def borg_permissions(permissions): + """Map a borg permissions string to a borgstore permissions dict (or None for "all"). + + The namespaces match the borg repository layout (see Repository.__init__ ns_config). + """ + if permissions == "all": + return None # permissions system will not be used + elif permissions == "no-delete": # mostly no delete, no overwrite + return { + "": "lr", + "archives": "lrw", + "cache": "lrwWD", # WD for chunks., last-key-checked, ... + "config": "lrW", # W for manifest + "keys": "lr", + "locks": "lrwD", # borg needs to create/delete a shared lock here + "packs": "lrw", + } + elif permissions == "write-only": # mostly no reading + return { + "": "l", + "archives": "lw", + "cache": "lrwWD", # read allowed, e.g. for chunks. cache + "config": "lrW", # W for manifest + "keys": "lr", + "locks": "lrwD", # borg needs to create/delete a shared lock here + "packs": "lw", # no r! + } + elif permissions == "read-only": # mostly r/o + return {"": "lr", "locks": "lrwD"} + else: + raise Error( + f"Invalid BORG_REPO_PERMISSIONS value: {permissions}, should be one of: " + f"all, no-delete, write-only, read-only." + ) + + +def rest_serve_command(location): + """Build the command line that serves a rest:// *location* via "borg serve --rest". + + For a local rest:// (no host) we run this borg directly (over stdio); if a host is + given, we prefix an ssh command (reusing borgstore's ssh_cmd / BORGSTORE_RSH). + """ + backend_arg = f"FILE:{location.path}" + if not location.host: + # run this borg locally, talking over stdio + borg_cmd = [sys.executable] if getattr(sys, "frozen", False) else [sys.executable, "-m", "borg"] + return borg_cmd + ["serve", "--rest", "--backend", backend_arg] + # reach the remote borg via ssh + remote_path = os.environ.get("BORG_REMOTE_PATH", "borg") + return ssh_cmd(location.user, location.host, location.port) + [ + remote_path, + "serve", + "--rest", + "--backend", + backend_arg, + ] + + +def build_rest_backend(location): + """Return a borgstore REST backend for a rest:// *location*, served by "borg serve --rest".""" + return REST(base_url="http://stdio-backend", command=rest_serve_command(location)) + + class Repository: """borgstore-based key/value store.""" @@ -125,39 +190,18 @@ class Repository: } # Get permissions from parameter or environment variable permissions = permissions if permissions is not None else os.environ.get("BORG_REPO_PERMISSIONS", "all") - - if permissions == "all": - permissions = None # permissions system will not be used - elif permissions == "no-delete": # mostly no delete, no overwrite - permissions = { - "": "lr", - "archives": "lrw", - "cache": "lrwWD", # WD for chunks., last-key-checked, ... - "config": "lrW", # W for manifest - "keys": "lr", - "locks": "lrwD", # borg needs to create/delete a shared lock here - "packs": "lrw", - } - elif permissions == "write-only": # mostly no reading - permissions = { - "": "l", - "archives": "lw", - "cache": "lrwWD", # read allowed, e.g. for chunks. cache - "config": "lrW", # W for manifest - "keys": "lr", - "locks": "lrwD", # borg needs to create/delete a shared lock here - "packs": "lw", # no r! - } - elif permissions == "read-only": # mostly r/o - permissions = {"": "lr", "locks": "lrwD"} - else: - raise Error( - f"Invalid BORG_REPO_PERMISSIONS value: {permissions}, should be one of: " - f"all, no-delete, write-only, read-only." - ) + permissions = borg_permissions(permissions) try: - self.store = Store(url, config=ns_config, permissions=permissions) + if location.proto == "rest": + # rest:// is served by "borg serve --rest" (reachable via ssh if a host is given), + # talking HTTP over stdio - rather than borgstore's own "borgstore-server-rest" command. + # permissions are not given to the (remote) backend here; they are enforced on the + # server side by "borg serve --rest --permissions ...". + backend = build_rest_backend(location) + self.store = Store(backend=backend, config=ns_config) + else: + self.store = Store(url, config=ns_config, permissions=permissions) except StoreBackendError as e: raise Error(str(e)) self.store_opened = False diff --git a/src/borg/testsuite/archiver/argparsing_test.py b/src/borg/testsuite/archiver/argparsing_test.py index 1819d9ca7..eaf7c4fb1 100644 --- a/src/borg/testsuite/archiver/argparsing_test.py +++ b/src/borg/testsuite/archiver/argparsing_test.py @@ -1,6 +1,7 @@ import pytest from . import Archiver, RK_ENCRYPTION, cmd +from ...helpers import PathNotAllowed from ...helpers.argparsing import ArgumentParser, flatten_namespace @@ -74,6 +75,45 @@ def test_get_args(): args = archiver.get_args(["borg", "serve"], "BORG_FOO=bar borg serve --info") assert args.func == archiver.do_serve + # rest server: the client chooses the backend (which repo), the forced command pins the + # restriction and the rest mode. The client's --backend must survive the merge so that + # do_serve_rest can validate it against the forced restrictions. + args = archiver.get_args( + ["borg", "serve", "--rest", "--restrict-to-path=/p1"], "borg serve --rest --backend=FILE:/p1/myrepo" + ) + assert args.func == archiver.do_serve + assert args.rest is True + assert args.restrict_to_paths == ["/p1"] + assert args.backend == "FILE:/p1/myrepo" + # the forced command pins the mode: a client cannot turn a forced legacy serve into a rest serve + args = archiver.get_args(["borg", "serve"], "borg serve --rest --backend=FILE:/p1/myrepo") + assert args.rest is False + + +def test_check_rest_restrictions(tmp_path): + archiver = Archiver() + check = archiver.check_rest_restrictions + allowed = tmp_path / "allowed" + allowed.mkdir() + repo = allowed / "myrepo" + + # no restrictions: anything is allowed + check(f"FILE:{repo}", None, None) + + # restrict-to-path: a backend below the allowed path is fine, outside is rejected + check(f"FILE:{repo}", [str(allowed)], None) + with pytest.raises(PathNotAllowed): + check(f"FILE:{tmp_path / 'other' / 'repo'}", [str(allowed)], None) + + # restrict-to-repository: only the exact repo path is allowed + check(f"FILE:{repo}", None, [str(repo)]) + with pytest.raises(PathNotAllowed): + check(f"FILE:{allowed / 'evil'}", None, [str(repo)]) + + # a non-FILE: backend cannot be restricted + with pytest.raises(PathNotAllowed): + check("sftp://host/path", [str(allowed)], None) + class TestCommonOptions: @staticmethod diff --git a/src/borg/testsuite/archiver/remote_repo_test.py b/src/borg/testsuite/archiver/remote_repo_test.py index 2374a9436..d91607292 100644 --- a/src/borg/testsuite/archiver/remote_repo_test.py +++ b/src/borg/testsuite/archiver/remote_repo_test.py @@ -2,6 +2,7 @@ import json import os import shutil import subprocess +import sys import pytest @@ -59,9 +60,16 @@ def test_rclone_repo_basics(archiver, tmp_path): @pytest.mark.skipif(not REST_URL, reason="BORG_TEST_REST_REPO not set.") -def test_rest_repo_basics(archiver): +def test_rest_repo_basics(archiver, monkeypatch): create_regular_file(archiver.input_path, "file1", size=100 * 1024) create_regular_file(archiver.input_path, "file2", size=10 * 1024) + # A rest:// repo over ssh starts "borg serve --rest" on the remote. For this test the remote is + # localhost (see CI BORG_TEST_REST_REPO), so point BORG_REMOTE_PATH at the borg under test + # (an absolute path that is valid locally) unless the caller already set it. + if not os.environ.get("BORG_REMOTE_PATH"): + borg_path = shutil.which("borg") or os.path.join(os.path.dirname(sys.executable), "borg") + if os.path.exists(borg_path): + monkeypatch.setenv("BORG_REMOTE_PATH", borg_path) archiver.repository_location = REST_URL archive_name = "test-archive" cmd(archiver, "repo-create", RK_ENCRYPTION) diff --git a/src/borg/testsuite/legacyrepository_test.py b/src/borg/testsuite/legacyrepository_test.py index 338a1f3fe..eae031535 100644 --- a/src/borg/testsuite/legacyrepository_test.py +++ b/src/borg/testsuite/legacyrepository_test.py @@ -8,10 +8,11 @@ import pytest from ..legacy.hashindex import NSIndex1 from ..helpers import Location from ..helpers import IntegrityError +from ..helpers import PathNotAllowed from ..helpers import msgpack from ..fslocking import Lock, LockFailed from ..platformflags import is_win32 -from ..legacy.remote import LegacyRemoteRepository, InvalidRPCMethod, PathNotAllowed +from ..legacy.remote import LegacyRemoteRepository, InvalidRPCMethod from ..legacy.repository import LegacyRepository, LoggedIO from ..legacy.repository import MAGIC, MAX_DATA_SIZE, TAG_DELETE, TAG_PUT, TAG_COMMIT from ..compress import CNONE diff --git a/src/borg/testsuite/repository_test.py b/src/borg/testsuite/repository_test.py index 601c1ecc0..2da91e5c5 100644 --- a/src/borg/testsuite/repository_test.py +++ b/src/borg/testsuite/repository_test.py @@ -1,15 +1,33 @@ import os +import sys import pytest from ..constants import ROBJ_FILE_STREAM -from ..helpers import IntegrityError -from ..repository import Repository, MAX_DATA_SIZE, cache_if_remote +from ..helpers import IntegrityError, Location +from ..repository import Repository, MAX_DATA_SIZE, cache_if_remote, rest_serve_command from ..repoobj import RepoObj, OBJ_MAGIC, OBJ_VERSION from ..crypto.key import PlaintextKey from .hashindex_test import H from .crypto.key_test import TestKey +def test_rest_serve_command_local(): + # rest:// without a host runs "borg serve --rest" locally, talking over stdio. + cmd = rest_serve_command(Location("rest:////tmp/repo")) + assert "ssh" not in cmd + assert cmd[0] == sys.executable + assert cmd[-4:] == ["serve", "--rest", "--backend", "FILE:/tmp/repo"] + + +def test_rest_serve_command_ssh(monkeypatch): + # rest:// with a host is reached via ssh, running "borg serve --rest" remotely. + monkeypatch.delenv("BORGSTORE_RSH", raising=False) + monkeypatch.delenv("BORG_REMOTE_PATH", raising=False) + cmd = rest_serve_command(Location("rest://user@host:2222/repo/path")) + assert cmd[:4] == ["ssh", "-p", "2222", "user@host"] + assert cmd[4:] == ["borg", "serve", "--rest", "--backend", "FILE:repo/path"] + + @pytest.fixture() def repository(tmp_path): repository_location = os.fspath(tmp_path / "repository")