Merge pull request #9734 from ThomasWaldmann/borg-serve-rest

borg serve --rest: serve rest:// repositories with borg

.github/workflows/ci.yml

@@ -243,10 +243,10 @@ jobs:
         # Start ssh-agent and add our key so paramiko can use the agent
         eval "$(ssh-agent -s)"
         ssh-add ~/.ssh/id_ed25519
-        sudo python3 -m venv /opt/borgstore-venv
-        sudo /opt/borgstore-venv/bin/pip install -U pip setuptools wheel
-        sudo /opt/borgstore-venv/bin/pip install "borgstore[rest]"
-        sudo ln -sf /opt/borgstore-venv/bin/borgstore-server-rest /usr/local/bin/borgstore-server-rest
+        # The rest test starts "borg serve --rest" over ssh as sftpuser, which runs the borg
+        # under test from the tox venv under $HOME. Allow sftpuser to traverse into the runner
+        # home so it can reach that borg (the venv dirs/files are created world-r/x by tox/pip).
+        sudo chmod o+x "$HOME"
         # Export SFTP test URL for tox via GITHUB_ENV
         echo "BORG_TEST_SFTP_REPO=sftp://sftpuser@localhost:22/borg/sftp-repo" >> $GITHUB_ENV
         echo "BORG_TEST_REST_REPO=rest://sftpuser@localhost:22/borg/rest-repo" >> $GITHUB_ENV

docs/changes.rst

@@ -171,6 +171,10 @@ New features:
 - WIP packs project, major repo format changes, you must create new repos! #8572
 - rest:// repository URLs - connect via ssh to remote borgstore REST server,
   talking http via stdio, #9593
+- ``borg serve --rest`` serves a (non-legacy) repository as the remote-side
+  component of a rest:// repository (HTTP over stdio). A rest:// client then
+  starts ``borg serve --rest`` on the remote.
+  ``borg serve`` (without --rest) serves legacy borg 1.x repositories.
 - removed ssh:// and socket:// support for current repositories; use a rest://
   repository instead (it can tunnel over ssh). ssh:// and ``borg serve`` remain
   available only for legacy (borg 1.x / v1) repositories, e.g. for

docs/quickstart.rst

@@ -65,7 +65,7 @@ If you only back up your own files, run it as your normal user (i.e. not root).
 For a local repository always use the same user to invoke borg.
 
 For a remote repository: always use e.g., rest://borg@remote_host (Borg connects
-via ssh and runs a borgstore REST server on the remote). You can use this
+via ssh and runs ``borg serve --rest`` on the remote). You can use this
 from different local users; the remote user running borg and accessing the
 repo will always be `borg`.
 
@@ -363,8 +363,8 @@ Remote repositories
 Borg can initialize and access repositories on remote hosts if the
 host is accessible using SSH.  This is fastest and easiest when Borg
 is installed on the remote host, in which case a ``rest://`` repository URL is
-used. Borg connects via SSH and runs a borgstore REST server on the remote host
-(talking HTTP over stdio)::
+used. Borg connects via SSH and runs ``borg serve --rest`` on the remote host,
+which serves the repository talking HTTP over stdio::
 
   $ borg -r rest://user@hostname:port/path/to/repo repo-create ...
 
@@ -533,8 +533,8 @@ Example with **borg extract**:
 Difference when using a **remote borg backup server**:
 
 It is basically all the same as with the local repository, but you need to
-refer to the repo using a ``rest://`` URL (Borg connects via ssh and runs a
-borgstore REST server on the remote host).
+refer to the repo using a ``rest://`` URL (Borg connects via ssh and runs
+``borg serve --rest`` on the remote host).
 
 In the given example, ``borg`` is the user name used to log into the machine
 ``backup.example.org`` which runs ssh on port ``2222`` and has the borg repo

src/borg/archiver/__init__.py

@@ -339,7 +339,15 @@ class Archiver(
                 # everything else comes from the forced "borg serve" command (or the defaults).
                 # stuff from denylist must never be used from the client.
                 denylist = {"restrict_to_paths", "restrict_to_repositories", "umask", "permissions"}
-                allowlist = {"debug_topics", "lock_wait", "log_level"}
+                # "backend" is given by the client to the REST server to contruct a posixfs backend
+                # that shall be used as a repository (borg serve --rest --backend FILE:<path>).
+                # Like the legacy repository path (transmitted via the RPC protocol),
+                # the client chooses *which* repository it wants to use; the ssh forced command pins the
+                # restrictions, and do_serve_rest validates the client backend against restrict_to_paths
+                # and restrict_to_repositories.
+                # The --rest mode flag itself is intentionally NOT allowlisted, so the forced command
+                # keeps pinning the mode (legacy/rpc vs. non-legacy/rest).
+                allowlist = {"debug_topics", "lock_wait", "log_level", "backend"}
                 not_present = object()
                 for attr_name in allowlist:
                     assert attr_name not in denylist, "allowlist has denylisted attribute name %s" % attr_name

src/borg/archiver/serve_cmd.py

@@ -1,4 +1,7 @@
+import os
+
 from ..constants import *  # NOQA
+from ..helpers import Error, PathNotAllowed
 from ..legacy.remote import RepositoryServer
 
 from ..logger import create_logger
@@ -10,30 +13,72 @@ logger = create_logger()
 class ServeMixIn:
     def do_serve(self, args):
         """Starts in server mode. This command is usually not used manually."""
-        RepositoryServer(
-            restrict_to_paths=args.restrict_to_paths,
-            restrict_to_repositories=args.restrict_to_repositories,
-            permissions=args.permissions,
-        ).serve()
+        if args.rest:
+            self.do_serve_rest(args)
+        else:
+            # note: legacy (borg 1.x) repositories have no permission system, so args.permissions
+            # is intentionally not forwarded here (it only applies to "borg serve --rest").
+            RepositoryServer(
+                restrict_to_paths=args.restrict_to_paths, restrict_to_repositories=args.restrict_to_repositories
+            ).serve()
+
+    def do_serve_rest(self, args):
+        """Serve a current (non-legacy) rest:// repository on stdio (borgstore REST server)."""
+        from borgstore.server.rest import serve as rest_serve
+        from ..repository import borg_permissions
+
+        if not args.backend:
+            raise Error("borg serve --rest requires --backend FILE:<path>.")
+        # enforce --restrict-to-path / --restrict-to-repository against the requested FILE: path
+        self.check_rest_restrictions(args.backend, args.restrict_to_paths, args.restrict_to_repositories)
+        permissions = (
+            args.permissions if args.permissions is not None else os.environ.get("BORG_REPO_PERMISSIONS", "all")
+        )
+        rest_serve(None, None, args.backend, permissions=borg_permissions(permissions), stdio=True)
+
+    @staticmethod
+    def check_rest_restrictions(backend, restrict_to_paths, restrict_to_repositories):
+        if not (restrict_to_paths or restrict_to_repositories):
+            return
+        if not backend.startswith("FILE:"):
+            raise PathNotAllowed("only FILE: backends can be restricted")
+        path = os.path.realpath(os.path.expanduser(backend[len("FILE:") :]))
+        path_with_sep = os.path.join(path, "")  # ensure trailing slash for prefix checks
+        if restrict_to_paths:
+            for p in restrict_to_paths:
+                if path_with_sep.startswith(os.path.join(os.path.realpath(p), "")):
+                    break
+            else:
+                raise PathNotAllowed(path)
+        if restrict_to_repositories:
+            for p in restrict_to_repositories:
+                if os.path.join(os.path.realpath(p), "") == path_with_sep:
+                    break
+            else:
+                raise PathNotAllowed(path)
 
     def build_parser_serve(self, subparsers, common_parser, mid_common_parser):
         from ._common import process_epilog
 
         serve_epilog = process_epilog(
             """
-        This command starts a repository server process.
+        This command starts a repository server process. It is usually started automatically via
+        SSH by a borg client and runs until that SSH connection is terminated.
 
-        `borg serve` is only used to serve legacy (borg 1.x / v1) repositories over SSH, so that
-        such repositories can still be accessed remotely (e.g. for `borg transfer --from-borg1`).
-        Current repositories are accessed via a rest:// repository instead (which can itself tunnel
-        over SSH), so they do not use `borg serve`.
+        It operates in one of two modes:
 
-        It is automatically started via SSH when a borg client uses an ssh://... repository.
-        In this mode, `borg serve` will run until that SSH connection is terminated.
+        - default (no option): serve a **legacy** (borg 1.x / v1) repository using the legacy
+          RPC protocol. This is used e.g. for ``borg transfer --from-borg1`` and is command-line
+          compatible with borg 1.x ``borg serve``.
 
-        Please note that `borg serve` does not support providing a specific repository via the
-        `--repo` option or the `BORG_REPO` environment variable. It is always the borg client that
-        specifies the repository to use when communicating with `borg serve`.
+        - ``--rest``: serve a **current** (non-legacy) repository as the server-side component of
+          a ``rest://`` repository, talking HTTP over stdio. The repository to serve is given via
+          ``--backend FILE:<path>``. A borg client using a ``rest://`` repository starts this
+          automatically (over SSH if a host is given).
+
+        Please note that, in legacy mode, `borg serve` does not support providing a specific
+        repository via the `--repo` option or the `BORG_REPO` environment variable - it is the
+        borg client that specifies the repository to use.
 
         The --permissions option enforces repository permissions:
 
@@ -50,6 +95,19 @@ class ServeMixIn:
         )
         subparser = ArgumentParser(parents=[common_parser], description=self.do_serve.__doc__, epilog=serve_epilog)
         subparsers.add_subcommand("serve", subparser, help="start the repository server process")
+        subparser.add_argument(
+            "--rest",
+            dest="rest",
+            action="store_true",
+            help="serve a current (non-legacy) repository as a rest:// server (HTTP over stdio). "
+            "Requires --backend. Without this option, a legacy (borg 1.x) repository is served.",
+        )
+        subparser.add_argument(
+            "--backend",
+            metavar="BACKEND_URL",
+            dest="backend",
+            help="(with --rest) backend URL of the repository to serve, e.g. FILE:/path/to/repo.",
+        )
         subparser.add_argument(
             "--restrict-to-path",
             metavar="PATH",

src/borg/helpers/__init__.py

@@ -13,7 +13,7 @@ from ..constants import *  # NOQA
 
 from .datastruct import StableDict, Buffer, EfficientCollectionQueue
 from .errors import Error, ErrorWithTraceback, IntegrityError, DecompressionError, CancelledByUser, CommandError
-from .errors import RTError, modern_ec
+from .errors import RTError, PathNotAllowed, modern_ec
 from .errors import BorgWarning, FileChangedWarning, BackupWarning, IncludePatternNeverMatchedWarning
 from .errors import BackupError, BackupOSError, BackupRaceConditionError, BackupItemExcluded
 from .errors import BackupPermissionError, BackupIOError, BackupFileNotFoundError

src/borg/helpers/errors.py

@@ -76,6 +76,12 @@ class CommandError(Error):
     exit_mcode = 4
 
 
+class PathNotAllowed(Error):
+    """Repository path not allowed: {}."""
+
+    exit_mcode = 83
+
+
 class BorgWarning:
     """Warning: {}"""
 

src/borg/legacy/remote.py

@@ -16,7 +16,7 @@ from subprocess import Popen, PIPE
 import borg.logger
 from .. import __version__
 from ..constants import *  # NOQA
-from ..helpers import Error, ErrorWithTraceback, IntegrityError
+from ..helpers import Error, ErrorWithTraceback, IntegrityError, PathNotAllowed
 from ..helpers import bin_to_hex
 from ..helpers import get_limited_unpacker
 from ..helpers import replace_placeholders
@@ -55,12 +55,6 @@ class ConnectionClosedWithHint(ConnectionClosed):
     exit_mcode = 81
 
 
-class PathNotAllowed(Error):
-    """Repository path not allowed: {}."""
-
-    exit_mcode = 83
-
-
 class InvalidRPCMethod(Error):
     """RPC method {} is not valid."""
 
@@ -758,13 +752,15 @@ class RepositoryServer:  # pragma: no cover
         "get_manifest",  # borg2 LegacyRepository has this
     )
 
-    def __init__(self, restrict_to_paths, restrict_to_repositories, permissions=None):
+    def __init__(self, restrict_to_paths, restrict_to_repositories):
         self.repository = None
         self.RepoCls = None
         self.rpc_methods = ("open", "close", "negotiate")
         self.restrict_to_paths = restrict_to_paths
         self.restrict_to_repositories = restrict_to_repositories
-        self.permissions = permissions
+        # note: legacy (borg 1.x / v1) repositories have no permission system, so borg serve
+        # does not accept/forward permissions here (the --permissions option only applies to
+        # the non-legacy "borg serve --rest" path).
         self.client_version = None  # we update this after client sends version information
 
     def filter_args(self, f, kwargs):

src/borg/repository.py

@@ -1,9 +1,11 @@
 import os
+import sys
 import time
 from pathlib import Path
 from hashlib import sha256
 
 from borgstore.store import Store
+from borgstore.backends.rest import REST, ssh_cmd
 from borgstore.store import ObjectNotFound as StoreObjectNotFound
 from borgstore.backends.errors import BackendError as StoreBackendError
 from borgstore.backends.errors import BackendDoesNotExist as StoreBackendDoesNotExist
@@ -34,6 +36,69 @@ def repo_lister(repository, *, limit=None):
         yield from result
 
 
+def borg_permissions(permissions):
+    """Map a borg permissions string to a borgstore permissions dict (or None for "all").
+
+    The namespaces match the borg repository layout (see Repository.__init__ ns_config).
+    """
+    if permissions == "all":
+        return None  # permissions system will not be used
+    elif permissions == "no-delete":  # mostly no delete, no overwrite
+        return {
+            "": "lr",
+            "archives": "lrw",
+            "cache": "lrwWD",  # WD for chunks.<HASH>, last-key-checked, ...
+            "config": "lrW",  # W for manifest
+            "keys": "lr",
+            "locks": "lrwD",  # borg needs to create/delete a shared lock here
+            "packs": "lrw",
+        }
+    elif permissions == "write-only":  # mostly no reading
+        return {
+            "": "l",
+            "archives": "lw",
+            "cache": "lrwWD",  # read allowed, e.g. for chunks.<HASH> cache
+            "config": "lrW",  # W for manifest
+            "keys": "lr",
+            "locks": "lrwD",  # borg needs to create/delete a shared lock here
+            "packs": "lw",  # no r!
+        }
+    elif permissions == "read-only":  # mostly r/o
+        return {"": "lr", "locks": "lrwD"}
+    else:
+        raise Error(
+            f"Invalid BORG_REPO_PERMISSIONS value: {permissions}, should be one of: "
+            f"all, no-delete, write-only, read-only."
+        )
+
+
+def rest_serve_command(location):
+    """Build the command line that serves a rest:// *location* via "borg serve --rest".
+
+    For a local rest:// (no host) we run this borg directly (over stdio); if a host is
+    given, we prefix an ssh command (reusing borgstore's ssh_cmd / BORGSTORE_RSH).
+    """
+    backend_arg = f"FILE:{location.path}"
+    if not location.host:
+        # run this borg locally, talking over stdio
+        borg_cmd = [sys.executable] if getattr(sys, "frozen", False) else [sys.executable, "-m", "borg"]
+        return borg_cmd + ["serve", "--rest", "--backend", backend_arg]
+    # reach the remote borg via ssh
+    remote_path = os.environ.get("BORG_REMOTE_PATH", "borg")
+    return ssh_cmd(location.user, location.host, location.port) + [
+        remote_path,
+        "serve",
+        "--rest",
+        "--backend",
+        backend_arg,
+    ]
+
+
+def build_rest_backend(location):
+    """Return a borgstore REST backend for a rest:// *location*, served by "borg serve --rest"."""
+    return REST(base_url="http://stdio-backend", command=rest_serve_command(location))
+
+
 class Repository:
     """borgstore-based key/value store."""
 
@@ -125,39 +190,18 @@ class Repository:
         }
         # Get permissions from parameter or environment variable
         permissions = permissions if permissions is not None else os.environ.get("BORG_REPO_PERMISSIONS", "all")
-
-        if permissions == "all":
-            permissions = None  # permissions system will not be used
-        elif permissions == "no-delete":  # mostly no delete, no overwrite
-            permissions = {
-                "": "lr",
-                "archives": "lrw",
-                "cache": "lrwWD",  # WD for chunks.<HASH>, last-key-checked, ...
-                "config": "lrW",  # W for manifest
-                "keys": "lr",
-                "locks": "lrwD",  # borg needs to create/delete a shared lock here
-                "packs": "lrw",
-            }
-        elif permissions == "write-only":  # mostly no reading
-            permissions = {
-                "": "l",
-                "archives": "lw",
-                "cache": "lrwWD",  # read allowed, e.g. for chunks.<HASH> cache
-                "config": "lrW",  # W for manifest
-                "keys": "lr",
-                "locks": "lrwD",  # borg needs to create/delete a shared lock here
-                "packs": "lw",  # no r!
-            }
-        elif permissions == "read-only":  # mostly r/o
-            permissions = {"": "lr", "locks": "lrwD"}
-        else:
-            raise Error(
-                f"Invalid BORG_REPO_PERMISSIONS value: {permissions}, should be one of: "
-                f"all, no-delete, write-only, read-only."
-            )
+        permissions = borg_permissions(permissions)
 
         try:
-            self.store = Store(url, config=ns_config, permissions=permissions)
+            if location.proto == "rest":
+                # rest:// is served by "borg serve --rest" (reachable via ssh if a host is given),
+                # talking HTTP over stdio - rather than borgstore's own "borgstore-server-rest" command.
+                # permissions are not given to the (remote) backend here; they are enforced on the
+                # server side by "borg serve --rest --permissions ...".
+                backend = build_rest_backend(location)
+                self.store = Store(backend=backend, config=ns_config)
+            else:
+                self.store = Store(url, config=ns_config, permissions=permissions)
         except StoreBackendError as e:
             raise Error(str(e))
         self.store_opened = False

src/borg/testsuite/archiver/argparsing_test.py

@@ -1,6 +1,7 @@
 import pytest
 
 from . import Archiver, RK_ENCRYPTION, cmd
+from ...helpers import PathNotAllowed
 from ...helpers.argparsing import ArgumentParser, flatten_namespace
 
 
@@ -74,6 +75,45 @@ def test_get_args():
     args = archiver.get_args(["borg", "serve"], "BORG_FOO=bar borg serve --info")
     assert args.func == archiver.do_serve
 
+    # rest server: the client chooses the backend (which repo), the forced command pins the
+    # restriction and the rest mode. The client's --backend must survive the merge so that
+    # do_serve_rest can validate it against the forced restrictions.
+    args = archiver.get_args(
+        ["borg", "serve", "--rest", "--restrict-to-path=/p1"], "borg serve --rest --backend=FILE:/p1/myrepo"
+    )
+    assert args.func == archiver.do_serve
+    assert args.rest is True
+    assert args.restrict_to_paths == ["/p1"]
+    assert args.backend == "FILE:/p1/myrepo"
+    # the forced command pins the mode: a client cannot turn a forced legacy serve into a rest serve
+    args = archiver.get_args(["borg", "serve"], "borg serve --rest --backend=FILE:/p1/myrepo")
+    assert args.rest is False
+
+
+def test_check_rest_restrictions(tmp_path):
+    archiver = Archiver()
+    check = archiver.check_rest_restrictions
+    allowed = tmp_path / "allowed"
+    allowed.mkdir()
+    repo = allowed / "myrepo"
+
+    # no restrictions: anything is allowed
+    check(f"FILE:{repo}", None, None)
+
+    # restrict-to-path: a backend below the allowed path is fine, outside is rejected
+    check(f"FILE:{repo}", [str(allowed)], None)
+    with pytest.raises(PathNotAllowed):
+        check(f"FILE:{tmp_path / 'other' / 'repo'}", [str(allowed)], None)
+
+    # restrict-to-repository: only the exact repo path is allowed
+    check(f"FILE:{repo}", None, [str(repo)])
+    with pytest.raises(PathNotAllowed):
+        check(f"FILE:{allowed / 'evil'}", None, [str(repo)])
+
+    # a non-FILE: backend cannot be restricted
+    with pytest.raises(PathNotAllowed):
+        check("sftp://host/path", [str(allowed)], None)
+
 
 class TestCommonOptions:
     @staticmethod

src/borg/testsuite/archiver/remote_repo_test.py

@@ -2,6 +2,7 @@ import json
 import os
 import shutil
 import subprocess
+import sys
 
 import pytest
 
@@ -59,9 +60,16 @@ def test_rclone_repo_basics(archiver, tmp_path):
 
 
 @pytest.mark.skipif(not REST_URL, reason="BORG_TEST_REST_REPO not set.")
-def test_rest_repo_basics(archiver):
+def test_rest_repo_basics(archiver, monkeypatch):
     create_regular_file(archiver.input_path, "file1", size=100 * 1024)
     create_regular_file(archiver.input_path, "file2", size=10 * 1024)
+    # A rest:// repo over ssh starts "borg serve --rest" on the remote. For this test the remote is
+    # localhost (see CI BORG_TEST_REST_REPO), so point BORG_REMOTE_PATH at the borg under test
+    # (an absolute path that is valid locally) unless the caller already set it.
+    if not os.environ.get("BORG_REMOTE_PATH"):
+        borg_path = shutil.which("borg") or os.path.join(os.path.dirname(sys.executable), "borg")
+        if os.path.exists(borg_path):
+            monkeypatch.setenv("BORG_REMOTE_PATH", borg_path)
     archiver.repository_location = REST_URL
     archive_name = "test-archive"
     cmd(archiver, "repo-create", RK_ENCRYPTION)

src/borg/testsuite/legacyrepository_test.py

@@ -8,10 +8,11 @@ import pytest
 from ..legacy.hashindex import NSIndex1
 from ..helpers import Location
 from ..helpers import IntegrityError
+from ..helpers import PathNotAllowed
 from ..helpers import msgpack
 from ..fslocking import Lock, LockFailed
 from ..platformflags import is_win32
-from ..legacy.remote import LegacyRemoteRepository, InvalidRPCMethod, PathNotAllowed
+from ..legacy.remote import LegacyRemoteRepository, InvalidRPCMethod
 from ..legacy.repository import LegacyRepository, LoggedIO
 from ..legacy.repository import MAGIC, MAX_DATA_SIZE, TAG_DELETE, TAG_PUT, TAG_COMMIT
 from ..compress import CNONE

src/borg/testsuite/repository_test.py

@@ -1,15 +1,33 @@
 import os
+import sys
 
 import pytest
 from ..constants import ROBJ_FILE_STREAM
-from ..helpers import IntegrityError
-from ..repository import Repository, MAX_DATA_SIZE, cache_if_remote
+from ..helpers import IntegrityError, Location
+from ..repository import Repository, MAX_DATA_SIZE, cache_if_remote, rest_serve_command
 from ..repoobj import RepoObj, OBJ_MAGIC, OBJ_VERSION
 from ..crypto.key import PlaintextKey
 from .hashindex_test import H
 from .crypto.key_test import TestKey
 
 
+def test_rest_serve_command_local():
+    # rest:// without a host runs "borg serve --rest" locally, talking over stdio.
+    cmd = rest_serve_command(Location("rest:////tmp/repo"))
+    assert "ssh" not in cmd
+    assert cmd[0] == sys.executable
+    assert cmd[-4:] == ["serve", "--rest", "--backend", "FILE:/tmp/repo"]
+
+
+def test_rest_serve_command_ssh(monkeypatch):
+    # rest:// with a host is reached via ssh, running "borg serve --rest" remotely.
+    monkeypatch.delenv("BORGSTORE_RSH", raising=False)
+    monkeypatch.delenv("BORG_REMOTE_PATH", raising=False)
+    cmd = rest_serve_command(Location("rest://user@host:2222/repo/path"))
+    assert cmd[:4] == ["ssh", "-p", "2222", "user@host"]
+    assert cmd[4:] == ["borg", "serve", "--rest", "--backend", "FILE:repo/path"]
+
+
 @pytest.fixture()
 def repository(tmp_path):
     repository_location = os.fspath(tmp_path / "repository")