upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-06-11 01:41:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

Repository3 / RemoteRepository3: implement a borgstore based repository

Browse source 
Simplify the repository a lot:

No repository transactions, no log-like appending, no append-only, no segments,
just using a key/value store for the individual chunks.

No locking yet.

Also:

mypy: ignore missing import
there are no library stubs for borgstore yet, so mypy errors without that option.

pyproject.toml: install borgstore directly from github
There is no pypi release yet.

use pip install -e . rather than python setup.py develop
The latter is deprecated and had issues installing the "borgstore from github" dependency.
This commit is contained in:
Thomas Waldmann 2024-08-04 15:57:37 +02:00
parent ea718b98f2
commit d30d5f4aec
No known key found for this signature in database
GPG key ID: 243ACFA951F78E01
37 changed files with 1967 additions and 601 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/ci.yml vendored
View file

@ -104,8 +104,7 @@ jobs:
pip install -r requirements.d/development.txt
- name: Install borgbackup
run: |
# pip install -e .
python setup.py -v develop
pip install -e .
- name: run tox env
env:
XDISTN: "4"

2
pyproject.toml
View file

@ -34,6 +34,8 @@ dependencies = [
"platformdirs >=3.0.0, <5.0.0; sys_platform == 'darwin'", # for macOS: breaking changes in 3.0.0,
"platformdirs >=2.6.0, <5.0.0; sys_platform != 'darwin'", # for others: 2.6+ works consistently.
"argon2-cffi",
"borgstore",
]
[project.optional-dependencies]

10
src/borg/archive.py
View file

@ -51,7 +51,7 @@ from .patterns import PathPrefixPattern, FnmatchPattern, IECommand
from .item import Item, ArchiveItem, ItemDiff
from .platform import acl_get, acl_set, set_flags, get_flags, swidth, hostname
from .remote import cache_if_remote
from .repository import Repository, LIST_SCAN_LIMIT
from .repository3 import Repository3, LIST_SCAN_LIMIT
from .repoobj import RepoObj
has_link = hasattr(os, "link")
@ -1046,7 +1046,7 @@ Duration: {0.duration}
def fetch_async_response(wait=True):
try:
return self.repository.async_response(wait=wait)
except Repository.ObjectNotFound:
except Repository3.ObjectNotFound:
nonlocal error
# object not in repo - strange, but we wanted to delete it anyway.
if forced == 0:
@ -1093,7 +1093,7 @@ Duration: {0.duration}
error = True
if progress:
pi.finish()
except (msgpack.UnpackException, Repository.ObjectNotFound):
except (msgpack.UnpackException, Repository3.ObjectNotFound):
# items metadata corrupted
if forced == 0:
raise
@ -1887,7 +1887,7 @@ class ArchiveChecker:
# Explicitly set the initial usable hash table capacity to avoid performance issues
# due to hash table "resonance".
# Since reconstruction of archive items can add some new chunks, add 10 % headroom.
self.chunks = ChunkIndex(usable=len(self.repository) * 1.1)
self.chunks = ChunkIndex()
marker = None
while True:
result = self.repository.list(limit=LIST_SCAN_LIMIT, marker=marker)
@ -1939,7 +1939,7 @@ class ArchiveChecker:
chunk_id = chunk_ids_revd.pop(-1) # better efficiency
try:
encrypted_data = next(chunk_data_iter)
except (Repository.ObjectNotFound, IntegrityErrorBase) as err:
except (Repository3.ObjectNotFound, IntegrityErrorBase) as err:
self.error_found = True
errors += 1
logger.error("chunk %s: %s", bin_to_hex(chunk_id), err)

25
src/borg/archiver/__init__.py
View file

@ -36,7 +36,7 @@ try:
from ..helpers import ErrorIgnoringTextIOWrapper
from ..helpers import msgpack
from ..helpers import sig_int
from ..remote import RemoteRepository
from ..remote3 import RemoteRepository3
from ..selftest import selftest
except BaseException:
# an unhandled exception in the try-block would cause the borg cli command to exit with rc 1 due to python's
@ -68,7 +68,6 @@ def get_func(args):
from .benchmark_cmd import BenchmarkMixIn
from .check_cmd import CheckMixIn
from .compact_cmd import CompactMixIn
from .config_cmd import ConfigMixIn
from .create_cmd import CreateMixIn
from .debug_cmd import DebugMixIn
from .delete_cmd import DeleteMixIn
@ -98,7 +97,6 @@ class Archiver(
BenchmarkMixIn,
CheckMixIn,
CompactMixIn,
ConfigMixIn,
CreateMixIn,
DebugMixIn,
DeleteMixIn,
@ -336,7 +334,6 @@ class Archiver(
self.build_parser_benchmarks(subparsers, common_parser, mid_common_parser)
self.build_parser_check(subparsers, common_parser, mid_common_parser)
self.build_parser_compact(subparsers, common_parser, mid_common_parser)
self.build_parser_config(subparsers, common_parser, mid_common_parser)
self.build_parser_create(subparsers, common_parser, mid_common_parser)
self.build_parser_debug(subparsers, common_parser, mid_common_parser)
self.build_parser_delete(subparsers, common_parser, mid_common_parser)
@ -412,22 +409,6 @@ class Archiver(
elif not args.paths_from_stdin:
# need at least 1 path but args.paths may also be populated from patterns
parser.error("Need at least one PATH argument.")
if not getattr(args, "lock", True): # Option --bypass-lock sets args.lock = False
bypass_allowed = {
self.do_check,
self.do_config,
self.do_diff,
self.do_export_tar,
self.do_extract,
self.do_info,
self.do_rinfo,
self.do_list,
self.do_rlist,
self.do_mount,
self.do_umount,
}
if func not in bypass_allowed:
raise Error("Not allowed to bypass locking mechanism for chosen command")
# we can only have a complete knowledge of placeholder replacements we should do **after** arg parsing,
# e.g. due to options like --timestamp that override the current time.
# thus we have to initialize replace_placeholders here and process all args that need placeholder replacement.
@ -581,7 +562,7 @@ def sig_trace_handler(sig_no, stack): # pragma: no cover
def format_tb(exc):
qualname = type(exc).__qualname__
remote = isinstance(exc, RemoteRepository.RPCError)
remote = isinstance(exc, RemoteRepository3.RPCError)
if remote:
prefix = "Borg server: "
trace_back = "\n".join(prefix + line for line in exc.exception_full.splitlines())
@ -659,7 +640,7 @@ def main(): # pragma: no cover
tb_log_level = logging.ERROR if e.traceback else logging.DEBUG
tb = format_tb(e)
exit_code = e.exit_code
except RemoteRepository.RPCError as e:
except RemoteRepository3.RPCError as e:
important = e.traceback
msg = e.exception_full if important else e.get_message()
msgid = e.exception_class

29
src/borg/archiver/_common.py
View file

@ -14,7 +14,9 @@ from ..helpers.nanorst import rst_to_terminal
from ..manifest import Manifest, AI_HUMAN_SORT_KEYS
from ..patterns import PatternMatcher
from ..remote import RemoteRepository
from ..remote3 import RemoteRepository3
from ..repository import Repository
from ..repository3 import Repository3
from ..repoobj import RepoObj, RepoObj1
from ..patterns import (
ArgparsePatternAction,
@ -29,9 +31,10 @@ from ..logger import create_logger
logger = create_logger(__name__)
def get_repository(location, *, create, exclusive, lock_wait, lock, append_only, make_parent_dirs, storage_quota, args):
def get_repository(location, *, create, exclusive, lock_wait, lock, append_only, make_parent_dirs, storage_quota, args, v1_or_v2):
if location.proto in ("ssh", "socket"):
repository = RemoteRepository(
RemoteRepoCls = RemoteRepository if v1_or_v2 else RemoteRepository3
repository = RemoteRepoCls(
location,
create=create,
exclusive=exclusive,
@ -43,7 +46,8 @@ def get_repository(location, *, create, exclusive, lock_wait, lock, append_only,
)
else:
repository = Repository(
RepoCls = Repository if v1_or_v2 else Repository3
repository = RepoCls(
location.path,
create=create,
exclusive=exclusive,
@ -98,8 +102,7 @@ def with_repository(
decorator_name="with_repository",
)
# To process the `--bypass-lock` option if specified, we need to
# modify `lock` inside `wrapper`. Therefore we cannot use the
# We may need to modify `lock` inside `wrapper`. Therefore we cannot use the
# `nonlocal` statement to access `lock` as modifications would also
# affect the scope outside of `wrapper`. Subsequent calls would
# only see the overwritten value of `lock`, not the original one.
@ -129,13 +132,15 @@ def with_repository(
make_parent_dirs=make_parent_dirs,
storage_quota=storage_quota,
args=args,
v1_or_v2=False,
)
with repository:
if repository.version not in (2,):
if repository.version not in (3,):
raise Error(
"This borg version only accepts version 2 repos for -r/--repo. "
"You can use 'borg transfer' to copy archives from old to new repos."
f"This borg version only accepts version 3 repos for -r/--repo, "
f"but not version {repository.version}. "
f"You can use 'borg transfer' to copy archives from old to new repos."
)
if manifest or cache:
manifest_ = Manifest.load(repository, compatibility)
@ -195,6 +200,7 @@ def with_other_repository(manifest=False, cache=False, compatibility=None):
make_parent_dirs=False,
storage_quota=None,
args=args,
v1_or_v2=True
)
with repository:
@ -504,13 +510,6 @@ def define_common_options(add_common_option):
action=Highlander,
help="wait at most SECONDS for acquiring a repository/cache lock (default: %(default)d).",
)
add_common_option(
"--bypass-lock",
dest="lock",
action="store_false",
default=argparse.SUPPRESS, # only create args attribute if option is specified
help="Bypass locking mechanism",
)
add_common_option("--show-version", dest="show_version", action="store_true", help="show/log the borg version")
add_common_option("--show-rc", dest="show_rc", action="store_true", help="show/log the return code (rc)")
add_common_option(

177
src/borg/archiver/config_cmd.py
View file

@ -1,177 +0,0 @@
import argparse
import configparser
from ._common import with_repository
from ..cache import Cache, assert_secure
from ..constants import * # NOQA
from ..helpers import Error, CommandError
from ..helpers import parse_file_size, hex_to_bin
from ..manifest import Manifest
from ..logger import create_logger
logger = create_logger()
class ConfigMixIn:
@with_repository(exclusive=True, manifest=False)
def do_config(self, args, repository):
"""get, set, and delete values in a repository or cache config file"""
def repo_validate(section, name, value=None, check_value=True):
if section not in ["repository"]:
raise ValueError("Invalid section")
if name in ["segments_per_dir", "last_segment_checked"]:
if check_value:
try:
int(value)
except ValueError:
raise ValueError("Invalid value") from None
elif name in ["max_segment_size", "additional_free_space", "storage_quota"]:
if check_value:
try:
parse_file_size(value)
except ValueError:
raise ValueError("Invalid value") from None
if name == "storage_quota":
if parse_file_size(value) < parse_file_size("10M"):
raise ValueError("Invalid value: storage_quota < 10M")
elif name == "max_segment_size":
if parse_file_size(value) >= MAX_SEGMENT_SIZE_LIMIT:
raise ValueError("Invalid value: max_segment_size >= %d" % MAX_SEGMENT_SIZE_LIMIT)
elif name in ["append_only"]:
if check_value and value not in ["0", "1"]:
raise ValueError("Invalid value")
elif name in ["id"]:
if check_value:
hex_to_bin(value, length=32)
else:
raise ValueError("Invalid name")
def cache_validate(section, name, value=None, check_value=True):
if section not in ["cache"]:
raise ValueError("Invalid section")
# currently, we do not support setting anything in the cache via borg config.
raise ValueError("Invalid name")
def list_config(config):
default_values = {
"version": "1",
"segments_per_dir": str(DEFAULT_SEGMENTS_PER_DIR),
"max_segment_size": str(MAX_SEGMENT_SIZE_LIMIT),
"additional_free_space": "0",
"storage_quota": repository.storage_quota,
"append_only": repository.append_only,
}
print("[repository]")
for key in [
"version",
"segments_per_dir",
"max_segment_size",
"storage_quota",
"additional_free_space",
"append_only",
"id",
]:
value = config.get("repository", key, fallback=False)
if value is None:
value = default_values.get(key)
if value is None:
raise Error("The repository config is missing the %s key which has no default value" % key)
print(f"{key} = {value}")
for key in ["last_segment_checked"]:
value = config.get("repository", key, fallback=None)
if value is None:
continue
print(f"{key} = {value}")
if not args.list:
if args.name is None:
raise CommandError("No config key name was provided.")
try:
section, name = args.name.split(".")
except ValueError:
section = args.cache and "cache" or "repository"
name = args.name
if args.cache:
manifest = Manifest.load(repository, (Manifest.Operation.WRITE,))
assert_secure(repository, manifest, self.lock_wait)
cache = Cache(repository, manifest, lock_wait=self.lock_wait)
try:
if args.cache:
cache.cache_config.load()
config = cache.cache_config._config
save = cache.cache_config.save
validate = cache_validate
else:
config = repository.config
save = lambda: repository.save_config(repository.path, repository.config) # noqa
validate = repo_validate
if args.delete:
validate(section, name, check_value=False)
config.remove_option(section, name)
if len(config.options(section)) == 0:
config.remove_section(section)
save()
elif args.list:
list_config(config)
elif args.value:
validate(section, name, args.value)
if section not in config.sections():
config.add_section(section)
config.set(section, name, args.value)
save()
else:
try:
print(config.get(section, name))
except (configparser.NoOptionError, configparser.NoSectionError) as e:
raise Error(e)
finally:
if args.cache:
cache.close()
def build_parser_config(self, subparsers, common_parser, mid_common_parser):
from ._common import process_epilog
config_epilog = process_epilog(
"""
This command gets and sets options in a local repository or cache config file.
For security reasons, this command only works on local repositories.
To delete a config value entirely, use ``--delete``. To list the values
of the configuration file or the default values, use ``--list``. To get an existing
key, pass only the key name. To set a key, pass both the key name and
the new value. Keys can be specified in the format "section.name" or
simply "name"; the section will default to "repository" and "cache" for
the repo and cache configs, respectively.
By default, borg config manipulates the repository config file. Using ``--cache``
edits the repository cache's config file instead.
"""
)
subparser = subparsers.add_parser(
"config",
parents=[common_parser],
add_help=False,
description=self.do_config.__doc__,
epilog=config_epilog,
formatter_class=argparse.RawDescriptionHelpFormatter,
help="get and set configuration values",
)
subparser.set_defaults(func=self.do_config)
subparser.add_argument(
"-c", "--cache", dest="cache", action="store_true", help="get and set values from the repo cache"
)
group = subparser.add_mutually_exclusive_group()
group.add_argument(
"-d", "--delete", dest="delete", action="store_true", help="delete the key from the config file"
)
group.add_argument("-l", "--list", dest="list", action="store_true", help="list the configuration of the repo")
subparser.add_argument("name", metavar="NAME", nargs="?", help="name of config key")
subparser.add_argument("value", metavar="VALUE", nargs="?", help="new value for key")

39
src/borg/archiver/debug_cmd.py
View file

@ -15,7 +15,8 @@ from ..helpers import positive_int_validator, archivename_validator
from ..helpers import CommandError, RTError
from ..manifest import Manifest
from ..platform import get_process_id
from ..repository import Repository, LIST_SCAN_LIMIT, TAG_PUT, TAG_DELETE, TAG_COMMIT
from ..repository import Repository, TAG_PUT, TAG_DELETE, TAG_COMMIT
from ..repository3 import Repository3, LIST_SCAN_LIMIT
from ..repoobj import RepoObj
from ._common import with_repository, Highlander
@ -330,7 +331,7 @@ class DebugMixIn:
repository.delete(id)
modified = True
print("object %s deleted." % hex_id)
except Repository.ObjectNotFound:
except Repository3.ObjectNotFound:
print("object %s not found." % hex_id)
if modified:
repository.commit(compact=False)
@ -351,23 +352,6 @@ class DebugMixIn:
except KeyError:
print("object %s not found [info from chunks cache]." % hex_id)
@with_repository(manifest=False, exclusive=True)
def do_debug_dump_hints(self, args, repository):
"""dump repository hints"""
if not repository._active_txn:
repository.prepare_txn(repository.get_transaction_id())
try:
hints = dict(
segments=repository.segments,
compact=repository.compact,
storage_quota_use=repository.storage_quota_use,
shadow_index={bin_to_hex(k): v for k, v in repository.shadow_index.items()},
)
with dash_open(args.path, "w") as fd:
json.dump(hints, fd, indent=4)
finally:
repository.rollback()
def do_debug_convert_profile(self, args):
"""convert Borg profile to Python profile"""
import marshal
@ -689,23 +673,6 @@ class DebugMixIn:
subparser.set_defaults(func=self.do_debug_refcount_obj)
subparser.add_argument("ids", metavar="IDs", nargs="+", type=str, help="hex object ID(s) to show refcounts for")
debug_dump_hints_epilog = process_epilog(
"""
This command dumps the repository hints data.
"""
)
subparser = debug_parsers.add_parser(
"dump-hints",
parents=[common_parser],
add_help=False,
description=self.do_debug_dump_hints.__doc__,
epilog=debug_dump_hints_epilog,
formatter_class=argparse.RawDescriptionHelpFormatter,
help="dump repo hints (debug)",
)
subparser.set_defaults(func=self.do_debug_dump_hints)
subparser.add_argument("path", metavar="PATH", type=str, help="file to dump data into")
debug_convert_profile_epilog = process_epilog(
"""
Convert a Borg profile to a Python cProfile compatible profile.

11
src/borg/archiver/rcompress_cmd.py
View file

@ -24,14 +24,7 @@ def find_chunks(repository, repo_objs, stats, ctype, clevel, olevel):
compr_keys = stats["compr_keys"] = set()
compr_wanted = ctype, clevel, olevel
state = None
chunks_count = len(repository)
chunks_limit = min(1000, max(100, chunks_count // 1000))
pi = ProgressIndicatorPercent(
total=chunks_count,
msg="Searching for recompression candidates %3.1f%%",
step=0.1,
msgid="rcompress.find_chunks",
)
chunks_limit = 1000
while True:
chunk_ids, state = repository.scan(limit=chunks_limit, state=state)
if not chunk_ids:
@ -44,8 +37,6 @@ def find_chunks(repository, repo_objs, stats, ctype, clevel, olevel):
compr_keys.add(compr_found)
stats[compr_found] += 1
stats["checked_count"] += 1
pi.show(increase=1)
pi.finish()
return recompress_ids

2
src/borg/archiver/serve_cmd.py
View file

@ -3,7 +3,7 @@ import argparse
from ._common import Highlander
from ..constants import * # NOQA
from ..helpers import parse_storage_quota
from ..remote import RepositoryServer
from ..remote3 import RepositoryServer
from ..logger import create_logger

4
src/borg/archiver/version_cmd.py
View file

@ -2,7 +2,7 @@ import argparse
from .. import __version__
from ..constants import * # NOQA
from ..remote import RemoteRepository
from ..remote3 import RemoteRepository3
from ..logger import create_logger
@ -16,7 +16,7 @@ class VersionMixIn:
client_version = parse_version(__version__)
if args.location.proto in ("ssh", "socket"):
with RemoteRepository(args.location, lock=False, args=args) as repository:
with RemoteRepository3(args.location, lock=False, args=args) as repository:
server_version = repository.server_version
else:
server_version = client_version

16
src/borg/cache.py
View file

@ -32,7 +32,7 @@ from .locking import Lock
from .manifest import Manifest
from .platform import SaveFile
from .remote import cache_if_remote
from .repository import LIST_SCAN_LIMIT
from .repository3 import LIST_SCAN_LIMIT
# note: cmtime might be either a ctime or a mtime timestamp, chunks is a list of ChunkListEntry
FileCacheEntry = namedtuple("FileCacheEntry", "age inode size cmtime chunks")
@ -718,35 +718,27 @@ class ChunksMixin:
return ChunkListEntry(id, size)
def _load_chunks_from_repo(self):
# Explicitly set the initial usable hash table capacity to avoid performance issues
# due to hash table "resonance".
# Since we're creating an archive, add 10 % from the start.
num_chunks = len(self.repository)
chunks = ChunkIndex(usable=num_chunks * 1.1)
pi = ProgressIndicatorPercent(
total=num_chunks, msg="Downloading chunk list... %3.0f%%", msgid="cache.download_chunks"
)
chunks = ChunkIndex()
t0 = perf_counter()
num_requests = 0
num_chunks = 0
marker = None
while True:
result = self.repository.list(limit=LIST_SCAN_LIMIT, marker=marker)
num_requests += 1
if not result:
break
pi.show(increase=len(result))
marker = result[-1]
# All chunks from the repository have a refcount of MAX_VALUE, which is sticky,
# therefore we can't/won't delete them. Chunks we added ourselves in this transaction
# (e.g. checkpoint archives) are tracked correctly.
init_entry = ChunkIndexEntry(refcount=ChunkIndex.MAX_VALUE, size=0)
for id_ in result:
num_chunks += 1
chunks[id_] = init_entry
assert len(chunks) == num_chunks
# LocalCache does not contain the manifest, either.
del chunks[self.manifest.MANIFEST_ID]
duration = perf_counter() - t0 or 0.01
pi.finish()
logger.debug(
"Cache: downloaded %d chunk IDs in %.2f s (%d requests), ~%s/s",
num_chunks,

4
src/borg/crypto/keymanager.py
View file

@ -5,7 +5,7 @@ from hashlib import sha256
from ..helpers import Error, yes, bin_to_hex, hex_to_bin, dash_open
from ..manifest import Manifest, NoManifestError
from ..repository import Repository
from ..repository3 import Repository3
from ..repoobj import RepoObj
@ -50,7 +50,7 @@ class KeyManager:
try:
manifest_chunk = self.repository.get(Manifest.MANIFEST_ID)
except Repository.ObjectNotFound:
except Repository3.ObjectNotFound:
raise NoManifestError
manifest_data = RepoObj.extract_crypted_data(manifest_chunk)

2
src/borg/fuse.py
View file

@ -46,7 +46,7 @@ from .helpers.lrucache import LRUCache
from .item import Item
from .platform import uid2user, gid2group
from .platformflags import is_darwin
from .remote import RemoteRepository
from .remote import RemoteRepository # TODO 3
def fuse_main():

2
src/borg/helpers/misc.py
View file

@ -2,7 +2,7 @@ import logging
import io
import os
import os.path
import platform
import platform # python stdlib import - if this fails, check that cwd != src/borg/
import sys
from collections import deque
from itertools import islice

4
src/borg/helpers/parseformat.py
View file

@ -1182,11 +1182,13 @@ def ellipsis_truncate(msg, space):
class BorgJsonEncoder(json.JSONEncoder):
def default(self, o):
from ..repository import Repository
from ..repository3 import Repository3
from ..remote import RemoteRepository
from ..remote3 import RemoteRepository3
from ..archive import Archive
from ..cache import LocalCache, AdHocCache, AdHocWithFilesCache
if isinstance(o, Repository) or isinstance(o, RemoteRepository):
if isinstance(o, (Repository, Repository3)) or isinstance(o, (RemoteRepository, RemoteRepository3)):
return {"id": bin_to_hex(o.id), "location": o._location.canonical_path()}
if isinstance(o, Archive):
return o.info()

4
src/borg/manifest.py
View file

@ -246,11 +246,11 @@ class Manifest:
def load(cls, repository, operations, key=None, *, ro_cls=RepoObj):
from .item import ManifestItem
from .crypto.key import key_factory
from .repository import Repository
from .repository3 import Repository3
try:
cdata = repository.get(cls.MANIFEST_ID)
except Repository.ObjectNotFound:
except Repository3.ObjectNotFound:
raise NoManifestError
if not key:
key = key_factory(repository, cdata, ro_cls=ro_cls)

4
src/borg/remote.py
View file

@ -640,6 +640,7 @@ class RemoteRepository:
exclusive=exclusive,
append_only=append_only,
make_parent_dirs=make_parent_dirs,
v1_or_v2=True, # make remote use Repository, not Repository3
)
info = self.info()
self.version = info["version"]
@ -939,9 +940,10 @@ class RemoteRepository:
since=parse_version("1.0.0"),
append_only={"since": parse_version("1.0.7"), "previously": False},
make_parent_dirs={"since": parse_version("1.1.9"), "previously": False},
v1_or_v2={"since": parse_version("2.0.0b8"), "previously": True}, # TODO fix version
)
def open(
self, path, create=False, lock_wait=None, lock=True, exclusive=False, append_only=False, make_parent_dirs=False
self, path, create=False, lock_wait=None, lock=True, exclusive=False, append_only=False, make_parent_dirs=False, v1_or_v2=False
):
"""actual remoting is done via self.call in the @api decorator"""

1269
src/borg/remote3.py Normal file
View file

File diff suppressed because it is too large Load diff

314
src/borg/repository3.py Normal file
View file

@ -0,0 +1,314 @@
import os
from borgstore.store import Store
from borgstore.store import ObjectNotFound as StoreObjectNotFound
from .constants import * # NOQA
from .helpers import Error, ErrorWithTraceback, IntegrityError
from .helpers import Location
from .helpers import bin_to_hex, hex_to_bin
from .logger import create_logger
from .repoobj import RepoObj
logger = create_logger(__name__)
class Repository3:
"""borgstore based key value store"""
class AlreadyExists(Error):
"""A repository already exists at {}."""
exit_mcode = 10
class CheckNeeded(ErrorWithTraceback):
"""Inconsistency detected. Please run "borg check {}"."""
exit_mcode = 12
class DoesNotExist(Error):
"""Repository {} does not exist."""
exit_mcode = 13
class InsufficientFreeSpaceError(Error):
"""Insufficient free space to complete transaction (required: {}, available: {})."""
exit_mcode = 14
class InvalidRepository(Error):
"""{} is not a valid repository. Check repo config."""
exit_mcode = 15
class InvalidRepositoryConfig(Error):
"""{} does not have a valid configuration. Check repo config [{}]."""
exit_mcode = 16
class ObjectNotFound(ErrorWithTraceback):
"""Object with key {} not found in repository {}."""
exit_mcode = 17
def __init__(self, id, repo):
if isinstance(id, bytes):
id = bin_to_hex(id)
super().__init__(id, repo)
class ParentPathDoesNotExist(Error):
"""The parent path of the repo directory [{}] does not exist."""
exit_mcode = 18
class PathAlreadyExists(Error):
"""There is already something at {}."""
exit_mcode = 19
class StorageQuotaExceeded(Error):
"""The storage quota ({}) has been exceeded ({}). Try deleting some archives."""
exit_mcode = 20
class PathPermissionDenied(Error):
"""Permission denied to {}."""
exit_mcode = 21
def __init__(
self,
path,
create=False,
exclusive=False,
lock_wait=None,
lock=True,
append_only=False,
storage_quota=None,
make_parent_dirs=False,
send_log_cb=None,
):
self.path = os.path.abspath(path)
url = "file://%s" % self.path
# use a Store with flat config storage and 2-levels-nested data storage
self.store = Store(url, levels={"config/": [0], "data/": [2]})
self._location = Location(url)
self.version = None
# long-running repository methods which emit log or progress output are responsible for calling
# the ._send_log method periodically to get log and progress output transferred to the borg client
# in a timely manner, in case we have a RemoteRepository.
# for local repositories ._send_log can be called also (it will just do nothing in that case).
self._send_log = send_log_cb or (lambda: None)
self.do_create = create
self.created = False
self.acceptable_repo_versions = (3, )
self.opened = False
self.append_only = append_only # XXX not implemented / not implementable
self.storage_quota = storage_quota # XXX not implemented
self.storage_quota_use = 0 # XXX not implemented
def __repr__(self):
return f"<{self.__class__.__name__} {self.path}>"
def __enter__(self):
if self.do_create:
self.do_create = False
self.create()
self.created = True
self.open()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
@property
def id_str(self):
return bin_to_hex(self.id)
def create(self):
"""Create a new empty repository"""
self.store.create()
self.store.open()
self.store.store("config/readme", REPOSITORY_README.encode())
self.version = 3
self.store.store("config/version", str(self.version).encode())
self.store.store("config/id", bin_to_hex(os.urandom(32)).encode())
self.store.close()
def _set_id(self, id):
# for testing: change the id of an existing repository
assert self.opened
assert isinstance(id, bytes) and len(id) == 32
self.id = id
self.store.store("config/id", bin_to_hex(id).encode())
def save_key(self, keydata):
# note: saving an empty key means that there is no repokey anymore
self.store.store("keys/repokey", keydata)
def load_key(self):
keydata = self.store.load("keys/repokey")
# note: if we return an empty string, it means there is no repo key
return keydata
def destroy(self):
"""Destroy the repository"""
self.close()
self.store.destroy()
def open(self):
self.store.open()
readme = self.store.load("config/readme").decode()
if readme != REPOSITORY_README:
raise self.InvalidRepository(self.path)
self.version = int(self.store.load("config/version").decode())
if self.version not in self.acceptable_repo_versions:
self.close()
raise self.InvalidRepositoryConfig(
self.path, "repository version %d is not supported by this borg version" % self.version
)
self.id = hex_to_bin(self.store.load("config/id").decode(), length=32)
self.opened = True
def close(self):
if self.opened:
self.store.close()
self.opened = False
def info(self):
"""return some infos about the repo (must be opened first)"""
info = dict(id=self.id, version=self.version, storage_quota_use=self.storage_quota_use, storage_quota=self.storage_quota, append_only=self.append_only)
return info
def commit(self, compact=True, threshold=0.1):
pass
def check(self, repair=False, max_duration=0):
"""Check repository consistency
This method verifies all segment checksums and makes sure
the index is consistent with the data stored in the segments.
"""
mode = "full"
logger.info("Starting repository check")
# XXX TODO
logger.info("Finished %s repository check, no problems found.", mode)
return True
def scan_low_level(self, segment=None, offset=None):
raise NotImplementedError
def __len__(self):
raise NotImplementedError
def __contains__(self, id):
raise NotImplementedError
def list(self, limit=None, marker=None, mask=0, value=0):
"""
list <limit> IDs starting from after id <marker> - in index (pseudo-random) order.
if mask and value are given, only return IDs where flags & mask == value (default: all IDs).
"""
infos = self.store.list("data") # XXX we can only get the full list from the store
ids = [hex_to_bin(info.name) for info in infos]
if marker is not None:
idx = ids.index(marker)
ids = ids[idx + 1:]
if limit is not None:
return ids[:limit]
return ids
def scan(self, limit=None, state=None):
"""
list (the next) <limit> chunk IDs from the repository.
state can either be None (initially, when starting to scan) or the object
returned from a previous scan call (meaning "continue scanning").
returns: list of chunk ids, state
"""
# we only have store.list() anyway, so just call .list() from here.
ids = self.list(limit=limit, marker=state)
state = ids[-1] if ids else None
return ids, state
def get(self, id, read_data=True):
id_hex = bin_to_hex(id)
key = "data/" + id_hex
try:
if read_data:
# read everything
return self.store.load(key)
else:
# RepoObj layout supports separately encrypted metadata and data.
# We return enough bytes so the client can decrypt the metadata.
meta_len_size = RepoObj.meta_len_hdr.size
extra_len = 1024 - meta_len_size # load a bit more, 1024b, reduces round trips
obj = self.store.load(key, size=meta_len_size + extra_len)
meta_len = obj[0:meta_len_size]
if len(meta_len) != meta_len_size:
raise IntegrityError(
f"Object too small [id {id_hex}]: expected {meta_len_size}, got {len(meta_len)} bytes"
)
ml = RepoObj.meta_len_hdr.unpack(meta_len)[0]
if ml > extra_len:
# we did not get enough, need to load more, but not all.
# this should be rare, as chunk metadata is rather small usually.
obj = self.store.load(key, size=meta_len_size + ml)
meta = obj[meta_len_size:meta_len_size + ml]
if len(meta) != ml:
raise IntegrityError(
f"Object too small [id {id_hex}]: expected {ml}, got {len(meta)} bytes"
)
return meta_len + meta
except StoreObjectNotFound:
raise self.ObjectNotFound(id, self.path) from None
def get_many(self, ids, read_data=True, is_preloaded=False):
for id_ in ids:
yield self.get(id_, read_data=read_data)
def put(self, id, data, wait=True):
"""put a repo object
Note: when doing calls with wait=False this gets async and caller must
deal with async results / exceptions later.
"""
data_size = len(data)
if data_size > MAX_DATA_SIZE:
raise IntegrityError(f"More than allowed put data [{data_size} > {MAX_DATA_SIZE}]")
key = "data/" + bin_to_hex(id)
self.store.store(key, data)
def delete(self, id, wait=True):
"""delete a repo object
Note: when doing calls with wait=False this gets async and caller must
deal with async results / exceptions later.
"""
key = "data/" + bin_to_hex(id)
try:
self.store.delete(key)
except StoreObjectNotFound:
raise self.ObjectNotFound(id, self.path) from None
def async_response(self, wait=True):
"""Get one async result (only applies to remote repositories).
async commands (== calls with wait=False, e.g. delete and put) have no results,
but may raise exceptions. These async exceptions must get collected later via
async_response() calls. Repeat the call until it returns None.
The previous calls might either return one (non-None) result or raise an exception.
If wait=True is given and there are outstanding responses, it will wait for them
to arrive. With wait=False, it will only return already received responses.
"""
def preload(self, ids):
"""Preload objects (only applies to remote repositories)"""
def break_lock(self):
pass

20
src/borg/testsuite/archiver/__init__.py
View file

@ -27,8 +27,8 @@ from ...helpers import init_ec_warnings
from ...logger import flush_logging
from ...manifest import Manifest
from ...platform import get_flags
from ...remote import RemoteRepository
from ...repository import Repository
from ...remote3 import RemoteRepository3
from ...repository3 import Repository3
from .. import has_lchflags, is_utime_fully_supported, have_fuse_mtime_ns, st_mtime_ns_round, no_selinux
from .. import changedir
from .. import are_symlinks_supported, are_hardlinks_supported, are_fifos_supported
@ -169,7 +169,7 @@ def create_src_archive(archiver, name, ts=None):
def open_archive(repo_path, name):
repository = Repository(repo_path, exclusive=True)
repository = Repository3(repo_path, exclusive=True)
with repository:
manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
archive = Archive(manifest, name)
@ -178,9 +178,9 @@ def open_archive(repo_path, name):
def open_repository(archiver):
if archiver.get_kind() == "remote":
return RemoteRepository(Location(archiver.repository_location))
return RemoteRepository3(Location(archiver.repository_location))
else:
return Repository(archiver.repository_path, exclusive=True)
return Repository3(archiver.repository_path, exclusive=True)
def create_regular_file(input_path, name, size=0, contents=None):
@ -256,17 +256,13 @@ def create_test_files(input_path, create_hardlinks=True):
def _extract_repository_id(repo_path):
with Repository(repo_path) as repository:
with Repository3(repo_path) as repository:
return repository.id
def _set_repository_id(repo_path, id):
config = ConfigParser(interpolation=None)
config.read(os.path.join(repo_path, "config"))
config.set("repository", "id", bin_to_hex(id))
with open(os.path.join(repo_path, "config"), "w") as fd:
config.write(fd)
with Repository(repo_path) as repository:
with Repository3(repo_path) as repository:
repository._set_id(id)
return repository.id

130
src/borg/testsuite/archiver/bypass_lock_option.py
View file

@ -1,130 +0,0 @@
import pytest
from ...constants import * # NOQA
from ...helpers import EXIT_ERROR
from ...locking import LockFailed
from ...remote import RemoteRepository
from .. import llfuse
from . import cmd, create_src_archive, RK_ENCRYPTION, read_only, fuse_mount
def test_readonly_check(archiver):
cmd(archiver, "rcreate", RK_ENCRYPTION)
create_src_archive(archiver, "test")
with read_only(archiver.repository_path):
# verify that command normally doesn't work with read-only repo
if archiver.FORK_DEFAULT:
cmd(archiver, "check", "--verify-data", exit_code=EXIT_ERROR)
else:
with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo:
cmd(archiver, "check", "--verify-data")
if isinstance(excinfo.value, RemoteRepository.RPCError):
assert excinfo.value.exception_class == "LockFailed"
# verify that command works with read-only repo when using --bypass-lock
cmd(archiver, "check", "--verify-data", "--bypass-lock")
def test_readonly_diff(archiver):
cmd(archiver, "rcreate", RK_ENCRYPTION)
create_src_archive(archiver, "a")
create_src_archive(archiver, "b")
with read_only(archiver.repository_path):
# verify that command normally doesn't work with read-only repo
if archiver.FORK_DEFAULT:
cmd(archiver, "diff", "a", "b", exit_code=EXIT_ERROR)
else:
with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo:
cmd(archiver, "diff", "a", "b")
if isinstance(excinfo.value, RemoteRepository.RPCError):
assert excinfo.value.exception_class == "LockFailed"
# verify that command works with read-only repo when using --bypass-lock
cmd(archiver, "diff", "a", "b", "--bypass-lock")
def test_readonly_export_tar(archiver):
cmd(archiver, "rcreate", RK_ENCRYPTION)
create_src_archive(archiver, "test")
with read_only(archiver.repository_path):
# verify that command normally doesn't work with read-only repo
if archiver.FORK_DEFAULT:
cmd(archiver, "export-tar", "test", "test.tar", exit_code=EXIT_ERROR)
else:
with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo:
cmd(archiver, "export-tar", "test", "test.tar")
if isinstance(excinfo.value, RemoteRepository.RPCError):
assert excinfo.value.exception_class == "LockFailed"
# verify that command works with read-only repo when using --bypass-lock
cmd(archiver, "export-tar", "test", "test.tar", "--bypass-lock")
def test_readonly_extract(archiver):
cmd(archiver, "rcreate", RK_ENCRYPTION)
create_src_archive(archiver, "test")
with read_only(archiver.repository_path):
# verify that command normally doesn't work with read-only repo
if archiver.FORK_DEFAULT:
cmd(archiver, "extract", "test", exit_code=EXIT_ERROR)
else:
with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo:
cmd(archiver, "extract", "test")
if isinstance(excinfo.value, RemoteRepository.RPCError):
assert excinfo.value.exception_class == "LockFailed"
# verify that command works with read-only repo when using --bypass-lock
cmd(archiver, "extract", "test", "--bypass-lock")
def test_readonly_info(archiver):
cmd(archiver, "rcreate", RK_ENCRYPTION)
create_src_archive(archiver, "test")
with read_only(archiver.repository_path):
# verify that command normally doesn't work with read-only repo
if archiver.FORK_DEFAULT:
cmd(archiver, "rinfo", exit_code=EXIT_ERROR)
else:
with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo:
cmd(archiver, "rinfo")
if isinstance(excinfo.value, RemoteRepository.RPCError):
assert excinfo.value.exception_class == "LockFailed"
# verify that command works with read-only repo when using --bypass-lock
cmd(archiver, "rinfo", "--bypass-lock")
def test_readonly_list(archiver):
cmd(archiver, "rcreate", RK_ENCRYPTION)
create_src_archive(archiver, "test")
with read_only(archiver.repository_path):
# verify that command normally doesn't work with read-only repo
if archiver.FORK_DEFAULT:
cmd(archiver, "rlist", exit_code=EXIT_ERROR)
else:
with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo:
cmd(archiver, "rlist")
if isinstance(excinfo.value, RemoteRepository.RPCError):
assert excinfo.value.exception_class == "LockFailed"
# verify that command works with read-only repo when using --bypass-lock
cmd(archiver, "rlist", "--bypass-lock")
@pytest.mark.skipif(not llfuse, reason="llfuse not installed")
def test_readonly_mount(archiver):
cmd(archiver, "rcreate", RK_ENCRYPTION)
create_src_archive(archiver, "test")
with read_only(archiver.repository_path):
# verify that command normally doesn't work with read-only repo
if archiver.FORK_DEFAULT:
with fuse_mount(archiver, exit_code=EXIT_ERROR):
pass
else:
with pytest.raises((LockFailed, RemoteRepository.RPCError)) as excinfo:
# self.fuse_mount always assumes fork=True, so for this test we have to set fork=False manually
with fuse_mount(archiver, fork=False):
pass
if isinstance(excinfo.value, RemoteRepository.RPCError):
assert excinfo.value.exception_class == "LockFailed"
# verify that command works with read-only repo when using --bypass-lock
with fuse_mount(archiver, None, "--bypass-lock"):
pass

8
src/borg/testsuite/archiver/check_cmd.py
View file

@ -8,7 +8,7 @@ from ...archive import ChunkBuffer
from ...constants import * # NOQA
from ...helpers import bin_to_hex, msgpack
from ...manifest import Manifest
from ...repository import Repository
from ...repository3 import Repository3
from . import cmd, src_file, create_src_archive, open_archive, generate_archiver_tests, RK_ENCRYPTION
pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA
@ -28,12 +28,10 @@ def test_check_usage(archivers, request):
output = cmd(archiver, "check", "-v", "--progress", exit_code=0)
assert "Starting repository check" in output
assert "Starting archive consistency check" in output
assert "Checking segments" in output
output = cmd(archiver, "check", "-v", "--repository-only", exit_code=0)
assert "Starting repository check" in output
assert "Starting archive consistency check" not in output
assert "Checking segments" not in output
output = cmd(archiver, "check", "-v", "--archives-only", exit_code=0)
assert "Starting repository check" not in output
@ -348,7 +346,7 @@ def test_extra_chunks(archivers, request):
pytest.skip("only works locally")
check_cmd_setup(archiver)
cmd(archiver, "check", exit_code=0)
with Repository(archiver.repository_location, exclusive=True) as repository:
with Repository3(archiver.repository_location, exclusive=True) as repository:
repository.put(b"01234567890123456789012345678901", b"xxxx")
repository.commit(compact=False)
output = cmd(archiver, "check", "-v", exit_code=0) # orphans are not considered warnings anymore
@ -391,7 +389,7 @@ def test_empty_repository(archivers, request):
if archiver.get_kind() == "remote":
pytest.skip("only works locally")
check_cmd_setup(archiver)
with Repository(archiver.repository_location, exclusive=True) as repository:
with Repository3(archiver.repository_location, exclusive=True) as repository:
for id_ in repository.list():
repository.delete(id_)
repository.commit(compact=False)

24
src/borg/testsuite/archiver/checks.py
View file

@ -9,8 +9,8 @@ from ...constants import * # NOQA
from ...helpers import Location, get_security_dir, bin_to_hex
from ...helpers import EXIT_ERROR
from ...manifest import Manifest, MandatoryFeatureUnsupported
from ...remote import RemoteRepository, PathNotAllowed
from ...repository import Repository
from ...remote3 import RemoteRepository3, PathNotAllowed
from ...repository3 import Repository3
from .. import llfuse
from .. import changedir
from . import cmd, _extract_repository_id, open_repository, check_cache, create_test_files
@ -25,7 +25,7 @@ def get_security_directory(repo_path):
def add_unknown_feature(repo_path, operation):
with Repository(repo_path, exclusive=True) as repository:
with Repository3(repo_path, exclusive=True) as repository:
manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
manifest.config["feature_flags"] = {operation.value: {"mandatory": ["unknown-feature"]}}
manifest.write()
@ -272,7 +272,7 @@ def test_unknown_mandatory_feature_in_cache(archivers, request):
remote_repo = archiver.get_kind() == "remote"
print(cmd(archiver, "rcreate", RK_ENCRYPTION))
with Repository(archiver.repository_path, exclusive=True) as repository:
with Repository3(archiver.repository_path, exclusive=True) as repository:
if remote_repo:
repository._location = Location(archiver.repository_location)
manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
@ -299,7 +299,7 @@ def test_unknown_mandatory_feature_in_cache(archivers, request):
if is_localcache:
assert called
with Repository(archiver.repository_path, exclusive=True) as repository:
with Repository3(archiver.repository_path, exclusive=True) as repository:
if remote_repo:
repository._location = Location(archiver.repository_location)
manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
@ -346,26 +346,26 @@ def test_env_use_chunks_archive(archivers, request, monkeypatch):
def test_remote_repo_restrict_to_path(remote_archiver):
original_location, repo_path = remote_archiver.repository_location, remote_archiver.repository_path
# restricted to repo directory itself:
with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-path", repo_path]):
with patch.object(RemoteRepository3, "extra_test_args", ["--restrict-to-path", repo_path]):
cmd(remote_archiver, "rcreate", RK_ENCRYPTION)
# restricted to repo directory itself, fail for other directories with same prefix:
with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-path", repo_path]):
with patch.object(RemoteRepository3, "extra_test_args", ["--restrict-to-path", repo_path]):
with pytest.raises(PathNotAllowed):
remote_archiver.repository_location = original_location + "_0"
cmd(remote_archiver, "rcreate", RK_ENCRYPTION)
# restricted to a completely different path:
with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-path", "/foo"]):
with patch.object(RemoteRepository3, "extra_test_args", ["--restrict-to-path", "/foo"]):
with pytest.raises(PathNotAllowed):
remote_archiver.repository_location = original_location + "_1"
cmd(remote_archiver, "rcreate", RK_ENCRYPTION)
path_prefix = os.path.dirname(repo_path)
# restrict to repo directory's parent directory:
with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-path", path_prefix]):
with patch.object(RemoteRepository3, "extra_test_args", ["--restrict-to-path", path_prefix]):
remote_archiver.repository_location = original_location + "_2"
cmd(remote_archiver, "rcreate", RK_ENCRYPTION)
# restrict to repo directory's parent directory and another directory:
with patch.object(
RemoteRepository, "extra_test_args", ["--restrict-to-path", "/foo", "--restrict-to-path", path_prefix]
RemoteRepository3, "extra_test_args", ["--restrict-to-path", "/foo", "--restrict-to-path", path_prefix]
):
remote_archiver.repository_location = original_location + "_3"
cmd(remote_archiver, "rcreate", RK_ENCRYPTION)
@ -374,10 +374,10 @@ def test_remote_repo_restrict_to_path(remote_archiver):
def test_remote_repo_restrict_to_repository(remote_archiver):
repo_path = remote_archiver.repository_path
# restricted to repo directory itself:
with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-repository", repo_path]):
with patch.object(RemoteRepository3, "extra_test_args", ["--restrict-to-repository", repo_path]):
cmd(remote_archiver, "rcreate", RK_ENCRYPTION)
parent_path = os.path.join(repo_path, "..")
with patch.object(RemoteRepository, "extra_test_args", ["--restrict-to-repository", parent_path]):
with patch.object(RemoteRepository3, "extra_test_args", ["--restrict-to-repository", parent_path]):
with pytest.raises(PathNotAllowed):
cmd(remote_archiver, "rcreate", RK_ENCRYPTION)

64
src/borg/testsuite/archiver/config_cmd.py
View file

@ -1,64 +0,0 @@
import os
import pytest
from ...constants import * # NOQA
from . import RK_ENCRYPTION, create_test_files, cmd, generate_archiver_tests
from ...helpers import CommandError, Error
pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,binary") # NOQA
def test_config(archivers, request):
archiver = request.getfixturevalue(archivers)
create_test_files(archiver.input_path)
os.unlink("input/flagfile")
cmd(archiver, "rcreate", RK_ENCRYPTION)
output = cmd(archiver, "config", "--list")
assert "[repository]" in output
assert "version" in output
assert "segments_per_dir" in output
assert "storage_quota" in output
assert "append_only" in output
assert "additional_free_space" in output
assert "id" in output
assert "last_segment_checked" not in output
if archiver.FORK_DEFAULT:
output = cmd(archiver, "config", "last_segment_checked", exit_code=2)
assert "No option " in output
else:
with pytest.raises(Error):
cmd(archiver, "config", "last_segment_checked")
cmd(archiver, "config", "last_segment_checked", "123")
output = cmd(archiver, "config", "last_segment_checked")
assert output == "123" + os.linesep
output = cmd(archiver, "config", "--list")
assert "last_segment_checked" in output
cmd(archiver, "config", "--delete", "last_segment_checked")
for cfg_key, cfg_value in [("additional_free_space", "2G"), ("repository.append_only", "1")]:
output = cmd(archiver, "config", cfg_key)
assert output == "0" + os.linesep
cmd(archiver, "config", cfg_key, cfg_value)
output = cmd(archiver, "config", cfg_key)
assert output == cfg_value + os.linesep
cmd(archiver, "config", "--delete", cfg_key)
if archiver.FORK_DEFAULT:
cmd(archiver, "config", cfg_key, exit_code=2)
else:
with pytest.raises(Error):
cmd(archiver, "config", cfg_key)
cmd(archiver, "config", "--list", "--delete", exit_code=2)
if archiver.FORK_DEFAULT:
expected_ec = CommandError().exit_code
cmd(archiver, "config", exit_code=expected_ec)
else:
with pytest.raises(CommandError):
cmd(archiver, "config")
if archiver.FORK_DEFAULT:
cmd(archiver, "config", "invalid-option", exit_code=2)
else:
with pytest.raises(Error):
cmd(archiver, "config", "invalid-option")

18
src/borg/testsuite/archiver/corruption.py
View file

@ -13,24 +13,6 @@ from ...hashindex import ChunkIndex
from ...cache import LocalCache
def test_check_corrupted_repository(archiver):
cmd(archiver, "rcreate", RK_ENCRYPTION)
create_src_archive(archiver, "test")
cmd(archiver, "extract", "test", "--dry-run")
cmd(archiver, "check")
name = sorted(os.listdir(os.path.join(archiver.tmpdir, "repository", "data", "0")), reverse=True)[1]
with open(os.path.join(archiver.tmpdir, "repository", "data", "0", name), "r+b") as fd:
fd.seek(100)
fd.write(b"XXXX")
if archiver.FORK_DEFAULT:
cmd(archiver, "check", exit_code=1)
else:
with pytest.raises(Error):
cmd(archiver, "check")
def corrupt_archiver(archiver):
create_test_files(archiver.input_path)
cmd(archiver, "rcreate", RK_ENCRYPTION)

4
src/borg/testsuite/archiver/create_cmd.py
View file

@ -16,7 +16,7 @@ from ...cache import get_cache_impl
from ...constants import * # NOQA
from ...manifest import Manifest
from ...platform import is_cygwin, is_win32, is_darwin
from ...repository import Repository
from ...repository3 import Repository3
from ...helpers import CommandError, BackupPermissionError
from .. import has_lchflags
from .. import changedir
@ -668,7 +668,7 @@ def test_create_dry_run(archivers, request):
cmd(archiver, "rcreate", RK_ENCRYPTION)
cmd(archiver, "create", "--dry-run", "test", "input")
# Make sure no archive has been created
with Repository(archiver.repository_path) as repository:
with Repository3(archiver.repository_path) as repository:
manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
assert len(manifest.archives) == 0

6
src/borg/testsuite/archiver/delete_cmd.py
View file

@ -1,7 +1,7 @@
from ...archive import Archive
from ...constants import * # NOQA
from ...manifest import Manifest
from ...repository import Repository
from ...repository3 import Repository3
from . import cmd, create_regular_file, src_file, create_src_archive, generate_archiver_tests, RK_ENCRYPTION
pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA
@ -47,7 +47,7 @@ def test_delete_force(archivers, request):
archiver = request.getfixturevalue(archivers)
cmd(archiver, "rcreate", "--encryption=none")
create_src_archive(archiver, "test")
with Repository(archiver.repository_path, exclusive=True) as repository:
with Repository3(archiver.repository_path, exclusive=True) as repository:
manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
archive = Archive(manifest, "test")
for item in archive.iter_items():
@ -69,7 +69,7 @@ def test_delete_double_force(archivers, request):
archiver = request.getfixturevalue(archivers)
cmd(archiver, "rcreate", "--encryption=none")
create_src_archive(archiver, "test")
with Repository(archiver.repository_path, exclusive=True) as repository:
with Repository3(archiver.repository_path, exclusive=True) as repository:
manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
archive = Archive(manifest, "test")
id = archive.metadata.items[0]

14
src/borg/testsuite/archiver/key_cmds.py
View file

@ -9,7 +9,7 @@ from ...crypto.keymanager import RepoIdMismatch, NotABorgKeyFile
from ...helpers import CommandError
from ...helpers import bin_to_hex, hex_to_bin
from ...helpers import msgpack
from ...repository import Repository
from ...repository3 import Repository3
from .. import key
from . import RK_ENCRYPTION, KF_ENCRYPTION, cmd, _extract_repository_id, _set_repository_id, generate_archiver_tests
@ -129,7 +129,7 @@ def test_key_export_repokey(archivers, request):
assert export_contents.startswith("BORG_KEY " + bin_to_hex(repo_id) + "\n")
with Repository(archiver.repository_path) as repository:
with Repository3(archiver.repository_path) as repository:
repo_key = AESOCBRepoKey(repository)
repo_key.load(None, Passphrase.env_passphrase())
@ -138,12 +138,12 @@ def test_key_export_repokey(archivers, request):
assert repo_key.crypt_key == backup_key.crypt_key
with Repository(archiver.repository_path) as repository:
with Repository3(archiver.repository_path) as repository:
repository.save_key(b"")
cmd(archiver, "key", "import", export_file)
with Repository(archiver.repository_path) as repository:
with Repository3(archiver.repository_path) as repository:
repo_key2 = AESOCBRepoKey(repository)
repo_key2.load(None, Passphrase.env_passphrase())
@ -302,7 +302,7 @@ def test_init_defaults_to_argon2(archivers, request):
"""https://github.com/borgbackup/borg/issues/747#issuecomment-1076160401"""
archiver = request.getfixturevalue(archivers)
cmd(archiver, "rcreate", RK_ENCRYPTION)
with Repository(archiver.repository_path) as repository:
with Repository3(archiver.repository_path) as repository:
key = msgpack.unpackb(binascii.a2b_base64(repository.load_key()))
assert key["algorithm"] == "argon2 chacha20-poly1305"
@ -313,7 +313,7 @@ def test_change_passphrase_does_not_change_algorithm_argon2(archivers, request):
os.environ["BORG_NEW_PASSPHRASE"] = "newpassphrase"
cmd(archiver, "key", "change-passphrase")
with Repository(archiver.repository_path) as repository:
with Repository3(archiver.repository_path) as repository:
key = msgpack.unpackb(binascii.a2b_base64(repository.load_key()))
assert key["algorithm"] == "argon2 chacha20-poly1305"
@ -323,6 +323,6 @@ def test_change_location_does_not_change_algorithm_argon2(archivers, request):
cmd(archiver, "rcreate", KF_ENCRYPTION)
cmd(archiver, "key", "change-location", "repokey")
with Repository(archiver.repository_path) as repository:
with Repository3(archiver.repository_path) as repository:
key = msgpack.unpackb(binascii.a2b_base64(repository.load_key()))
assert key["algorithm"] == "argon2 chacha20-poly1305"

4
src/borg/testsuite/archiver/rcompress_cmd.py
View file

@ -1,7 +1,7 @@
import os
from ...constants import * # NOQA
from ...repository import Repository
from ...repository3 import Repository3
from ...manifest import Manifest
from ...compress import ZSTD, ZLIB, LZ4, CNONE
from ...helpers import bin_to_hex
@ -12,7 +12,7 @@ from . import create_regular_file, cmd, RK_ENCRYPTION
def test_rcompress(archiver):
def check_compression(ctype, clevel, olevel):
"""check if all the chunks in the repo are compressed/obfuscated like expected"""
repository = Repository(archiver.repository_path, exclusive=True)
repository = Repository3(archiver.repository_path, exclusive=True)
with repository:
manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
state = None

29
src/borg/testsuite/archiver/rcreate_cmd.py
View file

@ -6,28 +6,11 @@ import pytest
from ...helpers.errors import Error, CancelledByUser
from ...constants import * # NOQA
from ...crypto.key import FlexiKey
from ...repository import Repository
from . import cmd, generate_archiver_tests, RK_ENCRYPTION, KF_ENCRYPTION
pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA
def test_rcreate_parent_dirs(archivers, request):
archiver = request.getfixturevalue(archivers)
if archiver.EXE:
pytest.skip("does not raise Exception, but sets rc==2")
remote_repo = archiver.get_kind() == "remote"
parent_path = os.path.join(archiver.tmpdir, "parent1", "parent2")
repository_path = os.path.join(parent_path, "repository")
archiver.repository_location = ("ssh://__testsuite__" + repository_path) if remote_repo else repository_path
with pytest.raises(Repository.ParentPathDoesNotExist):
# normal borg rcreate does NOT create missing parent dirs
cmd(archiver, "rcreate", "--encryption=none")
# but if told so, it does:
cmd(archiver, "rcreate", "--encryption=none", "--make-parent-dirs")
assert os.path.exists(parent_path)
def test_rcreate_interrupt(archivers, request):
archiver = request.getfixturevalue(archivers)
if archiver.EXE:
@ -51,18 +34,6 @@ def test_rcreate_requires_encryption_option(archivers, request):
cmd(archiver, "rcreate", exit_code=2)
def test_rcreate_nested_repositories(archivers, request):
archiver = request.getfixturevalue(archivers)
cmd(archiver, "rcreate", RK_ENCRYPTION)
archiver.repository_location += "/nested"
if archiver.FORK_DEFAULT:
expected_ec = Repository.AlreadyExists().exit_code
cmd(archiver, "rcreate", RK_ENCRYPTION, exit_code=expected_ec)
else:
with pytest.raises(Repository.AlreadyExists):
cmd(archiver, "rcreate", RK_ENCRYPTION)
def test_rcreate_refuse_to_overwrite_keyfile(archivers, request, monkeypatch):
# BORG_KEY_FILE=something borg rcreate should quit if "something" already exists.
# See: https://github.com/borgbackup/borg/pull/6046

4
src/borg/testsuite/archiver/rename_cmd.py
View file

@ -1,6 +1,6 @@
from ...constants import * # NOQA
from ...manifest import Manifest
from ...repository import Repository
from ...repository3 import Repository3
from . import cmd, create_regular_file, generate_archiver_tests, RK_ENCRYPTION
pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA
@ -21,7 +21,7 @@ def test_rename(archivers, request):
cmd(archiver, "extract", "test.3", "--dry-run")
cmd(archiver, "extract", "test.4", "--dry-run")
# Make sure both archives have been renamed
with Repository(archiver.repository_path) as repository:
with Repository3(archiver.repository_path) as repository:
manifest = Manifest.load(repository, Manifest.NO_OPERATION_CHECK)
assert len(manifest.archives) == 2
assert "test.3" in manifest.archives

2
src/borg/testsuite/archiver/return_codes.py
View file

@ -5,7 +5,7 @@ from . import cmd_fixture, changedir # NOQA
def test_return_codes(cmd_fixture, tmpdir):
repo = tmpdir.mkdir("repo")
repo = tmpdir / "repo" # borg creates the directory
input = tmpdir.mkdir("input")
output = tmpdir.mkdir("output")
input.join("test_file").write("content")

18
src/borg/testsuite/archiver/rinfo_cmd.py
View file

@ -35,21 +35,3 @@ def test_info_json(archivers, request):
stats = cache["stats"]
assert all(isinstance(o, int) for o in stats.values())
assert all(key in stats for key in ("total_chunks", "total_size", "total_unique_chunks", "unique_size"))
def test_info_on_repository_with_storage_quota(archivers, request):
archiver = request.getfixturevalue(archivers)
create_regular_file(archiver.input_path, "file1", contents=randbytes(1000 * 1000))
cmd(archiver, "rcreate", RK_ENCRYPTION, "--storage-quota=1G")
cmd(archiver, "create", "test", "input")
info_repo = cmd(archiver, "rinfo")
assert "Storage quota: 1.00 MB used out of 1.00 GB" in info_repo
def test_info_on_repository_without_storage_quota(archivers, request):
archiver = request.getfixturevalue(archivers)
create_regular_file(archiver.input_path, "file1", contents=randbytes(1000 * 1000))
cmd(archiver, "rcreate", RK_ENCRYPTION)
cmd(archiver, "create", "test", "input")
info_repo = cmd(archiver, "rinfo")
assert "Storage quota: 1.00 MB used" in info_repo

6
src/borg/testsuite/cache.py
View file

@ -12,7 +12,7 @@ from ..cache import AdHocCache
from ..crypto.key import AESOCBRepoKey
from ..hashindex import ChunkIndex, CacheSynchronizer
from ..manifest import Manifest
from ..repository import Repository
from ..repository3 import Repository3
class TestCacheSynchronizer:
@ -164,7 +164,7 @@ class TestAdHocCache:
@pytest.fixture
def repository(self, tmpdir):
self.repository_location = os.path.join(str(tmpdir), "repository")
with Repository(self.repository_location, exclusive=True, create=True) as repository:
with Repository3(self.repository_location, exclusive=True, create=True) as repository:
repository.put(H(1), b"1234")
repository.put(Manifest.MANIFEST_ID, b"5678")
yield repository
@ -201,7 +201,7 @@ class TestAdHocCache:
assert cache.seen_chunk(H(5)) == 1
cache.chunk_decref(H(5), 1, Statistics())
assert not cache.seen_chunk(H(5))
with pytest.raises(Repository.ObjectNotFound):
with pytest.raises(Repository3.ObjectNotFound):
repository.get(H(5))
def test_files_cache(self, cache):

4
src/borg/testsuite/repoobj.py
View file

@ -3,14 +3,14 @@ import pytest
from ..constants import ROBJ_FILE_STREAM, ROBJ_MANIFEST, ROBJ_ARCHIVE_META
from ..crypto.key import PlaintextKey
from ..helpers.errors import IntegrityError
from ..repository import Repository
from ..repository3 import Repository3
from ..repoobj import RepoObj, RepoObj1
from ..compress import LZ4
@pytest.fixture
def repository(tmpdir):
return Repository(tmpdir, create=True)
return Repository3(tmpdir, create=True)
@pytest.fixture

290
src/borg/testsuite/repository3.py Normal file
View file

@ -0,0 +1,290 @@
import logging
import os
import sys
from typing import Optional
import pytest
from ..helpers import Location
from ..helpers import IntegrityError
from ..platformflags import is_win32
from ..remote3 import RemoteRepository3, InvalidRPCMethod, PathNotAllowed
from ..repository3 import Repository3, MAX_DATA_SIZE
from ..repoobj import RepoObj
from .hashindex import H
@pytest.fixture()
def repository(tmp_path):
repository_location = os.fspath(tmp_path / "repository")
yield Repository3(repository_location, exclusive=True, create=True)
@pytest.fixture()
def remote_repository(tmp_path):
if is_win32:
pytest.skip("Remote repository does not yet work on Windows.")
repository_location = Location("ssh://__testsuite__" + os.fspath(tmp_path / "repository"))
yield RemoteRepository3(repository_location, exclusive=True, create=True)
def pytest_generate_tests(metafunc):
# Generates tests that run on both local and remote repos
if "repo_fixtures" in metafunc.fixturenames:
metafunc.parametrize("repo_fixtures", ["repository", "remote_repository"])
def get_repository_from_fixture(repo_fixtures, request):
# returns the repo object from the fixture for tests that run on both local and remote repos
return request.getfixturevalue(repo_fixtures)
def reopen(repository, exclusive: Optional[bool] = True, create=False):
if isinstance(repository, Repository3):
if repository.opened:
raise RuntimeError("Repo must be closed before a reopen. Cannot support nested repository contexts.")
return Repository3(repository.path, exclusive=exclusive, create=create)
if isinstance(repository, RemoteRepository3):
if repository.p is not None or repository.sock is not None:
raise RuntimeError("Remote repo must be closed before a reopen. Cannot support nested repository contexts.")
return RemoteRepository3(repository.location, exclusive=exclusive, create=create)
raise TypeError(
f"Invalid argument type. Expected 'Repository3' or 'RemoteRepository3', received '{type(repository).__name__}'."
)
def fchunk(data, meta=b""):
# format chunk: create a raw chunk that has valid RepoObj layout, but does not use encryption or compression.
meta_len = RepoObj.meta_len_hdr.pack(len(meta))
assert isinstance(data, bytes)
chunk = meta_len + meta + data
return chunk
def pchunk(chunk):
# parse chunk: parse data and meta from a raw chunk made by fchunk
meta_len_size = RepoObj.meta_len_hdr.size
meta_len = chunk[:meta_len_size]
meta_len = RepoObj.meta_len_hdr.unpack(meta_len)[0]
meta = chunk[meta_len_size : meta_len_size + meta_len]
data = chunk[meta_len_size + meta_len :]
return data, meta
def pdchunk(chunk):
# parse only data from a raw chunk made by fchunk
return pchunk(chunk)[0]
def test_basic_operations(repo_fixtures, request):
with get_repository_from_fixture(repo_fixtures, request) as repository:
for x in range(100):
repository.put(H(x), fchunk(b"SOMEDATA"))
key50 = H(50)
assert pdchunk(repository.get(key50)) == b"SOMEDATA"
repository.delete(key50)
with pytest.raises(Repository3.ObjectNotFound):
repository.get(key50)
with reopen(repository) as repository:
with pytest.raises(Repository3.ObjectNotFound):
repository.get(key50)
for x in range(100):
if x == 50:
continue
assert pdchunk(repository.get(H(x))) == b"SOMEDATA"
def test_read_data(repo_fixtures, request):
with get_repository_from_fixture(repo_fixtures, request) as repository:
meta, data = b"meta", b"data"
meta_len = RepoObj.meta_len_hdr.pack(len(meta))
chunk_complete = meta_len + meta + data
chunk_short = meta_len + meta
repository.put(H(0), chunk_complete)
assert repository.get(H(0)) == chunk_complete
assert repository.get(H(0), read_data=True) == chunk_complete
assert repository.get(H(0), read_data=False) == chunk_short
def test_consistency(repo_fixtures, request):
with get_repository_from_fixture(repo_fixtures, request) as repository:
repository.put(H(0), fchunk(b"foo"))
assert pdchunk(repository.get(H(0))) == b"foo"
repository.put(H(0), fchunk(b"foo2"))
assert pdchunk(repository.get(H(0))) == b"foo2"
repository.put(H(0), fchunk(b"bar"))
assert pdchunk(repository.get(H(0))) == b"bar"
repository.delete(H(0))
with pytest.raises(Repository3.ObjectNotFound):
repository.get(H(0))
def test_list(repo_fixtures, request):
with get_repository_from_fixture(repo_fixtures, request) as repository:
for x in range(100):
repository.put(H(x), fchunk(b"SOMEDATA"))
repo_list = repository.list()
assert len(repo_list) == 100
first_half = repository.list(limit=50)
assert len(first_half) == 50
assert first_half == repo_list[:50]
second_half = repository.list(marker=first_half[-1])
assert len(second_half) == 50
assert second_half == repo_list[50:]
assert len(repository.list(limit=50)) == 50
def test_scan(repo_fixtures, request):
with get_repository_from_fixture(repo_fixtures, request) as repository:
for x in range(100):
repository.put(H(x), fchunk(b"SOMEDATA"))
ids, _ = repository.scan()
assert len(ids) == 100
first_half, state = repository.scan(limit=50)
assert len(first_half) == 50
assert first_half == ids[:50]
second_half, _ = repository.scan(state=state)
assert len(second_half) == 50
assert second_half == ids[50:]
def test_max_data_size(repo_fixtures, request):
with get_repository_from_fixture(repo_fixtures, request) as repository:
max_data = b"x" * (MAX_DATA_SIZE - RepoObj.meta_len_hdr.size)
repository.put(H(0), fchunk(max_data))
assert pdchunk(repository.get(H(0))) == max_data
with pytest.raises(IntegrityError):
repository.put(H(1), fchunk(max_data + b"x"))
def check(repository, repo_path, repair=False, status=True):
assert repository.check(repair=repair) == status
# Make sure no tmp files are left behind
tmp_files = [name for name in os.listdir(repo_path) if "tmp" in name]
assert tmp_files == [], "Found tmp files"
def _get_mock_args():
class MockArgs:
remote_path = "borg"
umask = 0o077
debug_topics = []
rsh = None
def __contains__(self, item):
# to behave like argparse.Namespace
return hasattr(self, item)
return MockArgs()
def test_remote_invalid_rpc(remote_repository):
with remote_repository:
with pytest.raises(InvalidRPCMethod):
remote_repository.call("__init__", {})
def test_remote_rpc_exception_transport(remote_repository):
with remote_repository:
s1 = "test string"
try:
remote_repository.call("inject_exception", {"kind": "DoesNotExist"})
except Repository3.DoesNotExist as e:
assert len(e.args) == 1
assert e.args[0] == remote_repository.location.processed
try:
remote_repository.call("inject_exception", {"kind": "AlreadyExists"})
except Repository3.AlreadyExists as e:
assert len(e.args) == 1
assert e.args[0] == remote_repository.location.processed
try:
remote_repository.call("inject_exception", {"kind": "CheckNeeded"})
except Repository3.CheckNeeded as e:
assert len(e.args) == 1
assert e.args[0] == remote_repository.location.processed
try:
remote_repository.call("inject_exception", {"kind": "IntegrityError"})
except IntegrityError as e:
assert len(e.args) == 1
assert e.args[0] == s1
try:
remote_repository.call("inject_exception", {"kind": "PathNotAllowed"})
except PathNotAllowed as e:
assert len(e.args) == 1
assert e.args[0] == "foo"
try:
remote_repository.call("inject_exception", {"kind": "ObjectNotFound"})
except Repository3.ObjectNotFound as e:
assert len(e.args) == 2
assert e.args[0] == s1
assert e.args[1] == remote_repository.location.processed
try:
remote_repository.call("inject_exception", {"kind": "InvalidRPCMethod"})
except InvalidRPCMethod as e:
assert len(e.args) == 1
assert e.args[0] == s1
try:
remote_repository.call("inject_exception", {"kind": "divide"})
except RemoteRepository3.RPCError as e:
assert e.unpacked
assert e.get_message() == "ZeroDivisionError: integer division or modulo by zero\n"
assert e.exception_class == "ZeroDivisionError"
assert len(e.exception_full) > 0
def test_remote_ssh_cmd(remote_repository):
with remote_repository:
args = _get_mock_args()
remote_repository._args = args
assert remote_repository.ssh_cmd(Location("ssh://example.com/foo")) == ["ssh", "example.com"]
assert remote_repository.ssh_cmd(Location("ssh://user@example.com/foo")) == ["ssh", "user@example.com"]
assert remote_repository.ssh_cmd(Location("ssh://user@example.com:1234/foo")) == [
"ssh",
"-p",
"1234",
"user@example.com",
]
os.environ["BORG_RSH"] = "ssh --foo"
assert remote_repository.ssh_cmd(Location("ssh://example.com/foo")) == ["ssh", "--foo", "example.com"]
def test_remote_borg_cmd(remote_repository):
with remote_repository:
assert remote_repository.borg_cmd(None, testing=True) == [sys.executable, "-m", "borg", "serve"]
args = _get_mock_args()
# XXX without next line we get spurious test fails when using pytest-xdist, root cause unknown:
logging.getLogger().setLevel(logging.INFO)
# note: test logger is on info log level, so --info gets added automagically
assert remote_repository.borg_cmd(args, testing=False) == ["borg", "serve", "--info"]
args.remote_path = "borg-0.28.2"
assert remote_repository.borg_cmd(args, testing=False) == ["borg-0.28.2", "serve", "--info"]
args.debug_topics = ["something_client_side", "repository_compaction"]
assert remote_repository.borg_cmd(args, testing=False) == [
"borg-0.28.2",
"serve",
"--info",
"--debug-topic=borg.debug.repository_compaction",
]
args = _get_mock_args()
args.storage_quota = 0
assert remote_repository.borg_cmd(args, testing=False) == ["borg", "serve", "--info"]
args.storage_quota = 314159265
assert remote_repository.borg_cmd(args, testing=False) == [
"borg",
"serve",
"--info",
"--storage-quota=314159265",
]
args.rsh = "ssh -i foo"
remote_repository._args = args
assert remote_repository.ssh_cmd(Location("ssh://example.com/foo")) == ["ssh", "-i", "foo", "example.com"]

2
tox.ini
View file

@ -42,7 +42,7 @@ deps =
pytest
mypy
pkgconfig
commands = mypy
commands = mypy --ignore-missing-imports
[testenv:docs]
changedir = docs