upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-05-28 04:03:21 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #2154 from enkore/merge/1.0-maint

Browse source 
cherry pick changes back from 1.0-maint (one-off)
This commit is contained in:
enkore 2017-02-14 01:19:07 +01:00 committed by GitHub
commit a659d1bf5f
12 changed files with 2911 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
AUTHORS
View file

@ -8,6 +8,7 @@ Borg authors ("The Borg Collective")
- Michael Hanselmann <public@hansmi.ch>
- Teemu Toivanen <public@profnetti.fi>
- Marian Beermann <public@enkore.de>
- Martin Hostettler <textshell@uchuujin.de>
- Daniel Reichelt <hacking@nachtgeist.net>
- Lauri Niskanen <ape@ape3000.com>

30
docs/changes.rst
View file

@ -145,6 +145,36 @@ New features:
--keep-exclude-tags, to account for the change mentioned above.
Version 1.0.10 (2017-02-13)
---------------------------
Bug fixes:
- Manifest timestamps are now monotonically increasing,
this fixes issues when the system clock jumps backwards
or is set inconsistently across computers accessing the same repository, #2115
- Fixed testing regression in 1.0.10rc1 that lead to a hard dependency on
py.test >= 3.0, #2112
New features:
- "key export" can now generate a printable HTML page with both a QR code and
a human-readable "paperkey" representation (and custom text) through the
``--qr-html`` option.
The same functionality is also available through `paperkey.html <paperkey.html>`_,
which is the same HTML page generated by ``--qr-html``. It works with existing
"key export" files and key files.
Other changes:
- docs:
- language clarification - "borg create --one-file-system" option does not respect
mount points, but considers different file systems instead, #2141
- setup.py: build_api: sort file list for determinism
Version 1.1.0b3 (2017-01-15)
----------------------------

2
docs/conf.py
View file

@ -140,6 +140,8 @@ html_favicon = '_static/favicon.ico'
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['borg_theme']
html_extra_path = ['../src/borg/paperkey.html']
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
html_last_updated_fmt = '%Y-%m-%d'

5
docs/quickstart.rst
View file

@ -188,11 +188,14 @@ For automated backups the passphrase can be specified using the
You can make backups using :ref:`borg_key_export` subcommand.
If you want to print a backup of your key to paper use the ``--paper``
option of this command and print the result.
option of this command and print the result, or this print `template`_
if you need a version with QR-Code.
A backup inside of the backup that is encrypted with that key/passphrase
won't help you with that, of course.
.. _template: paperkey.html
.. _remote_repos:
Remote repositories

4
scripts/borg.exe.spec
View file

@ -10,7 +10,9 @@ block_cipher = None
a = Analysis([os.path.join(basepath, 'src/borg/__main__.py'), ],
pathex=[basepath, ],
binaries=[],
datas=[],
datas=[
('../src/borg/paperkey.html', 'borg'),
],
hiddenimports=['borg.platform.posix'],
hookspath=[],
runtime_hooks=[],

8
setup.py
View file

@ -584,10 +584,13 @@ class build_api(Command):
print("auto-generating API documentation")
with open("docs/api.rst", "w") as doc:
doc.write("""
.. IMPORTANT: this file is auto-generated by "setup.py build_api", do not edit!
API Documentation
=================
""")
for mod in glob('src/borg/*.py') + glob('src/borg/*.pyx'):
for mod in sorted(glob('src/borg/*.py') + glob('src/borg/*.pyx')):
print("examining module %s" % mod)
mod = mod.replace('.pyx', '').replace('.py', '').replace('/', '.')
if "._" not in mod:
@ -666,6 +669,9 @@ setup(
'borgfs = borg.archiver:main',
]
},
package_data={
'borg': ['paperkey.html']
},
cmdclass=cmdclass,
ext_modules=ext_modules,
setup_requires=['setuptools_scm>=1.7'],

200
src/borg/archiver.py
View file

@ -44,7 +44,8 @@ from .helpers import to_localtime, timestamp
from .helpers import get_cache_dir
from .helpers import Manifest
from .helpers import StableDict
from .helpers import update_excludes, check_extension_modules
from .helpers import check_extension_modules
from .helpers import ArgparsePatternAction, ArgparseExcludeFileAction, ArgparsePatternFileAction, parse_exclude_pattern
from .helpers import dir_is_tagged, is_slow_msgpack, yes, sysinfo
from .helpers import log_multi
from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern
@ -128,7 +129,7 @@ class Archiver:
def __init__(self, lock_wait=None, prog=None):
self.exit_code = EXIT_SUCCESS
self.lock_wait = lock_wait
self.parser = self.build_parser(prog)
self.prog = prog
def print_error(self, msg, *args):
msg = args and msg % args or msg
@ -172,10 +173,10 @@ class Archiver:
bi += slicelen
@staticmethod
def build_matcher(excludes, paths):
def build_matcher(inclexcl_patterns, paths):
matcher = PatternMatcher()
if excludes:
matcher.add(excludes, False)
if inclexcl_patterns:
matcher.add_inclexcl(inclexcl_patterns)
include_patterns = []
if paths:
include_patterns.extend(parse_pattern(i, PathPrefixPattern) for i in paths)
@ -271,7 +272,10 @@ class Archiver:
if not args.path:
self.print_error("output file to export key to expected")
return EXIT_ERROR
manager.export(args.path)
if args.qr:
manager.export_qr(args.path)
else:
manager.export(args.path)
return EXIT_SUCCESS
@with_repository(lock=False, exclusive=False, manifest=False, cache=False)
@ -313,8 +317,7 @@ class Archiver:
def do_create(self, args, repository, manifest=None, key=None):
"""Create new archive"""
matcher = PatternMatcher(fallback=True)
if args.excludes:
matcher.add(args.excludes, False)
matcher.add_inclexcl(args.patterns)
def create_inner(archive, cache):
# Add cache dir to inode_skip list
@ -520,7 +523,7 @@ class Archiver:
if sys.platform.startswith(('linux', 'freebsd', 'netbsd', 'openbsd', 'darwin', )):
logger.warning('Hint: You likely need to fix your locale setup. E.g. install locales and use: LANG=en_US.UTF-8')
matcher, include_patterns = self.build_matcher(args.excludes, args.paths)
matcher, include_patterns = self.build_matcher(args.patterns, args.paths)
progress = args.progress
output_list = args.output_list
@ -790,7 +793,7 @@ class Archiver:
'If you know for certain that they are the same, pass --same-chunker-params '
'to override this check.')
matcher, include_patterns = self.build_matcher(args.excludes, args.paths)
matcher, include_patterns = self.build_matcher(args.patterns, args.paths)
compare_archives(archive1, archive2, matcher)
@ -924,7 +927,7 @@ class Archiver:
return self._list_repository(args, manifest, write)
def _list_archive(self, args, repository, manifest, key, write):
matcher, _ = self.build_matcher(args.excludes, args.paths)
matcher, _ = self.build_matcher(args.patterns, args.paths)
with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache:
archive = Archive(repository, key, manifest, args.location.archive, cache=cache,
consider_part_files=args.consider_part_files)
@ -1154,7 +1157,7 @@ class Archiver:
env_var_override='BORG_RECREATE_I_KNOW_WHAT_I_AM_DOING'):
return EXIT_ERROR
matcher, include_patterns = self.build_matcher(args.excludes, args.paths)
matcher, include_patterns = self.build_matcher(args.patterns, args.paths)
self.output_list = args.output_list
self.output_filter = args.output_filter
@ -1398,8 +1401,9 @@ class Archiver:
helptext = collections.OrderedDict()
helptext['patterns'] = textwrap.dedent('''
Exclusion patterns support four separate styles, fnmatch, shell, regular
expressions and path prefixes. By default, fnmatch is used. If followed
File patterns support four separate styles: fnmatch, shell, regular
expressions and path prefixes. By default, fnmatch is used for
`--exclude` patterns and shell-style is used for `--pattern`. If followed
by a colon (':') the first two characters of a pattern are used as a
style selector. Explicit style selection is necessary when a
non-default style is desired or when the desired pattern starts with
@ -1407,12 +1411,12 @@ class Archiver:
`Fnmatch <https://docs.python.org/3/library/fnmatch.html>`_, selector `fm:`
This is the default style. These patterns use a variant of shell
pattern syntax, with '*' matching any number of characters, '?'
matching any single character, '[...]' matching any single
character specified, including ranges, and '[!...]' matching any
character not specified. For the purpose of these patterns, the
path separator ('\\' for Windows and '/' on other systems) is not
This is the default style for --exclude and --exclude-from.
These patterns use a variant of shell pattern syntax, with '*' matching
any number of characters, '?' matching any single character, '[...]'
matching any single character specified, including ranges, and '[!...]'
matching any character not specified. For the purpose of these patterns,
the path separator ('\\' for Windows and '/' on other systems) is not
treated specially. Wrap meta-characters in brackets for a literal
match (i.e. `[?]` to match the literal character `?`). For a path
to match a pattern, it must completely match from start to end, or
@ -1423,6 +1427,7 @@ class Archiver:
Shell-style patterns, selector `sh:`
This is the default style for --pattern and --patterns-from.
Like fnmatch patterns these are similar to shell patterns. The difference
is that the pattern may include `**/` for matching zero or more directory
levels, `*` for matching zero or more arbitrary characters with the
@ -1483,7 +1488,39 @@ class Archiver:
re:^/home/[^/]\.tmp/
sh:/home/*/.thumbnails
EOF
$ borg create --exclude-from exclude.txt backup /\n\n''')
$ borg create --exclude-from exclude.txt backup /
A more general and easier to use way to define filename matching patterns exists
with the `--pattern` and `--patterns-from` options. Using these, you may specify
the backup roots (starting points) and patterns for inclusion/exclusion. A
root path starts with the prefix `R`, followed by a path (a plain path, not a
file pattern). An include rule starts with the prefix +, an exclude rule starts
with the prefix -, both followed by a pattern.
Inclusion patterns are useful to include pathes that are contained in an excluded
path. The first matching pattern is used so if an include pattern matches before
an exclude pattern, the file is backed up.
Note that the default pattern style for `--pattern` and `--patterns-from` is
shell style (`sh:`), so those patterns behave similar to rsync include/exclude
patterns.
Patterns (`--pattern`) and excludes (`--exclude`) from the command line are
considered first (in the order of appearance). Then patterns from `--patterns-from`
are added. Exclusion patterns from `--exclude-from` files are appended last.
An example `--patterns-from` file could look like that::
R /
# can be rebuild
- /home/*/.cache
# they're downloads for a reason
- /home/*/Downloads
# susan is a nice person
# include susans home
+ /home/susan
# don't backup the other home directories
- /home/*\n\n''')
helptext['placeholders'] = textwrap.dedent('''
Repository (or Archive) URLs, --prefix and --remote-path values support these
placeholders:
@ -1714,6 +1751,9 @@ class Archiver:
help='show version number and exit')
subparsers = parser.add_subparsers(title='required arguments', metavar='<command>')
# some empty defaults for all subparsers
common_parser.set_defaults(paths=[], patterns=[])
serve_epilog = process_epilog("""
This command starts a repository server process. This command is usually not used manually.
""")
@ -1938,6 +1978,9 @@ class Archiver:
subparser.add_argument('--paper', dest='paper', action='store_true',
default=False,
help='Create an export suitable for printing and later type-in')
subparser.add_argument('--qr-html', dest='qr', action='store_true',
default=False,
help='Create an html file suitable for printing and later type-in or qr scan')
key_import_epilog = process_epilog("""
This command allows to restore a key previously backed up with the
@ -2108,11 +2151,10 @@ class Archiver:
help='only display items with the given status characters')
exclude_group = subparser.add_argument_group('Exclusion options')
exclude_group.add_argument('-e', '--exclude', dest='excludes',
type=parse_pattern, action='append',
exclude_group.add_argument('-e', '--exclude', dest='patterns',
type=parse_exclude_pattern, action='append',
metavar="PATTERN", help='exclude paths matching PATTERN')
exclude_group.add_argument('--exclude-from', dest='exclude_files',
type=argparse.FileType('r'), action='append',
exclude_group.add_argument('--exclude-from', action=ArgparseExcludeFileAction,
metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
exclude_group.add_argument('--exclude-caches', dest='exclude_caches',
action='store_true', default=False,
@ -2126,11 +2168,16 @@ class Archiver:
action='store_true', default=False,
help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise '
'excluded caches/directories')
exclude_group.add_argument('--pattern',
action=ArgparsePatternAction,
metavar="PATTERN", help='include/exclude paths matching PATTERN')
exclude_group.add_argument('--patterns-from', action=ArgparsePatternFileAction,
metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line')
fs_group = subparser.add_argument_group('Filesystem options')
fs_group.add_argument('-x', '--one-file-system', dest='one_file_system',
action='store_true', default=False,
help='stay in same file system, do not cross mount points')
help='stay in the same file system and do not store mount points of other file systems')
fs_group.add_argument('--numeric-owner', dest='numeric_owner',
action='store_true', default=False,
help='only store numeric user and group identifiers')
@ -2177,7 +2224,7 @@ class Archiver:
subparser.add_argument('location', metavar='ARCHIVE',
type=location_validator(archive=True),
help='name of archive to create (must be also a valid directory name)')
subparser.add_argument('paths', metavar='PATH', nargs='+', type=str,
subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
help='paths to archive')
extract_epilog = process_epilog("""
@ -2207,12 +2254,15 @@ class Archiver:
subparser.add_argument('-n', '--dry-run', dest='dry_run',
default=False, action='store_true',
help='do not actually change any files')
subparser.add_argument('-e', '--exclude', dest='excludes',
type=parse_pattern, action='append',
subparser.add_argument('-e', '--exclude', dest='patterns',
type=parse_exclude_pattern, action='append',
metavar="PATTERN", help='exclude paths matching PATTERN')
subparser.add_argument('--exclude-from', dest='exclude_files',
type=argparse.FileType('r'), action='append',
subparser.add_argument('--exclude-from', action=ArgparseExcludeFileAction,
metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
subparser.add_argument('--pattern', action=ArgparsePatternAction,
metavar="PATTERN", help='include/exclude paths matching PATTERN')
subparser.add_argument('--patterns-from', action=ArgparsePatternFileAction,
metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line')
subparser.add_argument('--numeric-owner', dest='numeric_owner',
action='store_true', default=False,
help='only obey numeric user and group identifiers')
@ -2255,12 +2305,6 @@ class Archiver:
formatter_class=argparse.RawDescriptionHelpFormatter,
help='find differences in archive contents')
subparser.set_defaults(func=self.do_diff)
subparser.add_argument('-e', '--exclude', dest='excludes',
type=parse_pattern, action='append',
metavar="PATTERN", help='exclude paths matching PATTERN')
subparser.add_argument('--exclude-from', dest='exclude_files',
type=argparse.FileType('r'), action='append',
metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
subparser.add_argument('--numeric-owner', dest='numeric_owner',
action='store_true', default=False,
help='only consider numeric user and group identifiers')
@ -2279,6 +2323,30 @@ class Archiver:
subparser.add_argument('paths', metavar='PATH', nargs='*', type=str,
help='paths of items inside the archives to compare; patterns are supported')
exclude_group = subparser.add_argument_group('Exclusion options')
exclude_group.add_argument('-e', '--exclude', dest='patterns',
type=parse_exclude_pattern, action='append',
metavar="PATTERN", help='exclude paths matching PATTERN')
exclude_group.add_argument('--exclude-from', action=ArgparseExcludeFileAction,
metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
exclude_group.add_argument('--exclude-caches', dest='exclude_caches',
action='store_true', default=False,
help='exclude directories that contain a CACHEDIR.TAG file ('
'http://www.brynosaurus.com/cachedir/spec.html)')
exclude_group.add_argument('--exclude-if-present', dest='exclude_if_present',
metavar='NAME', action='append', type=str,
help='exclude directories that are tagged by containing a filesystem object with '
'the given NAME')
exclude_group.add_argument('--keep-exclude-tags', '--keep-tag-files', dest='keep_exclude_tags',
action='store_true', default=False,
help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise '
'excluded caches/directories')
exclude_group.add_argument('--pattern',
action=ArgparsePatternAction,
metavar="PATTERN", help='include/exclude paths matching PATTERN')
exclude_group.add_argument('--patterns-from', action=ArgparsePatternFileAction,
metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line')
rename_epilog = process_epilog("""
This command renames an archive in the repository.
@ -2359,12 +2427,6 @@ class Archiver:
subparser.add_argument('--format', '--list-format', dest='format', type=str,
help="""specify format for file listing
(default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")""")
subparser.add_argument('-e', '--exclude', dest='excludes',
type=parse_pattern, action='append',
metavar="PATTERN", help='exclude paths matching PATTERN')
subparser.add_argument('--exclude-from', dest='exclude_files',
type=argparse.FileType('r'), action='append',
metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='',
type=location_validator(),
help='repository/archive to list contents of')
@ -2372,6 +2434,30 @@ class Archiver:
help='paths to list; patterns are supported')
self.add_archives_filters_args(subparser)
exclude_group = subparser.add_argument_group('Exclusion options')
exclude_group.add_argument('-e', '--exclude', dest='patterns',
type=parse_exclude_pattern, action='append',
metavar="PATTERN", help='exclude paths matching PATTERN')
exclude_group.add_argument('--exclude-from', action=ArgparseExcludeFileAction,
metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
exclude_group.add_argument('--exclude-caches', dest='exclude_caches',
action='store_true', default=False,
help='exclude directories that contain a CACHEDIR.TAG file ('
'http://www.brynosaurus.com/cachedir/spec.html)')
exclude_group.add_argument('--exclude-if-present', dest='exclude_if_present',
metavar='NAME', action='append', type=str,
help='exclude directories that are tagged by containing a filesystem object with '
'the given NAME')
exclude_group.add_argument('--keep-exclude-tags', '--keep-tag-files', dest='keep_exclude_tags',
action='store_true', default=False,
help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise '
'excluded caches/directories')
exclude_group.add_argument('--pattern',
action=ArgparsePatternAction,
metavar="PATTERN", help='include/exclude paths matching PATTERN')
exclude_group.add_argument('--patterns-from', action=ArgparsePatternFileAction,
metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line')
mount_epilog = process_epilog("""
This command mounts an archive as a FUSE filesystem. This can be useful for
browsing an archive or restoring individual files. Unless the ``--foreground``
@ -2712,11 +2798,10 @@ class Archiver:
help='print statistics at end')
exclude_group = subparser.add_argument_group('Exclusion options')
exclude_group.add_argument('-e', '--exclude', dest='excludes',
type=parse_pattern, action='append',
exclude_group.add_argument('-e', '--exclude', dest='patterns',
type=parse_exclude_pattern, action='append',
metavar="PATTERN", help='exclude paths matching PATTERN')
exclude_group.add_argument('--exclude-from', dest='exclude_files',
type=argparse.FileType('r'), action='append',
exclude_group.add_argument('--exclude-from', action=ArgparseExcludeFileAction,
metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line')
exclude_group.add_argument('--exclude-caches', dest='exclude_caches',
action='store_true', default=False,
@ -2724,12 +2809,17 @@ class Archiver:
'http://www.brynosaurus.com/cachedir/spec.html)')
exclude_group.add_argument('--exclude-if-present', dest='exclude_if_present',
metavar='NAME', action='append', type=str,
help='exclude directories that are tagged by containing a filesystem object with \
the given NAME')
help='exclude directories that are tagged by containing a filesystem object with '
'the given NAME')
exclude_group.add_argument('--keep-exclude-tags', '--keep-tag-files', dest='keep_exclude_tags',
action='store_true', default=False,
help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise \
excluded caches/directories')
help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise '
'excluded caches/directories')
exclude_group.add_argument('--pattern',
action=ArgparsePatternAction,
metavar="PATTERN", help='include/exclude paths matching PATTERN')
exclude_group.add_argument('--patterns-from', action=ArgparsePatternFileAction,
metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line')
archive_group = subparser.add_argument_group('Archive options')
archive_group.add_argument('--target', dest='target', metavar='TARGET', default=None,
@ -2992,8 +3082,12 @@ class Archiver:
# We can't use argparse for "serve" since we don't want it to show up in "Available commands"
if args:
args = self.preprocess_args(args)
args = self.parser.parse_args(args or ['-h'])
update_excludes(args)
parser = self.build_parser(self.prog)
args = parser.parse_args(args or ['-h'])
if args.func == self.do_create:
# need at least 1 path but args.paths may also be populated from patterns
if not args.paths:
parser.error('Need at least one PATH argument.')
return args
def prerun_checks(self, logger):

104
src/borg/helpers.py
View file

@ -200,6 +200,7 @@ class Manifest:
self.repository = repository
self.item_keys = frozenset(item_keys) if item_keys is not None else ITEM_KEYS
self.tam_verified = False
self.timestamp = None
@property
def id_str(self):
@ -245,7 +246,13 @@ class Manifest:
from .item import ManifestItem
if self.key.tam_required:
self.config[b'tam_required'] = True
self.timestamp = datetime.utcnow().isoformat()
# self.timestamp needs to be strictly monotonically increasing. Clocks often are not set correctly
if self.timestamp is None:
self.timestamp = datetime.utcnow().isoformat()
else:
prev_ts = datetime.strptime(self.timestamp, "%Y-%m-%dT%H:%M:%S.%f")
incremented = (prev_ts + timedelta(microseconds=1)).isoformat()
self.timestamp = max(incremented, datetime.utcnow().isoformat())
manifest = ManifestItem(
version=1,
archives=StableDict(self.archives.get_raw_dict()),
@ -355,21 +362,52 @@ def parse_timestamp(timestamp):
return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc)
def load_excludes(fh):
"""Load and parse exclude patterns from file object. Lines empty or starting with '#' after stripping whitespace on
both line ends are ignored.
"""
return [parse_pattern(pattern) for pattern in clean_lines(fh)]
def parse_add_pattern(patternstr, roots, patterns):
"""Parse a pattern string and add it to roots or patterns depending on the pattern type."""
pattern = parse_inclexcl_pattern(patternstr)
if pattern.ptype is RootPath:
roots.append(pattern.pattern)
else:
patterns.append(pattern)
def update_excludes(args):
"""Merge exclude patterns from files with those on command line."""
if hasattr(args, 'exclude_files') and args.exclude_files:
if not hasattr(args, 'excludes') or args.excludes is None:
args.excludes = []
for file in args.exclude_files:
args.excludes += load_excludes(file)
file.close()
def load_pattern_file(fileobj, roots, patterns):
for patternstr in clean_lines(fileobj):
parse_add_pattern(patternstr, roots, patterns)
def load_exclude_file(fileobj, patterns):
for patternstr in clean_lines(fileobj):
patterns.append(parse_exclude_pattern(patternstr))
class ArgparsePatternAction(argparse.Action):
def __init__(self, nargs=1, **kw):
super().__init__(nargs=nargs, **kw)
def __call__(self, parser, args, values, option_string=None):
parse_add_pattern(values[0], args.paths, args.patterns)
class ArgparsePatternFileAction(argparse.Action):
def __init__(self, nargs=1, **kw):
super().__init__(nargs=nargs, **kw)
def __call__(self, parser, args, values, option_string=None):
"""Load and parse patterns from a file.
Lines empty or starting with '#' after stripping whitespace on both line ends are ignored.
"""
filename = values[0]
with open(filename) as f:
self.parse(f, args)
def parse(self, fobj, args):
load_pattern_file(fobj, args.roots, args.patterns)
class ArgparseExcludeFileAction(ArgparsePatternFileAction):
def parse(self, fobj, args):
load_exclude_file(fobj, args.patterns)
class PatternMatcher:
@ -388,6 +426,12 @@ class PatternMatcher:
"""
self._items.extend((i, value) for i in patterns)
def add_inclexcl(self, patterns):
"""Add list of patterns (of type InclExclPattern) to internal list. The patterns ptype member is returned from
the match function when one of the given patterns matches.
"""
self._items.extend(patterns)
def match(self, path):
for (pattern, value) in self._items:
if pattern.match(path):
@ -539,6 +583,9 @@ _PATTERN_STYLES = set([
_PATTERN_STYLE_BY_PREFIX = dict((i.PREFIX, i) for i in _PATTERN_STYLES)
InclExclPattern = namedtuple('InclExclPattern', 'pattern ptype')
RootPath = object()
def parse_pattern(pattern, fallback=FnmatchPattern):
"""Read pattern from string and return an instance of the appropriate implementation class.
@ -556,6 +603,35 @@ def parse_pattern(pattern, fallback=FnmatchPattern):
return cls(pattern)
def parse_exclude_pattern(pattern, fallback=FnmatchPattern):
"""Read pattern from string and return an instance of the appropriate implementation class.
"""
epattern = parse_pattern(pattern, fallback)
return InclExclPattern(epattern, False)
def parse_inclexcl_pattern(pattern, fallback=ShellPattern):
"""Read pattern from string and return a InclExclPattern object."""
type_prefix_map = {
'-': False,
'+': True,
'R': RootPath,
'r': RootPath,
}
try:
ptype = type_prefix_map[pattern[0]]
pattern = pattern[1:].lstrip()
if not pattern:
raise ValueError("Missing pattern!")
except (IndexError, KeyError, ValueError):
raise argparse.ArgumentTypeError("Unable to parse pattern: {}".format(pattern))
if ptype is RootPath:
pobj = pattern
else:
pobj = parse_pattern(pattern, fallback)
return InclExclPattern(pobj, ptype)
def timestamp(s):
"""Convert a --timestamp=s argument to a datetime object"""
try:

20
src/borg/keymanager.py
View file

@ -2,6 +2,7 @@ from binascii import unhexlify, a2b_base64, b2a_base64
import binascii
import textwrap
from hashlib import sha256
import pkgutil
from .key import KeyfileKey, RepoKey, PassphraseKey, KeyfileNotFoundError, PlaintextKey
from .helpers import Manifest, NoManifestError, Error, yes, bin_to_hex
@ -77,16 +78,27 @@ class KeyManager:
elif self.keyblob_storage == KEYBLOB_REPO:
self.repository.save_key(self.keyblob.encode('utf-8'))
def get_keyfile_data(self):
data = '%s %s\n' % (KeyfileKey.FILE_ID, bin_to_hex(self.repository.id))
data += self.keyblob
if not self.keyblob.endswith('\n'):
data += '\n'
return data
def store_keyfile(self, target):
with open(target, 'w') as fd:
fd.write('%s %s\n' % (KeyfileKey.FILE_ID, bin_to_hex(self.repository.id)))
fd.write(self.keyblob)
if not self.keyblob.endswith('\n'):
fd.write('\n')
fd.write(self.get_keyfile_data())
def export(self, path):
self.store_keyfile(path)
def export_qr(self, path):
with open(path, 'wb') as fd:
key_data = self.get_keyfile_data()
html = pkgutil.get_data('borg', 'paperkey.html')
html = html.replace(b'</textarea>', key_data.encode() + b'</textarea>')
fd.write(html)
def export_paperkey(self, path):
def grouped(s):
ret = ''

2441
src/borg/paperkey.html Normal file
View file

File diff suppressed because it is too large Load diff

60
src/borg/testsuite/archiver.py
View file

@ -877,6 +877,53 @@ class ArchiverTestCase(ArchiverTestCaseBase):
os.mkdir('input/cache3')
os.link('input/cache1/%s' % CACHE_TAG_NAME, 'input/cache3/%s' % CACHE_TAG_NAME)
def test_create_without_root(self):
"""test create without a root"""
self.cmd('init', self.repository_location)
args = ['create', self.repository_location + '::test']
if self.FORK_DEFAULT:
self.cmd(*args, exit_code=2)
else:
self.assert_raises(SystemExit, lambda: self.cmd(*args))
def test_create_pattern_root(self):
"""test create with only a root pattern"""
self.cmd('init', self.repository_location)
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('file2', size=1024 * 80)
output = self.cmd('create', '-v', '--list', '--pattern=R input', self.repository_location + '::test')
self.assert_in("A input/file1", output)
self.assert_in("A input/file2", output)
def test_create_pattern(self):
"""test file patterns during create"""
self.cmd('init', self.repository_location)
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('file2', size=1024 * 80)
self.create_regular_file('file_important', size=1024 * 80)
output = self.cmd('create', '-v', '--list',
'--pattern=+input/file_important', '--pattern=-input/file*',
self.repository_location + '::test', 'input')
self.assert_in("A input/file_important", output)
self.assert_in("A input/file_important", output)
self.assert_in('x input/file1', output)
self.assert_in('x input/file2', output)
def test_extract_pattern_opt(self):
self.cmd('init', self.repository_location)
self.create_regular_file('file1', size=1024 * 80)
self.create_regular_file('file2', size=1024 * 80)
self.create_regular_file('file_important', size=1024 * 80)
self.cmd('create', self.repository_location + '::test', 'input')
with changedir('output'):
self.cmd('extract',
'--pattern=+input/file_important', '--pattern=-input/file*',
self.repository_location + '::test')
self.assert_equal(sorted(os.listdir('output/input')), ['file_important'])
def test_exclude_caches(self):
self.cmd('init', self.repository_location)
def _assert_test_caches(self):
with changedir('output'):
self.cmd('extract', self.repository_location + '::test')
@ -1973,6 +2020,19 @@ class ArchiverTestCase(ArchiverTestCaseBase):
assert repo_key2.enc_key == repo_key2.enc_key
def test_key_export_qr(self):
export_file = self.output_path + '/exported.html'
self.cmd('init', self.repository_location, '--encryption', 'repokey')
repo_id = self._extract_repository_id(self.repository_path)
self.cmd('key', 'export', '--qr-html', self.repository_location, export_file)
with open(export_file, 'r', encoding='utf-8') as fd:
export_contents = fd.read()
assert bin_to_hex(repo_id) in export_contents
assert export_contents.startswith('<!doctype html>')
assert export_contents.endswith('</html>')
def test_key_import_errors(self):
export_file = self.output_path + '/exported'
self.cmd('init', self.repository_location, '--encryption', 'keyfile')

114
src/borg/testsuite/helpers.py
View file

@ -1,11 +1,12 @@
import argparse
import hashlib
import logging
import os
import sys
from datetime import datetime, timezone, timedelta
from time import mktime, strptime, sleep
import pytest
import msgpack
import msgpack.fallback
@ -21,7 +22,7 @@ from ..helpers import yes, TRUISH, FALSISH, DEFAULTISH
from ..helpers import StableDict, bin_to_hex
from ..helpers import parse_timestamp, ChunkIteratorFileWrapper, ChunkerParams, Chunk
from ..helpers import ProgressIndicatorPercent, ProgressIndicatorEndless
from ..helpers import load_excludes
from ..helpers import load_exclude_file, load_pattern_file
from ..helpers import CompressionSpec, CompressionDecider1, CompressionDecider2
from ..helpers import parse_pattern, PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern
from ..helpers import swidth_slice
@ -431,8 +432,13 @@ def test_invalid_unicode_pattern(pattern):
(["pp:/"], [" #/wsfoobar", "\tstart/whitespace"]),
(["pp:aaabbb"], None),
(["pp:/data", "pp: #/", "pp:\tstart", "pp:/whitespace"], ["/more/data", "/home"]),
(["/nomatch", "/more/*"],
['/data/something00.txt', '/home', ' #/wsfoobar', '\tstart/whitespace', '/whitespace/end\t']),
# the order of exclude patterns shouldn't matter
(["/more/*", "/nomatch"],
['/data/something00.txt', '/home', ' #/wsfoobar', '\tstart/whitespace', '/whitespace/end\t']),
])
def test_patterns_from_file(tmpdir, lines, expected):
def test_exclude_patterns_from_file(tmpdir, lines, expected):
files = [
'/data/something00.txt', '/more/data', '/home',
' #/wsfoobar',
@ -441,8 +447,10 @@ def test_patterns_from_file(tmpdir, lines, expected):
]
def evaluate(filename):
patterns = []
load_exclude_file(open(filename, "rt"), patterns)
matcher = PatternMatcher(fallback=True)
matcher.add(load_excludes(open(filename, "rt")), False)
matcher.add_inclexcl(patterns)
return [path for path in files if matcher.match(path)]
exclfile = tmpdir.join("exclude.txt")
@ -453,6 +461,104 @@ def test_patterns_from_file(tmpdir, lines, expected):
assert evaluate(str(exclfile)) == (files if expected is None else expected)
@pytest.mark.parametrize("lines, expected_roots, expected_numpatterns", [
# "None" means all files, i.e. none excluded
([], [], 0),
(["# Comment only"], [], 0),
(["- *"], [], 1),
(["+fm:*/something00.txt",
"-/data"], [], 2),
(["R /"], ["/"], 0),
(["R /",
"# comment"], ["/"], 0),
(["# comment",
"- /data",
"R /home"], ["/home"], 1),
])
def test_load_patterns_from_file(tmpdir, lines, expected_roots, expected_numpatterns):
def evaluate(filename):
roots = []
inclexclpatterns = []
load_pattern_file(open(filename, "rt"), roots, inclexclpatterns)
return roots, len(inclexclpatterns)
patternfile = tmpdir.join("patterns.txt")
with patternfile.open("wt") as fh:
fh.write("\n".join(lines))
roots, numpatterns = evaluate(str(patternfile))
assert roots == expected_roots
assert numpatterns == expected_numpatterns
@pytest.mark.parametrize("lines", [
(["X /data"]), # illegal pattern type prefix
(["/data"]), # need a pattern type prefix
])
def test_load_invalid_patterns_from_file(tmpdir, lines):
patternfile = tmpdir.join("patterns.txt")
with patternfile.open("wt") as fh:
fh.write("\n".join(lines))
filename = str(patternfile)
with pytest.raises(argparse.ArgumentTypeError):
roots = []
inclexclpatterns = []
load_pattern_file(open(filename, "rt"), roots, inclexclpatterns)
@pytest.mark.parametrize("lines, expected", [
# "None" means all files, i.e. none excluded
([], None),
(["# Comment only"], None),
(["- *"], []),
# default match type is sh: for patterns -> * doesn't match a /
(["-*/something0?.txt"],
['/data', '/data/something00.txt', '/data/subdir/something01.txt',
'/home', '/home/leo', '/home/leo/t', '/home/other']),
(["-fm:*/something00.txt"],
['/data', '/data/subdir/something01.txt', '/home', '/home/leo', '/home/leo/t', '/home/other']),
(["-fm:*/something0?.txt"],
["/data", '/home', '/home/leo', '/home/leo/t', '/home/other']),
(["+/*/something0?.txt",
"-/data"],
["/data/something00.txt", '/home', '/home/leo', '/home/leo/t', '/home/other']),
(["+fm:*/something00.txt",
"-/data"],
["/data/something00.txt", '/home', '/home/leo', '/home/leo/t', '/home/other']),
# include /home/leo and exclude the rest of /home:
(["+/home/leo",
"-/home/*"],
['/data', '/data/something00.txt', '/data/subdir/something01.txt', '/home', '/home/leo', '/home/leo/t']),
# wrong order, /home/leo is already excluded by -/home/*:
(["-/home/*",
"+/home/leo"],
['/data', '/data/something00.txt', '/data/subdir/something01.txt', '/home']),
(["+fm:/home/leo",
"-/home/"],
['/data', '/data/something00.txt', '/data/subdir/something01.txt', '/home', '/home/leo', '/home/leo/t']),
])
def test_inclexcl_patterns_from_file(tmpdir, lines, expected):
files = [
'/data', '/data/something00.txt', '/data/subdir/something01.txt',
'/home', '/home/leo', '/home/leo/t', '/home/other'
]
def evaluate(filename):
matcher = PatternMatcher(fallback=True)
roots = []
inclexclpatterns = []
load_pattern_file(open(filename, "rt"), roots, inclexclpatterns)
matcher.add_inclexcl(inclexclpatterns)
return [path for path in files if matcher.match(path)]
patternfile = tmpdir.join("patterns.txt")
with patternfile.open("wt") as fh:
fh.write("\n".join(lines))
assert evaluate(str(patternfile)) == (files if expected is None else expected)
@pytest.mark.parametrize("pattern, cls", [
("", FnmatchPattern),