From 32e73e8c7e6a80bbae7a0371963ae8b865f1bb4f Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Mon, 30 Jan 2017 00:12:28 +0100 Subject: [PATCH 01/13] Manifest: Make sure manifest timestamp is strictly monotonically increasing. Computer clocks are often not set very accurately set, but borg assumes manifest timestamps are never going back in time. Ensure that this is actually the case. # Conflicts: # src/borg/helpers.py Original-Commit: 6b8cf0a --- src/borg/helpers.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index df2a136fd..20b0d1341 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -200,6 +200,7 @@ class Manifest: self.repository = repository self.item_keys = frozenset(item_keys) if item_keys is not None else ITEM_KEYS self.tam_verified = False + self.timestamp = None @property def id_str(self): @@ -245,7 +246,13 @@ class Manifest: from .item import ManifestItem if self.key.tam_required: self.config[b'tam_required'] = True - self.timestamp = datetime.utcnow().isoformat() + # self.timestamp needs to be strictly monotonically increasing. Clocks often are not set correctly + if self.timestamp is None: + self.timestamp = datetime.utcnow().isoformat() + else: + prev_ts = datetime.strptime(self.timestamp, "%Y-%m-%dT%H:%M:%S.%f") + incremented = (prev_ts + timedelta(microseconds=1)).isoformat() + self.timestamp = max(incremented, datetime.utcnow().isoformat()) manifest = ManifestItem( version=1, archives=StableDict(self.archives.get_raw_dict()), From fb44362c95c19b03c849621879147daf9bff3d3e Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Tue, 7 Feb 2017 00:32:39 +0100 Subject: [PATCH 02/13] Add myself to AUTHORS # Conflicts: # AUTHORS Original-Commit: bd8be26 --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 46cbc4474..051f54b42 100644 --- a/AUTHORS +++ b/AUTHORS @@ -8,6 +8,7 @@ Borg authors ("The Borg Collective") - Michael Hanselmann - Teemu Toivanen - Marian Beermann +- Martin Hostettler - Daniel Reichelt - Lauri Niskanen From b44882d10c8f28e1defde6b26a978be167188608 Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Sun, 25 Sep 2016 14:35:06 +0200 Subject: [PATCH 03/13] paperkey.html: Add interactive html template for printing key backups. --- docs/paperkey.html | 2438 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2438 insertions(+) create mode 100644 docs/paperkey.html diff --git a/docs/paperkey.html b/docs/paperkey.html new file mode 100644 index 000000000..82c7f98ac --- /dev/null +++ b/docs/paperkey.html @@ -0,0 +1,2438 @@ + + + + + + +BorgBackup Printable Key Template + + + + + + +
+
+

To create a printable key, either paste the contents of your keyfile or a key export in the text field + below, or select a key export file.

+

To create a key export use 

borg key export /path/to/repository exportfile.txt

+

If you are using keyfile mode, keyfiles are usually stored in $HOME/.config/borg/keys/

+

You can edit the parts with light blue border in the print preview below by click into them.

+

Key security: This print template will never send anything to remote servers. But keep in mind, that printing + might involve computers that can store the printed image, for example with cloud printing services, or + networked printers.

+

+
+ + +
+
+ QR error correction: +
+ QR code size
+ Text size
+ Text columns +
+ +
+
+ + +
+
+ +
+
BorgBackup Printable Key Backup
+
To restore either scan the QR code below, decode it and import it using +
borg key-import /path/to/repo scannedfile
+ +Or run +
borg key import --paper /path/to/repo
and type in the text below.

+
+
+
+
Notes:
+
+
+ + + + + \ No newline at end of file From e0f36e8613fd1015dabab45c6ea602a6aee0db35 Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Sun, 25 Sep 2016 17:08:40 +0200 Subject: [PATCH 04/13] quickstart.rst: Add link to paperkey template. --- docs/conf.py | 2 ++ docs/quickstart.rst | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/conf.py b/docs/conf.py index 8e51e4eac..608c69475 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -140,6 +140,8 @@ html_favicon = '_static/favicon.ico' # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['borg_theme'] +html_extra_path = ['paperkey.html'] + # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. html_last_updated_fmt = '%Y-%m-%d' diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 32770c2cb..e1e24e84c 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -188,11 +188,14 @@ For automated backups the passphrase can be specified using the You can make backups using :ref:`borg_key_export` subcommand. If you want to print a backup of your key to paper use the ``--paper`` - option of this command and print the result. + option of this command and print the result, or this print `template`_ + if you need a version with QR-Code. A backup inside of the backup that is encrypted with that key/passphrase won't help you with that, of course. +.. _template: paperkey.html + .. _remote_repos: Remote repositories From 179f1bc14794748aca4e0cd130336deb610582de Mon Sep 17 00:00:00 2001 From: Martin Hostettler Date: Wed, 8 Feb 2017 00:22:37 +0100 Subject: [PATCH 05/13] Add qr html export mode to `key export` command --- docs/conf.py | 2 +- setup.py | 3 +++ src/borg/archiver.py | 8 +++++++- src/borg/keymanager.py | 20 ++++++++++++++++---- {docs => src/borg}/paperkey.html | 0 5 files changed, 27 insertions(+), 6 deletions(-) rename {docs => src/borg}/paperkey.html (100%) diff --git a/docs/conf.py b/docs/conf.py index 608c69475..d1d64f9f4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -140,7 +140,7 @@ html_favicon = '_static/favicon.ico' # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['borg_theme'] -html_extra_path = ['paperkey.html'] +html_extra_path = ['../src/borg/paperkey.html'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/setup.py b/setup.py index 771f11b51..3864be5f7 100644 --- a/setup.py +++ b/setup.py @@ -666,6 +666,9 @@ setup( 'borgfs = borg.archiver:main', ] }, + package_data={ + 'borg': ['paperkey.html'] + }, cmdclass=cmdclass, ext_modules=ext_modules, setup_requires=['setuptools_scm>=1.7'], diff --git a/src/borg/archiver.py b/src/borg/archiver.py index feca6ccfe..38437b234 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -271,7 +271,10 @@ class Archiver: if not args.path: self.print_error("output file to export key to expected") return EXIT_ERROR - manager.export(args.path) + if args.qr: + manager.export_qr(args.path) + else: + manager.export(args.path) return EXIT_SUCCESS @with_repository(lock=False, exclusive=False, manifest=False, cache=False) @@ -1938,6 +1941,9 @@ class Archiver: subparser.add_argument('--paper', dest='paper', action='store_true', default=False, help='Create an export suitable for printing and later type-in') + subparser.add_argument('--qr-html', dest='qr', action='store_true', + default=False, + help='Create an html file suitable for printing and later type-in or qr scan') key_import_epilog = process_epilog(""" This command allows to restore a key previously backed up with the diff --git a/src/borg/keymanager.py b/src/borg/keymanager.py index 0b365e822..49799afc6 100644 --- a/src/borg/keymanager.py +++ b/src/borg/keymanager.py @@ -2,6 +2,7 @@ from binascii import unhexlify, a2b_base64, b2a_base64 import binascii import textwrap from hashlib import sha256 +import pkgutil from .key import KeyfileKey, RepoKey, PassphraseKey, KeyfileNotFoundError, PlaintextKey from .helpers import Manifest, NoManifestError, Error, yes, bin_to_hex @@ -77,16 +78,27 @@ class KeyManager: elif self.keyblob_storage == KEYBLOB_REPO: self.repository.save_key(self.keyblob.encode('utf-8')) + def get_keyfile_data(self): + data = '%s %s\n' % (KeyfileKey.FILE_ID, bin_to_hex(self.repository.id)) + data += self.keyblob + if not self.keyblob.endswith('\n'): + data += '\n' + return data + def store_keyfile(self, target): with open(target, 'w') as fd: - fd.write('%s %s\n' % (KeyfileKey.FILE_ID, bin_to_hex(self.repository.id))) - fd.write(self.keyblob) - if not self.keyblob.endswith('\n'): - fd.write('\n') + fd.write(self.get_keyfile_data()) def export(self, path): self.store_keyfile(path) + def export_qr(self, path): + with open(path, 'wb') as fd: + key_data = self.get_keyfile_data() + html = pkgutil.get_data('borg', 'paperkey.html') + html = html.replace(b'', key_data.encode() + b'') + fd.write(html) + def export_paperkey(self, path): def grouped(s): ret = '' diff --git a/docs/paperkey.html b/src/borg/paperkey.html similarity index 100% rename from docs/paperkey.html rename to src/borg/paperkey.html From 2cdb5838797a78fb460898e83202ec28f8275256 Mon Sep 17 00:00:00 2001 From: Benedikt Heine Date: Sun, 12 Feb 2017 17:18:08 +0100 Subject: [PATCH 06/13] clearify doc for same filesystems # Conflicts: # src/borg/archiver.py Original-Commit: d3a2f36b03 --- src/borg/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 38437b234..bca7a04c5 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -2136,7 +2136,7 @@ class Archiver: fs_group = subparser.add_argument_group('Filesystem options') fs_group.add_argument('-x', '--one-file-system', dest='one_file_system', action='store_true', default=False, - help='stay in same file system, do not cross mount points') + help='stay in the same file system and do not store mount points of other file systems') fs_group.add_argument('--numeric-owner', dest='numeric_owner', action='store_true', default=False, help='only store numeric user and group identifiers') From 79dd920661d9df607e70ec041eef8d2c89382479 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 12 Feb 2017 22:25:12 +0100 Subject: [PATCH 07/13] update CHANGES (1.0.10) # Conflicts: # docs/changes.rst Original-Commit: e635f219 --- docs/changes.rst | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 3c317ef4f..08f82e2e0 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -145,6 +145,35 @@ New features: --keep-exclude-tags, to account for the change mentioned above. +Version 1.0.10 (2017-02-13) +--------------------------- + +Bug fixes: + +- Manifest timestamps are now monotonically increasing, + this fixes issues when the system clock jumps backwards + or is set inconsistently across computers accessing the same repository, #2115 +- Fixed testing regression in 1.0.10rc1 that lead to a hard dependency on + py.test >= 3.0, #2112 + +New features: + +- "key export" can now generate a printable HTML page with both a QR code and + a human-readable "paperkey" representation (and custom text) through the + ``--qr-html`` option. + + The same functionality is also available through `paperkey.html `_, + which is the same HTML page generated by ``--qr-html``. It works with existing + "key export" files and key files. + +Other changes: + +- docs: + + - language clarification - "borg create --one-file-system" option does not respect + mount points, but considers different file systems instead, #2141 + + Version 1.1.0b3 (2017-01-15) ---------------------------- From 04bd6fb013f8139ccae40c5b98782cb55af3e465 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 12 Feb 2017 20:40:53 +0100 Subject: [PATCH 08/13] add test for export key --qr-html --- src/borg/testsuite/archiver.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index a9ad8ecf7..0636dfc8f 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1973,6 +1973,19 @@ class ArchiverTestCase(ArchiverTestCaseBase): assert repo_key2.enc_key == repo_key2.enc_key + def test_key_export_qr(self): + export_file = self.output_path + '/exported.html' + self.cmd('init', self.repository_location, '--encryption', 'repokey') + repo_id = self._extract_repository_id(self.repository_path) + self.cmd('key', 'export', '--qr-html', self.repository_location, export_file) + + with open(export_file, 'r') as fd: + export_contents = fd.read() + + assert bin_to_hex(repo_id) in export_contents + assert export_contents.startswith('') + assert export_contents.endswith('') + def test_key_import_errors(self): export_file = self.output_path + '/exported' self.cmd('init', self.repository_location, '--encryption', 'keyfile') From 1fabb2df5808d1427ef50b3879a3d509128faf7c Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 12 Feb 2017 20:45:41 +0100 Subject: [PATCH 09/13] key export: center QR code on the page --- src/borg/paperkey.html | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/borg/paperkey.html b/src/borg/paperkey.html index 82c7f98ac..4e1e859b3 100644 --- a/src/borg/paperkey.html +++ b/src/borg/paperkey.html @@ -2171,8 +2171,11 @@ if (typeof define == 'function' && define.amd) define([], function() { return Sh } } - - + /* center the QR code on the page */ + #qr { + width: 100%; + text-align: center; + } @@ -2217,7 +2220,7 @@ if (typeof define == 'function' && define.amd) define([], function() { return Sh
BorgBackup Printable Key Backup
To restore either scan the QR code below, decode it and import it using -
borg key-import /path/to/repo scannedfile
+
borg key import /path/to/repo scannedfile
Or run 
borg key import --paper /path/to/repo
and type in the text below.

From 44798e0edd62bd8c1df2544f0b71c4beefb9bd97 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 12 Feb 2017 22:36:24 +0100 Subject: [PATCH 10/13] setup.py: build_api: sort file list for determinism # Conflicts: # docs/api.rst # setup.py Original-Commit: e208d115 --- docs/changes.rst | 1 + setup.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/changes.rst b/docs/changes.rst index 08f82e2e0..d8d9a3519 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -172,6 +172,7 @@ Other changes: - language clarification - "borg create --one-file-system" option does not respect mount points, but considers different file systems instead, #2141 +- setup.py: build_api: sort file list for determinism Version 1.1.0b3 (2017-01-15) diff --git a/setup.py b/setup.py index 3864be5f7..d432bb164 100644 --- a/setup.py +++ b/setup.py @@ -584,10 +584,13 @@ class build_api(Command): print("auto-generating API documentation") with open("docs/api.rst", "w") as doc: doc.write(""" +.. IMPORTANT: this file is auto-generated by "setup.py build_api", do not edit! + + API Documentation ================= """) - for mod in glob('src/borg/*.py') + glob('src/borg/*.pyx'): + for mod in sorted(glob('src/borg/*.py') + glob('src/borg/*.pyx')): print("examining module %s" % mod) mod = mod.replace('.pyx', '').replace('.py', '').replace('/', '.') if "._" not in mod: From 11318c94dc79ab9086acd188f51980d466b7d0f8 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Mon, 13 Feb 2017 00:37:27 +0100 Subject: [PATCH 11/13] add paperkey.html to pyinstaller spec --- scripts/borg.exe.spec | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/borg.exe.spec b/scripts/borg.exe.spec index 9d165c74b..07dcdfbe1 100644 --- a/scripts/borg.exe.spec +++ b/scripts/borg.exe.spec @@ -10,7 +10,9 @@ block_cipher = None a = Analysis([os.path.join(basepath, 'src/borg/__main__.py'), ], pathex=[basepath, ], binaries=[], - datas=[], + datas=[ + ('../src/borg/paperkey.html', 'borg'), + ], hiddenimports=['borg.platform.posix'], hookspath=[], runtime_hooks=[], From 8d432b01e1914758c474f91b20fb95ff99855979 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 13 Feb 2017 04:12:12 +0100 Subject: [PATCH 12/13] paperkey.html - decode as utf-8, fixes #2150 hardcoded the encoding for reading it. while utf-8 is the default encoding on many systems, it does not work everywhere. and when it tries to decode with the ascii decoder, it fails. --- src/borg/testsuite/archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 0636dfc8f..5bfaca120 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1979,7 +1979,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): repo_id = self._extract_repository_id(self.repository_path) self.cmd('key', 'export', '--qr-html', self.repository_location, export_file) - with open(export_file, 'r') as fd: + with open(export_file, 'r', encoding='utf-8') as fd: export_contents = fd.read() assert bin_to_hex(repo_id) in export_contents From 30a5c5e44b185f0d9955d93913f38d6554c5fbed Mon Sep 17 00:00:00 2001 From: Alexander 'Leo' Bergolth Date: Tue, 2 Aug 2016 16:02:02 +0200 Subject: [PATCH 13/13] add two new options --pattern and --patterns-from as discussed in #1406 # Conflicts: # src/borg/archiver.py # src/borg/helpers.py # src/borg/testsuite/helpers.py Original-Commit: 876b670d --- src/borg/archiver.py | 190 ++++++++++++++++++++++++--------- src/borg/helpers.py | 95 ++++++++++++++--- src/borg/testsuite/archiver.py | 47 ++++++++ src/borg/testsuite/helpers.py | 114 +++++++++++++++++++- 4 files changed, 378 insertions(+), 68 deletions(-) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index bca7a04c5..f86fa7260 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -44,7 +44,8 @@ from .helpers import to_localtime, timestamp from .helpers import get_cache_dir from .helpers import Manifest from .helpers import StableDict -from .helpers import update_excludes, check_extension_modules +from .helpers import check_extension_modules +from .helpers import ArgparsePatternAction, ArgparseExcludeFileAction, ArgparsePatternFileAction, parse_exclude_pattern from .helpers import dir_is_tagged, is_slow_msgpack, yes, sysinfo from .helpers import log_multi from .helpers import parse_pattern, PatternMatcher, PathPrefixPattern @@ -128,7 +129,7 @@ class Archiver: def __init__(self, lock_wait=None, prog=None): self.exit_code = EXIT_SUCCESS self.lock_wait = lock_wait - self.parser = self.build_parser(prog) + self.prog = prog def print_error(self, msg, *args): msg = args and msg % args or msg @@ -172,10 +173,10 @@ class Archiver: bi += slicelen @staticmethod - def build_matcher(excludes, paths): + def build_matcher(inclexcl_patterns, paths): matcher = PatternMatcher() - if excludes: - matcher.add(excludes, False) + if inclexcl_patterns: + matcher.add_inclexcl(inclexcl_patterns) include_patterns = [] if paths: include_patterns.extend(parse_pattern(i, PathPrefixPattern) for i in paths) @@ -316,8 +317,7 @@ class Archiver: def do_create(self, args, repository, manifest=None, key=None): """Create new archive""" matcher = PatternMatcher(fallback=True) - if args.excludes: - matcher.add(args.excludes, False) + matcher.add_inclexcl(args.patterns) def create_inner(archive, cache): # Add cache dir to inode_skip list @@ -523,7 +523,7 @@ class Archiver: if sys.platform.startswith(('linux', 'freebsd', 'netbsd', 'openbsd', 'darwin', )): logger.warning('Hint: You likely need to fix your locale setup. E.g. install locales and use: LANG=en_US.UTF-8') - matcher, include_patterns = self.build_matcher(args.excludes, args.paths) + matcher, include_patterns = self.build_matcher(args.patterns, args.paths) progress = args.progress output_list = args.output_list @@ -793,7 +793,7 @@ class Archiver: 'If you know for certain that they are the same, pass --same-chunker-params ' 'to override this check.') - matcher, include_patterns = self.build_matcher(args.excludes, args.paths) + matcher, include_patterns = self.build_matcher(args.patterns, args.paths) compare_archives(archive1, archive2, matcher) @@ -927,7 +927,7 @@ class Archiver: return self._list_repository(args, manifest, write) def _list_archive(self, args, repository, manifest, key, write): - matcher, _ = self.build_matcher(args.excludes, args.paths) + matcher, _ = self.build_matcher(args.patterns, args.paths) with Cache(repository, key, manifest, lock_wait=self.lock_wait) as cache: archive = Archive(repository, key, manifest, args.location.archive, cache=cache, consider_part_files=args.consider_part_files) @@ -1157,7 +1157,7 @@ class Archiver: env_var_override='BORG_RECREATE_I_KNOW_WHAT_I_AM_DOING'): return EXIT_ERROR - matcher, include_patterns = self.build_matcher(args.excludes, args.paths) + matcher, include_patterns = self.build_matcher(args.patterns, args.paths) self.output_list = args.output_list self.output_filter = args.output_filter @@ -1401,8 +1401,9 @@ class Archiver: helptext = collections.OrderedDict() helptext['patterns'] = textwrap.dedent(''' - Exclusion patterns support four separate styles, fnmatch, shell, regular - expressions and path prefixes. By default, fnmatch is used. If followed + File patterns support four separate styles: fnmatch, shell, regular + expressions and path prefixes. By default, fnmatch is used for + `--exclude` patterns and shell-style is used for `--pattern`. If followed by a colon (':') the first two characters of a pattern are used as a style selector. Explicit style selection is necessary when a non-default style is desired or when the desired pattern starts with @@ -1410,12 +1411,12 @@ class Archiver: `Fnmatch `_, selector `fm:` - This is the default style. These patterns use a variant of shell - pattern syntax, with '*' matching any number of characters, '?' - matching any single character, '[...]' matching any single - character specified, including ranges, and '[!...]' matching any - character not specified. For the purpose of these patterns, the - path separator ('\\' for Windows and '/' on other systems) is not + This is the default style for --exclude and --exclude-from. + These patterns use a variant of shell pattern syntax, with '*' matching + any number of characters, '?' matching any single character, '[...]' + matching any single character specified, including ranges, and '[!...]' + matching any character not specified. For the purpose of these patterns, + the path separator ('\\' for Windows and '/' on other systems) is not treated specially. Wrap meta-characters in brackets for a literal match (i.e. `[?]` to match the literal character `?`). For a path to match a pattern, it must completely match from start to end, or @@ -1426,6 +1427,7 @@ class Archiver: Shell-style patterns, selector `sh:` + This is the default style for --pattern and --patterns-from. Like fnmatch patterns these are similar to shell patterns. The difference is that the pattern may include `**/` for matching zero or more directory levels, `*` for matching zero or more arbitrary characters with the @@ -1486,7 +1488,39 @@ class Archiver: re:^/home/[^/]\.tmp/ sh:/home/*/.thumbnails EOF - $ borg create --exclude-from exclude.txt backup /\n\n''') + $ borg create --exclude-from exclude.txt backup / + + + A more general and easier to use way to define filename matching patterns exists + with the `--pattern` and `--patterns-from` options. Using these, you may specify + the backup roots (starting points) and patterns for inclusion/exclusion. A + root path starts with the prefix `R`, followed by a path (a plain path, not a + file pattern). An include rule starts with the prefix +, an exclude rule starts + with the prefix -, both followed by a pattern. + Inclusion patterns are useful to include pathes that are contained in an excluded + path. The first matching pattern is used so if an include pattern matches before + an exclude pattern, the file is backed up. + + Note that the default pattern style for `--pattern` and `--patterns-from` is + shell style (`sh:`), so those patterns behave similar to rsync include/exclude + patterns. + + Patterns (`--pattern`) and excludes (`--exclude`) from the command line are + considered first (in the order of appearance). Then patterns from `--patterns-from` + are added. Exclusion patterns from `--exclude-from` files are appended last. + + An example `--patterns-from` file could look like that:: + + R / + # can be rebuild + - /home/*/.cache + # they're downloads for a reason + - /home/*/Downloads + # susan is a nice person + # include susans home + + /home/susan + # don't backup the other home directories + - /home/*\n\n''') helptext['placeholders'] = textwrap.dedent(''' Repository (or Archive) URLs, --prefix and --remote-path values support these placeholders: @@ -1717,6 +1751,9 @@ class Archiver: help='show version number and exit') subparsers = parser.add_subparsers(title='required arguments', metavar='') + # some empty defaults for all subparsers + common_parser.set_defaults(paths=[], patterns=[]) + serve_epilog = process_epilog(""" This command starts a repository server process. This command is usually not used manually. """) @@ -2114,11 +2151,10 @@ class Archiver: help='only display items with the given status characters') exclude_group = subparser.add_argument_group('Exclusion options') - exclude_group.add_argument('-e', '--exclude', dest='excludes', - type=parse_pattern, action='append', + exclude_group.add_argument('-e', '--exclude', dest='patterns', + type=parse_exclude_pattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') - exclude_group.add_argument('--exclude-from', dest='exclude_files', - type=argparse.FileType('r'), action='append', + exclude_group.add_argument('--exclude-from', action=ArgparseExcludeFileAction, metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line') exclude_group.add_argument('--exclude-caches', dest='exclude_caches', action='store_true', default=False, @@ -2132,6 +2168,11 @@ class Archiver: action='store_true', default=False, help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise ' 'excluded caches/directories') + exclude_group.add_argument('--pattern', + action=ArgparsePatternAction, + metavar="PATTERN", help='include/exclude paths matching PATTERN') + exclude_group.add_argument('--patterns-from', action=ArgparsePatternFileAction, + metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line') fs_group = subparser.add_argument_group('Filesystem options') fs_group.add_argument('-x', '--one-file-system', dest='one_file_system', @@ -2183,7 +2224,7 @@ class Archiver: subparser.add_argument('location', metavar='ARCHIVE', type=location_validator(archive=True), help='name of archive to create (must be also a valid directory name)') - subparser.add_argument('paths', metavar='PATH', nargs='+', type=str, + subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, help='paths to archive') extract_epilog = process_epilog(""" @@ -2213,12 +2254,15 @@ class Archiver: subparser.add_argument('-n', '--dry-run', dest='dry_run', default=False, action='store_true', help='do not actually change any files') - subparser.add_argument('-e', '--exclude', dest='excludes', - type=parse_pattern, action='append', + subparser.add_argument('-e', '--exclude', dest='patterns', + type=parse_exclude_pattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') - subparser.add_argument('--exclude-from', dest='exclude_files', - type=argparse.FileType('r'), action='append', + subparser.add_argument('--exclude-from', action=ArgparseExcludeFileAction, metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line') + subparser.add_argument('--pattern', action=ArgparsePatternAction, + metavar="PATTERN", help='include/exclude paths matching PATTERN') + subparser.add_argument('--patterns-from', action=ArgparsePatternFileAction, + metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line') subparser.add_argument('--numeric-owner', dest='numeric_owner', action='store_true', default=False, help='only obey numeric user and group identifiers') @@ -2261,12 +2305,6 @@ class Archiver: formatter_class=argparse.RawDescriptionHelpFormatter, help='find differences in archive contents') subparser.set_defaults(func=self.do_diff) - subparser.add_argument('-e', '--exclude', dest='excludes', - type=parse_pattern, action='append', - metavar="PATTERN", help='exclude paths matching PATTERN') - subparser.add_argument('--exclude-from', dest='exclude_files', - type=argparse.FileType('r'), action='append', - metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line') subparser.add_argument('--numeric-owner', dest='numeric_owner', action='store_true', default=False, help='only consider numeric user and group identifiers') @@ -2285,6 +2323,30 @@ class Archiver: subparser.add_argument('paths', metavar='PATH', nargs='*', type=str, help='paths of items inside the archives to compare; patterns are supported') + exclude_group = subparser.add_argument_group('Exclusion options') + exclude_group.add_argument('-e', '--exclude', dest='patterns', + type=parse_exclude_pattern, action='append', + metavar="PATTERN", help='exclude paths matching PATTERN') + exclude_group.add_argument('--exclude-from', action=ArgparseExcludeFileAction, + metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line') + exclude_group.add_argument('--exclude-caches', dest='exclude_caches', + action='store_true', default=False, + help='exclude directories that contain a CACHEDIR.TAG file (' + 'http://www.brynosaurus.com/cachedir/spec.html)') + exclude_group.add_argument('--exclude-if-present', dest='exclude_if_present', + metavar='NAME', action='append', type=str, + help='exclude directories that are tagged by containing a filesystem object with ' + 'the given NAME') + exclude_group.add_argument('--keep-exclude-tags', '--keep-tag-files', dest='keep_exclude_tags', + action='store_true', default=False, + help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise ' + 'excluded caches/directories') + exclude_group.add_argument('--pattern', + action=ArgparsePatternAction, + metavar="PATTERN", help='include/exclude paths matching PATTERN') + exclude_group.add_argument('--patterns-from', action=ArgparsePatternFileAction, + metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line') + rename_epilog = process_epilog(""" This command renames an archive in the repository. @@ -2365,12 +2427,6 @@ class Archiver: subparser.add_argument('--format', '--list-format', dest='format', type=str, help="""specify format for file listing (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")""") - subparser.add_argument('-e', '--exclude', dest='excludes', - type=parse_pattern, action='append', - metavar="PATTERN", help='exclude paths matching PATTERN') - subparser.add_argument('--exclude-from', dest='exclude_files', - type=argparse.FileType('r'), action='append', - metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line') subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', type=location_validator(), help='repository/archive to list contents of') @@ -2378,6 +2434,30 @@ class Archiver: help='paths to list; patterns are supported') self.add_archives_filters_args(subparser) + exclude_group = subparser.add_argument_group('Exclusion options') + exclude_group.add_argument('-e', '--exclude', dest='patterns', + type=parse_exclude_pattern, action='append', + metavar="PATTERN", help='exclude paths matching PATTERN') + exclude_group.add_argument('--exclude-from', action=ArgparseExcludeFileAction, + metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line') + exclude_group.add_argument('--exclude-caches', dest='exclude_caches', + action='store_true', default=False, + help='exclude directories that contain a CACHEDIR.TAG file (' + 'http://www.brynosaurus.com/cachedir/spec.html)') + exclude_group.add_argument('--exclude-if-present', dest='exclude_if_present', + metavar='NAME', action='append', type=str, + help='exclude directories that are tagged by containing a filesystem object with ' + 'the given NAME') + exclude_group.add_argument('--keep-exclude-tags', '--keep-tag-files', dest='keep_exclude_tags', + action='store_true', default=False, + help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise ' + 'excluded caches/directories') + exclude_group.add_argument('--pattern', + action=ArgparsePatternAction, + metavar="PATTERN", help='include/exclude paths matching PATTERN') + exclude_group.add_argument('--patterns-from', action=ArgparsePatternFileAction, + metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line') + mount_epilog = process_epilog(""" This command mounts an archive as a FUSE filesystem. This can be useful for browsing an archive or restoring individual files. Unless the ``--foreground`` @@ -2718,11 +2798,10 @@ class Archiver: help='print statistics at end') exclude_group = subparser.add_argument_group('Exclusion options') - exclude_group.add_argument('-e', '--exclude', dest='excludes', - type=parse_pattern, action='append', + exclude_group.add_argument('-e', '--exclude', dest='patterns', + type=parse_exclude_pattern, action='append', metavar="PATTERN", help='exclude paths matching PATTERN') - exclude_group.add_argument('--exclude-from', dest='exclude_files', - type=argparse.FileType('r'), action='append', + exclude_group.add_argument('--exclude-from', action=ArgparseExcludeFileAction, metavar='EXCLUDEFILE', help='read exclude patterns from EXCLUDEFILE, one per line') exclude_group.add_argument('--exclude-caches', dest='exclude_caches', action='store_true', default=False, @@ -2730,12 +2809,17 @@ class Archiver: 'http://www.brynosaurus.com/cachedir/spec.html)') exclude_group.add_argument('--exclude-if-present', dest='exclude_if_present', metavar='NAME', action='append', type=str, - help='exclude directories that are tagged by containing a filesystem object with \ - the given NAME') + help='exclude directories that are tagged by containing a filesystem object with ' + 'the given NAME') exclude_group.add_argument('--keep-exclude-tags', '--keep-tag-files', dest='keep_exclude_tags', action='store_true', default=False, - help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise \ - excluded caches/directories') + help='keep tag objects (i.e.: arguments to --exclude-if-present) in otherwise ' + 'excluded caches/directories') + exclude_group.add_argument('--pattern', + action=ArgparsePatternAction, + metavar="PATTERN", help='include/exclude paths matching PATTERN') + exclude_group.add_argument('--patterns-from', action=ArgparsePatternFileAction, + metavar='PATTERNFILE', help='read include/exclude patterns from PATTERNFILE, one per line') archive_group = subparser.add_argument_group('Archive options') archive_group.add_argument('--target', dest='target', metavar='TARGET', default=None, @@ -2998,8 +3082,12 @@ class Archiver: # We can't use argparse for "serve" since we don't want it to show up in "Available commands" if args: args = self.preprocess_args(args) - args = self.parser.parse_args(args or ['-h']) - update_excludes(args) + parser = self.build_parser(self.prog) + args = parser.parse_args(args or ['-h']) + if args.func == self.do_create: + # need at least 1 path but args.paths may also be populated from patterns + if not args.paths: + parser.error('Need at least one PATH argument.') return args def prerun_checks(self, logger): diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 20b0d1341..cf0af1e0e 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -362,21 +362,52 @@ def parse_timestamp(timestamp): return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc) -def load_excludes(fh): - """Load and parse exclude patterns from file object. Lines empty or starting with '#' after stripping whitespace on - both line ends are ignored. - """ - return [parse_pattern(pattern) for pattern in clean_lines(fh)] +def parse_add_pattern(patternstr, roots, patterns): + """Parse a pattern string and add it to roots or patterns depending on the pattern type.""" + pattern = parse_inclexcl_pattern(patternstr) + if pattern.ptype is RootPath: + roots.append(pattern.pattern) + else: + patterns.append(pattern) -def update_excludes(args): - """Merge exclude patterns from files with those on command line.""" - if hasattr(args, 'exclude_files') and args.exclude_files: - if not hasattr(args, 'excludes') or args.excludes is None: - args.excludes = [] - for file in args.exclude_files: - args.excludes += load_excludes(file) - file.close() +def load_pattern_file(fileobj, roots, patterns): + for patternstr in clean_lines(fileobj): + parse_add_pattern(patternstr, roots, patterns) + + +def load_exclude_file(fileobj, patterns): + for patternstr in clean_lines(fileobj): + patterns.append(parse_exclude_pattern(patternstr)) + + +class ArgparsePatternAction(argparse.Action): + def __init__(self, nargs=1, **kw): + super().__init__(nargs=nargs, **kw) + + def __call__(self, parser, args, values, option_string=None): + parse_add_pattern(values[0], args.paths, args.patterns) + + +class ArgparsePatternFileAction(argparse.Action): + def __init__(self, nargs=1, **kw): + super().__init__(nargs=nargs, **kw) + + def __call__(self, parser, args, values, option_string=None): + """Load and parse patterns from a file. + Lines empty or starting with '#' after stripping whitespace on both line ends are ignored. + """ + filename = values[0] + with open(filename) as f: + self.parse(f, args) + + def parse(self, fobj, args): + load_pattern_file(fobj, args.roots, args.patterns) + + +class ArgparseExcludeFileAction(ArgparsePatternFileAction): + def parse(self, fobj, args): + load_exclude_file(fobj, args.patterns) class PatternMatcher: @@ -395,6 +426,12 @@ class PatternMatcher: """ self._items.extend((i, value) for i in patterns) + def add_inclexcl(self, patterns): + """Add list of patterns (of type InclExclPattern) to internal list. The patterns ptype member is returned from + the match function when one of the given patterns matches. + """ + self._items.extend(patterns) + def match(self, path): for (pattern, value) in self._items: if pattern.match(path): @@ -546,6 +583,9 @@ _PATTERN_STYLES = set([ _PATTERN_STYLE_BY_PREFIX = dict((i.PREFIX, i) for i in _PATTERN_STYLES) +InclExclPattern = namedtuple('InclExclPattern', 'pattern ptype') +RootPath = object() + def parse_pattern(pattern, fallback=FnmatchPattern): """Read pattern from string and return an instance of the appropriate implementation class. @@ -563,6 +603,35 @@ def parse_pattern(pattern, fallback=FnmatchPattern): return cls(pattern) +def parse_exclude_pattern(pattern, fallback=FnmatchPattern): + """Read pattern from string and return an instance of the appropriate implementation class. + """ + epattern = parse_pattern(pattern, fallback) + return InclExclPattern(epattern, False) + + +def parse_inclexcl_pattern(pattern, fallback=ShellPattern): + """Read pattern from string and return a InclExclPattern object.""" + type_prefix_map = { + '-': False, + '+': True, + 'R': RootPath, + 'r': RootPath, + } + try: + ptype = type_prefix_map[pattern[0]] + pattern = pattern[1:].lstrip() + if not pattern: + raise ValueError("Missing pattern!") + except (IndexError, KeyError, ValueError): + raise argparse.ArgumentTypeError("Unable to parse pattern: {}".format(pattern)) + if ptype is RootPath: + pobj = pattern + else: + pobj = parse_pattern(pattern, fallback) + return InclExclPattern(pobj, ptype) + + def timestamp(s): """Convert a --timestamp=s argument to a datetime object""" try: diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 5bfaca120..2b0709be4 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -877,6 +877,53 @@ class ArchiverTestCase(ArchiverTestCaseBase): os.mkdir('input/cache3') os.link('input/cache1/%s' % CACHE_TAG_NAME, 'input/cache3/%s' % CACHE_TAG_NAME) + def test_create_without_root(self): + """test create without a root""" + self.cmd('init', self.repository_location) + args = ['create', self.repository_location + '::test'] + if self.FORK_DEFAULT: + self.cmd(*args, exit_code=2) + else: + self.assert_raises(SystemExit, lambda: self.cmd(*args)) + + def test_create_pattern_root(self): + """test create with only a root pattern""" + self.cmd('init', self.repository_location) + self.create_regular_file('file1', size=1024 * 80) + self.create_regular_file('file2', size=1024 * 80) + output = self.cmd('create', '-v', '--list', '--pattern=R input', self.repository_location + '::test') + self.assert_in("A input/file1", output) + self.assert_in("A input/file2", output) + + def test_create_pattern(self): + """test file patterns during create""" + self.cmd('init', self.repository_location) + self.create_regular_file('file1', size=1024 * 80) + self.create_regular_file('file2', size=1024 * 80) + self.create_regular_file('file_important', size=1024 * 80) + output = self.cmd('create', '-v', '--list', + '--pattern=+input/file_important', '--pattern=-input/file*', + self.repository_location + '::test', 'input') + self.assert_in("A input/file_important", output) + self.assert_in("A input/file_important", output) + self.assert_in('x input/file1', output) + self.assert_in('x input/file2', output) + + def test_extract_pattern_opt(self): + self.cmd('init', self.repository_location) + self.create_regular_file('file1', size=1024 * 80) + self.create_regular_file('file2', size=1024 * 80) + self.create_regular_file('file_important', size=1024 * 80) + self.cmd('create', self.repository_location + '::test', 'input') + with changedir('output'): + self.cmd('extract', + '--pattern=+input/file_important', '--pattern=-input/file*', + self.repository_location + '::test') + self.assert_equal(sorted(os.listdir('output/input')), ['file_important']) + + def test_exclude_caches(self): + self.cmd('init', self.repository_location) + def _assert_test_caches(self): with changedir('output'): self.cmd('extract', self.repository_location + '::test') diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index 49f32dfd4..8dca6a392 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -1,11 +1,12 @@ +import argparse import hashlib -import logging import os import sys from datetime import datetime, timezone, timedelta from time import mktime, strptime, sleep import pytest + import msgpack import msgpack.fallback @@ -21,7 +22,7 @@ from ..helpers import yes, TRUISH, FALSISH, DEFAULTISH from ..helpers import StableDict, bin_to_hex from ..helpers import parse_timestamp, ChunkIteratorFileWrapper, ChunkerParams, Chunk from ..helpers import ProgressIndicatorPercent, ProgressIndicatorEndless -from ..helpers import load_excludes +from ..helpers import load_exclude_file, load_pattern_file from ..helpers import CompressionSpec, CompressionDecider1, CompressionDecider2 from ..helpers import parse_pattern, PatternMatcher, RegexPattern, PathPrefixPattern, FnmatchPattern, ShellPattern from ..helpers import swidth_slice @@ -431,8 +432,13 @@ def test_invalid_unicode_pattern(pattern): (["pp:/"], [" #/wsfoobar", "\tstart/whitespace"]), (["pp:aaabbb"], None), (["pp:/data", "pp: #/", "pp:\tstart", "pp:/whitespace"], ["/more/data", "/home"]), + (["/nomatch", "/more/*"], + ['/data/something00.txt', '/home', ' #/wsfoobar', '\tstart/whitespace', '/whitespace/end\t']), + # the order of exclude patterns shouldn't matter + (["/more/*", "/nomatch"], + ['/data/something00.txt', '/home', ' #/wsfoobar', '\tstart/whitespace', '/whitespace/end\t']), ]) -def test_patterns_from_file(tmpdir, lines, expected): +def test_exclude_patterns_from_file(tmpdir, lines, expected): files = [ '/data/something00.txt', '/more/data', '/home', ' #/wsfoobar', @@ -441,8 +447,10 @@ def test_patterns_from_file(tmpdir, lines, expected): ] def evaluate(filename): + patterns = [] + load_exclude_file(open(filename, "rt"), patterns) matcher = PatternMatcher(fallback=True) - matcher.add(load_excludes(open(filename, "rt")), False) + matcher.add_inclexcl(patterns) return [path for path in files if matcher.match(path)] exclfile = tmpdir.join("exclude.txt") @@ -453,6 +461,104 @@ def test_patterns_from_file(tmpdir, lines, expected): assert evaluate(str(exclfile)) == (files if expected is None else expected) +@pytest.mark.parametrize("lines, expected_roots, expected_numpatterns", [ + # "None" means all files, i.e. none excluded + ([], [], 0), + (["# Comment only"], [], 0), + (["- *"], [], 1), + (["+fm:*/something00.txt", + "-/data"], [], 2), + (["R /"], ["/"], 0), + (["R /", + "# comment"], ["/"], 0), + (["# comment", + "- /data", + "R /home"], ["/home"], 1), +]) +def test_load_patterns_from_file(tmpdir, lines, expected_roots, expected_numpatterns): + def evaluate(filename): + roots = [] + inclexclpatterns = [] + load_pattern_file(open(filename, "rt"), roots, inclexclpatterns) + return roots, len(inclexclpatterns) + patternfile = tmpdir.join("patterns.txt") + + with patternfile.open("wt") as fh: + fh.write("\n".join(lines)) + + roots, numpatterns = evaluate(str(patternfile)) + assert roots == expected_roots + assert numpatterns == expected_numpatterns + + +@pytest.mark.parametrize("lines", [ + (["X /data"]), # illegal pattern type prefix + (["/data"]), # need a pattern type prefix +]) +def test_load_invalid_patterns_from_file(tmpdir, lines): + patternfile = tmpdir.join("patterns.txt") + with patternfile.open("wt") as fh: + fh.write("\n".join(lines)) + filename = str(patternfile) + with pytest.raises(argparse.ArgumentTypeError): + roots = [] + inclexclpatterns = [] + load_pattern_file(open(filename, "rt"), roots, inclexclpatterns) + + +@pytest.mark.parametrize("lines, expected", [ + # "None" means all files, i.e. none excluded + ([], None), + (["# Comment only"], None), + (["- *"], []), + # default match type is sh: for patterns -> * doesn't match a / + (["-*/something0?.txt"], + ['/data', '/data/something00.txt', '/data/subdir/something01.txt', + '/home', '/home/leo', '/home/leo/t', '/home/other']), + (["-fm:*/something00.txt"], + ['/data', '/data/subdir/something01.txt', '/home', '/home/leo', '/home/leo/t', '/home/other']), + (["-fm:*/something0?.txt"], + ["/data", '/home', '/home/leo', '/home/leo/t', '/home/other']), + (["+/*/something0?.txt", + "-/data"], + ["/data/something00.txt", '/home', '/home/leo', '/home/leo/t', '/home/other']), + (["+fm:*/something00.txt", + "-/data"], + ["/data/something00.txt", '/home', '/home/leo', '/home/leo/t', '/home/other']), + # include /home/leo and exclude the rest of /home: + (["+/home/leo", + "-/home/*"], + ['/data', '/data/something00.txt', '/data/subdir/something01.txt', '/home', '/home/leo', '/home/leo/t']), + # wrong order, /home/leo is already excluded by -/home/*: + (["-/home/*", + "+/home/leo"], + ['/data', '/data/something00.txt', '/data/subdir/something01.txt', '/home']), + (["+fm:/home/leo", + "-/home/"], + ['/data', '/data/something00.txt', '/data/subdir/something01.txt', '/home', '/home/leo', '/home/leo/t']), +]) +def test_inclexcl_patterns_from_file(tmpdir, lines, expected): + files = [ + '/data', '/data/something00.txt', '/data/subdir/something01.txt', + '/home', '/home/leo', '/home/leo/t', '/home/other' + ] + + def evaluate(filename): + matcher = PatternMatcher(fallback=True) + roots = [] + inclexclpatterns = [] + load_pattern_file(open(filename, "rt"), roots, inclexclpatterns) + matcher.add_inclexcl(inclexclpatterns) + return [path for path in files if matcher.match(path)] + + patternfile = tmpdir.join("patterns.txt") + + with patternfile.open("wt") as fh: + fh.write("\n".join(lines)) + + assert evaluate(str(patternfile)) == (files if expected is None else expected) + + @pytest.mark.parametrize("pattern, cls", [ ("", FnmatchPattern),