From 14f0fa6f6a21a68a91c7a753b3a4c1cd413cbc4c Mon Sep 17 00:00:00 2001 From: Teemu Toivanen Date: Fri, 5 Feb 2016 20:34:48 +0200 Subject: [PATCH 1/4] archive file listing output formatting Coding style fixup Add --list-format exmaple to usage examples --- borg/archiver.py | 59 ++++++++++++++++++++++++++++++++++++++++-------- borg/helpers.py | 22 ++++++++++++++++++ docs/usage.rst | 16 +++++++++++++ 3 files changed, 87 insertions(+), 10 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 242de2de4..097d5b137 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -15,7 +15,7 @@ import textwrap import traceback from . import __version__ -from .helpers import Error, location_validator, format_time, format_file_size, \ +from .helpers import Error, location_validator, format_time, format_line, safe_timestamp, format_file_size, \ parse_pattern, PathPrefixPattern, to_localtime, timestamp, \ get_cache_dir, get_keys_dir, prune_within, prune_split, \ Manifest, remove_surrogates, update_excludes, format_archive, check_extension_modules, Statistics, \ @@ -442,6 +442,13 @@ class Archiver: for item in archive.iter_items(): print(remove_surrogates(item[b'path'])) else: + longformat="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}" + userformat=longformat + if args.listformat: + userformat=args.listformat + + archive_name=archive.name + for item in archive.iter_items(): mode = stat.filemode(item[b'mode']) type = mode[0] @@ -451,12 +458,12 @@ class Archiver: size = sum(size for _, size, _ in item[b'chunks']) except KeyError: pass - try: - mtime = datetime.fromtimestamp(bigint_to_int(item[b'mtime']) / 1e9) - except OverflowError: - # likely a broken mtime and datetime did not want to go beyond year 9999 - mtime = datetime(9999, 12, 31, 23, 59, 59) + atime=safe_timestamp(item[b'atime']) + ctime=safe_timestamp(item[b'ctime']) + mtime=safe_timestamp(item[b'mtime']) + if b'source' in item: + source = item[b'source'] if type == 'l': extra = ' -> %s' % item[b'source'] else: @@ -464,10 +471,40 @@ class Archiver: extra = ' link to %s' % item[b'source'] else: extra = '' - print('%s %-6s %-6s %8d %s %s%s' % ( - mode, item[b'user'] or item[b'uid'], - item[b'group'] or item[b'gid'], size, format_time(mtime), - remove_surrogates(item[b'path']), extra)) + source = '' + + formatdata={ + 'mode': mode, + 'bmode': item[b'mode'], + 'type': type, + 'source': source, + 'linktarget': source, + 'user': item[b'user'] or item[b'uid'], + 'uid': item[b'uid'], + 'group': item[b'group'] or item[b'gid'], + 'gid': item[b'gid'], + 'size': size, + 'isomtime': format_time(mtime), + 'mtime': mtime, + 'isoctime': format_time(ctime), + 'ctime': ctime, + 'isoatime': format_time(atime), + 'atime': atime, + 'path': remove_surrogates(item[b'path']), + 'extra': extra, + 'archivename': archive_name, + 'SPACE': " ", + 'TAB': "\t", + 'LF': "\n", + 'CR': "\r", + 'NEWLINE': os.linesep, + 'formatkeys': () + } + formatdata["formatkeys"]=list(formatdata.keys()) + + + print(format_line(userformat, formatdata), end='') + else: for archive_info in manifest.list_archive_infos(sort_by='ts'): if args.prefix and not archive_info.name.startswith(args.prefix): @@ -1098,6 +1135,8 @@ class Archiver: subparser.add_argument('--short', dest='short', action='store_true', default=False, help='only print file/directory names, nothing else') + subparser.add_argument('--list-format', dest='listformat', type=str, + help='Format archive listing line') subparser.add_argument('-P', '--prefix', dest='prefix', type=str, help='only consider archive names starting with this prefix') subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', diff --git a/borg/helpers.py b/borg/helpers.py index 4f230e6d1..a7b5be260 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -518,6 +518,28 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present): tag_paths.append(tag_path) return tag_paths +def format_line(formatstr, data): + """Filter out unwanted properties of str.format(), because "formatstr" + is user provided. + """ + try: + return(formatstr.format(**data)) + except KeyError as e: + print('Error in lineformat: "{}" - reason "{}"'.format(formatstr, str(e))) + except ValueError as e: + print('Error in lineformat: "{}" - reason "{}"'.format(formatstr, str(e))) + except: + print('Line format error') + raise + return '' + +def safe_timestamp(timedata): + try: + return datetime.fromtimestamp(bigint_to_int(timedata) / 1e9) + except OverflowError: + # likely a broken file time and datetime did not want to go beyond year 9999 + return datetime(9999, 12, 31, 23, 59, 59) + def format_time(t): """use ISO-8601 date and time format diff --git a/docs/usage.rst b/docs/usage.rst index 519834d79..128d92680 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -334,7 +334,23 @@ Examples -rw-r--r-- root root 1383 May 22 22:25 etc/ImageMagick-6/colors.xml ... + $ borg list /mnt/backup::archiveA --list-format="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}" + drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 . + drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 code + drwxrwxr-x user user 0 Sun, 2015-02-01 11:00:00 code/myproject + -rw-rw-r-- user user 1416192 Sun, 2015-02-01 11:00:00 code/myproject/file.ext + ... + # see what is change between archives, based on file modification time, size and file path + $ borg list /mnt/backup::archiveA --list-format="{mtime:%s}{TAB}{size}{TAB}{path}{LF}" |sort -n > /tmp/list.archiveA + $ borg list /mnt/backup::archiveB --list-format="{mtime:%s}{TAB}{size}{TAB}{path}{LF}" |sort -n > /tmp/list.archiveB + $ diff -y /tmp/list.archiveA /tmp/list.archiveB + 1422781200 0 . 1422781200 0 . + 1422781200 0 code 1422781200 0 code + 1422781200 0 code/myproject 1422781200 0 code/myproject + 1422781200 1416192 code/myproject/file.ext | 1454664653 1416192 code/myproject/file.ext + ... + .. include:: usage/prune.rst.inc Examples From 1b76c3902f25a8afbd52d7defa7b0a8e7999a7fd Mon Sep 17 00:00:00 2001 From: Teemu Toivanen Date: Wed, 3 Feb 2016 11:34:56 +0200 Subject: [PATCH 2/4] Optimize default listing --- borg/archiver.py | 85 ++++++++++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 097d5b137..1ee728b0a 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -442,10 +442,16 @@ class Archiver: for item in archive.iter_items(): print(remove_surrogates(item[b'path'])) else: - longformat="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}" + longformat="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}" userformat=longformat + """use_user_format flag is used to speed up default listing. + When user issues format options, listing is a bit slower, but more keys are available and + precalculated + """ + use_user_format=False if args.listformat: - userformat=args.listformat + userformat=args.listformat + use_user_format=True archive_name=archive.name @@ -458,10 +464,13 @@ class Archiver: size = sum(size for _, size, _ in item[b'chunks']) except KeyError: pass - atime=safe_timestamp(item[b'atime']) - ctime=safe_timestamp(item[b'ctime']) mtime=safe_timestamp(item[b'mtime']) + if use_user_format: + atime=safe_timestamp(item[b'atime']) + ctime=safe_timestamp(item[b'ctime']) + + if b'source' in item: source = item[b'source'] if type == 'l': @@ -472,38 +481,44 @@ class Archiver: else: extra = '' source = '' - - formatdata={ - 'mode': mode, - 'bmode': item[b'mode'], - 'type': type, - 'source': source, - 'linktarget': source, - 'user': item[b'user'] or item[b'uid'], - 'uid': item[b'uid'], - 'group': item[b'group'] or item[b'gid'], - 'gid': item[b'gid'], - 'size': size, - 'isomtime': format_time(mtime), - 'mtime': mtime, - 'isoctime': format_time(ctime), - 'ctime': ctime, - 'isoatime': format_time(atime), - 'atime': atime, - 'path': remove_surrogates(item[b'path']), - 'extra': extra, - 'archivename': archive_name, - 'SPACE': " ", - 'TAB': "\t", - 'LF': "\n", - 'CR': "\r", - 'NEWLINE': os.linesep, - 'formatkeys': () - } - formatdata["formatkeys"]=list(formatdata.keys()) - - print(format_line(userformat, formatdata), end='') + formatdata={ + 'mode': mode, + 'user': item[b'user'] or item[b'uid'], + 'group': item[b'group'] or item[b'gid'], + 'size': size, + 'isomtime': format_time(mtime), + 'path': remove_surrogates(item[b'path']), + 'extra': extra, + } + if use_user_format: + formatdata_user_format={ + 'bmode': item[b'mode'], + 'type': type, + 'source': source, + 'linktarget': source, + 'uid': item[b'uid'], + 'gid': item[b'gid'], + 'mtime': mtime, + 'isoctime': format_time(ctime), + 'ctime': ctime, + 'isoatime': format_time(atime), + 'atime': atime, + 'archivename': archive_name, + 'SPACE': " ", + 'TAB': "\t", + 'LF': "\n", + 'CR': "\r", + 'NEWLINE': os.linesep, + 'formatkeys': () + } + formatdata_user_format["formatkeys"]=list(formatdata.keys()) + formatdata.update(formatdata_user_format) + + if use_user_format: + print(format_line(userformat, formatdata), end='') + else: + print(format_line(userformat, formatdata)) else: for archive_info in manifest.list_archive_infos(sort_by='ts'): From 4a82d696f5f733c5a606c26c7b22554cef110473 Mon Sep 17 00:00:00 2001 From: Teemu Toivanen Date: Wed, 3 Feb 2016 20:14:15 +0200 Subject: [PATCH 3/4] Fix coding style and whitespaces in code, travis/flake8 --- borg/archiver.py | 53 ++++++++++++++++++++++++------------------------ borg/helpers.py | 27 ++++++++++++------------ 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 1ee728b0a..f3d90bb02 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -442,18 +442,18 @@ class Archiver: for item in archive.iter_items(): print(remove_surrogates(item[b'path'])) else: - longformat="{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}" - userformat=longformat + long_format = "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}" + user_format = long_format """use_user_format flag is used to speed up default listing. When user issues format options, listing is a bit slower, but more keys are available and precalculated - """ - use_user_format=False + """ + use_user_format = False if args.listformat: - userformat=args.listformat - use_user_format=True - - archive_name=archive.name + user_format = args.listformat + use_user_format = True + + archive_name = archive.name for item in archive.iter_items(): mode = stat.filemode(item[b'mode']) @@ -464,13 +464,12 @@ class Archiver: size = sum(size for _, size, _ in item[b'chunks']) except KeyError: pass - mtime=safe_timestamp(item[b'mtime']) - + mtime = safe_timestamp(item[b'mtime']) + if use_user_format: - atime=safe_timestamp(item[b'atime']) - ctime=safe_timestamp(item[b'ctime']) - - + atime = safe_timestamp(item[b'atime']) + ctime = safe_timestamp(item[b'ctime']) + if b'source' in item: source = item[b'source'] if type == 'l': @@ -481,18 +480,18 @@ class Archiver: else: extra = '' source = '' - - formatdata={ + + item_data = { 'mode': mode, 'user': item[b'user'] or item[b'uid'], - 'group': item[b'group'] or item[b'gid'], + 'group': item[b'group'] or item[b'gid'], 'size': size, - 'isomtime': format_time(mtime), - 'path': remove_surrogates(item[b'path']), + 'isomtime': format_time(mtime), + 'path': remove_surrogates(item[b'path']), 'extra': extra, } if use_user_format: - formatdata_user_format={ + item_data_advanced = { 'bmode': item[b'mode'], 'type': type, 'source': source, @@ -512,13 +511,13 @@ class Archiver: 'NEWLINE': os.linesep, 'formatkeys': () } - formatdata_user_format["formatkeys"]=list(formatdata.keys()) - formatdata.update(formatdata_user_format) - + item_data_advanced["formatkeys"] = list(item_data.keys()) + item_data.update(item_data_advanced) + if use_user_format: - print(format_line(userformat, formatdata), end='') + print(format_line(user_format, item_data), end='') else: - print(format_line(userformat, formatdata)) + print(format_line(user_format, item_data)) else: for archive_info in manifest.list_archive_infos(sort_by='ts'): @@ -1151,7 +1150,9 @@ class Archiver: action='store_true', default=False, help='only print file/directory names, nothing else') subparser.add_argument('--list-format', dest='listformat', type=str, - help='Format archive listing line') + help="""specify format for archive file listing + (default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}") + Special "{formatkeys}" exists to list available keys""") subparser.add_argument('-P', '--prefix', dest='prefix', type=str, help='only consider archive names starting with this prefix') subparser.add_argument('location', metavar='REPOSITORY_OR_ARCHIVE', nargs='?', default='', diff --git a/borg/helpers.py b/borg/helpers.py index a7b5be260..a8152c3a3 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -518,24 +518,25 @@ def dir_is_tagged(path, exclude_caches, exclude_if_present): tag_paths.append(tag_path) return tag_paths -def format_line(formatstr, data): - """Filter out unwanted properties of str.format(), because "formatstr" - is user provided. - """ + +def format_line(format, data): + # TODO: Filter out unwanted properties of str.format(), because "format" is user provided. + try: - return(formatstr.format(**data)) - except KeyError as e: - print('Error in lineformat: "{}" - reason "{}"'.format(formatstr, str(e))) - except ValueError as e: - print('Error in lineformat: "{}" - reason "{}"'.format(formatstr, str(e))) + return format.format(**data) + except (KeyError, ValueError) as e: + # this should catch format errors + print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e))) except: - print('Line format error') - raise + # something unexpected, print error and raise exception + print('Error in lineformat: "{}" - reason "{}"'.format(format, str(e))) + raise return '' -def safe_timestamp(timedata): + +def safe_timestamp(item_timestamp_ns): try: - return datetime.fromtimestamp(bigint_to_int(timedata) / 1e9) + return datetime.fromtimestamp(bigint_to_int(item_timestamp_ns) / 1e9) except OverflowError: # likely a broken file time and datetime did not want to go beyond year 9999 return datetime(9999, 12, 31, 23, 59, 59) From 61a40cbeacb3592d86c3ee69f79af4d83533122a Mon Sep 17 00:00:00 2001 From: Teemu Toivanen Date: Fri, 5 Feb 2016 11:52:02 +0200 Subject: [PATCH 4/4] add test for list --list-format --- borg/testsuite/archiver.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index d9eede408..be25a0da3 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -892,6 +892,16 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.assert_in('test-2', output) self.assert_not_in('something-else', output) + def test_list_list_format(self): + self.cmd('init', self.repository_location) + test_archive = self.repository_location + '::test' + self.cmd('create', test_archive, src_dir) + output_1 = self.cmd('list', test_archive) + output_2 = self.cmd('list', '--list-format', '{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}', test_archive) + output_not_equel = self.cmd('list', '--list-format', '{mtime:%s} {path}{NL}', test_archive) + self.assertEqual(output_1, output_2) + self.assertNotEqual(output_1, output_not_equel) + def test_break_lock(self): self.cmd('init', self.repository_location) self.cmd('break-lock', self.repository_location)