From b73786690587a77e93411d57fa9dcb64e38cd891 Mon Sep 17 00:00:00 2001 From: Abogical Date: Sun, 13 Nov 2016 23:34:15 +0200 Subject: [PATCH 1/8] Improve extract progress display and ProgressIndicatorPercent --- src/borg/archive.py | 12 ++++-------- src/borg/archiver.py | 4 +++- src/borg/helpers.py | 31 +++++++++++++++++++++++++++---- src/borg/testsuite/archiver.py | 2 +- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/borg/archive.py b/src/borg/archive.py index e831be225..5cff04565 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -31,7 +31,7 @@ from .helpers import format_time, format_timedelta, format_file_size, file_statu from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates, swidth_slice from .helpers import decode_dict, StableDict from .helpers import int_to_bigint, bigint_to_int, bin_to_hex -from .helpers import ProgressIndicatorPercent, log_multi +from .helpers import ellipsis_truncate, ProgressIndicatorPercent, log_multi from .helpers import PathPrefixPattern, FnmatchPattern from .helpers import consume, chunkit from .helpers import CompressionDecider1, CompressionDecider2, CompressionSpec @@ -93,11 +93,7 @@ class Statistics: msg = '' space = columns - swidth(msg) if space >= 8: - if space < swidth('...') + swidth(path): - path = '%s...%s' % (swidth_slice(path, space // 2 - swidth('...')), - swidth_slice(path, -space // 2)) - space -= swidth(path) - msg += path + ' ' * space + msg += ellipsis_truncate(path, space) else: msg = ' ' * columns print(msg, file=stream or sys.stderr, end="\r", flush=True) @@ -448,7 +444,7 @@ Number of files: {0.stats.nfiles}'''.format( if 'chunks' in item: for _, data in self.pipeline.fetch_many([c.id for c in item.chunks], is_preloaded=True): if pi: - pi.show(increase=len(data)) + pi.show(increase=len(data), info=[remove_surrogates(item.path)]) if stdout: sys.stdout.buffer.write(data) if stdout: @@ -501,7 +497,7 @@ Number of files: {0.stats.nfiles}'''.format( ids = [c.id for c in item.chunks] for _, data in self.pipeline.fetch_many(ids, is_preloaded=True): if pi: - pi.show(increase=len(data)) + pi.show(increase=len(data), info=[remove_surrogates(item.path)]) with backup_io(): if sparse and self.zeros.startswith(data): # all-zero chunk: create a hole in a sparse file diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 6f3fa61d1..00db79b3a 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -501,7 +501,7 @@ class Archiver: filter = self.build_filter(matcher, peek_and_store_hardlink_masters, strip_components) if progress: - pi = ProgressIndicatorPercent(msg='Extracting files %5.1f%%', step=0.1) + pi = ProgressIndicatorPercent(msg='%5.1f%% Extracting: %s', step=0.1) pi.output('Calculating size') extracted_size = sum(item.file_size(hardlink_masters) for item in archive.iter_items(filter)) pi.total = extracted_size @@ -546,6 +546,8 @@ class Archiver: for pattern in include_patterns: if pattern.match_count == 0: self.print_warning("Include pattern '%s' never matched.", pattern) + if pi: + pi.finish() return self.exit_code @with_repository() diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 0b322685a..a0562b13a 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -26,6 +26,7 @@ from functools import wraps, partial, lru_cache from itertools import islice from operator import attrgetter from string import Formatter +from shutil import get_terminal_size import msgpack import msgpack.fallback @@ -1191,6 +1192,20 @@ def yes(msg=None, false_msg=None, true_msg=None, default_msg=None, env_var_override = None +def ellipsis_truncate(msg, space): + from .platform import swidth + """ + shorten a long string by adding ellipsis between it and return it, example: + this_is_a_very_long_string -------> this_is..._string + """ + ellipsis_width = swidth('...') + msg_width = swidth(msg) + if space < ellipsis_width + msg_width: + return '%s...%s' % (swidth_slice(msg, space // 2 - ellipsis_width), + swidth_slice(msg, -space // 2)) + return msg + ' ' * (space - msg_width) + + class ProgressIndicatorPercent: LOGGER = 'borg.output.progress' @@ -1208,7 +1223,6 @@ class ProgressIndicatorPercent: self.trigger_at = start # output next percentage value when reaching (at least) this self.step = step self.msg = msg - self.output_len = len(self.msg % 100.0) self.handler = None self.logger = logging.getLogger(self.LOGGER) @@ -1239,14 +1253,23 @@ class ProgressIndicatorPercent: self.trigger_at += self.step return pct - def show(self, current=None, increase=1): + def show(self, current=None, increase=1, info=[]): pct = self.progress(current, increase) if pct is not None: + # truncate the last argument, if space is available + if info != []: + msg = self.msg % (pct, *info[:-1], '') + space = get_terminal_size()[0] - len(msg) + if space < 8: + info[-1] = '' + else: + info[-1] = ellipsis_truncate(info[-1], space) + return self.output(self.msg % (pct, *info)) + return self.output(self.msg % pct) def output(self, message): - self.output_len = max(len(message), self.output_len) - message = message.ljust(self.output_len) + message = message.ljust(get_terminal_size(fallback=(4, 1))[0]) self.logger.info(message) def finish(self): diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index a978fa0f1..d720af46f 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -774,7 +774,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): with changedir('output'): output = self.cmd('extract', self.repository_location + '::test', '--progress') - assert 'Extracting files' in output + assert 'Extracting:' in output def _create_test_caches(self): self.cmd('init', self.repository_location) From 467fe38b1589bb9bc7b33c3e68eb82498f18fade Mon Sep 17 00:00:00 2001 From: Abogical Date: Mon, 14 Nov 2016 00:05:42 +0200 Subject: [PATCH 2/8] Compatibility with python 3.4 --- src/borg/helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index a0562b13a..e70db7234 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1258,13 +1258,13 @@ class ProgressIndicatorPercent: if pct is not None: # truncate the last argument, if space is available if info != []: - msg = self.msg % (pct, *info[:-1], '') + msg = self.msg % tuple([pct] + info[:-1] + ['']) space = get_terminal_size()[0] - len(msg) if space < 8: info[-1] = '' else: info[-1] = ellipsis_truncate(info[-1], space) - return self.output(self.msg % (pct, *info)) + return self.output(self.msg % tuple([pct] + info)) return self.output(self.msg % pct) From 1362d2e90f48465396212abbc4b006acac64f304 Mon Sep 17 00:00:00 2001 From: Abogical Date: Mon, 14 Nov 2016 00:45:54 +0200 Subject: [PATCH 3/8] Set COLUMNS & LINES as if it was a terminal --- src/borg/testsuite/helpers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index bd4865f0d..db90b1fcf 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -904,6 +904,8 @@ def test_yes_env_output(capfd, monkeypatch): def test_progress_percentage_sameline(capfd): + os.environ['COLUMNS'] = '4' + os.environ['LINES'] = '1' pi = ProgressIndicatorPercent(1000, step=5, start=0, msg="%3.0f%%") pi.logger.setLevel('INFO') pi.show(0) From 3232769ffc815241d2560ca71017ef9c9328f51a Mon Sep 17 00:00:00 2001 From: Abogical Date: Mon, 14 Nov 2016 11:43:46 +0200 Subject: [PATCH 4/8] Respond to feedback --- src/borg/helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index e70db7234..f65ebc053 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1193,11 +1193,11 @@ def yes(msg=None, false_msg=None, true_msg=None, default_msg=None, def ellipsis_truncate(msg, space): - from .platform import swidth """ shorten a long string by adding ellipsis between it and return it, example: this_is_a_very_long_string -------> this_is..._string """ + from .platform import swidth ellipsis_width = swidth('...') msg_width = swidth(msg) if space < ellipsis_width + msg_width: @@ -1253,11 +1253,11 @@ class ProgressIndicatorPercent: self.trigger_at += self.step return pct - def show(self, current=None, increase=1, info=[]): + def show(self, current=None, increase=1, info=None): pct = self.progress(current, increase) if pct is not None: - # truncate the last argument, if space is available - if info != []: + # truncate the last argument, if no space is available + if info is not None: msg = self.msg % tuple([pct] + info[:-1] + ['']) space = get_terminal_size()[0] - len(msg) if space < 8: From af925d272312ede6abe1a61e9e786a95bdfe359b Mon Sep 17 00:00:00 2001 From: Abogical Date: Mon, 14 Nov 2016 14:49:24 +0200 Subject: [PATCH 5/8] do not justify if we're not outputing to a terminal --- src/borg/helpers.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index f65ebc053..853d3625b 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1258,18 +1258,24 @@ class ProgressIndicatorPercent: if pct is not None: # truncate the last argument, if no space is available if info is not None: - msg = self.msg % tuple([pct] + info[:-1] + ['']) - space = get_terminal_size()[0] - len(msg) - if space < 8: - info[-1] = '' - else: - info[-1] = ellipsis_truncate(info[-1], space) - return self.output(self.msg % tuple([pct] + info)) + # no need to truncate if we're not outputing to a terminal + terminal_space = get_terminal_size(fallback=(-1, -1))[0] + if terminal_space != -1: + space = terminal_space - len(self.msg % tuple([pct] + info[:-1] + [''])) + if space < 8: + info[-1] = ' ' * space + else: + info[-1] = ellipsis_truncate(info[-1], space) + return self.output(self.msg % tuple([pct] + info), justify=False) return self.output(self.msg % pct) - def output(self, message): - message = message.ljust(get_terminal_size(fallback=(4, 1))[0]) + def output(self, message, justify=True): + if justify: + terminal_space = get_terminal_size(fallback=(-1, -1))[0] + # no need to ljust if we're not outputing to a terminal + if terminal_space != -1: + message = message.ljust(terminal_space) self.logger.info(message) def finish(self): From 76638d0562794c571b085e775b2ed5442494073e Mon Sep 17 00:00:00 2001 From: Abogical Date: Mon, 14 Nov 2016 14:56:34 +0200 Subject: [PATCH 6/8] If there is a small space for ellipsis_truncate, show '...' only --- src/borg/helpers.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 853d3625b..f2dd59ad0 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1200,6 +1200,8 @@ def ellipsis_truncate(msg, space): from .platform import swidth ellipsis_width = swidth('...') msg_width = swidth(msg) + if space < 8: + return '...' + ' ' * (space - ellipsis_width) if space < ellipsis_width + msg_width: return '%s...%s' % (swidth_slice(msg, space // 2 - ellipsis_width), swidth_slice(msg, -space // 2)) @@ -1262,10 +1264,7 @@ class ProgressIndicatorPercent: terminal_space = get_terminal_size(fallback=(-1, -1))[0] if terminal_space != -1: space = terminal_space - len(self.msg % tuple([pct] + info[:-1] + [''])) - if space < 8: - info[-1] = ' ' * space - else: - info[-1] = ellipsis_truncate(info[-1], space) + info[-1] = ellipsis_truncate(info[-1], space) return self.output(self.msg % tuple([pct] + info), justify=False) return self.output(self.msg % pct) From 3896f26ab24bbfaf47656ec582558076b08dfa96 Mon Sep 17 00:00:00 2001 From: Abogical Date: Mon, 14 Nov 2016 15:10:01 +0200 Subject: [PATCH 7/8] use monkeypatch --- src/borg/testsuite/helpers.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index db90b1fcf..f41cb4425 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -903,9 +903,10 @@ def test_yes_env_output(capfd, monkeypatch): assert 'yes' in err -def test_progress_percentage_sameline(capfd): - os.environ['COLUMNS'] = '4' - os.environ['LINES'] = '1' +def test_progress_percentage_sameline(capfd, monkeypatch): + # run the test as if it was in a 4x1 terminal + monkeypatch.setenv('COLUMNS', '4') + monkeypatch.setenv('LINES', '1') pi = ProgressIndicatorPercent(1000, step=5, start=0, msg="%3.0f%%") pi.logger.setLevel('INFO') pi.show(0) @@ -923,7 +924,10 @@ def test_progress_percentage_sameline(capfd): assert err == ' ' * 4 + '\r' -def test_progress_percentage_step(capfd): +def test_progress_percentage_step(capfd, monkeypatch): + # run the test as if it was in a 4x1 terminal + monkeypatch.setenv('COLUMNS', '4') + monkeypatch.setenv('LINES', '1') pi = ProgressIndicatorPercent(100, step=2, start=0, msg="%3.0f%%") pi.logger.setLevel('INFO') pi.show() From 34f529c7df63b52a30da8cb639d02c1d9004d810 Mon Sep 17 00:00:00 2001 From: Abogical Date: Mon, 14 Nov 2016 17:37:49 +0200 Subject: [PATCH 8/8] satisfy codecov --- src/borg/archiver.py | 1 + src/borg/helpers.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 00db79b3a..5e764344f 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -547,6 +547,7 @@ class Archiver: if pattern.match_count == 0: self.print_warning("Include pattern '%s' never matched.", pattern) if pi: + # clear progress output pi.finish() return self.exit_code diff --git a/src/borg/helpers.py b/src/borg/helpers.py index f2dd59ad0..ab2e1271a 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1201,6 +1201,7 @@ def ellipsis_truncate(msg, space): ellipsis_width = swidth('...') msg_width = swidth(msg) if space < 8: + # if there is very little space, just show ... return '...' + ' ' * (space - ellipsis_width) if space < ellipsis_width + msg_width: return '%s...%s' % (swidth_slice(msg, space // 2 - ellipsis_width), @@ -1256,6 +1257,13 @@ class ProgressIndicatorPercent: return pct def show(self, current=None, increase=1, info=None): + """ + Show and output the progress message + + :param current: set the current percentage [None] + :param increase: increase the current percentage [None] + :param info: array of strings to be formatted with msg [None] + """ pct = self.progress(current, increase) if pct is not None: # truncate the last argument, if no space is available