From d5515b69529720d1686ff298ad340e3ab4ce5795 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Mon, 27 Feb 2017 20:38:02 +0100 Subject: [PATCH] add msgid to progress output --- docs/internals/frontends.rst | 12 ++++++++---- src/borg/archive.py | 5 +++-- src/borg/archiver.py | 5 +++-- src/borg/cache.py | 7 ++++--- src/borg/helpers.py | 17 ++++++++++------- src/borg/repository.py | 9 ++++++--- 6 files changed, 34 insertions(+), 21 deletions(-) diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index b6bc18dc0..e11a1a3f4 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -44,7 +44,9 @@ progress_message saying what is currently worked on. operation - integer ID of the operation + unique, opaque integer ID of the operation + msgid + Message ID of the operation (may be *none*) finished boolean indicating whether the operation has finished, only the last object for an *operation* can have this property set to *true*. @@ -52,10 +54,12 @@ progress_message current progress message (may be empty/absent) progress_percent - Absolute progress display with defined end/total and current value. + Absolute progress information with defined end/total and current value. operation - integer ID of the operation + unique, opaque integer ID of the operation + msgid + Message ID of the operation (may be *none*) finished boolean indicating whether the operation has finished, only the last object for an *operation* can have this property set to *true*. @@ -81,7 +85,7 @@ log_message created Unix timestamp (float) levelname - Upper-case log level name (also called severity). Defined levels are: DEBUG, INFO, WARNING, CRITICAL + Upper-case log level name (also called severity). Defined levels are: DEBUG, INFO, WARNING, ERROR, CRITICAL name Name of the emitting entity message diff --git a/src/borg/archive.py b/src/borg/archive.py index 1df1401de..6fc0739e2 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -734,7 +734,7 @@ Utilization of max. archive size: {csize_max:.0%} try: unpacker = msgpack.Unpacker(use_list=False) items_ids = self.metadata.items - pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%") + pi = ProgressIndicatorPercent(total=len(items_ids), msg="Decrementing references %3.0f%%", msgid='archive.delete') for (i, (items_id, data)) in enumerate(zip(items_ids, self.repository.get_many(items_ids))): if progress: pi.show(i) @@ -1153,7 +1153,8 @@ class ArchiveChecker: chunks_count_segments = 0 errors = 0 defect_chunks = [] - pi = ProgressIndicatorPercent(total=chunks_count_index, msg="Verifying data %6.2f%%", step=0.01) + pi = ProgressIndicatorPercent(total=chunks_count_index, msg="Verifying data %6.2f%%", step=0.01, + msgid='check.verify_data') marker = None while True: chunk_ids = self.repository.scan(limit=100, marker=marker) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 243e91d2d..ad8a286d8 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -555,7 +555,7 @@ class Archiver: filter = self.build_filter(matcher, peek_and_store_hardlink_masters, strip_components) if progress: - pi = ProgressIndicatorPercent(msg='%5.1f%% Extracting: %s', step=0.1) + pi = ProgressIndicatorPercent(msg='%5.1f%% Extracting: %s', step=0.1, msgid='extract') pi.output('Calculating size') extracted_size = sum(item.get_size(hardlink_masters) for item in archive.iter_items(filter)) pi.total = extracted_size @@ -589,7 +589,8 @@ class Archiver: self.print_warning('%s: %s', remove_surrogates(orig_path), e) if not args.dry_run: - pi = ProgressIndicatorPercent(total=len(dirs), msg='Setting directory permissions %3.0f%%') + pi = ProgressIndicatorPercent(total=len(dirs), msg='Setting directory permissions %3.0f%%', + msgid='extract.permissions') while dirs: pi.show() dir_item = dirs.pop(-1) diff --git a/src/borg/cache.py b/src/borg/cache.py index 504001e7a..56e165609 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -320,7 +320,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" def begin_txn(self): # Initialize transaction snapshot - pi = ProgressIndicatorMessage() + pi = ProgressIndicatorMessage(msgid='cache.begin_transaction') txn_dir = os.path.join(self.path, 'txn.tmp') os.mkdir(txn_dir) pi.output('Initializing cache transaction: Reading config') @@ -340,7 +340,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" if not self.txn_active: return self.security_manager.save(self.manifest, self.key, self) - pi = ProgressIndicatorMessage() + pi = ProgressIndicatorMessage(msgid='cache.commit') if self.files is not None: if self._newest_mtime is None: # was never set because no files were modified/added @@ -468,7 +468,8 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" chunk_idx = None if self.progress: pi = ProgressIndicatorPercent(total=len(archive_ids), step=0.1, - msg='%3.0f%% Syncing chunks cache. Processing archive %s') + msg='%3.0f%% Syncing chunks cache. Processing archive %s', + msgid='cache.sync') for archive_id in archive_ids: archive_name = lookup_name(archive_id) if self.progress: diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 7f2a4e670..b4efd7e70 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1384,13 +1384,15 @@ class ProgressIndicatorBase: @classmethod def operation_id(cls): + """Unique number, can be used by receiving applications to distinguish different operations.""" cls.operation_id_counter += 1 return cls.operation_id_counter - def __init__(self): + def __init__(self, msgid=None): self.handler = None self.logger = logging.getLogger(self.LOGGER) self.id = self.operation_id() + self.msgid = msgid # If there are no handlers, set one up explicitly because the # terminator and propagation needs to be set. If there are, @@ -1426,12 +1428,13 @@ class ProgressIndicatorBase: assert self.json if not self.emit: return - print(json.dumps(dict( + kwargs.update(dict( operation=self.id, + msgid=self.msgid, type=self.JSON_TYPE, - finished=finished, - **kwargs, - )), file=sys.stderr) + finished=finished + )) + print(json.dumps(kwargs), file=sys.stderr) def finish(self): if self.json: @@ -1461,7 +1464,7 @@ class ProgressIndicatorMessage(ProgressIndicatorBase): class ProgressIndicatorPercent(ProgressIndicatorBase): JSON_TYPE = 'progress_percent' - def __init__(self, total=0, step=5, start=0, msg="%3.0f%%"): + def __init__(self, total=0, step=5, start=0, msg="%3.0f%%", msgid=None): """ Percentage-based progress indicator @@ -1476,7 +1479,7 @@ class ProgressIndicatorPercent(ProgressIndicatorBase): self.step = step self.msg = msg - super().__init__() + super().__init__(msgid=msgid) def progress(self, current=None, increase=1): if current is not None: diff --git a/src/borg/repository.py b/src/borg/repository.py index 81935f4a9..41d865d54 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -483,7 +483,8 @@ class Repository: unused = [] logger.debug('compaction started.') - pi = ProgressIndicatorPercent(total=len(self.compact), msg='Compacting segments %3.0f%%', step=1) + pi = ProgressIndicatorPercent(total=len(self.compact), msg='Compacting segments %3.0f%%', step=1, + msgid='repository.compact_segments') for segment, freeable_space in sorted(self.compact.items()): if not self.io.segment_exists(segment): logger.warning('segment %d not found, but listed in compaction data', segment) @@ -584,7 +585,8 @@ class Repository: self.prepare_txn(index_transaction_id, do_cleanup=False) try: segment_count = sum(1 for _ in self.io.segment_iterator()) - pi = ProgressIndicatorPercent(total=segment_count, msg="Replaying segments %3.0f%%") + pi = ProgressIndicatorPercent(total=segment_count, msg='Replaying segments %3.0f%%', + msgid='repository.replay_segments') for i, (segment, filename) in enumerate(self.io.segment_iterator()): pi.show(i) if index_transaction_id is not None and segment <= index_transaction_id: @@ -694,7 +696,8 @@ class Repository: self.prepare_txn(None) # self.index, self.compact, self.segments all empty now! segment_count = sum(1 for _ in self.io.segment_iterator()) logger.debug('Found %d segments', segment_count) - pi = ProgressIndicatorPercent(total=segment_count, msg="Checking segments %3.1f%%", step=0.1) + pi = ProgressIndicatorPercent(total=segment_count, msg='Checking segments %3.1f%%', step=0.1, + msgid='repository.check') for i, (segment, filename) in enumerate(self.io.segment_iterator()): pi.show(i) if segment > transaction_id: