From 420c984f05ff8bc1b16c3b4b661bdaebe285a421 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 30 Oct 2016 18:18:05 +0100 Subject: [PATCH 1/6] fix wrong duration if clock jumps during create --- borg/archive.py | 9 ++++++--- borg/archiver.py | 6 +++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index ebfa091f9..f93e0a8c9 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -1,5 +1,5 @@ from contextlib import contextmanager -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta from getpass import getuser from itertools import groupby import errno @@ -186,7 +186,7 @@ class Archive: def __init__(self, repository, key, manifest, name, cache=None, create=False, checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, progress=False, - chunker_params=CHUNKER_PARAMS, start=None, end=None): + chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None): self.cwd = os.getcwd() self.key = key self.repository = repository @@ -200,9 +200,12 @@ class Archive: self.numeric_owner = numeric_owner self.noatime = noatime self.noctime = noctime + assert (start is None) == (start_monotonic is None), 'Logic error: if start is given, start_monotonic must be given as well and vice versa.' if start is None: start = datetime.utcnow() + start_monotonic = time.monotonic() self.start = start + self.start_monotonic = start_monotonic if end is None: end = datetime.utcnow() self.end = end @@ -302,7 +305,7 @@ Number of files: {0.stats.nfiles}'''.format( raise self.AlreadyExists(name) self.items_buffer.flush(flush=True) if timestamp is None: - self.end = datetime.utcnow() + self.end = self.start + timedelta(seconds=time.monotonic() - self.start_monotonic) start = self.start end = self.end else: diff --git a/borg/archiver.py b/borg/archiver.py index cb02d4d8a..5ee612972 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -5,7 +5,6 @@ from operator import attrgetter import argparse import functools import inspect -import io import os import re import shlex @@ -13,6 +12,7 @@ import signal import stat import sys import textwrap +import time import traceback import collections @@ -263,7 +263,6 @@ class Archiver: if args.progress: archive.stats.show_progress(final=True) if args.stats: - archive.end = datetime.utcnow() log_multi(DASHES, str(archive), DASHES, @@ -276,6 +275,7 @@ class Archiver: self.ignore_inode = args.ignore_inode dry_run = args.dry_run t0 = datetime.utcnow() + t0_monotonic = time.monotonic() if not dry_run: key.compressor = Compressor(**args.compression) with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache: @@ -283,7 +283,7 @@ class Archiver: create=True, checkpoint_interval=args.checkpoint_interval, numeric_owner=args.numeric_owner, noatime=args.noatime, noctime=args.noctime, progress=args.progress, - chunker_params=args.chunker_params, start=t0) + chunker_params=args.chunker_params, start=t0, start_monotonic=t0_monotonic) create_inner(archive, cache) else: create_inner(None, None) From 2dc558a02e24461f0cda9869cb49bcd73038a815 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sun, 30 Oct 2016 18:18:23 +0100 Subject: [PATCH 2/6] fix create progress not updating if clock jumps --- borg/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/borg/helpers.py b/borg/helpers.py index ff7d16075..a8106aeef 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -219,7 +219,7 @@ class Statistics: return format_file_size(self.csize) def show_progress(self, item=None, final=False, stream=None, dt=None): - now = time.time() + now = time.monotonic() if dt is None or now - self.last_progress > dt: self.last_progress = now columns, lines = get_terminal_size() From a8d921a54cbb9a5e7b86668bd04a050c23110ad4 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sat, 17 Dec 2016 11:37:56 +0100 Subject: [PATCH 3/6] base archive timestamps on end time The assumption is that if the clock jumps during the Borg run that it was jump-corrected and is now correct, while the start timestamp would be wrong. --- borg/archive.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/borg/archive.py b/borg/archive.py index f93e0a8c9..5001d3108 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -305,7 +305,8 @@ Number of files: {0.stats.nfiles}'''.format( raise self.AlreadyExists(name) self.items_buffer.flush(flush=True) if timestamp is None: - self.end = self.start + timedelta(seconds=time.monotonic() - self.start_monotonic) + self.end = datetime.utcnow() + self.start = self.end - timedelta(seconds=time.monotonic() - self.start_monotonic) start = self.start end = self.end else: From f5d6093ccc0b41ea7465dfded9091b0d95d01403 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sat, 17 Dec 2016 11:46:20 +0100 Subject: [PATCH 4/6] fix checkpoints when clock jumps --- borg/archive.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 5001d3108..64be0f4d8 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -215,7 +215,7 @@ class Archive: self.chunker = Chunker(self.key.chunk_seed, *chunker_params) if name in manifest.archives: raise self.AlreadyExists(name) - self.last_checkpoint = time.time() + self.last_checkpoint = time.monotonic() i = 0 while True: self.checkpoint_name = '%s.checkpoint%s' % (name, i and ('.%d' % i) or '') @@ -290,9 +290,9 @@ Number of files: {0.stats.nfiles}'''.format( if self.show_progress: self.stats.show_progress(item=item, dt=0.2) self.items_buffer.add(item) - if time.time() - self.last_checkpoint > self.checkpoint_interval: + if time.monotonic() - self.last_checkpoint > self.checkpoint_interval: self.write_checkpoint() - self.last_checkpoint = time.time() + self.last_checkpoint = time.monotonic() def write_checkpoint(self): self.save(self.checkpoint_name) From 445365b3ff19830aefe2c7b1332cba05703fcb97 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sat, 17 Dec 2016 12:00:25 +0100 Subject: [PATCH 5/6] update changes --- docs/changes.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changes.rst b/docs/changes.rst index 4ff900e7d..71677bcb3 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -82,6 +82,9 @@ Bug fixes: - skip corrupted chunks during manifest rebuild - fix TypeError in integrity error handler, #1903, #1894 - fix location parser for archives with @ char (regression introduced in 1.0.8), #1930 +- fix wrong duration/timestamps if system clock jumped during a create +- fix progress display not updating if system clock jumps backwards +- fix checkpoint interval being incorrect if system clock jumps Other changes: From baa8baafdbd342bdaec0e39aa59fc160bea09e2c Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Sat, 17 Dec 2016 12:55:16 +0100 Subject: [PATCH 6/6] create: fix duration if --timestamp is given --- borg/archive.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 64be0f4d8..641fd08dd 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -304,15 +304,17 @@ Number of files: {0.stats.nfiles}'''.format( if name in self.manifest.archives: raise self.AlreadyExists(name) self.items_buffer.flush(flush=True) + duration = timedelta(seconds=time.monotonic() - self.start_monotonic) if timestamp is None: self.end = datetime.utcnow() - self.start = self.end - timedelta(seconds=time.monotonic() - self.start_monotonic) + self.start = self.end - duration start = self.start end = self.end else: self.end = timestamp - start = timestamp - end = timestamp # we only have 1 value + self.start = timestamp - duration + end = timestamp + start = self.start metadata = StableDict({ 'version': 1, 'name': name,