Merge pull request #1941 from enkore/issue/1936

Fix some clock problems

borg/archive.py

@@ -1,5 +1,5 @@
 from contextlib import contextmanager
-from datetime import datetime, timezone
+from datetime import datetime, timezone, timedelta
 from getpass import getuser
 from itertools import groupby
 import errno
@@ -186,7 +186,7 @@ class Archive:
 
     def __init__(self, repository, key, manifest, name, cache=None, create=False,
                  checkpoint_interval=300, numeric_owner=False, noatime=False, noctime=False, progress=False,
-                 chunker_params=CHUNKER_PARAMS, start=None, end=None):
+                 chunker_params=CHUNKER_PARAMS, start=None, start_monotonic=None, end=None):
         self.cwd = os.getcwd()
         self.key = key
         self.repository = repository
@@ -200,9 +200,12 @@ class Archive:
         self.numeric_owner = numeric_owner
         self.noatime = noatime
         self.noctime = noctime
+        assert (start is None) == (start_monotonic is None), 'Logic error: if start is given, start_monotonic must be given as well and vice versa.'
         if start is None:
             start = datetime.utcnow()
+            start_monotonic = time.monotonic()
         self.start = start
+        self.start_monotonic = start_monotonic
         if end is None:
             end = datetime.utcnow()
         self.end = end
@@ -212,7 +215,7 @@ class Archive:
             self.chunker = Chunker(self.key.chunk_seed, *chunker_params)
             if name in manifest.archives:
                 raise self.AlreadyExists(name)
-            self.last_checkpoint = time.time()
+            self.last_checkpoint = time.monotonic()
             i = 0
             while True:
                 self.checkpoint_name = '%s.checkpoint%s' % (name, i and ('.%d' % i) or '')
@@ -287,9 +290,9 @@ Number of files: {0.stats.nfiles}'''.format(
         if self.show_progress:
             self.stats.show_progress(item=item, dt=0.2)
         self.items_buffer.add(item)
-        if time.time() - self.last_checkpoint > self.checkpoint_interval:
+        if time.monotonic() - self.last_checkpoint > self.checkpoint_interval:
             self.write_checkpoint()
-            self.last_checkpoint = time.time()
+            self.last_checkpoint = time.monotonic()
 
     def write_checkpoint(self):
         self.save(self.checkpoint_name)
@@ -301,14 +304,17 @@ Number of files: {0.stats.nfiles}'''.format(
         if name in self.manifest.archives:
             raise self.AlreadyExists(name)
         self.items_buffer.flush(flush=True)
+        duration = timedelta(seconds=time.monotonic() - self.start_monotonic)
         if timestamp is None:
             self.end = datetime.utcnow()
+            self.start = self.end - duration
             start = self.start
             end = self.end
         else:
             self.end = timestamp
-            start = timestamp
-            end = timestamp  # we only have 1 value
+            self.start = timestamp - duration
+            end = timestamp
+            start = self.start
         metadata = StableDict({
             'version': 1,
             'name': name,

borg/archiver.py

@@ -5,7 +5,6 @@ from operator import attrgetter
 import argparse
 import functools
 import inspect
-import io
 import os
 import re
 import shlex
@@ -13,6 +12,7 @@ import signal
 import stat
 import sys
 import textwrap
+import time
 import traceback
 import collections
 
@@ -263,7 +263,6 @@ class Archiver:
                 if args.progress:
                     archive.stats.show_progress(final=True)
                 if args.stats:
-                    archive.end = datetime.utcnow()
                     log_multi(DASHES,
                               str(archive),
                               DASHES,
@@ -276,6 +275,7 @@ class Archiver:
         self.ignore_inode = args.ignore_inode
         dry_run = args.dry_run
         t0 = datetime.utcnow()
+        t0_monotonic = time.monotonic()
         if not dry_run:
             key.compressor = Compressor(**args.compression)
             with Cache(repository, key, manifest, do_files=args.cache_files, lock_wait=self.lock_wait) as cache:
@@ -283,7 +283,7 @@ class Archiver:
                                   create=True, checkpoint_interval=args.checkpoint_interval,
                                   numeric_owner=args.numeric_owner, noatime=args.noatime, noctime=args.noctime,
                                   progress=args.progress,
-                                  chunker_params=args.chunker_params, start=t0)
+                                  chunker_params=args.chunker_params, start=t0, start_monotonic=t0_monotonic)
                 create_inner(archive, cache)
         else:
             create_inner(None, None)

borg/helpers.py

@@ -219,7 +219,7 @@ class Statistics:
         return format_file_size(self.csize)
 
     def show_progress(self, item=None, final=False, stream=None, dt=None):
-        now = time.time()
+        now = time.monotonic()
         if dt is None or now - self.last_progress > dt:
             self.last_progress = now
             columns, lines = get_terminal_size()

docs/changes.rst

@@ -82,6 +82,9 @@ Bug fixes:
   - skip corrupted chunks during manifest rebuild
 - fix TypeError in integrity error handler, #1903, #1894
 - fix location parser for archives with @ char (regression introduced in 1.0.8), #1930
+- fix wrong duration/timestamps if system clock jumped during a create
+- fix progress display not updating if system clock jumps backwards
+- fix checkpoint interval being incorrect if system clock jumps
 
 Other changes: