From ade08ce842b355025388c6dd884ffc052e20a795 Mon Sep 17 00:00:00 2001
From: Thomas Waldmann <tw@waldmann-edv.de>
Date: Thu, 11 Aug 2022 22:35:18 +0200
Subject: [PATCH] use timezones

- timezone aware timestamps
- str representation with +HHMM or +HH:MM
- get rid of to_locatime
- fix with_timestamp
- have archive start/end time always in local time with tz or as given
- idea: do not lose tz information

then we know when a backup was made and even from
which timezone it was made. if we want to compute
utc, we can do that using these infos.

this makes a quite nice archives list, with timestamps
as expected (in local time with timezone info).

at some places we just enforce utc, like for the
repo manifest timestamp or for the transaction log,
these are usually not looked at by the user.
---
 src/borg/archive.py             | 20 ++++++++++----------
 src/borg/archiver/create.py     |  6 +++---
 src/borg/archiver/help.py       |  2 +-
 src/borg/archiver/recreate.py   |  4 ++--
 src/borg/archiver/tar.py        |  6 +++---
 src/borg/helpers/__init__.py    |  2 +-
 src/borg/helpers/manifest.py    | 15 ++++++++-------
 src/borg/helpers/misc.py        |  4 ++--
 src/borg/helpers/parseformat.py | 14 +++++++++-----
 src/borg/helpers/time.py        | 27 ++++-----------------------
 src/borg/repository.py          |  4 ++--
 src/borg/testsuite/archiver.py  | 15 ++++++++-------
 12 files changed, 53 insertions(+), 66 deletions(-)

diff --git a/src/borg/archive.py b/src/borg/archive.py
index bed8b7012..e9fd05d12 100644
--- a/src/borg/archive.py
+++ b/src/borg/archive.py
@@ -6,7 +6,7 @@ import sys
 import time
 from collections import OrderedDict
 from contextlib import contextmanager
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, timedelta
 from functools import partial
 from getpass import getuser
 from io import BytesIO
@@ -479,13 +479,13 @@ class Archive:
             start_monotonic is None
         ), "Logic error: if start is given, start_monotonic must be given as well and vice versa."
         if start is None:
-            start = datetime.utcnow()
+            start = datetime.now().astimezone()  # local time with local timezone
             start_monotonic = time.monotonic()
         self.chunker_params = chunker_params
         self.start = start
         self.start_monotonic = start_monotonic
         if end is None:
-            end = datetime.utcnow()
+            end = datetime.now().astimezone()  # local time with local timezone
         self.end = end
         self.consider_part_files = consider_part_files
         self.pipeline = DownloadPipeline(self.repository, self.key)
@@ -549,8 +549,8 @@ class Archive:
     def info(self):
         if self.create:
             stats = self.stats
-            start = self.start.replace(tzinfo=timezone.utc)
-            end = self.end.replace(tzinfo=timezone.utc)
+            start = self.start
+            end = self.end
         else:
             stats = self.calc_stats(self.cache)
             start = self.ts
@@ -587,8 +587,8 @@ Time (end):   {end}
 Duration: {0.duration}
 """.format(
             self,
-            start=OutputTimestamp(self.start.replace(tzinfo=timezone.utc)),
-            end=OutputTimestamp(self.end.replace(tzinfo=timezone.utc)),
+            start=OutputTimestamp(self.start),
+            end=OutputTimestamp(self.end),
             location=self.repository._location.canonical_path(),
         )
 
@@ -629,11 +629,11 @@ Duration: {0.duration}
         item_ptrs = archive_put_items(self.items_buffer.chunks, key=self.key, cache=self.cache, stats=self.stats)
         duration = timedelta(seconds=time.monotonic() - self.start_monotonic)
         if timestamp is None:
-            end = datetime.utcnow()
+            end = datetime.now().astimezone()  # local time with local timezone
             start = end - duration
         else:
-            end = timestamp + duration
             start = timestamp
+            end = start + duration
         self.start = start
         self.end = end
         metadata = {
@@ -2314,7 +2314,7 @@ class ArchiveRecreater:
             target.rename(archive.name)
         if self.stats:
             target.start = _start
-            target.end = datetime.utcnow()
+            target.end = datetime.now().astimezone()  # local time with local timezone
             log_multi(str(target), str(target.stats))
 
     def matcher_add_tagged_dirs(self, archive):
diff --git a/src/borg/archiver/create.py b/src/borg/archiver/create.py
index 42b9d52bd..0030fabce 100644
--- a/src/borg/archiver/create.py
+++ b/src/borg/archiver/create.py
@@ -204,7 +204,7 @@ class CreateMixIn:
         self.noxattrs = args.noxattrs
         self.exclude_nodump = args.exclude_nodump
         dry_run = args.dry_run
-        t0 = datetime.utcnow()
+        t0 = datetime.now().astimezone()  # local time with local timezone
         t0_monotonic = time.monotonic()
         logger.info('Creating archive at "%s"' % args.location.processed)
         if not dry_run:
@@ -821,8 +821,8 @@ class CreateMixIn:
             dest="timestamp",
             type=timestamp,
             default=None,
-            help="manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). "
-            "Alternatively, give a reference file/directory.",
+            help="manually specify the archive creation date/time (yyyy-mm-ddThh:mm:ss[(+|-)HH:MM] format, "
+            "(+|-)HH:MM is the UTC offset, default: +00:00). Alternatively, give a reference file/directory.",
         )
         archive_group.add_argument(
             "-c",
diff --git a/src/borg/archiver/help.py b/src/borg/archiver/help.py
index 4b5de0650..899cda3c6 100644
--- a/src/borg/archiver/help.py
+++ b/src/borg/archiver/help.py
@@ -291,7 +291,7 @@ class HelpMixIn:
         Examples::
 
             borg create /path/to/repo::{hostname}-{user}-{utcnow} ...
-            borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ...
+            borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S%z} ...
             borg prune -a '{hostname}-*' ...
 
         .. note::
diff --git a/src/borg/archiver/recreate.py b/src/borg/archiver/recreate.py
index 3a0a64dbd..2b10366ca 100644
--- a/src/borg/archiver/recreate.py
+++ b/src/borg/archiver/recreate.py
@@ -177,8 +177,8 @@ class RecreateMixIn:
             dest="timestamp",
             type=timestamp,
             default=None,
-            help="manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). "
-            "alternatively, give a reference file/directory.",
+            help="manually specify the archive creation date/time (yyyy-mm-ddThh:mm:ss[(+|-)HH:MM] format, "
+            "(+|-)HH:MM is the UTC offset, default: +00:00). Alternatively, give a reference file/directory.",
         )
         archive_group.add_argument(
             "-C",
diff --git a/src/borg/archiver/tar.py b/src/borg/archiver/tar.py
index be35f58b6..f9ecb91a9 100644
--- a/src/borg/archiver/tar.py
+++ b/src/borg/archiver/tar.py
@@ -238,7 +238,7 @@ class TarMixIn:
         return self.exit_code
 
     def _import_tar(self, args, repository, manifest, key, cache, tarstream):
-        t0 = datetime.utcnow()
+        t0 = datetime.now().astimezone()  # local time with local timezone
         t0_monotonic = time.monotonic()
 
         archive = Archive(
@@ -485,8 +485,8 @@ class TarMixIn:
             type=timestamp,
             default=None,
             metavar="TIMESTAMP",
-            help="manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). "
-            "alternatively, give a reference file/directory.",
+            help="manually specify the archive creation date/time (yyyy-mm-ddThh:mm:ss[(+|-)HH:MM] format, "
+            "(+|-)HH:MM is the UTC offset, default: +00:00). Alternatively, give a reference file/directory.",
         )
         archive_group.add_argument(
             "-c",
diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py
index cea471506..b694d5bf4 100644
--- a/src/borg/helpers/__init__.py
+++ b/src/borg/helpers/__init__.py
@@ -36,7 +36,7 @@ from .process import signal_handler, raising_signal_handler, sig_int, ignore_sig
 from .process import popen_with_error_handling, is_terminal, prepare_subprocess_env, create_filter_process
 from .progress import ProgressIndicatorPercent, ProgressIndicatorEndless, ProgressIndicatorMessage
 from .time import parse_timestamp, timestamp, safe_timestamp, safe_s, safe_ns, MAX_S, SUPPORT_32BIT_PLATFORMS
-from .time import format_time, format_timedelta, isoformat_time, to_localtime, OutputTimestamp
+from .time import format_time, format_timedelta, OutputTimestamp
 from .yes_no import yes, TRUISH, FALSISH, DEFAULTISH
 
 from .msgpack import is_slow_msgpack, is_supported_msgpack, get_limited_unpacker
diff --git a/src/borg/helpers/manifest.py b/src/borg/helpers/manifest.py
index c28c1ee73..704902136 100644
--- a/src/borg/helpers/manifest.py
+++ b/src/borg/helpers/manifest.py
@@ -3,7 +3,7 @@ import os
 import os.path
 import re
 from collections import abc, namedtuple
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from operator import attrgetter
 from typing import Sequence, FrozenSet
 
@@ -65,7 +65,7 @@ class Archives(abc.MutableMapping):
         id, ts = info
         assert isinstance(id, bytes)
         if isinstance(ts, datetime):
-            ts = ts.replace(tzinfo=None).isoformat(timespec="microseconds")
+            ts = ts.isoformat(timespec="microseconds")
         assert isinstance(ts, str)
         self._archives[name] = {"id": id, "time": ts}
 
@@ -180,7 +180,7 @@ class Manifest:
 
     @property
     def last_timestamp(self):
-        return parse_timestamp(self.timestamp, tzinfo=None)
+        return parse_timestamp(self.timestamp)
 
     @classmethod
     def load(cls, repository, operations, key=None, force_tam_not_required=False):
@@ -254,11 +254,12 @@ class Manifest:
             self.config["tam_required"] = True
         # self.timestamp needs to be strictly monotonically increasing. Clocks often are not set correctly
         if self.timestamp is None:
-            self.timestamp = datetime.utcnow().isoformat(timespec="microseconds")
+            self.timestamp = datetime.now(tz=timezone.utc).isoformat(timespec="microseconds")
         else:
-            prev_ts = self.last_timestamp
-            incremented = (prev_ts + timedelta(microseconds=1)).isoformat(timespec="microseconds")
-            self.timestamp = max(incremented, datetime.utcnow().isoformat(timespec="microseconds"))
+            incremented_ts = self.last_timestamp + timedelta(microseconds=1)
+            now_ts = datetime.now(tz=timezone.utc)
+            max_ts = max(incremented_ts, now_ts)
+            self.timestamp = max_ts.isoformat(timespec="microseconds")
         # include checks for limits as enforced by limited unpacker (used by load())
         assert len(self.archives) <= MAX_ARCHIVES
         assert all(len(name) <= 255 for name in self.archives)
diff --git a/src/borg/helpers/misc.py b/src/borg/helpers/misc.py
index 3143540a4..60760598a 100644
--- a/src/borg/helpers/misc.py
+++ b/src/borg/helpers/misc.py
@@ -13,7 +13,6 @@ from ..logger import create_logger
 
 logger = create_logger()
 
-from .time import to_localtime
 from . import msgpack
 from .. import __version__ as borg_version
 from .. import chunker
@@ -55,7 +54,8 @@ def prune_split(archives, rule, n, kept_because=None):
 
     a = None
     for a in sorted(archives, key=attrgetter("ts"), reverse=True):
-        period = to_localtime(a.ts).strftime(pattern)
+        # we compute the pruning in local time zone
+        period = a.ts.astimezone().strftime(pattern)
         if period != last:
             last = period
             if a.id not in kept_because:
diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py
index a561e90cd..5f40cbe26 100644
--- a/src/borg/helpers/parseformat.py
+++ b/src/borg/helpers/parseformat.py
@@ -21,7 +21,7 @@ logger = create_logger()
 from .errors import Error
 from .fs import get_keys_dir
 from .msgpack import Timestamp
-from .time import OutputTimestamp, format_time, to_localtime, safe_timestamp
+from .time import OutputTimestamp, format_time, safe_timestamp
 from .. import __version__ as borg_version
 from .. import __version_tuple__ as borg_version_tuple
 from ..constants import *  # NOQA
@@ -196,7 +196,7 @@ def replace_placeholders(text, overrides={}):
         "fqdn": fqdn,
         "reverse-fqdn": ".".join(reversed(fqdn.split("."))),
         "hostname": hostname,
-        "now": DatetimeWrapper(current_time.astimezone(None)),
+        "now": DatetimeWrapper(current_time.astimezone()),
         "utcnow": DatetimeWrapper(current_time),
         "user": getosusername(),
         "uuid4": str(uuid.uuid4()),
@@ -303,7 +303,7 @@ def sizeof_fmt_decimal(num, suffix="B", sep="", precision=2, sign=False):
 
 
 def format_archive(archive):
-    return "%-36s %s [%s]" % (archive.name, format_time(to_localtime(archive.ts)), bin_to_hex(archive.id))
+    return "%-36s %s [%s]" % (archive.name, format_time(archive.ts), bin_to_hex(archive.id))
 
 
 def parse_stringified_list(s):
@@ -500,9 +500,13 @@ class Location:
             )
 
     def with_timestamp(self, timestamp):
+        # note: this only affects the repository URL/path, not the archive name!
         return Location(
             self.raw,
-            overrides={"now": DatetimeWrapper(timestamp.astimezone(None)), "utcnow": DatetimeWrapper(timestamp)},
+            overrides={
+                "now": DatetimeWrapper(timestamp),
+                "utcnow": DatetimeWrapper(timestamp.astimezone(timezone.utc)),
+            },
         )
 
 
@@ -973,7 +977,7 @@ def basic_json_data(manifest, *, cache=None, extra=None):
     key = manifest.key
     data = extra or {}
     data.update({"repository": BorgJsonEncoder().default(manifest.repository), "encryption": {"mode": key.ARG_NAME}})
-    data["repository"]["last_modified"] = OutputTimestamp(manifest.last_timestamp.replace(tzinfo=timezone.utc))
+    data["repository"]["last_modified"] = OutputTimestamp(manifest.last_timestamp)
     if key.NAME.startswith("key file"):
         data["encryption"]["keyfile"] = key.find_key()
     if cache:
diff --git a/src/borg/helpers/time.py b/src/borg/helpers/time.py
index 0b18de14a..ac1506f30 100644
--- a/src/borg/helpers/time.py
+++ b/src/borg/helpers/time.py
@@ -1,17 +1,11 @@
 import os
-import time
 from datetime import datetime, timezone
 
 
-def to_localtime(ts):
-    """Convert datetime object from UTC to local time zone"""
-    return datetime(*time.localtime((ts - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds())[:6])
-
-
 def parse_timestamp(timestamp, tzinfo=timezone.utc):
     """Parse a ISO 8601 timestamp string"""
     dt = datetime.fromisoformat(timestamp)
-    if tzinfo is not None:
+    if dt.tzinfo is None:
         dt = dt.replace(tzinfo=tzinfo)
     return dt
 
@@ -24,10 +18,7 @@ def timestamp(s):
         return datetime.fromtimestamp(ts, tz=timezone.utc)
     except OSError:
         # didn't work, try parsing as a ISO timestamp. if no TZ is given, we assume UTC.
-        dt = datetime.fromisoformat(s)
-        if dt.tzinfo is None:
-            dt = dt.replace(tzinfo=timezone.utc)
-        return dt
+        return parse_timestamp(s)
 
 
 # Not too rarely, we get crappy timestamps from the fs, that overflow some computations.
@@ -84,15 +75,7 @@ def format_time(ts: datetime, format_spec=""):
     """
     Convert *ts* to a human-friendly format with textual weekday.
     """
-    return ts.strftime("%a, %Y-%m-%d %H:%M:%S" if format_spec == "" else format_spec)
-
-
-def isoformat_time(ts: datetime):
-    """
-    Format *ts* according to ISO 8601.
-    """
-    # note: first make all datetime objects tz aware before adding %z here.
-    return ts.isoformat(timespec="microseconds")
+    return ts.strftime("%a, %Y-%m-%d %H:%M:%S %z" if format_spec == "" else format_spec)
 
 
 def format_timedelta(td):
@@ -113,8 +96,6 @@ def format_timedelta(td):
 
 class OutputTimestamp:
     def __init__(self, ts: datetime):
-        if ts.tzinfo == timezone.utc:
-            ts = to_localtime(ts)
         self.ts = ts
 
     def __format__(self, format_spec):
@@ -124,6 +105,6 @@ class OutputTimestamp:
         return f"{self}"
 
     def isoformat(self):
-        return isoformat_time(self.ts)
+        return self.ts.isoformat(timespec="microseconds")
 
     to_json = isoformat
diff --git a/src/borg/repository.py b/src/borg/repository.py
index d55e2d9e6..6518c0174 100644
--- a/src/borg/repository.py
+++ b/src/borg/repository.py
@@ -8,7 +8,7 @@ import time
 from binascii import hexlify, unhexlify
 from collections import defaultdict
 from configparser import ConfigParser
-from datetime import datetime
+from datetime import datetime, timezone
 from functools import partial
 from itertools import islice
 
@@ -657,7 +657,7 @@ class Repository:
             with open(os.path.join(self.path, "transactions"), "a") as log:
                 print(
                     "transaction %d, UTC time %s"
-                    % (transaction_id, datetime.utcnow().isoformat(timespec="microseconds")),
+                    % (transaction_id, datetime.now(tz=timezone.utc).isoformat(timespec="microseconds")),
                     file=log,
                 )
 
diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py
index 33eb48122..dd8fa48e3 100644
--- a/src/borg/testsuite/archiver.py
+++ b/src/borg/testsuite/archiver.py
@@ -18,9 +18,7 @@ import time
 import unittest
 from binascii import unhexlify, b2a_base64, a2b_base64
 from configparser import ConfigParser
-from datetime import datetime
-from datetime import timezone
-from datetime import timedelta
+from datetime import datetime, timezone, timedelta
 from hashlib import sha256
 from io import BytesIO, StringIO
 from unittest.mock import patch
@@ -251,7 +249,7 @@ def test_disk_full(cmd):
 
 def checkts(ts):
     # check if the timestamp is in the expected format
-    assert datetime.strptime(ts, ISO_FORMAT)  # must not raise
+    assert datetime.strptime(ts, ISO_FORMAT + "%z")  # must not raise
 
 
 class ArchiverTestCaseBase(BaseTestCase):
@@ -2596,7 +2594,6 @@ class ArchiverTestCase(ArchiverTestCaseBase):
         file1 = items[1]
         assert file1["path"] == "input/file1"
         assert file1["size"] == 81920
-        checkts(file1["mtime"])
 
         list_archive = self.cmd(
             f"--repo={self.repository_location}", "list", "test", "--json-lines", "--format={sha256}"
@@ -4063,7 +4060,9 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
                             "version": 1,
                             "archives": {},
                             "config": {},
-                            "timestamp": (datetime.utcnow() + timedelta(days=1)).isoformat(timespec="microseconds"),
+                            "timestamp": (datetime.now(tz=timezone.utc) + timedelta(days=1)).isoformat(
+                                timespec="microseconds"
+                            ),
                         }
                     ),
                 ),
@@ -4083,7 +4082,9 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
                         {
                             "version": 1,
                             "archives": {},
-                            "timestamp": (datetime.utcnow() + timedelta(days=1)).isoformat(timespec="microseconds"),
+                            "timestamp": (datetime.now(tz=timezone.utc) + timedelta(days=1)).isoformat(
+                                timespec="microseconds"
+                            ),
                         }
                     ),
                 ),