upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-06-11 09:59:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

use timezones

Browse source 
- timezone aware timestamps
- str representation with +HHMM or +HH:MM
- get rid of to_locatime
- fix with_timestamp
- have archive start/end time always in local time with tz or as given
- idea: do not lose tz information

then we know when a backup was made and even from
which timezone it was made. if we want to compute
utc, we can do that using these infos.

this makes a quite nice archives list, with timestamps
as expected (in local time with timezone info).

at some places we just enforce utc, like for the
repo manifest timestamp or for the transaction log,
these are usually not looked at by the user.
This commit is contained in:
Thomas Waldmann 2022-08-11 22:35:18 +02:00
parent bab68a8d25
commit ade08ce842
12 changed files with 53 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
src/borg/archive.py
View file

@ -6,7 +6,7 @@ import sys
import time
from collections import OrderedDict
from contextlib import contextmanager
from datetime import datetime, timezone, timedelta
from datetime import datetime, timedelta
from functools import partial
from getpass import getuser
from io import BytesIO
@ -479,13 +479,13 @@ class Archive:
start_monotonic is None
), "Logic error: if start is given, start_monotonic must be given as well and vice versa."
if start is None:
start = datetime.utcnow()
start = datetime.now().astimezone() # local time with local timezone
start_monotonic = time.monotonic()
self.chunker_params = chunker_params
self.start = start
self.start_monotonic = start_monotonic
if end is None:
end = datetime.utcnow()
end = datetime.now().astimezone() # local time with local timezone
self.end = end
self.consider_part_files = consider_part_files
self.pipeline = DownloadPipeline(self.repository, self.key)
@ -549,8 +549,8 @@ class Archive:
def info(self):
if self.create:
stats = self.stats
start = self.start.replace(tzinfo=timezone.utc)
end = self.end.replace(tzinfo=timezone.utc)
start = self.start
end = self.end
else:
stats = self.calc_stats(self.cache)
start = self.ts
@ -587,8 +587,8 @@ Time (end): {end}
Duration: {0.duration}
""".format(
self,
start=OutputTimestamp(self.start.replace(tzinfo=timezone.utc)),
end=OutputTimestamp(self.end.replace(tzinfo=timezone.utc)),
start=OutputTimestamp(self.start),
end=OutputTimestamp(self.end),
location=self.repository._location.canonical_path(),
)
@ -629,11 +629,11 @@ Duration: {0.duration}
item_ptrs = archive_put_items(self.items_buffer.chunks, key=self.key, cache=self.cache, stats=self.stats)
duration = timedelta(seconds=time.monotonic() - self.start_monotonic)
if timestamp is None:
end = datetime.utcnow()
end = datetime.now().astimezone() # local time with local timezone
start = end - duration
else:
end = timestamp + duration
start = timestamp
end = start + duration
self.start = start
self.end = end
metadata = {
@ -2314,7 +2314,7 @@ class ArchiveRecreater:
target.rename(archive.name)
if self.stats:
target.start = _start
target.end = datetime.utcnow()
target.end = datetime.now().astimezone() # local time with local timezone
log_multi(str(target), str(target.stats))
def matcher_add_tagged_dirs(self, archive):

6
src/borg/archiver/create.py
View file

@ -204,7 +204,7 @@ class CreateMixIn:
self.noxattrs = args.noxattrs
self.exclude_nodump = args.exclude_nodump
dry_run = args.dry_run
t0 = datetime.utcnow()
t0 = datetime.now().astimezone() # local time with local timezone
t0_monotonic = time.monotonic()
logger.info('Creating archive at "%s"' % args.location.processed)
if not dry_run:
@ -821,8 +821,8 @@ class CreateMixIn:
dest="timestamp",
type=timestamp,
default=None,
help="manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). "
"Alternatively, give a reference file/directory.",
help="manually specify the archive creation date/time (yyyy-mm-ddThh:mm:ss[(+|-)HH:MM] format, "
"(+|-)HH:MM is the UTC offset, default: +00:00). Alternatively, give a reference file/directory.",
)
archive_group.add_argument(
"-c",

2
src/borg/archiver/help.py
View file

@ -291,7 +291,7 @@ class HelpMixIn:
Examples::
borg create /path/to/repo::{hostname}-{user}-{utcnow} ...
borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S} ...
borg create /path/to/repo::{hostname}-{now:%Y-%m-%d_%H:%M:%S%z} ...
borg prune -a '{hostname}-*' ...
.. note::

4
src/borg/archiver/recreate.py
View file

@ -177,8 +177,8 @@ class RecreateMixIn:
dest="timestamp",
type=timestamp,
default=None,
help="manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). "
"alternatively, give a reference file/directory.",
help="manually specify the archive creation date/time (yyyy-mm-ddThh:mm:ss[(+|-)HH:MM] format, "
"(+|-)HH:MM is the UTC offset, default: +00:00). Alternatively, give a reference file/directory.",
)
archive_group.add_argument(
"-C",

6
src/borg/archiver/tar.py
View file

@ -238,7 +238,7 @@ class TarMixIn:
return self.exit_code
def _import_tar(self, args, repository, manifest, key, cache, tarstream):
t0 = datetime.utcnow()
t0 = datetime.now().astimezone() # local time with local timezone
t0_monotonic = time.monotonic()
archive = Archive(
@ -485,8 +485,8 @@ class TarMixIn:
type=timestamp,
default=None,
metavar="TIMESTAMP",
help="manually specify the archive creation date/time (UTC, yyyy-mm-ddThh:mm:ss format). "
"alternatively, give a reference file/directory.",
help="manually specify the archive creation date/time (yyyy-mm-ddThh:mm:ss[(+|-)HH:MM] format, "
"(+|-)HH:MM is the UTC offset, default: +00:00). Alternatively, give a reference file/directory.",
)
archive_group.add_argument(
"-c",

2
src/borg/helpers/__init__.py
View file

@ -36,7 +36,7 @@ from .process import signal_handler, raising_signal_handler, sig_int, ignore_sig
from .process import popen_with_error_handling, is_terminal, prepare_subprocess_env, create_filter_process
from .progress import ProgressIndicatorPercent, ProgressIndicatorEndless, ProgressIndicatorMessage
from .time import parse_timestamp, timestamp, safe_timestamp, safe_s, safe_ns, MAX_S, SUPPORT_32BIT_PLATFORMS
from .time import format_time, format_timedelta, isoformat_time, to_localtime, OutputTimestamp
from .time import format_time, format_timedelta, OutputTimestamp
from .yes_no import yes, TRUISH, FALSISH, DEFAULTISH
from .msgpack import is_slow_msgpack, is_supported_msgpack, get_limited_unpacker

15
src/borg/helpers/manifest.py
View file

@ -3,7 +3,7 @@ import os
import os.path
import re
from collections import abc, namedtuple
from datetime import datetime, timedelta
from datetime import datetime, timedelta, timezone
from operator import attrgetter
from typing import Sequence, FrozenSet
@ -65,7 +65,7 @@ class Archives(abc.MutableMapping):
id, ts = info
assert isinstance(id, bytes)
if isinstance(ts, datetime):
ts = ts.replace(tzinfo=None).isoformat(timespec="microseconds")
ts = ts.isoformat(timespec="microseconds")
assert isinstance(ts, str)
self._archives[name] = {"id": id, "time": ts}
@ -180,7 +180,7 @@ class Manifest:
@property
def last_timestamp(self):
return parse_timestamp(self.timestamp, tzinfo=None)
return parse_timestamp(self.timestamp)
@classmethod
def load(cls, repository, operations, key=None, force_tam_not_required=False):
@ -254,11 +254,12 @@ class Manifest:
self.config["tam_required"] = True
# self.timestamp needs to be strictly monotonically increasing. Clocks often are not set correctly
if self.timestamp is None:
self.timestamp = datetime.utcnow().isoformat(timespec="microseconds")
self.timestamp = datetime.now(tz=timezone.utc).isoformat(timespec="microseconds")
else:
prev_ts = self.last_timestamp
incremented = (prev_ts + timedelta(microseconds=1)).isoformat(timespec="microseconds")
self.timestamp = max(incremented, datetime.utcnow().isoformat(timespec="microseconds"))
incremented_ts = self.last_timestamp + timedelta(microseconds=1)
now_ts = datetime.now(tz=timezone.utc)
max_ts = max(incremented_ts, now_ts)
self.timestamp = max_ts.isoformat(timespec="microseconds")
# include checks for limits as enforced by limited unpacker (used by load())
assert len(self.archives) <= MAX_ARCHIVES
assert all(len(name) <= 255 for name in self.archives)

4
src/borg/helpers/misc.py
View file

@ -13,7 +13,6 @@ from ..logger import create_logger
logger = create_logger()
from .time import to_localtime
from . import msgpack
from .. import __version__ as borg_version
from .. import chunker
@ -55,7 +54,8 @@ def prune_split(archives, rule, n, kept_because=None):
a = None
for a in sorted(archives, key=attrgetter("ts"), reverse=True):
period = to_localtime(a.ts).strftime(pattern)
# we compute the pruning in local time zone
period = a.ts.astimezone().strftime(pattern)
if period != last:
last = period
if a.id not in kept_because:

14
src/borg/helpers/parseformat.py
View file

@ -21,7 +21,7 @@ logger = create_logger()
from .errors import Error
from .fs import get_keys_dir
from .msgpack import Timestamp
from .time import OutputTimestamp, format_time, to_localtime, safe_timestamp
from .time import OutputTimestamp, format_time, safe_timestamp
from .. import __version__ as borg_version
from .. import __version_tuple__ as borg_version_tuple
from ..constants import * # NOQA
@ -196,7 +196,7 @@ def replace_placeholders(text, overrides={}):
"fqdn": fqdn,
"reverse-fqdn": ".".join(reversed(fqdn.split("."))),
"hostname": hostname,
"now": DatetimeWrapper(current_time.astimezone(None)),
"now": DatetimeWrapper(current_time.astimezone()),
"utcnow": DatetimeWrapper(current_time),
"user": getosusername(),
"uuid4": str(uuid.uuid4()),
@ -303,7 +303,7 @@ def sizeof_fmt_decimal(num, suffix="B", sep="", precision=2, sign=False):
def format_archive(archive):
return "%-36s %s [%s]" % (archive.name, format_time(to_localtime(archive.ts)), bin_to_hex(archive.id))
return "%-36s %s [%s]" % (archive.name, format_time(archive.ts), bin_to_hex(archive.id))
def parse_stringified_list(s):
@ -500,9 +500,13 @@ class Location:
)
def with_timestamp(self, timestamp):
# note: this only affects the repository URL/path, not the archive name!
return Location(
self.raw,
overrides={"now": DatetimeWrapper(timestamp.astimezone(None)), "utcnow": DatetimeWrapper(timestamp)},
overrides={
"now": DatetimeWrapper(timestamp),
"utcnow": DatetimeWrapper(timestamp.astimezone(timezone.utc)),
},
)
@ -973,7 +977,7 @@ def basic_json_data(manifest, *, cache=None, extra=None):
key = manifest.key
data = extra or {}
data.update({"repository": BorgJsonEncoder().default(manifest.repository), "encryption": {"mode": key.ARG_NAME}})
data["repository"]["last_modified"] = OutputTimestamp(manifest.last_timestamp.replace(tzinfo=timezone.utc))
data["repository"]["last_modified"] = OutputTimestamp(manifest.last_timestamp)
if key.NAME.startswith("key file"):
data["encryption"]["keyfile"] = key.find_key()
if cache:

27
src/borg/helpers/time.py
View file

@ -1,17 +1,11 @@
import os
import time
from datetime import datetime, timezone
def to_localtime(ts):
"""Convert datetime object from UTC to local time zone"""
return datetime(*time.localtime((ts - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds())[:6])
def parse_timestamp(timestamp, tzinfo=timezone.utc):
"""Parse a ISO 8601 timestamp string"""
dt = datetime.fromisoformat(timestamp)
if tzinfo is not None:
if dt.tzinfo is None:
dt = dt.replace(tzinfo=tzinfo)
return dt
@ -24,10 +18,7 @@ def timestamp(s):
return datetime.fromtimestamp(ts, tz=timezone.utc)
except OSError:
# didn't work, try parsing as a ISO timestamp. if no TZ is given, we assume UTC.
dt = datetime.fromisoformat(s)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt
return parse_timestamp(s)
# Not too rarely, we get crappy timestamps from the fs, that overflow some computations.
@ -84,15 +75,7 @@ def format_time(ts: datetime, format_spec=""):
"""
Convert *ts* to a human-friendly format with textual weekday.
"""
return ts.strftime("%a, %Y-%m-%d %H:%M:%S" if format_spec == "" else format_spec)
def isoformat_time(ts: datetime):
"""
Format *ts* according to ISO 8601.
"""
# note: first make all datetime objects tz aware before adding %z here.
return ts.isoformat(timespec="microseconds")
return ts.strftime("%a, %Y-%m-%d %H:%M:%S %z" if format_spec == "" else format_spec)
def format_timedelta(td):
@ -113,8 +96,6 @@ def format_timedelta(td):
class OutputTimestamp:
def __init__(self, ts: datetime):
if ts.tzinfo == timezone.utc:
ts = to_localtime(ts)
self.ts = ts
def __format__(self, format_spec):
@ -124,6 +105,6 @@ class OutputTimestamp:
return f"{self}"
def isoformat(self):
return isoformat_time(self.ts)
return self.ts.isoformat(timespec="microseconds")
to_json = isoformat

4
src/borg/repository.py
View file

@ -8,7 +8,7 @@ import time
from binascii import hexlify, unhexlify
from collections import defaultdict
from configparser import ConfigParser
from datetime import datetime
from datetime import datetime, timezone
from functools import partial
from itertools import islice
@ -657,7 +657,7 @@ class Repository:
with open(os.path.join(self.path, "transactions"), "a") as log:
print(
"transaction %d, UTC time %s"
% (transaction_id, datetime.utcnow().isoformat(timespec="microseconds")),
% (transaction_id, datetime.now(tz=timezone.utc).isoformat(timespec="microseconds")),
file=log,
)

15
src/borg/testsuite/archiver.py
View file

@ -18,9 +18,7 @@ import time
import unittest
from binascii import unhexlify, b2a_base64, a2b_base64
from configparser import ConfigParser
from datetime import datetime
from datetime import timezone
from datetime import timedelta
from datetime import datetime, timezone, timedelta
from hashlib import sha256
from io import BytesIO, StringIO
from unittest.mock import patch
@ -251,7 +249,7 @@ def test_disk_full(cmd):
def checkts(ts):
# check if the timestamp is in the expected format
assert datetime.strptime(ts, ISO_FORMAT) # must not raise
assert datetime.strptime(ts, ISO_FORMAT + "%z") # must not raise
class ArchiverTestCaseBase(BaseTestCase):
@ -2596,7 +2594,6 @@ class ArchiverTestCase(ArchiverTestCaseBase):
file1 = items[1]
assert file1["path"] == "input/file1"
assert file1["size"] == 81920
checkts(file1["mtime"])
list_archive = self.cmd(
f"--repo={self.repository_location}", "list", "test", "--json-lines", "--format={sha256}"
@ -4063,7 +4060,9 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
"version": 1,
"archives": {},
"config": {},
"timestamp": (datetime.utcnow() + timedelta(days=1)).isoformat(timespec="microseconds"),
"timestamp": (datetime.now(tz=timezone.utc) + timedelta(days=1)).isoformat(
timespec="microseconds"
),
}
),
),
@ -4083,7 +4082,9 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase):
{
"version": 1,
"archives": {},
"timestamp": (datetime.utcnow() + timedelta(days=1)).isoformat(timespec="microseconds"),
"timestamp": (datetime.now(tz=timezone.utc) + timedelta(days=1)).isoformat(
timespec="microseconds"
),
}
),
),