upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-05-28 04:03:21 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

archive: use 3 timestamps, fixes #9400

Browse source 
time: the nominal ts, used for prune, list, sorting, ...
start: operation start time (informative)
end: operation end time (informative)

Often, "time" is the same as "start" (normal borg create).

But it can make sense to have a different "time":
- borg create --timestamp=...
- borg recreate --timestamp=...
- borg recreate (will keep "time" as in original archive)
- borg transfer (will keep "time" as in original archive)

recreate and transfer produce new archives, "start" and "end"
will reflect the recreate/transfer operation.

Also: remove start_monotonic. start and end are just what the
clock shows (including tz), so should be ok to compute duration
from that, even for dst switching times.
This commit is contained in:
Thomas Waldmann 2026-03-03 19:21:42 +01:00
parent 57af68e1e1
commit 9b633cdb68
No known key found for this signature in database
GPG key ID: 243ACFA951F78E01
11 changed files with 99 additions and 78 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

88
src/borg/archive.py
View file

@ -500,8 +500,8 @@ class Archive:
noxattrs=False,
progress=False,
chunker_params=CHUNKER_PARAMS,
timestamp=None,
start=None,
start_monotonic=None,
end=None,
log_json=False,
iec=False,
@ -530,18 +530,10 @@ class Archive:
self.noflags = noflags
self.noacls = noacls
self.noxattrs = noxattrs
assert (start is None) == (
start_monotonic is None
), "Logic error: if start is given, start_monotonic must be given as well and vice versa."
if start is None:
start = archive_ts_now()
start_monotonic = time.monotonic()
self.chunker_params = chunker_params
self.start = start
self.start_monotonic = start_monotonic
if end is None:
end = archive_ts_now()
self.end = end
self.start = start if start is not None else archive_ts_now()
self.end = end if end is not None else self.start
self.timestamp = timestamp if timestamp is not None else self.start
self.pipeline = DownloadPipeline(self.repository, self.repo_objs)
self.create = create
if self.create:
@ -578,15 +570,22 @@ class Archive:
@property
def ts(self):
"""Timestamp of archive creation (start) in UTC"""
"""Nominal archive timestamp in UTC."""
ts = self.metadata.time
return parse_timestamp(ts)
@property
def ts_start(self):
"""Timestamp of archive operation start in UTC."""
# fall back to "time" in case "start" is not found
ts = self.metadata.get("start") or self.metadata.time
return parse_timestamp(ts)
@property
def ts_end(self):
"""Timestamp of archive creation (end) in UTC"""
# fall back to time if there is no time_end present in metadata
ts = self.metadata.get("time_end") or self.metadata.time
"""Timestamp of archive operation end in UTC."""
# fall back to "time" in case "end" or "time_end" are not found
ts = self.metadata.get("end") or self.metadata.get("time_end") or self.metadata.time
return parse_timestamp(ts)
@property
@ -604,15 +603,18 @@ class Archive:
def info(self):
if self.create:
stats = self.stats
ts = self.timestamp
start = self.start
end = self.end
else:
stats = self.calc_stats(self.cache)
start = self.ts
ts = self.ts
start = self.ts_start
end = self.ts_end
info = {
"name": self.name,
"id": self.fpr,
"time": OutputTimestamp(ts),
"start": OutputTimestamp(start),
"end": OutputTimestamp(end),
"duration": (end - start).total_seconds(),
@ -637,11 +639,13 @@ class Archive:
Repository: {location}
Archive name: {0.name}
Archive fingerprint: {0.fpr}
Time (start): {start}
Time (end): {end}
Time (nominal): {time}
Time (start): {start}
Time (end): {end}
Duration: {0.duration}
""".format(
self,
time=OutputTimestamp(self.timestamp),
start=OutputTimestamp(self.start),
end=OutputTimestamp(self.end),
location=self.repository._location.canonical_path(),
@ -678,15 +682,10 @@ Duration: {0.duration}
item_ptrs = archive_put_items(
self.items_buffer.chunks, repo_objs=self.repo_objs, cache=self.cache, stats=self.stats
) # this adds the sizes of the item ptrs chunks to stats.osize
duration = timedelta(seconds=time.monotonic() - self.start_monotonic)
if timestamp is None:
end = archive_ts_now()
start = end - duration
else:
start = timestamp
end = start + duration
self.start = start
self.end = end
start = self.start
end = self.end = archive_ts_now()
nominal = start if timestamp is None else timestamp
self.timestamp = nominal
metadata = {
"version": 2,
"name": name,
@ -697,8 +696,9 @@ Duration: {0.duration}
"cwd": self.cwd,
"hostname": hostname,
"username": getuser(),
"time": start.isoformat(timespec="microseconds"),
"time_end": end.isoformat(timespec="microseconds"),
"time": nominal.isoformat(timespec="microseconds"),
"start": start.isoformat(timespec="microseconds"),
"end": end.isoformat(timespec="microseconds"),
"chunker_params": self.chunker_params,
}
# we always want to create archives with the addtl. metadata (nfiles, etc.),
@ -2305,32 +2305,18 @@ class ArchiveRecreater:
return
if comment is None:
comment = archive.metadata.get("comment", "")
# Keep for the statistics if necessary
if self.stats:
_start = target.start
additional_metadata = {
"command_line": archive.metadata.command_line,
# but also remember recreate metadata:
"recreate_command_line": join_cmd(sys.argv),
}
if self.timestamp is None:
additional_metadata = {
"time": archive.metadata.time,
"time_end": archive.metadata.get("time_end") or archive.metadata.time,
"command_line": archive.metadata.command_line,
# but also remember recreate metadata:
"recreate_command_line": join_cmd(sys.argv),
}
else:
additional_metadata = {
"command_line": archive.metadata.command_line,
# but also remember recreate metadata:
"recreate_command_line": join_cmd(sys.argv),
}
# if no timestamp is specified, keep the original timestamp
additional_metadata["time"] = archive.metadata.time
target.save(comment=comment, timestamp=self.timestamp, additional_metadata=additional_metadata)
if delete_original:
archive.delete()
if self.stats:
target.start = _start
target.end = archive_ts_now()
log_multi(str(target), str(target.stats))
def matcher_add_tagged_dirs(self, archive):

2
src/borg/archiver/create_cmd.py
View file

@ -216,7 +216,6 @@ class CreateMixIn:
dry_run = args.dry_run
self.start_backup = time.time_ns()
t0 = archive_ts_now()
t0_monotonic = time.monotonic()
logger.info('Creating archive at "%s"' % args.location.processed)
if not dry_run:
with Cache(
@ -238,7 +237,6 @@ class CreateMixIn:
progress=args.progress,
chunker_params=args.chunker_params,
start=t0,
start_monotonic=t0_monotonic,
log_json=args.log_json,
iec=args.iec,
)

1
src/borg/archiver/info_cmd.py
View file

@ -42,6 +42,7 @@ class InfoMixIn:
Hostname: {hostname}
Username: {username}
Tags: {tags}
Time (nominal): {time}
Time (start): {start}
Time (end): {end}
Duration: {duration}

3
src/borg/archiver/tar_cmds.py
View file

@ -4,7 +4,6 @@ import logging
import os
import stat
import tarfile
import time
from ..archive import Archive, TarfileObjectProcessors, ChunksProcessor
from ..compress import CompressionSpec
@ -269,7 +268,6 @@ class TarMixIn:
def _import_tar(self, args, repository, manifest, key, cache, tarstream):
t0 = archive_ts_now()
t0_monotonic = time.monotonic()
archive = Archive(
manifest,
@ -279,7 +277,6 @@ class TarMixIn:
progress=args.progress,
chunker_params=args.chunker_params,
start=t0,
start_monotonic=t0_monotonic,
log_json=args.log_json,
)
cp = ChunksProcessor(cache=cache, key=key, add_item=archive.add_item, rechunkify=False)

5
src/borg/constants.py
View file

@ -11,7 +11,10 @@ REQUIRED_ITEM_KEYS = frozenset(["path", "mtime"])
# this set must be kept complete, otherwise rebuild_manifest might malfunction:
# fmt: off
ARCHIVE_KEYS = frozenset(['version', 'name', 'hostname', 'username', 'time', 'time_end',
ARCHIVE_KEYS = frozenset(['version', 'name', 'hostname', 'username',
'time', # v2+ archives AND borg 1.x archives
'time_end', # only legacy borg 1.x
'start', 'end', # v2+ archives
'tags', # v2+ archives
'items', # legacy v1 archives
'item_ptrs', # v2+ archives

14
src/borg/helpers/parseformat.py
View file

@ -765,10 +765,9 @@ class ArchiveFormatter(BaseFormatter):
"name": 'alias of "archive"',
"comment": "archive comment",
"tags": "archive tags",
# *start* is the key used by borg-info for this timestamp, this makes the formats more compatible
"start": "time (start) of creation of the archive",
"time": 'alias of "start"',
"end": "time (end) of creation of the archive",
"time": "nominal time of the archive",
"start": "start time of the archive operation",
"end": "end time of the archive operation",
"command_line": "command line which was used to create the archive",
"id": "internal ID of the archive",
"hostname": "hostname of host on which this archive was created",
@ -778,7 +777,7 @@ class ArchiveFormatter(BaseFormatter):
}
KEY_GROUPS = (
("archive", "name", "comment", "id", "tags"),
("start", "time", "end", "command_line"),
("time", "start", "end", "command_line"),
("hostname", "username"),
("size", "nfiles"),
)
@ -802,6 +801,7 @@ class ArchiveFormatter(BaseFormatter):
"command_line": partial(self.get_meta, "command_line", ""),
"size": partial(self.get_meta, "size", 0),
"nfiles": partial(self.get_meta, "nfiles", 0),
"start": self.get_ts_start,
"end": self.get_ts_end,
"tags": self.get_tags,
}
@ -817,7 +817,6 @@ class ArchiveFormatter(BaseFormatter):
"archive": archive_info.name,
"id": bin_to_hex(archive_info.id),
"time": self.format_time(archive_info.ts),
"start": self.format_time(archive_info.ts),
}
for key in self.used_call_keys:
item_data[key] = self.call_keys[key]()
@ -841,6 +840,9 @@ class ArchiveFormatter(BaseFormatter):
def get_meta(self, key, default=None):
return self.archive.metadata.get(key, default)
def get_ts_start(self):
return self.format_time(self.archive.ts_start)
def get_ts_end(self):
return self.format_time(self.archive.ts_end)

8
src/borg/item.pyi
View file

@ -25,6 +25,14 @@ class ArchiveItem(PropDict):
@name.setter
def name(self, val: str) -> None: ...
@property
def start(self) -> str: ...
@start.setter
def start(self, val: str) -> None: ...
@property
def end(self) -> str: ...
@end.setter
def end(self, val: str) -> None: ...
@property
def time(self) -> str: ...
@time.setter
def time(self, val: str) -> None: ...

8
src/borg/item.pyx
View file

@ -508,8 +508,10 @@ cdef class ArchiveItem(PropDict):
command_line = PropDictProperty(str, 'surrogate-escaped str')
hostname = PropDictProperty(str, 'surrogate-escaped str')
username = PropDictProperty(str, 'surrogate-escaped str')
time = PropDictProperty(str)
time_end = PropDictProperty(str)
start = PropDictProperty(str) # new in borg2 (was: time)
end = PropDictProperty(str) # new in borg2 (was: time_end)
time = PropDictProperty(str) # borg2: nominal archive time, borg 1.x: same + start time
time_end = PropDictProperty(str) # legacy borg 1.x (now: end)
comment = PropDictProperty(str, 'surrogate-escaped str')
tags = PropDictProperty(list) # list of s-e-str
chunker_params = PropDictProperty(tuple)
@ -533,7 +535,7 @@ cdef class ArchiveItem(PropDict):
assert isinstance(v, int)
if k in ('name', 'hostname', 'username', 'comment', 'cwd'):
v = fix_str_value(d, k)
if k in ('time', 'time_end'):
if k in ('start', 'end', 'time', 'time_end'):
v = fix_str_value(d, k, 'replace')
if k == 'chunker_params':
v = fix_tuple_of_str_and_int(v)

25
src/borg/testsuite/archiver/recreate_cmd_test.py
View file

@ -1,5 +1,6 @@
import os
import re
import time
from datetime import datetime
import pytest
@ -186,17 +187,37 @@ def test_recreate_no_rechunkify(archivers, request):
assert num_chunks == num_chunks_after_recreate
def test_recreate_timestamp(archivers, request):
def test_recreate_keep_original_timestamp(archivers, request):
archiver = request.getfixturevalue(archivers)
create_test_files(archiver.input_path)
cmd(archiver, "repo-create", RK_ENCRYPTION)
cmd(archiver, "create", "test0", "input")
info_orig = cmd(archiver, "info", "-a", "test0").splitlines()
# this shall recreate the archive and keep the nominal timestamp
time.sleep(1)
cmd(archiver, "recreate", "test0", "--comment", "test")
info_recreated = cmd(archiver, "info", "-a", "test0").splitlines()
nominal_orig = next(item for item in info_orig if item.startswith("Time (nominal):"))
nominal_recreated = next(item for item in info_recreated if item.startswith("Time (nominal):"))
assert nominal_orig == nominal_recreated
def test_recreate_with_given_timestamp(archivers, request):
archiver = request.getfixturevalue(archivers)
create_test_files(archiver.input_path)
cmd(archiver, "repo-create", RK_ENCRYPTION)
cmd(archiver, "create", "test0", "input")
# this shall recreate the archive with a different nominal timestamp
cmd(archiver, "recreate", "test0", "--timestamp", "1970-01-02T00:00:00", "--comment", "test")
info = cmd(archiver, "info", "-a", "test0").splitlines()
dtime = datetime(1970, 1, 2, 0, 0, 0).astimezone() # local time in local timezone
s_time = dtime.strftime("%Y-%m-%d %H:%M:.. %z").replace("+", r"\+")
assert any([re.search(r"Time \(start\).+ %s" % s_time, item) for item in info])
assert any([re.search(r"Time \(nominal\).+ %s" % s_time, item) for item in info])
# start/end time are just from the recreate operation
dtime = datetime.now().astimezone() # current local time
s_time = dtime.strftime("%Y-%m-%d %H:%M:.. %z").replace("+", r"\+")
assert any([re.search(r"Time \(end\).+ %s" % s_time, item) for item in info])
assert any([re.search(r"Time \(start\).+ %s" % s_time, item) for item in info])
def test_recreate_dry_run(archivers, request):

14
src/borg/testsuite/archiver/transfer_cmd_test.py
View file

@ -80,12 +80,14 @@ def test_transfer_upgrade(archivers, request, monkeypatch):
# timestamps:
# borg 1.2 transformed to local time and had microseconds = 0, no tzoffset
# borg 2 uses local time, with microseconds and with tzoffset
for key in "start", "time":
# fix expectation: local time meant +01:00, so we convert that to whatever local tz is here.
expected_archive[key] = convert_tz(expected_archive[key], repo12_tzoffset, None)
# set microseconds to 0, so we can compare got with expected.
got_ts = parse_timestamp(got_archive[key])
got_archive[key] = got_ts.replace(microsecond=0).isoformat(timespec="microseconds")
# the only important timestamp is "time", which has the nominal timestamp of the archive.
del expected_archive["start"]
key = "time"
# fix expectation: local time meant +01:00, so we convert that to whatever local tz is here.
expected_archive[key] = convert_tz(expected_archive[key], repo12_tzoffset, None)
# set microseconds to 0, so we can compare got with expected.
got_ts = parse_timestamp(got_archive[key])
got_archive[key] = got_ts.replace(microsecond=0).isoformat(timespec="microseconds")
assert got == expected
for archive in got["archives"]:

9
src/borg/upgrade.py
View file

@ -31,7 +31,8 @@ class UpgraderNoOp:
"hostname",
"username",
"time",
"time_end",
"start",
"end",
"comment",
"chunker_params",
"recreate_command_line",
@ -161,9 +162,9 @@ class UpgraderFrom12To20:
# this is a borg < 1.2 chunker_params tuple, no chunker algo specified, but we only had buzhash:
new_metadata["chunker_params"] = (CH_BUZHASH,) + chunker_params
# old borg used UTC timestamps, but did not have the explicit tz offset in them.
for attr in ("time", "time_end"):
if hasattr(metadata, attr):
new_metadata[attr] = getattr(metadata, attr) + "+00:00"
# the only important timestamp is "time", which has the nominal timestamp of the archive.
if hasattr(metadata, "time"):
new_metadata["time"] = getattr(metadata, "time") + "+00:00"
# borg 1: cmdline, recreate_cmdline: a copy of sys.argv
# borg 2: command_line, recreate_command_line: a single string
if hasattr(metadata, "cmdline"):