diff --git a/src/borg/archive.py b/src/borg/archive.py index a4cdea93a..509c46ddb 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -500,8 +500,8 @@ class Archive: noxattrs=False, progress=False, chunker_params=CHUNKER_PARAMS, + timestamp=None, start=None, - start_monotonic=None, end=None, log_json=False, iec=False, @@ -530,18 +530,10 @@ class Archive: self.noflags = noflags self.noacls = noacls self.noxattrs = noxattrs - assert (start is None) == ( - start_monotonic is None - ), "Logic error: if start is given, start_monotonic must be given as well and vice versa." - if start is None: - start = archive_ts_now() - start_monotonic = time.monotonic() self.chunker_params = chunker_params - self.start = start - self.start_monotonic = start_monotonic - if end is None: - end = archive_ts_now() - self.end = end + self.start = start if start is not None else archive_ts_now() + self.end = end if end is not None else self.start + self.timestamp = timestamp if timestamp is not None else self.start self.pipeline = DownloadPipeline(self.repository, self.repo_objs) self.create = create if self.create: @@ -578,15 +570,22 @@ class Archive: @property def ts(self): - """Timestamp of archive creation (start) in UTC""" + """Nominal archive timestamp in UTC.""" ts = self.metadata.time return parse_timestamp(ts) + @property + def ts_start(self): + """Timestamp of archive operation start in UTC.""" + # fall back to "time" in case "start" is not found + ts = self.metadata.get("start") or self.metadata.time + return parse_timestamp(ts) + @property def ts_end(self): - """Timestamp of archive creation (end) in UTC""" - # fall back to time if there is no time_end present in metadata - ts = self.metadata.get("time_end") or self.metadata.time + """Timestamp of archive operation end in UTC.""" + # fall back to "time" in case "end" or "time_end" are not found + ts = self.metadata.get("end") or self.metadata.get("time_end") or self.metadata.time return parse_timestamp(ts) @property @@ -604,15 +603,18 @@ class Archive: def info(self): if self.create: stats = self.stats + ts = self.timestamp start = self.start end = self.end else: stats = self.calc_stats(self.cache) - start = self.ts + ts = self.ts + start = self.ts_start end = self.ts_end info = { "name": self.name, "id": self.fpr, + "time": OutputTimestamp(ts), "start": OutputTimestamp(start), "end": OutputTimestamp(end), "duration": (end - start).total_seconds(), @@ -637,11 +639,13 @@ class Archive: Repository: {location} Archive name: {0.name} Archive fingerprint: {0.fpr} -Time (start): {start} -Time (end): {end} +Time (nominal): {time} +Time (start): {start} +Time (end): {end} Duration: {0.duration} """.format( self, + time=OutputTimestamp(self.timestamp), start=OutputTimestamp(self.start), end=OutputTimestamp(self.end), location=self.repository._location.canonical_path(), @@ -678,15 +682,10 @@ Duration: {0.duration} item_ptrs = archive_put_items( self.items_buffer.chunks, repo_objs=self.repo_objs, cache=self.cache, stats=self.stats ) # this adds the sizes of the item ptrs chunks to stats.osize - duration = timedelta(seconds=time.monotonic() - self.start_monotonic) - if timestamp is None: - end = archive_ts_now() - start = end - duration - else: - start = timestamp - end = start + duration - self.start = start - self.end = end + start = self.start + end = self.end = archive_ts_now() + nominal = start if timestamp is None else timestamp + self.timestamp = nominal metadata = { "version": 2, "name": name, @@ -697,8 +696,9 @@ Duration: {0.duration} "cwd": self.cwd, "hostname": hostname, "username": getuser(), - "time": start.isoformat(timespec="microseconds"), - "time_end": end.isoformat(timespec="microseconds"), + "time": nominal.isoformat(timespec="microseconds"), + "start": start.isoformat(timespec="microseconds"), + "end": end.isoformat(timespec="microseconds"), "chunker_params": self.chunker_params, } # we always want to create archives with the addtl. metadata (nfiles, etc.), @@ -2305,32 +2305,18 @@ class ArchiveRecreater: return if comment is None: comment = archive.metadata.get("comment", "") - - # Keep for the statistics if necessary - if self.stats: - _start = target.start - + additional_metadata = { + "command_line": archive.metadata.command_line, + # but also remember recreate metadata: + "recreate_command_line": join_cmd(sys.argv), + } if self.timestamp is None: - additional_metadata = { - "time": archive.metadata.time, - "time_end": archive.metadata.get("time_end") or archive.metadata.time, - "command_line": archive.metadata.command_line, - # but also remember recreate metadata: - "recreate_command_line": join_cmd(sys.argv), - } - else: - additional_metadata = { - "command_line": archive.metadata.command_line, - # but also remember recreate metadata: - "recreate_command_line": join_cmd(sys.argv), - } - + # if no timestamp is specified, keep the original timestamp + additional_metadata["time"] = archive.metadata.time target.save(comment=comment, timestamp=self.timestamp, additional_metadata=additional_metadata) if delete_original: archive.delete() if self.stats: - target.start = _start - target.end = archive_ts_now() log_multi(str(target), str(target.stats)) def matcher_add_tagged_dirs(self, archive): diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index cc8996073..426adc0ed 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -216,7 +216,6 @@ class CreateMixIn: dry_run = args.dry_run self.start_backup = time.time_ns() t0 = archive_ts_now() - t0_monotonic = time.monotonic() logger.info('Creating archive at "%s"' % args.location.processed) if not dry_run: with Cache( @@ -238,7 +237,6 @@ class CreateMixIn: progress=args.progress, chunker_params=args.chunker_params, start=t0, - start_monotonic=t0_monotonic, log_json=args.log_json, iec=args.iec, ) diff --git a/src/borg/archiver/info_cmd.py b/src/borg/archiver/info_cmd.py index 5209bae15..dbb8b3ab6 100644 --- a/src/borg/archiver/info_cmd.py +++ b/src/borg/archiver/info_cmd.py @@ -42,6 +42,7 @@ class InfoMixIn: Hostname: {hostname} Username: {username} Tags: {tags} + Time (nominal): {time} Time (start): {start} Time (end): {end} Duration: {duration} diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py index 4a0dd4c22..af67d6d10 100644 --- a/src/borg/archiver/tar_cmds.py +++ b/src/borg/archiver/tar_cmds.py @@ -4,7 +4,6 @@ import logging import os import stat import tarfile -import time from ..archive import Archive, TarfileObjectProcessors, ChunksProcessor from ..compress import CompressionSpec @@ -269,7 +268,6 @@ class TarMixIn: def _import_tar(self, args, repository, manifest, key, cache, tarstream): t0 = archive_ts_now() - t0_monotonic = time.monotonic() archive = Archive( manifest, @@ -279,7 +277,6 @@ class TarMixIn: progress=args.progress, chunker_params=args.chunker_params, start=t0, - start_monotonic=t0_monotonic, log_json=args.log_json, ) cp = ChunksProcessor(cache=cache, key=key, add_item=archive.add_item, rechunkify=False) diff --git a/src/borg/constants.py b/src/borg/constants.py index bbbbc5610..f128cc2f8 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -11,7 +11,10 @@ REQUIRED_ITEM_KEYS = frozenset(["path", "mtime"]) # this set must be kept complete, otherwise rebuild_manifest might malfunction: # fmt: off -ARCHIVE_KEYS = frozenset(['version', 'name', 'hostname', 'username', 'time', 'time_end', +ARCHIVE_KEYS = frozenset(['version', 'name', 'hostname', 'username', + 'time', # v2+ archives AND borg 1.x archives + 'time_end', # only legacy borg 1.x + 'start', 'end', # v2+ archives 'tags', # v2+ archives 'items', # legacy v1 archives 'item_ptrs', # v2+ archives diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 5132446b5..9d3151235 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -765,10 +765,9 @@ class ArchiveFormatter(BaseFormatter): "name": 'alias of "archive"', "comment": "archive comment", "tags": "archive tags", - # *start* is the key used by borg-info for this timestamp, this makes the formats more compatible - "start": "time (start) of creation of the archive", - "time": 'alias of "start"', - "end": "time (end) of creation of the archive", + "time": "nominal time of the archive", + "start": "start time of the archive operation", + "end": "end time of the archive operation", "command_line": "command line which was used to create the archive", "id": "internal ID of the archive", "hostname": "hostname of host on which this archive was created", @@ -778,7 +777,7 @@ class ArchiveFormatter(BaseFormatter): } KEY_GROUPS = ( ("archive", "name", "comment", "id", "tags"), - ("start", "time", "end", "command_line"), + ("time", "start", "end", "command_line"), ("hostname", "username"), ("size", "nfiles"), ) @@ -802,6 +801,7 @@ class ArchiveFormatter(BaseFormatter): "command_line": partial(self.get_meta, "command_line", ""), "size": partial(self.get_meta, "size", 0), "nfiles": partial(self.get_meta, "nfiles", 0), + "start": self.get_ts_start, "end": self.get_ts_end, "tags": self.get_tags, } @@ -817,7 +817,6 @@ class ArchiveFormatter(BaseFormatter): "archive": archive_info.name, "id": bin_to_hex(archive_info.id), "time": self.format_time(archive_info.ts), - "start": self.format_time(archive_info.ts), } for key in self.used_call_keys: item_data[key] = self.call_keys[key]() @@ -841,6 +840,9 @@ class ArchiveFormatter(BaseFormatter): def get_meta(self, key, default=None): return self.archive.metadata.get(key, default) + def get_ts_start(self): + return self.format_time(self.archive.ts_start) + def get_ts_end(self): return self.format_time(self.archive.ts_end) diff --git a/src/borg/item.pyi b/src/borg/item.pyi index a5047da1b..4289c9e77 100644 --- a/src/borg/item.pyi +++ b/src/borg/item.pyi @@ -25,6 +25,14 @@ class ArchiveItem(PropDict): @name.setter def name(self, val: str) -> None: ... @property + def start(self) -> str: ... + @start.setter + def start(self, val: str) -> None: ... + @property + def end(self) -> str: ... + @end.setter + def end(self, val: str) -> None: ... + @property def time(self) -> str: ... @time.setter def time(self, val: str) -> None: ... diff --git a/src/borg/item.pyx b/src/borg/item.pyx index 8e39d346a..184379b30 100644 --- a/src/borg/item.pyx +++ b/src/borg/item.pyx @@ -508,8 +508,10 @@ cdef class ArchiveItem(PropDict): command_line = PropDictProperty(str, 'surrogate-escaped str') hostname = PropDictProperty(str, 'surrogate-escaped str') username = PropDictProperty(str, 'surrogate-escaped str') - time = PropDictProperty(str) - time_end = PropDictProperty(str) + start = PropDictProperty(str) # new in borg2 (was: time) + end = PropDictProperty(str) # new in borg2 (was: time_end) + time = PropDictProperty(str) # borg2: nominal archive time, borg 1.x: same + start time + time_end = PropDictProperty(str) # legacy borg 1.x (now: end) comment = PropDictProperty(str, 'surrogate-escaped str') tags = PropDictProperty(list) # list of s-e-str chunker_params = PropDictProperty(tuple) @@ -533,7 +535,7 @@ cdef class ArchiveItem(PropDict): assert isinstance(v, int) if k in ('name', 'hostname', 'username', 'comment', 'cwd'): v = fix_str_value(d, k) - if k in ('time', 'time_end'): + if k in ('start', 'end', 'time', 'time_end'): v = fix_str_value(d, k, 'replace') if k == 'chunker_params': v = fix_tuple_of_str_and_int(v) diff --git a/src/borg/testsuite/archiver/recreate_cmd_test.py b/src/borg/testsuite/archiver/recreate_cmd_test.py index ec7d6ea5e..42dc39500 100644 --- a/src/borg/testsuite/archiver/recreate_cmd_test.py +++ b/src/borg/testsuite/archiver/recreate_cmd_test.py @@ -1,5 +1,6 @@ import os import re +import time from datetime import datetime import pytest @@ -186,17 +187,37 @@ def test_recreate_no_rechunkify(archivers, request): assert num_chunks == num_chunks_after_recreate -def test_recreate_timestamp(archivers, request): +def test_recreate_keep_original_timestamp(archivers, request): archiver = request.getfixturevalue(archivers) create_test_files(archiver.input_path) cmd(archiver, "repo-create", RK_ENCRYPTION) cmd(archiver, "create", "test0", "input") + info_orig = cmd(archiver, "info", "-a", "test0").splitlines() + # this shall recreate the archive and keep the nominal timestamp + time.sleep(1) + cmd(archiver, "recreate", "test0", "--comment", "test") + info_recreated = cmd(archiver, "info", "-a", "test0").splitlines() + nominal_orig = next(item for item in info_orig if item.startswith("Time (nominal):")) + nominal_recreated = next(item for item in info_recreated if item.startswith("Time (nominal):")) + assert nominal_orig == nominal_recreated + + +def test_recreate_with_given_timestamp(archivers, request): + archiver = request.getfixturevalue(archivers) + create_test_files(archiver.input_path) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test0", "input") + # this shall recreate the archive with a different nominal timestamp cmd(archiver, "recreate", "test0", "--timestamp", "1970-01-02T00:00:00", "--comment", "test") info = cmd(archiver, "info", "-a", "test0").splitlines() dtime = datetime(1970, 1, 2, 0, 0, 0).astimezone() # local time in local timezone s_time = dtime.strftime("%Y-%m-%d %H:%M:.. %z").replace("+", r"\+") - assert any([re.search(r"Time \(start\).+ %s" % s_time, item) for item in info]) + assert any([re.search(r"Time \(nominal\).+ %s" % s_time, item) for item in info]) + # start/end time are just from the recreate operation + dtime = datetime.now().astimezone() # current local time + s_time = dtime.strftime("%Y-%m-%d %H:%M:.. %z").replace("+", r"\+") assert any([re.search(r"Time \(end\).+ %s" % s_time, item) for item in info]) + assert any([re.search(r"Time \(start\).+ %s" % s_time, item) for item in info]) def test_recreate_dry_run(archivers, request): diff --git a/src/borg/testsuite/archiver/transfer_cmd_test.py b/src/borg/testsuite/archiver/transfer_cmd_test.py index 68a29728a..7255b683b 100644 --- a/src/borg/testsuite/archiver/transfer_cmd_test.py +++ b/src/borg/testsuite/archiver/transfer_cmd_test.py @@ -80,12 +80,14 @@ def test_transfer_upgrade(archivers, request, monkeypatch): # timestamps: # borg 1.2 transformed to local time and had microseconds = 0, no tzoffset # borg 2 uses local time, with microseconds and with tzoffset - for key in "start", "time": - # fix expectation: local time meant +01:00, so we convert that to whatever local tz is here. - expected_archive[key] = convert_tz(expected_archive[key], repo12_tzoffset, None) - # set microseconds to 0, so we can compare got with expected. - got_ts = parse_timestamp(got_archive[key]) - got_archive[key] = got_ts.replace(microsecond=0).isoformat(timespec="microseconds") + # the only important timestamp is "time", which has the nominal timestamp of the archive. + del expected_archive["start"] + key = "time" + # fix expectation: local time meant +01:00, so we convert that to whatever local tz is here. + expected_archive[key] = convert_tz(expected_archive[key], repo12_tzoffset, None) + # set microseconds to 0, so we can compare got with expected. + got_ts = parse_timestamp(got_archive[key]) + got_archive[key] = got_ts.replace(microsecond=0).isoformat(timespec="microseconds") assert got == expected for archive in got["archives"]: diff --git a/src/borg/upgrade.py b/src/borg/upgrade.py index 9f6f653f0..a7b3f00df 100644 --- a/src/borg/upgrade.py +++ b/src/borg/upgrade.py @@ -31,7 +31,8 @@ class UpgraderNoOp: "hostname", "username", "time", - "time_end", + "start", + "end", "comment", "chunker_params", "recreate_command_line", @@ -161,9 +162,9 @@ class UpgraderFrom12To20: # this is a borg < 1.2 chunker_params tuple, no chunker algo specified, but we only had buzhash: new_metadata["chunker_params"] = (CH_BUZHASH,) + chunker_params # old borg used UTC timestamps, but did not have the explicit tz offset in them. - for attr in ("time", "time_end"): - if hasattr(metadata, attr): - new_metadata[attr] = getattr(metadata, attr) + "+00:00" + # the only important timestamp is "time", which has the nominal timestamp of the archive. + if hasattr(metadata, "time"): + new_metadata["time"] = getattr(metadata, "time") + "+00:00" # borg 1: cmdline, recreate_cmdline: a copy of sys.argv # borg 2: command_line, recreate_command_line: a single string if hasattr(metadata, "cmdline"):