From fa65c9b1434e4299639ee2e3c7e4439a5f8d20db Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Mon, 7 Aug 2017 13:01:33 +0200 Subject: [PATCH 1/3] list: fix weird mixup of mtime/isomtime (cherry picked from commit 2ff29891f197623c54d7f40147b7411ece67524c) --- docs/changes.rst | 13 +++++++++++++ src/borg/archiver.py | 4 ++-- src/borg/testsuite/archiver.py | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 25909cf30..bed3b26d1 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -131,6 +131,19 @@ The best check that everything is ok is to run a dry-run extraction:: Changelog ========= +Version 1.1.0rc2 (not released yet) +------------------------------------ + +Compatibility notes: + +- list: corrected mix-up of "isomtime" and "mtime" formats. Previously, + "isomtime" was the default but produced a verbose human format, + while "mtime" produced a ISO-8601-like format. + The behaviours have been swapped (so "mtime" is human, "isomtime" is ISO-like), + and the default is now "mtime". + "isomtime" is now a real ISO-8601 format ("T" between date and time, not a space). + + Version 1.1.0rc1 (2017-07-24) ----------------------------- diff --git a/src/borg/archiver.py b/src/borg/archiver.py index a3db174ac..0065cd820 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -1334,7 +1334,7 @@ class Archiver: elif args.short: format = "{path}{NL}" else: - format = "{mode} {user:6} {group:6} {size:8} {isomtime} {path}{extra}{NL}" + format = "{mode} {user:6} {group:6} {size:8} {mtime} {path}{extra}{NL}" def _list_inner(cache): archive = Archive(repository, key, manifest, args.location.archive, cache=cache, @@ -3119,7 +3119,7 @@ class Archiver: help='only print file/directory names, nothing else') subparser.add_argument('--format', '--list-format', metavar='FORMAT', dest='format', help='specify format for file listing ' - '(default: "{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NL}")') + '(default: "{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NL}")') subparser.add_argument('--json', action='store_true', help='Only valid for listing repository contents. Format output as JSON. ' 'The form of ``--format`` is ignored, ' diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index a26bc47cf..6ae7d4710 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1787,7 +1787,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): output_warn = self.cmd('list', '--list-format', '-', test_archive) self.assert_in('--list-format" has been deprecated.', output_warn) output_1 = self.cmd('list', test_archive) - output_2 = self.cmd('list', '--format', '{mode} {user:6} {group:6} {size:8d} {isomtime} {path}{extra}{NEWLINE}', test_archive) + output_2 = self.cmd('list', '--format', '{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NEWLINE}', test_archive) output_3 = self.cmd('list', '--format', '{mtime:%s} {path}{NL}', test_archive) self.assertEqual(output_1, output_2) self.assertNotEqual(output_1, output_3) From 008571228f52d89a6235bf20b1e4a9644c479d4c Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Wed, 16 Aug 2017 17:57:08 +0200 Subject: [PATCH 2/3] one datetime formatter to rule them all (cherry picked from commit a836f451ab239da516fa9232c11005c62f7e04a3) --- docs/internals/frontends.rst | 21 +++++++------ src/borg/archive.py | 10 +++--- src/borg/helpers.py | 56 +++++++++++++++++++--------------- src/borg/testsuite/archiver.py | 2 ++ 4 files changed, 49 insertions(+), 40 deletions(-) diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index c41d427eb..0441caa53 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -209,8 +209,9 @@ Standard output *stdout* is different and more command-dependent than logging. Commands like :ref:`borg_info`, :ref:`borg_create` and :ref:`borg_list` implement a ``--json`` option which turns their regular output into a single JSON object. -Dates are formatted according to ISO-8601 with the strftime format string '%a, %Y-%m-%d %H:%M:%S', -e.g. *Sat, 2016-02-25 23:50:06*. +Dates are formatted according to ISO-8601 in local time. Neither an explicit time zone nor microseconds +are specified *at this time* (subject to change). The equivalent strftime format string is '%Y-%m-%dT%H:%M:%S', +e.g. 2017-08-07T12:27:20. The root object at least contains a *repository* key with an object containing: @@ -267,7 +268,7 @@ Example *borg info* output:: }, "repository": { "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", - "last_modified": "Mon, 2017-02-27 21:21:58", + "last_modified": "2017-08-07T12:27:20", "location": "/home/user/testrepo" }, "security_dir": "/home/user/.config/borg/security/0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", @@ -328,7 +329,7 @@ Example of a simple archive listing (``borg list --last 1 --json``):: { "id": "80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a", "name": "host-system-backup-2017-02-27", - "start": "Mon, 2017-02-27 21:21:52" + "start": "2017-08-07T12:27:20" } ], "encryption": { @@ -336,7 +337,7 @@ Example of a simple archive listing (``borg list --last 1 --json``):: }, "repository": { "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", - "last_modified": "Mon, 2017-02-27 21:21:58", + "last_modified": "2017-08-07T12:27:20", "location": "/home/user/repository" } } @@ -354,14 +355,14 @@ The same archive with more information (``borg info --last 1 --json``):: ], "comment": "", "duration": 5.641542, - "end": "Mon, 2017-02-27 21:21:58", + "end": "2017-02-27T12:27:20", "hostname": "host", "id": "80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a", "limits": { "max_archive_size": 0.0001330855110409714 }, "name": "host-system-backup-2017-02-27", - "start": "Mon, 2017-02-27 21:21:52", + "start": "2017-02-27T12:27:20", "stats": { "compressed_size": 1880961894, "deduplicated_size": 2791, @@ -387,7 +388,7 @@ The same archive with more information (``borg info --last 1 --json``):: }, "repository": { "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", - "last_modified": "Mon, 2017-02-27 21:21:58", + "last_modified": "2017-08-07T12:27:20", "location": "/home/user/repository" } } @@ -405,8 +406,8 @@ Refer to the *borg list* documentation for the available keys and their meaning. Example (excerpt) of ``borg list --json-lines``:: - {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "Sat, 2016-05-07 19:46:01", "size": 0} - {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux/baz", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "Sat, 2016-05-07 19:46:01", "size": 0} + {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "2017-02-27T12:27:20", "size": 0} + {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux/baz", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "2017-02-27T12:27:20", "size": 0} .. _msgid: diff --git a/src/borg/archive.py b/src/borg/archive.py index ff8f8729e..f28c7f128 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -32,7 +32,7 @@ from .helpers import ChunkIteratorFileWrapper, open_item from .helpers import Error, IntegrityError, set_ec from .helpers import uid2user, user2uid, gid2group, group2gid from .helpers import parse_timestamp, to_localtime -from .helpers import format_time, format_timedelta, format_file_size, file_status, FileSize +from .helpers import OutputTimestamp, format_timedelta, format_file_size, file_status, FileSize from .helpers import safe_encode, safe_decode, make_path_safe, remove_surrogates from .helpers import StableDict from .helpers import bin_to_hex @@ -381,8 +381,8 @@ class Archive: info = { 'name': self.name, 'id': self.fpr, - 'start': format_time(to_localtime(start)), - 'end': format_time(to_localtime(end)), + 'start': OutputTimestamp(start), + 'end': OutputTimestamp(end), 'duration': (end - start).total_seconds(), 'stats': stats.as_dict(), 'limits': { @@ -411,8 +411,8 @@ Number of files: {0.stats.nfiles} Utilization of max. archive size: {csize_max:.0%} '''.format( self, - start=format_time(to_localtime(self.start.replace(tzinfo=timezone.utc))), - end=format_time(to_localtime(self.end.replace(tzinfo=timezone.utc))), + start=OutputTimestamp(self.start.replace(tzinfo=timezone.utc)), + end=OutputTimestamp(self.end.replace(tzinfo=timezone.utc)), csize_max=self.cache.chunks[self.id].csize / MAX_DATA_SIZE) def __repr__(self): diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 86368a9d5..4b68eabe3 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -744,6 +744,22 @@ def format_timedelta(td): return txt +class OutputTimestamp: + def __init__(self, ts: datetime): + if ts.tzinfo == timezone.utc: + ts = to_localtime(ts) + self.ts = ts + + def __format__(self, format_spec): + return format_time(self.ts) + + def __str__(self): + return '{}'.format(self) + + def to_json(self): + return isoformat_time(self.ts) + + def format_file_size(v, precision=2, sign=False): """Format file size into a human friendly format """ @@ -1664,12 +1680,11 @@ class ArchiveFormatter(BaseFormatter): if self.json: self.item_data = {} self.format_item = self.format_item_json - self.format_time = self.format_time_json else: self.item_data = static_keys def format_item_json(self, item): - return json.dumps(self.get_item_data(item)) + '\n' + return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + '\n' def get_item_data(self, archive_info): self.name = archive_info.name @@ -1703,12 +1718,7 @@ class ArchiveFormatter(BaseFormatter): return self.format_time(self.archive.ts_end) def format_time(self, ts): - t = to_localtime(ts) - return format_time(t) - - def format_time_json(self, ts): - t = to_localtime(ts) - return isoformat_time(t) + return OutputTimestamp(ts) class ItemFormatter(BaseFormatter): @@ -1784,7 +1794,6 @@ class ItemFormatter(BaseFormatter): if self.json_lines: self.item_data = {} self.format_item = self.format_item_json - self.format_time = self.format_time_json else: self.item_data = static_keys self.format = partial_format(format, static_keys) @@ -1796,19 +1805,19 @@ class ItemFormatter(BaseFormatter): 'dcsize': partial(self.sum_unique_chunks_metadata, lambda chunk: chunk.csize), 'num_chunks': self.calculate_num_chunks, 'unique_chunks': partial(self.sum_unique_chunks_metadata, lambda chunk: 1), - 'isomtime': partial(self.format_time, 'mtime'), - 'isoctime': partial(self.format_time, 'ctime'), - 'isoatime': partial(self.format_time, 'atime'), - 'mtime': partial(self.time, 'mtime'), - 'ctime': partial(self.time, 'ctime'), - 'atime': partial(self.time, 'atime'), + 'isomtime': partial(self.format_iso_time, 'mtime'), + 'isoctime': partial(self.format_iso_time, 'ctime'), + 'isoatime': partial(self.format_iso_time, 'atime'), + 'mtime': partial(self.format_time, 'mtime'), + 'ctime': partial(self.format_time, 'ctime'), + 'atime': partial(self.format_time, 'atime'), } for hash_function in hashlib.algorithms_guaranteed: self.add_key(hash_function, partial(self.hash_item, hash_function)) self.used_call_keys = set(self.call_keys) & self.format_keys def format_item_json(self, item): - return json.dumps(self.get_item_data(item)) + '\n' + return json.dumps(self.get_item_data(item), cls=BorgJsonEncoder) + '\n' def add_key(self, key, callable_with_item): self.call_keys[key] = callable_with_item @@ -1883,15 +1892,10 @@ class ItemFormatter(BaseFormatter): return hash.hexdigest() def format_time(self, key, item): - t = self.time(key, item) - return format_time(t) + return OutputTimestamp(safe_timestamp(item.get(key) or item.mtime)) - def format_time_json(self, key, item): - t = self.time(key, item) - return isoformat_time(t) - - def time(self, key, item): - return safe_timestamp(item.get(key) or item.mtime) + def format_iso_time(self, key, item): + return self.format_time(key, item).to_json() class ChunkIteratorFileWrapper: @@ -2204,6 +2208,8 @@ class BorgJsonEncoder(json.JSONEncoder): return { 'stats': o.stats(), } + if callable(getattr(o, 'to_json', None)): + return o.to_json() return super().default(o) @@ -2216,7 +2222,7 @@ def basic_json_data(manifest, *, cache=None, extra=None): 'mode': key.ARG_NAME, }, }) - data['repository']['last_modified'] = isoformat_time(to_localtime(manifest.last_timestamp.replace(tzinfo=timezone.utc))) + data['repository']['last_modified'] = OutputTimestamp(manifest.last_timestamp.replace(tzinfo=timezone.utc)) if key.NAME.startswith('key file'): data['encryption']['keyfile'] = key.find_key() if cache: diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 6ae7d4710..a32cd6f8d 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1325,6 +1325,8 @@ class ArchiverTestCase(ArchiverTestCaseBase): assert isinstance(archive['duration'], float) assert len(archive['id']) == 64 assert 'stats' in archive + assert datetime.strptime(archive['start'], ISO_FORMAT) + assert datetime.strptime(archive['end'], ISO_FORMAT) def test_comment(self): self.create_regular_file('file1', size=1024 * 80) From d5697fb4a8b9d3f21f76a11b224783ec36668f52 Mon Sep 17 00:00:00 2001 From: Marian Beermann Date: Mon, 7 Aug 2017 13:08:25 +0200 Subject: [PATCH 3/3] always use microseconds for ISO 8601 output (cherry picked from commit ab4981eff65984f23dceb2adcfb29d2d74d70e35) --- docs/internals/frontends.rst | 22 +++++++++++----------- src/borg/helpers.py | 21 +++++++++++++-------- src/borg/testsuite/archiver.py | 4 ++-- 3 files changed, 26 insertions(+), 21 deletions(-) diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index 0441caa53..b4ebf5eb1 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -209,9 +209,9 @@ Standard output *stdout* is different and more command-dependent than logging. Commands like :ref:`borg_info`, :ref:`borg_create` and :ref:`borg_list` implement a ``--json`` option which turns their regular output into a single JSON object. -Dates are formatted according to ISO-8601 in local time. Neither an explicit time zone nor microseconds -are specified *at this time* (subject to change). The equivalent strftime format string is '%Y-%m-%dT%H:%M:%S', -e.g. 2017-08-07T12:27:20. +Dates are formatted according to ISO 8601 in local time. No explicit time zone is specified *at this time* +(subject to change). The equivalent strftime format string is '%Y-%m-%dT%H:%M:%S.%f', +e.g. ``2017-08-07T12:27:20.123456``. The root object at least contains a *repository* key with an object containing: @@ -268,7 +268,7 @@ Example *borg info* output:: }, "repository": { "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", - "last_modified": "2017-08-07T12:27:20", + "last_modified": "2017-08-07T12:27:20.789123", "location": "/home/user/testrepo" }, "security_dir": "/home/user/.config/borg/security/0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", @@ -329,7 +329,7 @@ Example of a simple archive listing (``borg list --last 1 --json``):: { "id": "80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a", "name": "host-system-backup-2017-02-27", - "start": "2017-08-07T12:27:20" + "start": "2017-08-07T12:27:20.789123" } ], "encryption": { @@ -337,7 +337,7 @@ Example of a simple archive listing (``borg list --last 1 --json``):: }, "repository": { "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", - "last_modified": "2017-08-07T12:27:20", + "last_modified": "2017-08-07T12:27:20.789123", "location": "/home/user/repository" } } @@ -355,14 +355,14 @@ The same archive with more information (``borg info --last 1 --json``):: ], "comment": "", "duration": 5.641542, - "end": "2017-02-27T12:27:20", + "end": "2017-02-27T12:27:20.789123", "hostname": "host", "id": "80cd07219ad725b3c5f665c1dcf119435c4dee1647a560ecac30f8d40221a46a", "limits": { "max_archive_size": 0.0001330855110409714 }, "name": "host-system-backup-2017-02-27", - "start": "2017-02-27T12:27:20", + "start": "2017-02-27T12:27:20.789123", "stats": { "compressed_size": 1880961894, "deduplicated_size": 2791, @@ -388,7 +388,7 @@ The same archive with more information (``borg info --last 1 --json``):: }, "repository": { "id": "0cbe6166b46627fd26b97f8831e2ca97584280a46714ef84d2b668daf8271a23", - "last_modified": "2017-08-07T12:27:20", + "last_modified": "2017-08-07T12:27:20.789123", "location": "/home/user/repository" } } @@ -406,8 +406,8 @@ Refer to the *borg list* documentation for the available keys and their meaning. Example (excerpt) of ``borg list --json-lines``:: - {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "2017-02-27T12:27:20", "size": 0} - {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux/baz", "healthy": true, "source": "", "linktarget": "", "flags": null, "isomtime": "2017-02-27T12:27:20", "size": 0} + {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux", "healthy": true, "source": "", "linktarget": "", "flags": null, "mtime": "2017-02-27T12:27:20.023407", "size": 0} + {"type": "d", "mode": "drwxr-xr-x", "user": "user", "group": "user", "uid": 1000, "gid": 1000, "path": "linux/baz", "healthy": true, "source": "", "linktarget": "", "flags": null, "mtime": "2017-02-27T12:27:20.585407", "size": 0} .. _msgid: diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 4b68eabe3..64ee4c82a 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -715,16 +715,19 @@ def safe_timestamp(item_timestamp_ns): return datetime.fromtimestamp(t_ns / 1e9) -def format_time(t): - """use ISO-8601-like date and time format (human readable, with wkday and blank date/time separator) +def format_time(ts: datetime): """ - return t.strftime('%a, %Y-%m-%d %H:%M:%S') + Convert *ts* to a human-friendly format with textual weekday. + """ + return ts.strftime('%a, %Y-%m-%d %H:%M:%S') -def isoformat_time(t): - """use ISO-8601 date and time format (machine readable, no wkday, no microseconds either) +def isoformat_time(ts: datetime): """ - return t.strftime('%Y-%m-%dT%H:%M:%S') # note: first make all datetime objects tz aware before adding %z here. + Format *ts* according to ISO 8601. + """ + # note: first make all datetime objects tz aware before adding %z here. + return ts.strftime('%Y-%m-%dT%H:%M:%S.%f') def format_timedelta(td): @@ -756,9 +759,11 @@ class OutputTimestamp: def __str__(self): return '{}'.format(self) - def to_json(self): + def isoformat(self): return isoformat_time(self.ts) + to_json = isoformat + def format_file_size(v, precision=2, sign=False): """Format file size into a human friendly format @@ -1895,7 +1900,7 @@ class ItemFormatter(BaseFormatter): return OutputTimestamp(safe_timestamp(item.get(key) or item.mtime)) def format_iso_time(self, key, item): - return self.format_time(key, item).to_json() + return self.format_time(key, item).isoformat() class ChunkIteratorFileWrapper: diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index a32cd6f8d..a333eca17 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -60,7 +60,7 @@ from . import key src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) -ISO_FORMAT = '%Y-%m-%dT%H:%M:%S' +ISO_FORMAT = '%Y-%m-%dT%H:%M:%S.%f' def exec_cmd(*args, archiver=None, fork=False, exe=None, input=b'', binary_output=False, **kw): @@ -1863,7 +1863,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): file1 = items[1] assert file1['path'] == 'input/file1' assert file1['size'] == 81920 - assert datetime.strptime(file1['isomtime'], ISO_FORMAT) # must not raise + assert datetime.strptime(file1['mtime'], ISO_FORMAT) # must not raise list_archive = self.cmd('list', '--json-lines', '--format={sha256}', self.repository_location + '::test') items = [json.loads(s) for s in list_archive.splitlines()]