From d17553add64d0875a81ec87021c95c2140be946a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 6 Aug 2017 01:59:55 +0200 Subject: [PATCH 1/4] archive listing: use iso8601 timestamp format with --json-lines like yyyy-mm-ddThh:mm:ss - no tz yet, this likely needs more refactoring to tz aware and utc datetime objects everywhere, currently there are naive datetime objects and also localtime at quite some places. (cherry picked from commit 043d794b91a6c3b3ce18fef18cdbd256df398d04) --- src/borg/helpers.py | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index 1b6d38d83..b28ac7c5b 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -717,11 +717,17 @@ def safe_timestamp(item_timestamp_ns): def format_time(t): - """use ISO-8601 date and time format + """use ISO-8601-like date and time format (human readable, with wkday and blank date/time separator) """ return t.strftime('%a, %Y-%m-%d %H:%M:%S') +def isoformat_time(t): + """use ISO-8601 date and time format (machine readable, no wkday, no microseconds either) + """ + return t.strftime('%Y-%m-%dT%H:%M:%S') # note: first make all datetime objects tz aware before adding %z here. + + def format_timedelta(td): """Format timedelta in a human friendly format """ @@ -1762,6 +1768,12 @@ class ItemFormatter(BaseFormatter): 'archiveid': archive.fpr, } static_keys.update(self.FIXED_KEYS) + if self.json_lines: + self.item_data = {} + self.format_item = self.format_item_json + self.format_time = self.format_time_json + else: + self.item_data = static_keys self.format = partial_format(format, static_keys) self.format_keys = {f[1] for f in Formatter().parse(format)} self.call_keys = { @@ -1781,11 +1793,6 @@ class ItemFormatter(BaseFormatter): for hash_function in hashlib.algorithms_guaranteed: self.add_key(hash_function, partial(self.hash_item, hash_function)) self.used_call_keys = set(self.call_keys) & self.format_keys - if self.json_lines: - self.item_data = {} - self.format_item = self.format_item_json - else: - self.item_data = static_keys def format_item_json(self, item): return json.dumps(self.get_item_data(item)) + '\n' @@ -1863,7 +1870,12 @@ class ItemFormatter(BaseFormatter): return hash.hexdigest() def format_time(self, key, item): - return format_time(safe_timestamp(item.get(key) or item.mtime)) + t = self.time(key, item) + return format_time(t) + + def format_time_json(self, key, item): + t = self.time(key, item) + return isoformat_time(t) def time(self, key, item): return safe_timestamp(item.get(key) or item.mtime) From 65940be21a3212f3c676fa7e7667c3b69ea5d312 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 6 Aug 2017 02:13:17 +0200 Subject: [PATCH 2/4] archives list: use iso8601 timestamp format with --json like yyyy-mm-ddThh:mm:ss - no tz yet, this likely needs more refactoring to tz aware and utc datetime objects everywhere, currently there are naive datetime objects and also localtime at quite some places. (cherry picked from commit b64561fe6f067ec28ccd2b89d00f8e38e3397814) --- src/borg/helpers.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index b28ac7c5b..d6720854c 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -1660,6 +1660,7 @@ class ArchiveFormatter(BaseFormatter): if self.json: self.item_data = {} self.format_item = self.format_item_json + self.format_time = self.format_time_json else: self.item_data = static_keys @@ -1676,8 +1677,8 @@ class ArchiveFormatter(BaseFormatter): 'archive': remove_surrogates(archive_info.name), 'barchive': archive_info.name, 'id': bin_to_hex(archive_info.id), - 'time': format_time(to_localtime(archive_info.ts)), - 'start': format_time(to_localtime(archive_info.ts)), + 'time': self.format_time(archive_info.ts), + 'start': self.format_time(archive_info.ts), }) for key in self.used_call_keys: item_data[key] = self.call_keys[key]() @@ -1695,7 +1696,15 @@ class ArchiveFormatter(BaseFormatter): return remove_surrogates(self.archive.comment) if rs else self.archive.comment def get_ts_end(self): - return format_time(to_localtime(self.archive.ts_end)) + return self.format_time(self.archive.ts_end) + + def format_time(self, ts): + t = to_localtime(ts) + return format_time(t) + + def format_time_json(self, ts): + t = to_localtime(ts) + return isoformat_time(t) class ItemFormatter(BaseFormatter): From 74c9277d7649d9168b4b40660cbef6ec97b478f2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 6 Aug 2017 02:22:38 +0200 Subject: [PATCH 3/4] repo last_modified: use iso8601 timestamp format with --json like yyyy-mm-ddThh:mm:ss - no tz yet, this likely needs more refactoring to tz aware and utc datetime objects everywhere, currently there are naive datetime objects and also localtime at quite some places. (cherry picked from commit 32174dd9c885e2267a939cb81fe2d4130308d487) --- src/borg/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/borg/helpers.py b/src/borg/helpers.py index d6720854c..fed3b769b 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -2212,7 +2212,7 @@ def basic_json_data(manifest, *, cache=None, extra=None): 'mode': key.ARG_NAME, }, }) - data['repository']['last_modified'] = format_time(to_localtime(manifest.last_timestamp.replace(tzinfo=timezone.utc))) + data['repository']['last_modified'] = isoformat_time(to_localtime(manifest.last_timestamp.replace(tzinfo=timezone.utc))) if key.NAME.startswith('key file'): data['encryption']['keyfile'] = key.find_key() if cache: From eb7d473dc79f601b07f49b4a87b3b1ba30c91046 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 6 Aug 2017 23:49:18 +0200 Subject: [PATCH 4/4] test json timestamps for iso format (cherry picked from commit c3b0214e89e99b4cb2ae1fd6e0070603dabdffef) --- src/borg/testsuite/archiver.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index c342ee8cb..a26bc47cf 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -60,6 +60,8 @@ from . import key src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +ISO_FORMAT = '%Y-%m-%dT%H:%M:%S' + def exec_cmd(*args, archiver=None, fork=False, exe=None, input=b'', binary_output=False, **kw): if fork: @@ -1304,6 +1306,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): repository = info_repo['repository'] assert len(repository['id']) == 64 assert 'last_modified' in repository + assert datetime.strptime(repository['last_modified'], ISO_FORMAT) # must not raise assert info_repo['encryption']['mode'] == 'repokey' assert 'keyfile' not in info_repo['encryption'] cache = info_repo['cache'] @@ -1846,9 +1849,11 @@ class ArchiverTestCase(ArchiverTestCaseBase): list_repo = json.loads(self.cmd('list', '--json', self.repository_location)) repository = list_repo['repository'] assert len(repository['id']) == 64 - assert 'last_modified' in repository + assert datetime.strptime(repository['last_modified'], ISO_FORMAT) # must not raise assert list_repo['encryption']['mode'] == 'repokey' assert 'keyfile' not in list_repo['encryption'] + archive0 = list_repo['archives'][0] + assert datetime.strptime(archive0['time'], ISO_FORMAT) # must not raise list_archive = self.cmd('list', '--json-lines', self.repository_location + '::test') items = [json.loads(s) for s in list_archive.splitlines()] @@ -1856,6 +1861,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): file1 = items[1] assert file1['path'] == 'input/file1' assert file1['size'] == 81920 + assert datetime.strptime(file1['isomtime'], ISO_FORMAT) # must not raise list_archive = self.cmd('list', '--json-lines', '--format={sha256}', self.repository_location + '::test') items = [json.loads(s) for s in list_archive.splitlines()]