From 909c68a0d78db1f55cfeed0612f1cf0cbd0d3779 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 10 Sep 2017 05:17:40 +0200 Subject: [PATCH 1/4] add ISO_FORMAT* constants to helpers module no constants module here, so put it into helpers. --- borg/helpers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/borg/helpers.py b/borg/helpers.py index a0d29011b..b13ac8fde 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -39,6 +39,12 @@ import msgpack.fallback import socket +# never use datetime.isoformat(), it is evil. always use one of these: +# datetime.strftime(ISO_FORMAT) # output always includes .microseconds +# datetime.strftime(ISO_FORMAT_NO_USECS) # output never includes microseconds +ISO_FORMAT_NO_USECS = '%Y-%m-%dT%H:%M:%S' +ISO_FORMAT = ISO_FORMAT_NO_USECS + '.%f' + # 20 MiB minus 41 bytes for a Repository header (because the "size" field in the Repository includes # the header, and the total size was set to 20 MiB). MAX_DATA_SIZE = 20971479 From da30e54fc4d49f1d9bad6bba806902d04276ddb3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Sep 2017 05:02:44 +0200 Subject: [PATCH 2/4] use ISO_FORMAT* constants (cherry picked from commit 457f5ceb30c1817556c1423ef42be4a63dee9ff1) --- borg/helpers.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index b13ac8fde..f50379fea 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -314,7 +314,7 @@ class Manifest: if self.timestamp is None: self.timestamp = datetime.utcnow().isoformat() else: - prev_ts = datetime.strptime(self.timestamp, "%Y-%m-%dT%H:%M:%S.%f") + prev_ts = datetime.strptime(self.timestamp, ISO_FORMAT) incremented = (prev_ts + timedelta(microseconds=1)).isoformat() self.timestamp = max(incremented, datetime.utcnow().isoformat()) # include checks for limits as enforced by limited unpacker (used by load()) @@ -493,10 +493,8 @@ def to_localtime(ts): def parse_timestamp(timestamp): """Parse a ISO 8601 timestamp string""" - if '.' in timestamp: # microseconds might not be present - return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f').replace(tzinfo=timezone.utc) - else: - return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc) + fmt = ISO_FORMAT if '.' in timestamp else ISO_FORMAT_NO_USECS + return datetime.strptime(timestamp, fmt).replace(tzinfo=timezone.utc) def load_excludes(fh): From 9c3bae46e0bf82396ee31986837b150f0a5d6031 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Sep 2017 05:51:00 +0200 Subject: [PATCH 3/4] use safe parse_timestamp to parse timestamps, fixes #2994 also: refactor so it is possible to get tz-unaware datetime objects from parse_timestamp. (cherry picked from commit a9aa3c5f340eb3d8baca1871aa93579a81ef5f9f) --- borg/helpers.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/borg/helpers.py b/borg/helpers.py index f50379fea..0689aa2dc 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -314,7 +314,7 @@ class Manifest: if self.timestamp is None: self.timestamp = datetime.utcnow().isoformat() else: - prev_ts = datetime.strptime(self.timestamp, ISO_FORMAT) + prev_ts = parse_timestamp(self.timestamp, tzinfo=None) incremented = (prev_ts + timedelta(microseconds=1)).isoformat() self.timestamp = max(incremented, datetime.utcnow().isoformat()) # include checks for limits as enforced by limited unpacker (used by load()) @@ -491,10 +491,13 @@ def to_localtime(ts): return datetime(*time.localtime((ts - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds())[:6]) -def parse_timestamp(timestamp): +def parse_timestamp(timestamp, tzinfo=timezone.utc): """Parse a ISO 8601 timestamp string""" fmt = ISO_FORMAT if '.' in timestamp else ISO_FORMAT_NO_USECS - return datetime.strptime(timestamp, fmt).replace(tzinfo=timezone.utc) + dt = datetime.strptime(timestamp, fmt) + if tzinfo is not None: + dt = dt.replace(tzinfo=tzinfo) + return dt def load_excludes(fh): From 9529895ed419c6a6a77b99470f629527f3064978 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Sep 2017 06:13:47 +0200 Subject: [PATCH 4/4] get rid of datetime.isoformat to avoid bugs like #2994 (cherry picked from commit da2f8dbe81cd88aff973c58b5dd69193625335bc) --- borg/archive.py | 6 +++--- borg/helpers.py | 6 +++--- borg/repository.py | 5 +++-- borg/testsuite/archiver.py | 6 +++--- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/borg/archive.py b/borg/archive.py index 6ed9121f3..b9d3c40f7 100644 --- a/borg/archive.py +++ b/borg/archive.py @@ -17,7 +17,7 @@ import time from io import BytesIO from . import xattr from .helpers import Error, uid2user, user2uid, gid2group, group2gid, bin_to_hex, \ - parse_timestamp, to_localtime, format_time, format_timedelta, remove_surrogates, \ + parse_timestamp, to_localtime, ISO_FORMAT, format_time, format_timedelta, remove_surrogates, \ Manifest, Statistics, decode_dict, make_path_safe, StableDict, int_to_bigint, bigint_to_int, \ ProgressIndicatorPercent, IntegrityError, set_ec, EXIT_WARNING, safe_ns from .platform import acl_get, acl_set @@ -321,8 +321,8 @@ Number of files: {0.stats.nfiles}'''.format( 'cmdline': sys.argv, 'hostname': socket.gethostname(), 'username': getuser(), - 'time': start.isoformat(), - 'time_end': end.isoformat(), + 'time': start.strftime(ISO_FORMAT), + 'time_end': end.strftime(ISO_FORMAT), }) data = self.key.pack_and_authenticate_metadata(metadata, context=b'archive') self.id = self.key.id_hash(data) diff --git a/borg/helpers.py b/borg/helpers.py index 0689aa2dc..6d193bbff 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -312,11 +312,11 @@ class Manifest: self.config[b'tam_required'] = True # self.timestamp needs to be strictly monotonically increasing. Clocks often are not set correctly if self.timestamp is None: - self.timestamp = datetime.utcnow().isoformat() + self.timestamp = datetime.utcnow().strftime(ISO_FORMAT) else: prev_ts = parse_timestamp(self.timestamp, tzinfo=None) - incremented = (prev_ts + timedelta(microseconds=1)).isoformat() - self.timestamp = max(incremented, datetime.utcnow().isoformat()) + incremented = (prev_ts + timedelta(microseconds=1)).strftime(ISO_FORMAT) + self.timestamp = max(incremented, datetime.utcnow().strftime(ISO_FORMAT)) # include checks for limits as enforced by limited unpacker (used by load()) assert len(self.archives) <= MAX_ARCHIVES assert all(len(name) <= 255 for name in self.archives) diff --git a/borg/repository.py b/borg/repository.py index cd4957c8c..8e9f8d949 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -14,7 +14,7 @@ from .logger import create_logger logger = create_logger() from .helpers import Error, ErrorWithTraceback, IntegrityError, Location, ProgressIndicatorPercent, bin_to_hex -from .helpers import LIST_SCAN_LIMIT, MAX_OBJECT_SIZE, MAX_DATA_SIZE +from .helpers import LIST_SCAN_LIMIT, MAX_OBJECT_SIZE, MAX_DATA_SIZE, ISO_FORMAT from .hashindex import NSIndex from .locking import Lock, LockError, LockErrorT from .lrucache import LRUCache @@ -273,7 +273,8 @@ class Repository: os.path.join(self.path, 'index.%d' % transaction_id)) if self.append_only: with open(os.path.join(self.path, 'transactions'), 'a') as log: - print('transaction %d, UTC time %s' % (transaction_id, datetime.utcnow().isoformat()), file=log) + print('transaction %d, UTC time %s' % ( + transaction_id, datetime.utcnow().strftime(ISO_FORMAT)), file=log) # Remove old indices current = '.%d' % transaction_id for name in os.listdir(self.path): diff --git a/borg/testsuite/archiver.py b/borg/testsuite/archiver.py index 290a73f54..d1aea8765 100644 --- a/borg/testsuite/archiver.py +++ b/borg/testsuite/archiver.py @@ -25,7 +25,7 @@ from ..archiver import Archiver from ..cache import Cache from ..crypto import bytes_to_long, num_aes_blocks from ..helpers import Manifest, PatternMatcher, parse_pattern, EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR, bin_to_hex, \ - get_security_dir, MAX_S, MandatoryFeatureUnsupported, Location + get_security_dir, MAX_S, MandatoryFeatureUnsupported, Location, ISO_FORMAT from ..key import RepoKey, KeyfileKey, Passphrase, TAMRequiredError from ..keymanager import RepoIdMismatch, NotABorgKeyFile from ..remote import RemoteRepository, PathNotAllowed @@ -1766,7 +1766,7 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase): 'version': 1, 'archives': {}, 'config': {}, - 'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(), + 'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT), }))) repository.commit() @@ -1778,7 +1778,7 @@ class ManifestAuthenticationTest(ArchiverTestCaseBase): repository.put(Manifest.MANIFEST_ID, key.encrypt(msgpack.packb({ 'version': 1, 'archives': {}, - 'timestamp': (datetime.utcnow() + timedelta(days=1)).isoformat(), + 'timestamp': (datetime.utcnow() + timedelta(days=1)).strftime(ISO_FORMAT), }))) repository.commit()