From af9f1c24c494ca374cdfe0364f1ebe1fb35d19fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Borgstr=C3=B6m?= Date: Sun, 19 Apr 2015 22:42:52 +0200 Subject: [PATCH 1/7] Fix parsing of iso8601 timestamps with zero microseconds Closes #282 --- CHANGES | 6 ++++++ attic/archive.py | 3 +-- attic/helpers.py | 8 ++++++++ attic/testsuite/helpers.py | 9 ++++++++- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 6e64c5807..9ed6944c9 100644 --- a/CHANGES +++ b/CHANGES @@ -3,6 +3,12 @@ Attic Changelog Here you can see the full list of changes between each Attic release. +Version 0.16 +------------ + +(bugfix release, released on X) +- Fix parsing of iso 8601 timestamps with zero microseconds (#282) + Version 0.15 ------------ diff --git a/attic/archive.py b/attic/archive.py index d78a7fdb3..f5726ea27 100644 --- a/attic/archive.py +++ b/attic/archive.py @@ -163,8 +163,7 @@ class Archive: @property def ts(self): """Timestamp of archive creation in UTC""" - t, f = self.metadata[b'time'].split('.', 1) - return datetime.strptime(t, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc) + timedelta(seconds=float('.' + f)) + return parse_timestamp(self.metadata[b'time']) def __repr__(self): return 'Archive(%r)' % self.name diff --git a/attic/helpers.py b/attic/helpers.py index 646ba2571..2e380cd68 100644 --- a/attic/helpers.py +++ b/attic/helpers.py @@ -185,6 +185,14 @@ def to_localtime(ts): return datetime(*time.localtime((ts - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds())[:6]) +def parse_timestamp(timestamp): + """Parse a ISO 8601 timestamp string""" + if '.' in timestamp: # microseconds might not be pressent + return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%f').replace(tzinfo=timezone.utc) + else: + return datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc) + + def update_excludes(args): """Merge exclude patterns from files with those on command line. Empty lines and lines starting with '#' are ignored, but whitespace diff --git a/attic/testsuite/helpers.py b/attic/testsuite/helpers.py index e8c9ee402..0b5a75836 100644 --- a/attic/testsuite/helpers.py +++ b/attic/testsuite/helpers.py @@ -5,7 +5,7 @@ import os import tempfile import unittest from attic.helpers import adjust_patterns, exclude_path, Location, format_timedelta, IncludePattern, ExcludePattern, make_path_safe, UpgradableLock, prune_within, prune_split, to_localtime, \ - StableDict, int_to_bigint, bigint_to_int + StableDict, int_to_bigint, bigint_to_int, parse_timestamp from attic.testsuite import AtticTestCase import msgpack @@ -209,3 +209,10 @@ class StableDictTestCase(AtticTestCase): d = StableDict(foo=1, bar=2, boo=3, baz=4) self.assert_equal(list(d.items()), [('bar', 2), ('baz', 4), ('boo', 3), ('foo', 1)]) self.assert_equal(hashlib.md5(msgpack.packb(d)).hexdigest(), 'fc78df42cd60691b3ac3dd2a2b39903f') + + +class TestParseTimestamp(AtticTestCase): + + def test(self): + self.assert_equal(parse_timestamp('2015-04-19T20:25:00.226410'), datetime(2015, 4, 19, 20, 25, 0, 226410, timezone.utc)) + self.assert_equal(parse_timestamp('2015-04-19T20:25:00'), datetime(2015, 4, 19, 20, 25, 0, 0, timezone.utc)) From b61ba51c0d1f7b7fc0f65d281fdf9d7b88857c68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Borgstr=C3=B6m?= Date: Sun, 19 Apr 2015 23:09:10 +0200 Subject: [PATCH 2/7] More user friendly error message when repository key file is not found Closes #236 --- CHANGES | 1 + attic/key.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 9ed6944c9..b0d2b749d 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,7 @@ Version 0.16 ------------ (bugfix release, released on X) +- More user friendly error message when repository key file is not found (#236) - Fix parsing of iso 8601 timestamps with zero microseconds (#282) Version 0.15 diff --git a/attic/key.py b/attic/key.py index ef623f36c..42f8420af 100644 --- a/attic/key.py +++ b/attic/key.py @@ -17,6 +17,11 @@ class UnsupportedPayloadError(Error): """Unsupported payload type {}. A newer version is required to access this repository. """ +class KeyfileNotFoundError(Error): + """No key file for repository {} found in {}. + """ + + class HMAC(hmac.HMAC): """Workaround a bug in Python < 3.4 Where HMAC does not accept memoryviews """ @@ -221,7 +226,7 @@ class KeyfileKey(AESKeyBase): line = fd.readline().strip() if line and line.startswith(cls.FILE_ID) and line[10:] == id: return filename - raise Exception('Key file for repository with ID %s not found' % id) + raise KeyfileNotFoundError(repository._location.canonical_path(), get_keys_dir()) def load(self, filename, passphrase): with open(filename, 'r') as fd: From b7e6efbfc578e5ccc3b0997be67a73e1cba39cc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Borgstr=C3=B6m?= Date: Sun, 19 Apr 2015 23:21:46 +0200 Subject: [PATCH 3/7] Added missing import of parse_timestamp --- attic/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attic/archive.py b/attic/archive.py index f5726ea27..68acf866c 100644 --- a/attic/archive.py +++ b/attic/archive.py @@ -17,7 +17,7 @@ from attic import xattr from attic.platform import acl_get, acl_set from attic.chunker import Chunker from attic.hashindex import ChunkIndex -from attic.helpers import Error, uid2user, user2uid, gid2group, group2gid, \ +from attic.helpers import parse_timestamp, Error, uid2user, user2uid, gid2group, group2gid, \ Manifest, Statistics, decode_dict, st_mtime_ns, make_path_safe, StableDict, int_to_bigint, bigint_to_int ITEMS_BUFFER = 1024 * 1024 From 04d50c7141a47920ece9daa5cc2b052b45ac6ba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Borgstr=C3=B6m?= Date: Sun, 19 Apr 2015 23:24:14 +0200 Subject: [PATCH 4/7] Removed some unused imports --- attic/archive.py | 2 +- attic/helpers.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/attic/archive.py b/attic/archive.py index 68acf866c..245e4d7d6 100644 --- a/attic/archive.py +++ b/attic/archive.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta, timezone +from datetime import datetime from getpass import getuser from itertools import groupby import errno diff --git a/attic/helpers.py b/attic/helpers.py index 2e380cd68..14483027d 100644 --- a/attic/helpers.py +++ b/attic/helpers.py @@ -5,7 +5,6 @@ import msgpack import os import pwd import re -import stat import sys import time from datetime import datetime, timezone, timedelta From dd0c69ac32d6c66b44040d0cae0445490c5995a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Borgstr=C3=B6m?= Date: Sun, 19 Apr 2015 23:45:05 +0200 Subject: [PATCH 5/7] Fix "All archives" output for attic info Closes #183 --- CHANGES | 1 + attic/archive.py | 2 +- attic/cache.py | 13 ++++++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGES b/CHANGES index b0d2b749d..f703b9b12 100644 --- a/CHANGES +++ b/CHANGES @@ -7,6 +7,7 @@ Version 0.16 ------------ (bugfix release, released on X) +- Fix "All archives" output for attic info. (#183) - More user friendly error message when repository key file is not found (#236) - Fix parsing of iso 8601 timestamps with zero microseconds (#282) diff --git a/attic/archive.py b/attic/archive.py index 245e4d7d6..dd734b256 100644 --- a/attic/archive.py +++ b/attic/archive.py @@ -207,7 +207,7 @@ class Archive: def calc_stats(self, cache): def add(id): - count, size, csize = self.cache.chunks[id] + count, size, csize = cache.chunks[id] stats.update(size, csize, count == 1) self.cache.chunks[id] = count - 1, size, csize def add_file_chunks(chunks): diff --git a/attic/cache.py b/attic/cache.py index 3e0e50e08..95017b61b 100644 --- a/attic/cache.py +++ b/attic/cache.py @@ -93,11 +93,7 @@ class Cache(object): with open(os.path.join(self.path, 'files'), 'w') as fd: pass # empty file - def open(self): - if not os.path.isdir(self.path): - raise Exception('%s Does not look like an Attic cache' % self.path) - self.lock = UpgradableLock(os.path.join(self.path, 'config'), exclusive=True) - self.rollback() + def _do_open(self): self.config = RawConfigParser() self.config.read(os.path.join(self.path, 'config')) if self.config.getint('cache', 'version') != 1: @@ -110,6 +106,12 @@ class Cache(object): self.chunks = ChunkIndex.read(os.path.join(self.path, 'chunks').encode('utf-8')) self.files = None + def open(self): + if not os.path.isdir(self.path): + raise Exception('%s Does not look like an Attic cache' % self.path) + self.lock = UpgradableLock(os.path.join(self.path, 'config'), exclusive=True) + self.rollback() + def close(self): if self.lock: self.lock.release() @@ -181,6 +183,7 @@ class Cache(object): if os.path.exists(os.path.join(self.path, 'txn.tmp')): shutil.rmtree(os.path.join(self.path, 'txn.tmp')) self.txn_active = False + self._do_open() def sync(self): """Initializes cache by fetching and reading all archive indicies From 80dad2df61e75b1749b8fcec4828361d11c9cbb7 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 19 Apr 2015 23:54:44 +0200 Subject: [PATCH 6/7] use sys.platform everywhere, fixes #188 os.uname is UNIX-only, sys.platform is portable. note: - this doesn't implicate attic will now work on windows. - windows is untested / unsupported and there might be a lot of other issues left. - attic's xattr module already used sys.platform, so this is better for internal consistency also. --- attic/platform.py | 10 ++++------ setup.py | 8 +++----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/attic/platform.py b/attic/platform.py index 346d6b750..185dd2825 100644 --- a/attic/platform.py +++ b/attic/platform.py @@ -1,12 +1,10 @@ -import os +import sys -platform = os.uname()[0] - -if platform == 'Linux': +if sys.platform.startswith('linux'): from attic.platform_linux import acl_get, acl_set, API_VERSION -elif platform == 'FreeBSD': +elif sys.platform.startswith('freebsd'): from attic.platform_freebsd import acl_get, acl_set, API_VERSION -elif platform == 'Darwin': +elif sys.platform == 'darwin': from attic.platform_darwin import acl_get, acl_set, API_VERSION else: API_VERSION = 2 diff --git a/setup.py b/setup.py index 2c1432b10..4828fd812 100644 --- a/setup.py +++ b/setup.py @@ -9,8 +9,6 @@ versioneer.versionfile_build = 'attic/_version.py' versioneer.tag_prefix = '' versioneer.parentdir_prefix = 'Attic-' # dirname like 'myproject-1.2.0' -platform = os.uname()[0] - min_python = (3, 2) if sys.version_info < min_python: print("Attic requires Python %d.%d or later" % min_python) @@ -89,11 +87,11 @@ ext_modules = [ Extension('attic.chunker', [chunker_source]), Extension('attic.hashindex', [hashindex_source]) ] -if platform == 'Linux': +if sys.platform.startswith('linux'): ext_modules.append(Extension('attic.platform_linux', [platform_linux_source], libraries=['acl'])) -elif platform == 'FreeBSD': +elif sys.platform.startswith('freebsd'): ext_modules.append(Extension('attic.platform_freebsd', [platform_freebsd_source])) -elif platform == 'Darwin': +elif sys.platform == 'darwin': ext_modules.append(Extension('attic.platform_darwin', [platform_darwin_source])) setup( From ed9475d7cd1b3c516d6fe4e699b80160d33f4927 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Borgstr=C3=B6m?= Date: Mon, 20 Apr 2015 00:07:26 +0200 Subject: [PATCH 7/7] Fix cache object reference --- attic/archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attic/archive.py b/attic/archive.py index dd734b256..1b3818bc8 100644 --- a/attic/archive.py +++ b/attic/archive.py @@ -209,7 +209,7 @@ class Archive: def add(id): count, size, csize = cache.chunks[id] stats.update(size, csize, count == 1) - self.cache.chunks[id] = count - 1, size, csize + cache.chunks[id] = count - 1, size, csize def add_file_chunks(chunks): for id, _, _ in chunks: add(id)