From 3bb4745be319bdbc1dbc18bc77a8895fad44d25e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Borgstr=C3=B6m?= Date: Sat, 23 Oct 2010 21:38:42 +0200 Subject: [PATCH] More cryptography work. --- dedupestore/archive.py | 18 +++---- dedupestore/cache.py | 6 +-- dedupestore/crypto.py | 107 ++++++++++++++++++----------------------- dedupestore/helpers.py | 5 ++ 4 files changed, 63 insertions(+), 73 deletions(-) diff --git a/dedupestore/archive.py b/dedupestore/archive.py index 04d9f4e9d..38300d81b 100644 --- a/dedupestore/archive.py +++ b/dedupestore/archive.py @@ -1,5 +1,6 @@ from datetime import datetime import logging +import msgpack import os import stat import sys @@ -7,7 +8,7 @@ import sys from .cache import NS_ARCHIVES, NS_CHUNKS, NS_CINDEX from .chunkifier import chunkify from .crypto import CryptoManager -from .helpers import uid2user, user2uid, gid2group, group2gid +from .helpers import uid2user, user2uid, gid2group, group2gid, IntegrityError CHUNK_SIZE = 55001 @@ -26,12 +27,12 @@ class Archive(object): def load(self, id): self.id = id - archive = self.crypto.unpack_read(self.store.get(NS_ARCHIVES, self.id)) + archive = msgpack.unpackb(self.crypto.decrypt(self.store.get(NS_ARCHIVES, self.id))) if archive['version'] != 1: raise Exception('Archive version %r not supported' % archive['version']) self.items = archive['items'] self.name = archive['name'] - cindex = self.crypto.unpack_create(self.store.get(NS_CINDEX, self.id)) + cindex = msgpack.unpackb(self.crypto.decrypt(self.store.get(NS_CINDEX, self.id))) assert cindex['version'] == 1 self.chunks = cindex['chunks'] for i, chunk in enumerate(self.chunks): @@ -46,15 +47,14 @@ class Archive(object): 'ts': datetime.utcnow().isoformat(), 'items': self.items, } - data = self.crypto.pack_read(archive) + data = self.crypto.encrypt_read(msgpack.packb(archive)) self.store.put(NS_ARCHIVES, self.id, data) cindex = { 'version': 1, 'chunks': self.chunks, } - data = self.crypto.pack_create(cindex) + data = self.crypto.encrypt_create(msgpack.packb(cindex)) self.store.put(NS_CINDEX, self.id, data) - self.crypto.store_key() self.store.commit() def add_chunk(self, id, size): @@ -118,7 +118,7 @@ class Archive(object): for chunk in item['chunks']: id = self.chunk_idx[chunk] try: - fd.write(self.crypto.unpack_read(self.store.get(NS_CHUNKS, id))) + fd.write(self.crypto.decrypt(self.store.get(NS_CHUNKS, id))) except ValueError: raise Exception('Invalid chunk checksum') self.restore_stat(path, item) @@ -146,8 +146,8 @@ class Archive(object): for chunk in item['chunks']: id = self.chunk_idx[chunk] try: - self.crypto.unpack_read(self.store.get(NS_CHUNKS, id)) - except ValueError: + self.crypto.decrypt(self.store.get(NS_CHUNKS, id)) + except IntegrityError: logging.error('%s ... ERROR', item['path']) break else: diff --git a/dedupestore/cache.py b/dedupestore/cache.py index c0b16506a..f520d1edd 100644 --- a/dedupestore/cache.py +++ b/dedupestore/cache.py @@ -42,7 +42,7 @@ class Cache(object): if self.store.tid == 0: return for id in list(self.store.list(NS_CINDEX)): - cindex = crypto.unpack_create(self.store.get(NS_CINDEX, id)) + cindex = msgpack.unpackb(crypto.decrypt(self.store.get(NS_CINDEX, id))) for id, size in cindex['chunks']: try: count, size = self.chunkmap[id] @@ -65,10 +65,10 @@ class Cache(object): with open(self.path, 'wb') as fd: fd.write(data) - def add_chunk(self, id, data, crypt): + def add_chunk(self, id, data, crypto): if self.seen_chunk(id): return self.chunk_incref(id) - data = crypt.pack_read(data) + data = crypto.encrypt_read(data) csize = len(data) self.store.put(NS_CHUNKS, id, data) self.chunkmap[id] = (1, csize) diff --git a/dedupestore/crypto.py b/dedupestore/crypto.py index 981f2f6c4..34cb0f6e3 100644 --- a/dedupestore/crypto.py +++ b/dedupestore/crypto.py @@ -1,19 +1,18 @@ -import hashlib -import hmac -import msgpack import os import zlib from Crypto.Cipher import AES +from Crypto.Hash import SHA256, HMAC +from Crypto.Util.number import bytes_to_long, long_to_bytes + +from .helpers import IntegrityError +from .oaep import OAEP class CryptoManager(object): - KEY_CREATE = 1 - KEY_READ = 2 - KEY_ID = 3 - KEY_ARCHIVE = 4 - KEY_CINDEX = 5 + CREATE = '\1' + READ = '\2' def __init__(self, store): self.key_cache = {} @@ -22,61 +21,47 @@ class CryptoManager(object): self.id_key = '0' * 32 self.read_key = os.urandom(32) self.create_key = os.urandom(32) - - def get_key(self, tid): - try: - return self.key_cache[tid] - except KeyError: - keys = self.load_key(tid) - self.key_cache[tid] = keys - return keys - - def load_key(self, tid): - data = self.store.get('K', str(tid)) - id = data[:32] - if self.id_hash(data[32:]) != id: - raise Exception('Invalid key object found') - key = msgpack.unpackb(data[32:]) - return key['create'], key['read'] - - def store_key(self): - key = { - 'version': 1, - 'read': self.read_key, - 'create': self.create_key, - } - data = msgpack.packb(key) - id = self.id_hash(data) - self.store.put('K', str(self.tid), id + data) + self.read_encrypted = OAEP(256, hash=SHA256).encode(self.read_key, os.urandom(32)) + self.create_encrypted = OAEP(256, hash=SHA256).encode(self.create_key, os.urandom(32)) def id_hash(self, data): - return hmac.new(self.id_key, data, hashlib.sha256).digest() + return HMAC.new(self.id_key, data, SHA256).digest() - def pack(self, data, key): - data = zlib.compress(msgpack.packb(data)) - id = hmac.new(key, data, hashlib.sha256).digest() - data = AES.new(key, AES.MODE_CFB, id[:16]).encrypt(data) - return id + msgpack.packb((1, self.tid, data)) + def encrypt_read(self, data): + key_data = OAEP(256, hash=SHA256).encode(self.read_key, os.urandom(32)) + #key_data = self.rsa_create.encrypt(key_data) + data = zlib.compress(data) + hash = SHA256.new(data).digest() + data = AES.new(self.read_key, AES.MODE_CFB, hash[:16]).encrypt(data) + return ''.join((self.READ, self.read_encrypted, hash, data)) - def pack_read(self, data): - return self.pack(data, self.read_key) - - def pack_create(self, data): - return self.pack(data, self.create_key) - - def unpack(self, data, key_idx): - id = data[:32] - version, tid, data = msgpack.unpackb(data[32:]) - assert version == 1 - key = self.get_key(tid)[key_idx] - data = AES.new(key, AES.MODE_CFB, id[:16]).decrypt(data) - if hmac.new(key, data, hashlib.sha256).digest() != id: - raise ValueError - return msgpack.unpackb(zlib.decompress(data)) - - def unpack_read(self, data): - return self.unpack(data, 1) - - def unpack_create(self, data): - return self.unpack(data, 0) + def encrypt_create(self, data): + key_data = OAEP(256, hash=SHA256).encode(self.create_key, os.urandom(32)) + #key_data = self.rsa_create.encrypt(key_data) + data = zlib.compress(data) + hash = SHA256.new(data).digest() + data = AES.new(self.create_key, AES.MODE_CFB, hash[:16]).encrypt(data) + return ''.join((self.CREATE, self.create_encrypted, hash, data)) + def decrypt(self, data): + type = data[0] + if type == self.READ: + key_data = data[1:257] + hash = data[257:289] + #key_data = self.rsa_create.decrypt(key_data) + key = OAEP(256, hash=SHA256).decode(key_data) + data = AES.new(key, AES.MODE_CFB, hash[:16]).decrypt(data[289:]) + if SHA256.new(data).digest() != hash: + raise IntegrityError('decryption failed') + return zlib.decompress(data) + elif type == self.CREATE: + key_data = data[1:257] + hash = data[257:289] + #key_data = self.rsa_create.decrypt(key_data) + key = OAEP(256, hash=SHA256).decode(key_data) + data = AES.new(key, AES.MODE_CFB, hash[:16]).decrypt(data[289:]) + if SHA256.new(data).digest() != hash: + raise IntegrityError('decryption failed') + return zlib.decompress(data) + else: + raise Exception('Unknown pack type %d found' % ord(type)) diff --git a/dedupestore/helpers.py b/dedupestore/helpers.py index 8a4267c64..0be819312 100644 --- a/dedupestore/helpers.py +++ b/dedupestore/helpers.py @@ -5,6 +5,11 @@ import pwd import re +class IntegrityError(Exception): + """ + """ + + def memoize(function): cache = {} def decorated_function(*args):