From b6445655460e8c61326410d024a30a377e8cd882 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Wed, 15 Jul 2015 00:01:07 +0200 Subject: [PATCH] repo key mode (and deprecate passphrase mode), fixes #85 see usage.rst change for a description and why this is needed --- borg/archiver.py | 2 +- borg/key.py | 274 +++++++++++++++++++++++++------------- borg/remote.py | 11 ++ borg/repository.py | 19 ++- borg/testsuite/archive.py | 4 +- docs/usage.rst | 39 +++++- 6 files changed, 249 insertions(+), 100 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 8ddebd210..6275edf22 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -539,7 +539,7 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") type=location_validator(archive=False), help='repository to create') subparser.add_argument('-e', '--encryption', dest='encryption', - choices=('none', 'passphrase', 'keyfile'), default='none', + choices=('none', 'passphrase', 'keyfile', 'repokey'), default='none', help='select encryption method') check_epilog = textwrap.dedent(""" diff --git a/borg/key.py b/borg/key.py index 31267d0d9..fabdae5b3 100644 --- a/borg/key.py +++ b/borg/key.py @@ -1,5 +1,6 @@ from binascii import hexlify, a2b_base64, b2a_base64 -from getpass import getpass +import configparser +import getpass import os import msgpack import textwrap @@ -23,6 +24,11 @@ class KeyfileNotFoundError(Error): """ +class RepoKeyNotFoundError(Error): + """No key entry found in the config of repository {}. + """ + + class HMAC(hmac.HMAC): """Workaround a bug in Python < 3.4 Where HMAC does not accept memoryviews """ @@ -33,27 +39,35 @@ class HMAC(hmac.HMAC): def key_creator(repository, args): if args.encryption == 'keyfile': return KeyfileKey.create(repository, args) - elif args.encryption == 'passphrase': + elif args.encryption == 'repokey': + return RepoKey.create(repository, args) + elif args.encryption == 'passphrase': # deprecated, kill in 1.x return PassphraseKey.create(repository, args) else: return PlaintextKey.create(repository, args) def key_factory(repository, manifest_data): - if manifest_data[0] == KeyfileKey.TYPE: + key_type = manifest_data[0] + if key_type == KeyfileKey.TYPE: return KeyfileKey.detect(repository, manifest_data) - elif manifest_data[0] == PassphraseKey.TYPE: + elif key_type == RepoKey.TYPE: + return RepoKey.detect(repository, manifest_data) + elif key_type == PassphraseKey.TYPE: # deprecated, kill in 1.x return PassphraseKey.detect(repository, manifest_data) - elif manifest_data[0] == PlaintextKey.TYPE: + elif key_type == PlaintextKey.TYPE: return PlaintextKey.detect(repository, manifest_data) else: - raise UnsupportedPayloadError(manifest_data[0]) + raise UnsupportedPayloadError(key_type) class KeyBase: + TYPE = None # override in subclasses - def __init__(self): + def __init__(self, repository): self.TYPE_STR = bytes([self.TYPE]) + self.repository = repository + self.target = None # key location file path / repo obj self.compression_level = 0 def id_hash(self, data): @@ -74,12 +88,12 @@ class PlaintextKey(KeyBase): @classmethod def create(cls, repository, args): - print('Encryption NOT enabled.\nUse the "--encryption=passphrase|keyfile" to enable encryption.') - return cls() + print('Encryption NOT enabled.\nUse the "--encryption=repokey|keyfile|passphrase" to enable encryption.') + return cls(repository) @classmethod def detect(cls, repository, manifest_data): - return cls() + return cls(repository) def id_hash(self, data): return sha256(data).digest() @@ -155,38 +169,65 @@ class AESKeyBase(KeyBase): self.dec_cipher = AES(is_encrypt=False, key=self.enc_key) +class Passphrase(str): + @classmethod + def env_passphrase(cls, default=None): + passphrase = os.environ.get('BORG_PASSPHRASE', default) + if passphrase is not None: + return cls(passphrase) + + @classmethod + def getpass(cls, prompt): + return cls(getpass.getpass(prompt)) + + @classmethod + def new(cls, allow_empty=False): + passphrase = cls.env_passphrase() + if passphrase is not None: + return passphrase + while True: + passphrase = cls.getpass('Enter new passphrase: ') + if allow_empty or passphrase: + passphrase2 = cls.getpass('Enter same passphrase again: ') + if passphrase == passphrase2: + print('Remember your passphrase. Your data will be inaccessible without it.') + return passphrase + else: + print('Passphrases do not match') + else: + print('Passphrase must not be blank') + + def __repr__(self): + return '' + + def kdf(self, salt, iterations, length): + return pbkdf2_sha256(self.encode('utf-8'), salt, iterations, length) + + class PassphraseKey(AESKeyBase): + # This mode is DEPRECATED and will be killed at 1.0 release. + # With this mode: + # - you can never ever change your passphrase for existing repos. + # - you can never ever use a different iterations count for existing repos. TYPE = 0x01 - iterations = 100000 + iterations = 100000 # must not be changed ever! @classmethod def create(cls, repository, args): - key = cls() - passphrase = os.environ.get('BORG_PASSPHRASE') - if passphrase is not None: - passphrase2 = passphrase - else: - passphrase, passphrase2 = 1, 2 - while passphrase != passphrase2: - passphrase = getpass('Enter passphrase: ') - if not passphrase: - print('Passphrase must not be blank') - continue - passphrase2 = getpass('Enter same passphrase again: ') - if passphrase != passphrase2: - print('Passphrases do not match') + key = cls(repository) + print('WARNING: "passphrase" mode is deprecated and will be removed in 1.0.') + print('If you want something similar (but with less issues), use "repokey" mode.') + passphrase = Passphrase.new(allow_empty=False) key.init(repository, passphrase) - if passphrase: - print('Remember your passphrase. Your data will be inaccessible without it.') return key @classmethod def detect(cls, repository, manifest_data): prompt = 'Enter passphrase for %s: ' % repository._location.orig - key = cls() - passphrase = os.environ.get('BORG_PASSPHRASE') + key = cls(repository) + passphrase = Passphrase.env_passphrase() if passphrase is None: - passphrase = getpass(prompt) + passphrase = Passphrase.getpass(prompt) while True: key.init(repository, passphrase) try: @@ -195,7 +236,7 @@ class PassphraseKey(AESKeyBase): key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks)) return key except IntegrityError: - passphrase = getpass(prompt) + passphrase = Passphrase.getpass(prompt) def change_passphrase(self): class ImmutablePassphraseError(Error): @@ -204,41 +245,31 @@ class PassphraseKey(AESKeyBase): raise ImmutablePassphraseError def init(self, repository, passphrase): - self.init_from_random_data(pbkdf2_sha256(passphrase.encode('utf-8'), repository.id, self.iterations, 100)) + self.init_from_random_data(passphrase.kdf(repository.id, self.iterations, 100)) self.init_ciphers() -class KeyfileKey(AESKeyBase): - FILE_ID = 'BORG_KEY' - TYPE = 0x00 - +class KeyfileKeyBase(AESKeyBase): @classmethod def detect(cls, repository, manifest_data): - key = cls() - path = cls.find_key_file(repository) - prompt = 'Enter passphrase for key file %s: ' % path - passphrase = os.environ.get('BORG_PASSPHRASE', '') - while not key.load(path, passphrase): - passphrase = getpass(prompt) + key = cls(repository) + target = key.find_key() + prompt = 'Enter passphrase for key %s: ' % target + passphrase = Passphrase.env_passphrase(default='') + while not key.load(target, passphrase): + passphrase = Passphrase.getpass(prompt) num_blocks = num_aes_blocks(len(manifest_data) - 41) key.init_ciphers(PREFIX + long_to_bytes(key.extract_nonce(manifest_data) + num_blocks)) return key - @classmethod - def find_key_file(cls, repository): - id = hexlify(repository.id).decode('ascii') - keys_dir = get_keys_dir() - for name in os.listdir(keys_dir): - filename = os.path.join(keys_dir, name) - with open(filename, 'r') as fd: - line = fd.readline().strip() - if line and line.startswith(cls.FILE_ID) and line[len(cls.FILE_ID)+1:] == id: - return filename - raise KeyfileNotFoundError(repository._location.canonical_path(), get_keys_dir()) + def find_key(self): + raise NotImplementedError - def load(self, filename, passphrase): - with open(filename, 'r') as fd: - cdata = a2b_base64(''.join(fd.readlines()[1:]).encode('ascii')) # .encode needed for Python 3.[0-2] + def load(self, target, passphrase): + raise NotImplementedError + + def _load(self, key_data, passphrase): + cdata = a2b_base64(key_data.encode('ascii')) # .encode needed for Python 3.[0-2] data = self.decrypt_key_file(cdata, passphrase) if data: key = msgpack.unpackb(data) @@ -249,23 +280,22 @@ class KeyfileKey(AESKeyBase): self.enc_hmac_key = key[b'enc_hmac_key'] self.id_key = key[b'id_key'] self.chunk_seed = key[b'chunk_seed'] - self.path = filename return True + return False def decrypt_key_file(self, data, passphrase): d = msgpack.unpackb(data) assert d[b'version'] == 1 assert d[b'algorithm'] == b'sha256' - key = pbkdf2_sha256(passphrase.encode('utf-8'), d[b'salt'], d[b'iterations'], 32) + key = passphrase.kdf(d[b'salt'], d[b'iterations'], 32) data = AES(is_encrypt=False, key=key).decrypt(d[b'data']) - if HMAC(key, data, sha256).digest() != d[b'hash']: - return None - return data + if HMAC(key, data, sha256).digest() == d[b'hash']: + return data def encrypt_key_file(self, data, passphrase): salt = get_random_bytes(32) iterations = 100000 - key = pbkdf2_sha256(passphrase.encode('utf-8'), salt, iterations, 32) + key = passphrase.kdf(salt, iterations, 32) hash = HMAC(key, data, sha256).digest() cdata = AES(is_encrypt=True, key=key).encrypt(data) d = { @@ -278,7 +308,7 @@ class KeyfileKey(AESKeyBase): } return msgpack.packb(d) - def save(self, path, passphrase): + def _save(self, passphrase): key = { 'version': 1, 'repository_id': self.repository_id, @@ -288,45 +318,101 @@ class KeyfileKey(AESKeyBase): 'chunk_seed': self.chunk_seed, } data = self.encrypt_key_file(msgpack.packb(key), passphrase) - with open(path, 'w') as fd: - fd.write('%s %s\n' % (self.FILE_ID, hexlify(self.repository_id).decode('ascii'))) - fd.write('\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii')))) - fd.write('\n') - self.path = path + key_data = '\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii'))) + return key_data def change_passphrase(self): - passphrase, passphrase2 = 1, 2 - while passphrase != passphrase2: - passphrase = getpass('New passphrase: ') - passphrase2 = getpass('Enter same passphrase again: ') - if passphrase != passphrase2: - print('Passphrases do not match') - self.save(self.path, passphrase) - print('Key file "%s" updated' % self.path) + passphrase = Passphrase.new(allow_empty=True) + self.save(self.target, passphrase) + print('Key updated') @classmethod def create(cls, repository, args): + passphrase = Passphrase.new(allow_empty=True) + key = cls(repository) + key.repository_id = repository.id + key.init_from_random_data(get_random_bytes(100)) + key.init_ciphers() + target = key.get_new_target(args) + key.save(target, passphrase) + print('Key in "%s" created.' % target) + print('Keep this key safe. Your data will be inaccessible without it.') + return key + + def save(self, target, passphrase): + raise NotImplementedError + + def get_new_target(self, args): + raise NotImplementedError + + +class KeyfileKey(KeyfileKeyBase): + TYPE = 0x00 + FILE_ID = 'BORG_KEY' + + def find_key(self): + id = hexlify(self.repository.id).decode('ascii') + keys_dir = get_keys_dir() + for name in os.listdir(keys_dir): + filename = os.path.join(keys_dir, name) + with open(filename, 'r') as fd: + line = fd.readline().strip() + if line.startswith(self.FILE_ID) and line[len(self.FILE_ID)+1:] == id: + return filename + raise KeyfileNotFoundError(self.repository._location.canonical_path(), get_keys_dir()) + + def get_new_target(self, args): filename = args.repository.to_key_filename() path = filename i = 1 while os.path.exists(path): i += 1 path = filename + '.%d' % i - passphrase = os.environ.get('BORG_PASSPHRASE') - if passphrase is not None: - passphrase2 = passphrase - else: - passphrase, passphrase2 = 1, 2 - while passphrase != passphrase2: - passphrase = getpass('Enter passphrase (empty for no passphrase):') - passphrase2 = getpass('Enter same passphrase again: ') - if passphrase != passphrase2: - print('Passphrases do not match') - key = cls() - key.repository_id = repository.id - key.init_from_random_data(get_random_bytes(100)) - key.init_ciphers() - key.save(path, passphrase) - print('Key file "%s" created.' % key.path) - print('Keep this file safe. Your data will be inaccessible without it.') - return key + return path + + def load(self, target, passphrase): + with open(target, 'r') as fd: + key_data = ''.join(fd.readlines()[1:]) + success = self._load(key_data, passphrase) + if success: + self.target = target + return success + + def save(self, target, passphrase): + key_data = self._save(passphrase) + with open(target, 'w') as fd: + fd.write('%s %s\n' % (self.FILE_ID, hexlify(self.repository_id).decode('ascii'))) + fd.write(key_data) + fd.write('\n') + self.target = target + + +class RepoKey(KeyfileKeyBase): + TYPE = 0x03 + + def find_key(self): + loc = self.repository._location.canonical_path() + try: + self.repository.load_key() + return loc + except configparser.NoOptionError: + raise RepoKeyNotFoundError(loc) + + def get_new_target(self, args): + return self.repository + + def load(self, target, passphrase): + # what we get in target is just a repo location, but we already have the repo obj: + target = self.repository + key_data = target.load_key() + key_data = key_data.decode('utf-8') # remote repo: msgpack issue #99, getting bytes + success = self._load(key_data, passphrase) + if success: + self.target = target + return success + + def save(self, target, passphrase): + key_data = self._save(passphrase) + key_data = key_data.encode('utf-8') # remote repo: msgpack issue #99, giving bytes + target.save_key(key_data) + self.target = target diff --git a/borg/remote.py b/borg/remote.py index 0ad91d76e..fe859ac1f 100644 --- a/borg/remote.py +++ b/borg/remote.py @@ -41,6 +41,8 @@ class RepositoryServer: 'put', 'repair', 'rollback', + 'save_key', + 'load_key', ) def __init__(self, restrict_to_paths): @@ -151,6 +153,9 @@ class RemoteRepository: def __del__(self): self.close() + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self.location.canonical_path()) + def call(self, cmd, *args, **kw): for resp in self.call_many(cmd, [args], **kw): return resp @@ -276,6 +281,12 @@ class RemoteRepository: def delete(self, id_, wait=True): return self.call('delete', id_, wait=wait) + def save_key(self, keydata): + return self.call('save_key', keydata) + + def load_key(self): + return self.call('load_key') + def close(self): if self.p: self.p.stdin.close() diff --git a/borg/repository.py b/borg/repository.py index 60831c8ee..97cdeac04 100644 --- a/borg/repository.py +++ b/borg/repository.py @@ -62,6 +62,9 @@ class Repository: def __del__(self): self.close() + def __repr__(self): + return '<%s %s>' % (self.__class__.__name__, self.path) + def create(self, path): """Create a new empty repository at `path` """ @@ -78,9 +81,23 @@ class Repository: config.set('repository', 'segments_per_dir', self.DEFAULT_SEGMENTS_PER_DIR) config.set('repository', 'max_segment_size', self.DEFAULT_MAX_SEGMENT_SIZE) config.set('repository', 'id', hexlify(os.urandom(32)).decode('ascii')) - with open(os.path.join(path, 'config'), 'w') as fd: + self.save_config(path, config) + + def save_config(self, path, config): + config_path = os.path.join(path, 'config') + with open(config_path, 'w') as fd: config.write(fd) + def save_key(self, keydata): + assert self.config + keydata = keydata.decode('utf-8') # remote repo: msgpack issue #99, getting bytes + self.config.set('repository', 'key', keydata) + self.save_config(self.path, self.config) + + def load_key(self): + keydata = self.config.get('repository', 'key') + return keydata.encode('utf-8') # remote repo: msgpack issue #99, returning bytes + def destroy(self): """Destroy the repository at `self.path` """ diff --git a/borg/testsuite/archive.py b/borg/testsuite/archive.py index abb5bccb9..9a20e9f6e 100644 --- a/borg/testsuite/archive.py +++ b/borg/testsuite/archive.py @@ -23,7 +23,7 @@ class ArchiveTimestampTestCase(BaseTestCase): def _test_timestamp_parsing(self, isoformat, expected): repository = Mock() - key = PlaintextKey() + key = PlaintextKey(repository) manifest = Manifest(repository, key) a = Archive(repository, key, manifest, 'test', create=True) a.metadata = {b'time': isoformat} @@ -45,7 +45,7 @@ class ChunkBufferTestCase(BaseTestCase): def test(self): data = [{b'foo': 1}, {b'bar': 2}] cache = MockCache() - key = PlaintextKey() + key = PlaintextKey(None) chunks = CacheChunkBuffer(cache, key, None) for d in data: chunks.add(d) diff --git a/docs/usage.rst b/docs/usage.rst index d5d3a94a9..c90cf97bc 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -78,8 +78,43 @@ Examples # Remote repository (accesses a remote borg via ssh) $ borg init user@hostname:backup - # Encrypted remote repository - $ borg init --encryption=passphrase user@hostname:backup + # Encrypted remote repository, store the key in the repo + $ borg init --encryption=repokey user@hostname:backup + + # Encrypted remote repository, store the key your home dir + $ borg init --encryption=keyfile user@hostname:backup + +Important notes about encryption: + +Use encryption! Repository encryption protects you e.g. against the case that +an attacker has access to your backup repository. + +But be careful with the key / the passphrase: + +``--encryption=passphrase`` is DEPRECATED and will be removed in next major release. +This mode has very fundamental, unfixable problems (like you can never change +your passphrase or the pbkdf2 iteration count for an existing repository, because +the encryption / decryption key is directly derived from the passphrase). + +If you want "passphrase-only" security, just use the ``repokey`` mode. The key will +be stored inside the repository (in its "config" file). In above mentioned +attack scenario, the attacker will have the key (but not the passphrase). + +If you want "passphrase and having-the-key" security, use the ``keyfile`` mode. +The key will be stored in your home directory (in ``.borg/keys``). In the attack +scenario, the attacker who has just access to your repo won't have the key (and +also not the passphrase). + +Make a backup copy of the key file (``keyfile`` mode) or repo config file +(``repokey`` mode) and keep it at a safe place, so you still have the key in +case it gets corrupted or lost. +The backup that is encrypted with that key won't help you with that, of course. + +Make sure you use a good passphrase. Not too short, not too simple. The real +encryption / decryption key is encrypted with / locked by your passphrase. +If an attacker gets your key, he can't unlock and use it without knowing the +passphrase. In ``repokey`` and ``keyfile`` modes, you can change your passphrase +for existing repos. .. include:: usage/create.rst.inc