diff --git a/docs/usage.rst b/docs/usage.rst index 51fa600a8..8410d1156 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -189,9 +189,10 @@ Directories and files: Default to '~/.config/borg/keys'. This directory contains keys for encrypted repositories. BORG_KEY_FILE When set, use the given filename as repository key file. - BORG_NONCES_DIR - Default to '~/.config/borg/key-nonces'. This directory contains information borg uses to - track its usage of NONCES ("numbers used once" - usually in encryption context). + BORG_SECURITY_DIR + Default to '~/.config/borg/security'. This directory contains information borg uses to + track its usage of NONCES ("numbers used once" - usually in encryption context) and other + security relevant data. BORG_CACHE_DIR Default to '~/.cache/borg'. This directory contains the local cache and might need a lot of space for dealing with big repositories). diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 6b443257d..996eb9453 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -970,6 +970,7 @@ class Archiver: if key.NAME.startswith('key file'): print('Key file: %s' % key.find_key()) print('Cache: %s' % cache.path) + print('Security dir: %s' % cache.security_manager.dir) print(DASHES) print(STATS_HEADER) print(str(cache)) diff --git a/src/borg/cache.py b/src/borg/cache.py index aa91e7b3d..813c94698 100644 --- a/src/borg/cache.py +++ b/src/borg/cache.py @@ -14,7 +14,7 @@ from .constants import CACHE_README from .hashindex import ChunkIndex, ChunkIndexEntry from .helpers import Location from .helpers import Error -from .helpers import get_cache_dir +from .helpers import get_cache_dir, get_security_dir from .helpers import decode_dict, int_to_bigint, bigint_to_int, bin_to_hex from .helpers import format_file_size from .helpers import yes @@ -29,6 +29,113 @@ ChunkListEntry = namedtuple('ChunkListEntry', 'id size csize') FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size mtime chunk_ids') +class SecurityManager: + def __init__(self, repository): + self.repository = repository + self.dir = get_security_dir(repository.id_str) + self.key_type_file = os.path.join(self.dir, 'key-type') + self.location_file = os.path.join(self.dir, 'location') + self.manifest_ts_file = os.path.join(self.dir, 'manifest-timestamp') + + def known(self): + return os.path.exists(self.key_type_file) + + def key_matches(self, key): + if not self.known(): + return False + try: + with open(self.key_type_file, 'r') as fd: + type = fd.read() + return type == str(key.TYPE) + except OSError as exc: + logger.warning('Could not read/parse key type file: %s', exc) + + def save(self, manifest, key, cache): + logger.debug('security: saving state for %s to %s', self.repository.id_str, self.dir) + current_location = cache.repository._location.canonical_path() + logger.debug('security: current location %s', current_location) + logger.debug('security: key type %s', str(key.TYPE)) + logger.debug('security: manifest timestamp %s', manifest.timestamp) + with open(self.location_file, 'w') as fd: + fd.write(current_location) + with open(self.key_type_file, 'w') as fd: + fd.write(str(key.TYPE)) + with open(self.manifest_ts_file, 'w') as fd: + fd.write(manifest.timestamp) + + def assert_location_matches(self, cache): + # Warn user before sending data to a relocated repository + try: + with open(self.location_file) as fd: + previous_location = fd.read() + logger.debug('security: read previous_location %r', previous_location) + except FileNotFoundError: + logger.debug('security: previous_location file %s not found', self.location_file) + previous_location = None + except OSError as exc: + logger.warning('Could not read previous location file: %s', exc) + previous_location = None + if cache.previous_location and previous_location != cache.previous_location: + # Reconcile cache and security dir; we take the cache location. + previous_location = cache.previous_location + logger.debug('security: using previous_location of cache: %r', previous_location) + if previous_location and previous_location != self.repository._location.canonical_path(): + msg = ("Warning: The repository at location {} was previously located at {}\n".format( + self.repository._location.canonical_path(), previous_location) + + "Do you want to continue? [yN] ") + if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.", + retry=False, env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'): + raise Cache.RepositoryAccessAborted() + # adapt on-disk config immediately if the new location was accepted + logger.debug('security: updating location stored in cache and security dir') + with open(self.location_file, 'w') as fd: + fd.write(cache.repository._location.canonical_path()) + cache.begin_txn() + cache.commit() + + def assert_no_manifest_replay(self, manifest, key, cache): + try: + with open(self.manifest_ts_file) as fd: + timestamp = fd.read() + logger.debug('security: read manifest timestamp %r', timestamp) + except FileNotFoundError: + logger.debug('security: manifest timestamp file %s not found', self.manifest_ts_file) + timestamp = '' + except OSError as exc: + logger.warning('Could not read previous location file: %s', exc) + timestamp = '' + timestamp = max(timestamp, cache.timestamp or '') + logger.debug('security: determined newest manifest timestamp as %s', timestamp) + # If repository is older than the cache or security dir something fishy is going on + if timestamp and timestamp > manifest.timestamp: + if isinstance(key, PlaintextKey): + raise Cache.RepositoryIDNotUnique() + else: + raise Cache.RepositoryReplay() + + def assert_key_type(self, key, cache): + # Make sure an encrypted repository has not been swapped for an unencrypted repository + if cache.key_type is not None and cache.key_type != str(key.TYPE): + raise Cache.EncryptionMethodMismatch() + if self.known() and not self.key_matches(key): + raise Cache.EncryptionMethodMismatch() + + def assert_secure(self, manifest, key, cache): + self.assert_location_matches(cache) + self.assert_key_type(key, cache) + self.assert_no_manifest_replay(manifest, key, cache) + if not self.known(): + self.save(manifest, key, cache) + + def assert_access_unknown(self, warn_if_unencrypted, key): + if warn_if_unencrypted and isinstance(key, PlaintextKey) and not self.known(): + msg = ("Warning: Attempting to access a previously unknown unencrypted repository!\n" + + "Do you want to continue? [yN] ") + if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.", + retry=False, env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'): + raise Cache.CacheInitAbortedError() + + class Cache: """Client Side cache """ @@ -77,44 +184,19 @@ class Cache: self.key = key self.manifest = manifest self.path = path or os.path.join(get_cache_dir(), repository.id_str) + self.security_manager = SecurityManager(repository) self.hostname_is_unique = yes(env_var_override='BORG_HOSTNAME_IS_UNIQUE', prompt=False, env_msg=None) if self.hostname_is_unique: logger.info('Enabled removal of stale cache locks') self.do_files = do_files # Warn user before sending data to a never seen before unencrypted repository if not os.path.exists(self.path): - if warn_if_unencrypted and isinstance(key, PlaintextKey): - msg = ("Warning: Attempting to access a previously unknown unencrypted repository!" + - "\n" + - "Do you want to continue? [yN] ") - if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.", - retry=False, env_var_override='BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK'): - raise self.CacheInitAbortedError() + self.security_manager.assert_access_unknown(warn_if_unencrypted, key) self.create() self.open(lock_wait=lock_wait) try: - # Warn user before sending data to a relocated repository - if self.previous_location and self.previous_location != repository._location.canonical_path(): - msg = ("Warning: The repository at location {} was previously located at {}".format(repository._location.canonical_path(), self.previous_location) + - "\n" + - "Do you want to continue? [yN] ") - if not yes(msg, false_msg="Aborting.", invalid_msg="Invalid answer, aborting.", - retry=False, env_var_override='BORG_RELOCATED_REPO_ACCESS_IS_OK'): - raise self.RepositoryAccessAborted() - # adapt on-disk config immediately if the new location was accepted - self.begin_txn() - self.commit() - + self.security_manager.assert_secure(manifest, key, self) if sync and self.manifest.id != self.manifest_id: - # If repository is older than the cache something fishy is going on - if self.timestamp and self.timestamp > manifest.timestamp: - if isinstance(key, PlaintextKey): - raise self.RepositoryIDNotUnique() - else: - raise self.RepositoryReplay() - # Make sure an encrypted repository has not been swapped for an unencrypted repository - if self.key_type is not None and self.key_type != str(key.TYPE): - raise self.EncryptionMethodMismatch() self.sync() self.commit() except: @@ -252,6 +334,7 @@ Chunk index: {0.total_unique_chunks:20d} {0.total_chunks:20d}""" """ if not self.txn_active: return + self.security_manager.save(self.manifest, self.key, self) pi = ProgressIndicatorMessage() if self.files is not None: if self._newest_mtime is None: diff --git a/src/borg/helpers.py b/src/borg/helpers.py index b3a48d16e..93413845b 100644 --- a/src/borg/helpers.py +++ b/src/borg/helpers.py @@ -288,15 +288,17 @@ def get_keys_dir(): return keys_dir -def get_nonces_dir(): - """Determine where to store the local nonce high watermark""" +def get_security_dir(repository_id=None): + """Determine where to store local security information.""" xdg_config = os.environ.get('XDG_CONFIG_HOME', os.path.join(get_home_dir(), '.config')) - nonces_dir = os.environ.get('BORG_NONCES_DIR', os.path.join(xdg_config, 'borg', 'key-nonces')) - if not os.path.exists(nonces_dir): - os.makedirs(nonces_dir) - os.chmod(nonces_dir, stat.S_IRWXU) - return nonces_dir + security_dir = os.environ.get('BORG_SECURITY_DIR', os.path.join(xdg_config, 'borg', 'security')) + if repository_id: + security_dir = os.path.join(security_dir, repository_id) + if not os.path.exists(security_dir): + os.makedirs(security_dir) + os.chmod(security_dir, stat.S_IRWXU) + return security_dir def get_cache_dir(): diff --git a/src/borg/nonces.py b/src/borg/nonces.py index 4f9299581..e6eb7a2c7 100644 --- a/src/borg/nonces.py +++ b/src/borg/nonces.py @@ -3,7 +3,7 @@ import sys from binascii import unhexlify from .crypto import bytes_to_long, long_to_bytes -from .helpers import get_nonces_dir +from .helpers import get_security_dir from .helpers import bin_to_hex from .platform import SaveFile from .remote import InvalidRPCMethod @@ -19,7 +19,7 @@ class NonceManager: self.enc_cipher = enc_cipher self.end_of_nonce_reservation = None self.manifest_nonce = manifest_nonce - self.nonce_file = os.path.join(get_nonces_dir(), self.repository.id_str) + self.nonce_file = os.path.join(get_security_dir(self.repository.id_str), 'nonce') def get_local_free_nonce(self): try: diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 6e8aa9c7a..75f67ecbf 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -29,7 +29,7 @@ from ..archiver import Archiver from ..cache import Cache from ..constants import * # NOQA from ..crypto import bytes_to_long, num_aes_blocks -from ..helpers import PatternMatcher, parse_pattern, Location +from ..helpers import PatternMatcher, parse_pattern, Location, get_security_dir from ..helpers import Chunk, Manifest from ..helpers import EXIT_SUCCESS, EXIT_WARNING, EXIT_ERROR from ..helpers import bin_to_hex @@ -382,8 +382,7 @@ class ArchiverTestCase(ArchiverTestCaseBase): item_count = 4 if has_lchflags else 5 # one file is UF_NODUMP self.assert_in('Number of files: %d' % item_count, info_output) shutil.rmtree(self.cache_path) - with environment_variable(BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK='yes'): - info_output2 = self.cmd('info', self.repository_location + '::test') + info_output2 = self.cmd('info', self.repository_location + '::test') def filter(output): # filter for interesting "info" output, ignore cache rebuilding related stuff @@ -563,6 +562,89 @@ class ArchiverTestCase(ArchiverTestCaseBase): else: self.assert_raises(Cache.RepositoryAccessAborted, lambda: self.cmd('create', self.repository_location + '_encrypted::test.2', 'input')) + def test_repository_swap_detection_no_cache(self): + self.create_test_files() + os.environ['BORG_PASSPHRASE'] = 'passphrase' + self.cmd('init', '--encryption=repokey', self.repository_location) + repository_id = self._extract_repository_id(self.repository_path) + self.cmd('create', self.repository_location + '::test', 'input') + shutil.rmtree(self.repository_path) + self.cmd('init', '--encryption=none', self.repository_location) + self._set_repository_id(self.repository_path, repository_id) + self.assert_equal(repository_id, self._extract_repository_id(self.repository_path)) + self.cmd('delete', '--cache-only', self.repository_location) + if self.FORK_DEFAULT: + self.cmd('create', self.repository_location + '::test.2', 'input', exit_code=EXIT_ERROR) + else: + self.assert_raises(Cache.EncryptionMethodMismatch, lambda: self.cmd('create', self.repository_location + '::test.2', 'input')) + + def test_repository_swap_detection2_no_cache(self): + self.create_test_files() + self.cmd('init', '--encryption=none', self.repository_location + '_unencrypted') + os.environ['BORG_PASSPHRASE'] = 'passphrase' + self.cmd('init', '--encryption=repokey', self.repository_location + '_encrypted') + self.cmd('create', self.repository_location + '_encrypted::test', 'input') + self.cmd('delete', '--cache-only', self.repository_location + '_unencrypted') + self.cmd('delete', '--cache-only', self.repository_location + '_encrypted') + shutil.rmtree(self.repository_path + '_encrypted') + os.rename(self.repository_path + '_unencrypted', self.repository_path + '_encrypted') + if self.FORK_DEFAULT: + self.cmd('create', self.repository_location + '_encrypted::test.2', 'input', exit_code=EXIT_ERROR) + else: + with pytest.raises(Cache.RepositoryAccessAborted): + self.cmd('create', self.repository_location + '_encrypted::test.2', 'input') + + def test_repository_move(self): + self.cmd('init', self.repository_location) + repository_id = bin_to_hex(self._extract_repository_id(self.repository_path)) + os.rename(self.repository_path, self.repository_path + '_new') + with environment_variable(BORG_RELOCATED_REPO_ACCESS_IS_OK='yes'): + self.cmd('info', self.repository_location + '_new') + security_dir = get_security_dir(repository_id) + with open(os.path.join(security_dir, 'location')) as fd: + location = fd.read() + assert location == Location(self.repository_location + '_new').canonical_path() + # Needs no confirmation anymore + self.cmd('info', self.repository_location + '_new') + shutil.rmtree(self.cache_path) + self.cmd('info', self.repository_location + '_new') + shutil.rmtree(security_dir) + self.cmd('info', self.repository_location + '_new') + for file in ('location', 'key-type', 'manifest-timestamp'): + assert os.path.exists(os.path.join(security_dir, file)) + + def test_security_dir_compat(self): + self.cmd('init', self.repository_location) + repository_id = bin_to_hex(self._extract_repository_id(self.repository_path)) + security_dir = get_security_dir(repository_id) + with open(os.path.join(security_dir, 'location'), 'w') as fd: + fd.write('something outdated') + # This is fine, because the cache still has the correct information. security_dir and cache can disagree + # if older versions are used to confirm a renamed repository. + self.cmd('info', self.repository_location) + + def test_unknown_unencrypted(self): + self.cmd('init', '--encryption=none', self.repository_location) + repository_id = bin_to_hex(self._extract_repository_id(self.repository_path)) + security_dir = get_security_dir(repository_id) + # Ok: repository is known + self.cmd('info', self.repository_location) + + # Ok: repository is still known (through security_dir) + shutil.rmtree(self.cache_path) + self.cmd('info', self.repository_location) + + # Needs confirmation: cache and security dir both gone (eg. another host or rm -rf ~) + shutil.rmtree(self.cache_path) + shutil.rmtree(security_dir) + if self.FORK_DEFAULT: + self.cmd('info', self.repository_location, exit_code=EXIT_ERROR) + else: + with pytest.raises(Cache.CacheInitAbortedError): + self.cmd('info', self.repository_location) + with environment_variable(BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK='yes'): + self.cmd('info', self.repository_location) + def test_strip_components(self): self.cmd('init', self.repository_location) self.create_regular_file('dir/file') diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index f41cb4425..a66d681f7 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -15,7 +15,7 @@ from ..helpers import Buffer from ..helpers import partial_format, format_file_size, parse_file_size, format_timedelta, format_line, PlaceholderError, replace_placeholders from ..helpers import make_path_safe, clean_lines from ..helpers import prune_within, prune_split -from ..helpers import get_cache_dir, get_keys_dir, get_nonces_dir +from ..helpers import get_cache_dir, get_keys_dir, get_security_dir from ..helpers import is_slow_msgpack from ..helpers import yes, TRUISH, FALSISH, DEFAULTISH from ..helpers import StableDict, int_to_bigint, bigint_to_int, bin_to_hex @@ -660,14 +660,15 @@ def test_get_keys_dir(monkeypatch): assert get_keys_dir() == '/var/tmp' -def test_get_nonces_dir(monkeypatch): - """test that get_nonces_dir respects environment""" +def test_get_security_dir(monkeypatch): + """test that get_security_dir respects environment""" monkeypatch.delenv('XDG_CONFIG_HOME', raising=False) - assert get_nonces_dir() == os.path.join(os.path.expanduser('~'), '.config', 'borg', 'key-nonces') + assert get_security_dir() == os.path.join(os.path.expanduser('~'), '.config', 'borg', 'security') + assert get_security_dir(repository_id='1234') == os.path.join(os.path.expanduser('~'), '.config', 'borg', 'security', '1234') monkeypatch.setenv('XDG_CONFIG_HOME', '/var/tmp/.config') - assert get_nonces_dir() == os.path.join('/var/tmp/.config', 'borg', 'key-nonces') - monkeypatch.setenv('BORG_NONCES_DIR', '/var/tmp') - assert get_nonces_dir() == '/var/tmp' + assert get_security_dir() == os.path.join('/var/tmp/.config', 'borg', 'security') + monkeypatch.setenv('BORG_SECURITY_DIR', '/var/tmp') + assert get_security_dir() == '/var/tmp' def test_file_size(): diff --git a/src/borg/testsuite/key.py b/src/borg/testsuite/key.py index 5f456b0bf..0702301e2 100644 --- a/src/borg/testsuite/key.py +++ b/src/borg/testsuite/key.py @@ -10,7 +10,7 @@ from ..crypto import bytes_to_long, num_aes_blocks from ..helpers import Location from ..helpers import Chunk from ..helpers import IntegrityError -from ..helpers import get_nonces_dir +from ..helpers import get_security_dir from ..key import PlaintextKey, PassphraseKey, KeyfileKey, RepoKey, Blake2KeyfileKey, Blake2RepoKey, AuthenticatedKey from ..key import Passphrase, PasswordRetriesExceeded, bin_to_hex @@ -118,7 +118,7 @@ class TestKey: def test_keyfile_nonce_rollback_protection(self, monkeypatch, keys_dir): monkeypatch.setenv('BORG_PASSPHRASE', 'test') repository = self.MockRepository() - with open(os.path.join(get_nonces_dir(), repository.id_str), "w") as fd: + with open(os.path.join(get_security_dir(repository.id_str), 'nonce'), "w") as fd: fd.write("0000000000002000") key = KeyfileKey.create(repository, self.MockArgs()) data = key.encrypt(Chunk(b'ABC')) diff --git a/src/borg/testsuite/nonces.py b/src/borg/testsuite/nonces.py index 14d1f52d5..d88d260aa 100644 --- a/src/borg/testsuite/nonces.py +++ b/src/borg/testsuite/nonces.py @@ -2,7 +2,7 @@ import os.path import pytest -from ..helpers import get_nonces_dir +from ..helpers import get_security_dir from ..key import bin_to_hex from ..nonces import NonceManager from ..remote import InvalidRPCMethod @@ -61,11 +61,11 @@ class TestNonceManager: self.repository = None def cache_nonce(self): - with open(os.path.join(get_nonces_dir(), self.repository.id_str), "r") as fd: + with open(os.path.join(get_security_dir(self.repository.id_str), 'nonce'), "r") as fd: return fd.read() def set_cache_nonce(self, nonce): - with open(os.path.join(get_nonces_dir(), self.repository.id_str), "w") as fd: + with open(os.path.join(get_security_dir(self.repository.id_str), 'nonce'), "w") as fd: assert fd.write(nonce) def test_empty_cache_and_old_server(self, monkeypatch):