From c30df4e033834c4d96be67fe4bcedb75014dc115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Antoine=20Beaupr=C3=A9?= Date: Wed, 30 Sep 2015 23:18:03 -0400 Subject: [PATCH] move converter code out of test suite --- borg/converter.py | 147 +++++++++++++++++++++++++++++++++++++ borg/testsuite/convert.py | 151 ++------------------------------------ 2 files changed, 152 insertions(+), 146 deletions(-) create mode 100644 borg/converter.py diff --git a/borg/converter.py b/borg/converter.py new file mode 100644 index 000000000..b558af883 --- /dev/null +++ b/borg/converter.py @@ -0,0 +1,147 @@ +from binascii import hexlify +import os + +from .helpers import IntegrityError, get_keys_dir +from .repository import Repository, MAGIC +from .key import KeyfileKey, KeyfileNotFoundError + +class NotImplementedException(Exception): + pass + +class AtticRepositoryConverter(Repository): + def convert(self): + '''convert an attic repository to a borg repository + + those are the files that need to be converted here, from most + important to least important: segments, key files, and various + caches, the latter being optional, as they will be rebuilt if + missing.''' + print("reading segments from attic repository using borg") + segments = [ filename for i, filename in self.io.segment_iterator() ] + try: + keyfile = self.find_attic_keyfile() + except KeyfileNotFoundError: + print("no key file found for repository") + else: + self.convert_keyfiles(keyfile) + self.close() + self.convert_segments(segments) + self.convert_cache() + + @staticmethod + def convert_segments(segments): + '''convert repository segments from attic to borg + + replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in + `$ATTIC_REPO/data/**`. + + luckily the segment length didn't change so we can just + replace the 8 first bytes of all regular files in there.''' + for filename in segments: + print("converting segment %s in place" % filename) + with open(filename, 'r+b') as segment: + segment.seek(0) + segment.write(MAGIC) + + def find_attic_keyfile(self): + '''find the attic keyfiles + + the keyfiles are loaded by `KeyfileKey.find_key_file()`. that + finds the keys with the right identifier for the repo + + this is expected to look into $HOME/.attic/keys or + $ATTIC_KEYS_DIR for key files matching the given Borg + repository. + + it is expected to raise an exception (KeyfileNotFoundError) if + no key is found. whether that exception is from Borg or Attic + is unclear. + + this is split in a separate function in case we want to use + the attic code here directly, instead of our local + implementation.''' + return AtticKeyfileKey.find_key_file(self) + + @staticmethod + def convert_keyfiles(keyfile): + + '''convert key files from attic to borg + + replacement pattern is `s/ATTIC KEY/BORG_KEY/` in + `get_keys_dir()`, that is `$ATTIC_KEYS_DIR` or + `$HOME/.attic/keys`, and moved to `$BORG_KEYS_DIR` or + `$HOME/.borg/keys`. + + no need to decrypt to convert. we need to rewrite the whole + key file because magic number length changed, but that's not a + problem because the keyfiles are small (compared to, say, + all the segments).''' + print("converting keyfile %s" % keyfile) + with open(keyfile, 'r') as f: + data = f.read() + data = data.replace(AtticKeyfileKey.FILE_ID, + KeyfileKey.FILE_ID, + 1) + keyfile = os.path.join(get_keys_dir(), + os.path.basename(keyfile)) + print("writing borg keyfile to %s" % keyfile) + with open(keyfile, 'w') as f: + f.write(data) + with open(keyfile, 'r') as f: + data = f.read() + assert data.startswith(KeyfileKey.FILE_ID) + + def convert_cache(self): + '''convert caches from attic to borg + + those are all hash indexes, so we need to + `s/ATTICIDX/BORG_IDX/` in a few locations: + + * the repository index (in `$ATTIC_REPO/index.%d`, where `%d` + is the `Repository.get_index_transaction_id()`), which we + should probably update, with a lock, see + `Repository.open()`, which i'm not sure we should use + because it may write data on `Repository.close()`... + + * the `files` and `chunks` cache (in + `$HOME/.cache/attic//`), which we could just drop, + but if we'd want to convert, we could open it with the + `Cache.open()`, edit in place and then `Cache.close()` to + make sure we have locking right + ''' + raise NotImplementedException('not implemented') + +class AtticKeyfileKey(KeyfileKey): + '''backwards compatible Attick key file parser''' + FILE_ID = 'ATTIC KEY' + + # verbatim copy from attic + @staticmethod + def get_keys_dir(): + """Determine where to repository keys and cache""" + return os.environ.get('ATTIC_KEYS_DIR', + os.path.join(os.path.expanduser('~'), '.attic', 'keys')) + + @classmethod + def find_key_file(cls, repository): + '''copy of attic's `find_key_file`_ + + this has two small modifications: + + 1. it uses the above `get_keys_dir`_ instead of the global one, + assumed to be borg's + + 2. it uses `repository.path`_ instead of + `repository._location.canonical_path`_ because we can't + assume the repository has been opened by the archiver yet + ''' + get_keys_dir = cls.get_keys_dir + id = hexlify(repository.id).decode('ascii') + keys_dir = get_keys_dir() + for name in os.listdir(keys_dir): + filename = os.path.join(keys_dir, name) + with open(filename, 'r') as fd: + line = fd.readline().strip() + if line and line.startswith(cls.FILE_ID) and line[10:] == id: + return filename + raise KeyfileNotFoundError(repository.path, get_keys_dir()) diff --git a/borg/testsuite/convert.py b/borg/testsuite/convert.py index 9d5f95142..74196063e 100644 --- a/borg/testsuite/convert.py +++ b/borg/testsuite/convert.py @@ -1,4 +1,3 @@ -from binascii import hexlify import os import pytest import shutil @@ -8,14 +7,12 @@ import attic.repository import attic.key import attic.helpers -from ..helpers import IntegrityError, get_keys_dir +from ..converter import AtticRepositoryConverter, NotImplementedException +from ..helpers import get_keys_dir +from ..key import KeyfileKey from ..repository import Repository, MAGIC -from ..key import KeyfileKey, KeyfileNotFoundError from . import BaseTestCase -class NotImplementedException(Exception): - pass - class ConversionTestCase(BaseTestCase): class MockArgs: @@ -56,7 +53,8 @@ class ConversionTestCase(BaseTestCase): assert not self.repository.check() # can't check raises() because check() handles the error self.repository.close() print("opening attic repository with borg and converting") - self.open(self.tmppath, repo_type = AtticRepositoryConverter).convert() + with pytest.raises(NotImplementedException): + self.open(self.tmppath, repo_type = AtticRepositoryConverter).convert() # check that the new keyfile is alright keyfile = os.path.join(get_keys_dir(), os.path.basename(self.key.path)) @@ -65,142 +63,3 @@ class ConversionTestCase(BaseTestCase): self.repository = self.open(self.tmppath) assert self.repository.check() self.repository.close() - -class AtticRepositoryConverter(Repository): - def convert(self): - '''convert an attic repository to a borg repository - - those are the files that need to be converted here, from most - important to least important: segments, key files, and various - caches, the latter being optional, as they will be rebuilt if - missing.''' - print("reading segments from attic repository using borg") - segments = [ filename for i, filename in self.io.segment_iterator() ] - try: - keyfile = self.find_attic_keyfile() - except KeyfileNotFoundError: - print("no key file found for repository") - else: - self.convert_keyfiles(keyfile) - self.close() - self.convert_segments(segments) - with pytest.raises(NotImplementedException): - self.convert_cache() - - @staticmethod - def convert_segments(segments): - '''convert repository segments from attic to borg - - replacement pattern is `s/ATTICSEG/BORG_SEG/` in files in - `$ATTIC_REPO/data/**`. - - luckily the segment length didn't change so we can just - replace the 8 first bytes of all regular files in there.''' - for filename in segments: - print("converting segment %s in place" % filename) - with open(filename, 'r+b') as segment: - segment.seek(0) - segment.write(MAGIC) - - def find_attic_keyfile(self): - '''find the attic keyfiles - - the keyfiles are loaded by `KeyfileKey.find_key_file()`. that - finds the keys with the right identifier for the repo - - this is expected to look into $HOME/.attic/keys or - $ATTIC_KEYS_DIR for key files matching the given Borg - repository. - - it is expected to raise an exception (KeyfileNotFoundError) if - no key is found. whether that exception is from Borg or Attic - is unclear. - - this is split in a separate function in case we want to use - the attic code here directly, instead of our local - implementation.''' - return AtticKeyfileKey.find_key_file(self) - - @staticmethod - def convert_keyfiles(keyfile): - - '''convert key files from attic to borg - - replacement pattern is `s/ATTIC KEY/BORG_KEY/` in - `get_keys_dir()`, that is `$ATTIC_KEYS_DIR` or - `$HOME/.attic/keys`, and moved to `$BORG_KEYS_DIR` or - `$HOME/.borg/keys`. - - no need to decrypt to convert. we need to rewrite the whole - key file because magic number length changed, but that's not a - problem because the keyfiles are small (compared to, say, - all the segments).''' - print("converting keyfile %s" % keyfile) - with open(keyfile, 'r') as f: - data = f.read() - data = data.replace(AtticKeyfileKey.FILE_ID, - KeyfileKey.FILE_ID, - 1) - keyfile = os.path.join(get_keys_dir(), - os.path.basename(keyfile)) - print("writing borg keyfile to %s" % keyfile) - with open(keyfile, 'w') as f: - f.write(data) - with open(keyfile, 'r') as f: - data = f.read() - assert data.startswith(KeyfileKey.FILE_ID) - - def convert_cache(self): - '''convert caches from attic to borg - - those are all hash indexes, so we need to - `s/ATTICIDX/BORG_IDX/` in a few locations: - - * the repository index (in `$ATTIC_REPO/index.%d`, where `%d` - is the `Repository.get_index_transaction_id()`), which we - should probably update, with a lock, see - `Repository.open()`, which i'm not sure we should use - because it may write data on `Repository.close()`... - - * the `files` and `chunks` cache (in - `$HOME/.cache/attic//`), which we could just drop, - but if we'd want to convert, we could open it with the - `Cache.open()`, edit in place and then `Cache.close()` to - make sure we have locking right - ''' - raise NotImplementedException('not implemented') - -class AtticKeyfileKey(KeyfileKey): - '''backwards compatible Attick key file parser''' - FILE_ID = 'ATTIC KEY' - - # verbatim copy from attic - @staticmethod - def get_keys_dir(): - """Determine where to repository keys and cache""" - return os.environ.get('ATTIC_KEYS_DIR', - os.path.join(os.path.expanduser('~'), '.attic', 'keys')) - - @classmethod - def find_key_file(cls, repository): - '''copy of attic's `find_key_file`_ - - this has two small modifications: - - 1. it uses the above `get_keys_dir`_ instead of the global one, - assumed to be borg's - - 2. it uses `repository.path`_ instead of - `repository._location.canonical_path`_ because we can't - assume the repository has been opened by the archiver yet - ''' - get_keys_dir = cls.get_keys_dir - id = hexlify(repository.id).decode('ascii') - keys_dir = get_keys_dir() - for name in os.listdir(keys_dir): - filename = os.path.join(keys_dir, name) - with open(filename, 'r') as fd: - line = fd.readline().strip() - if line and line.startswith(cls.FILE_ID) and line[10:] == id: - return filename - raise KeyfileNotFoundError(repository.path, get_keys_dir())