diff --git a/docs/global.rst.inc b/docs/global.rst.inc index d34f09651..88c659a36 100644 --- a/docs/global.rst.inc +++ b/docs/global.rst.inc @@ -15,6 +15,7 @@ .. _libacl: https://savannah.nongnu.org/projects/acl/ .. _libattr: https://savannah.nongnu.org/projects/attr/ .. _liblz4: https://github.com/Cyan4973/lz4 +.. _libb2: https://github.com/BLAKE2/libb2 .. _OpenSSL: https://www.openssl.org/ .. _`Python 3`: https://www.python.org/ .. _Buzhash: https://en.wikipedia.org/wiki/Buzhash diff --git a/docs/installation.rst b/docs/installation.rst index 9f71525ce..cf0e3aae5 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -213,6 +213,7 @@ following dependencies first: * some Python dependencies, pip will automatically install them for you * optionally, the llfuse_ Python package is required if you wish to mount an archive as a FUSE filesystem. See setup.py about the version requirements. +* optionally libb2_. If it is not found a bundled implementation is used instead. If you have troubles finding the right package names, have a look at the distribution specific sections below and also at the Vagrantfile in our repo. diff --git a/docs/usage.rst b/docs/usage.rst index b05b81c48..bfeb6ef0f 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -194,6 +194,9 @@ Building: Adds given OpenSSL header file directory to the default locations (setup.py). BORG_LZ4_PREFIX Adds given LZ4 header file directory to the default locations (setup.py). + BORG_LIBB2_PREFIX + Adds given prefix directory to the default locations. If a 'include/blake2.h' is found Borg + will be linked against the system libb2 instead of a bundled implementation. (setup.py) Please note: diff --git a/scripts/py36-blake2.py b/scripts/py36-blake2.py new file mode 100644 index 000000000..758a4f346 --- /dev/null +++ b/scripts/py36-blake2.py @@ -0,0 +1,36 @@ + +""" +This script checks compatibility of crypto.blake2b_256 against hashlib.blake2b in CPython 3.6. +""" + +import hashlib +import sys + + +def test_b2(b2_input, b2_output): + digest = hashlib.blake2b(b2_input, digest_size=32).digest() + identical = b2_output == digest + + print('Input: ', b2_input.hex()) + print('Expected: ', b2_output.hex()) + print('Calculated:', digest.hex()) + print('Identical: ', identical) + print() + if not identical: + sys.exit(1) + + +test_b2( + bytes.fromhex('037fb9b75b20d623f1d5a568050fccde4a1b7c5f5047432925e941a17c7a2d0d7061796c6f6164'), + bytes.fromhex('a22d4fc81bb61c3846c334a09eaf28d22dd7df08c9a7a41e713ef28d80eebd45') +) + +test_b2( + b'abc', + bytes.fromhex('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319') +) + +test_b2( + bytes.fromhex('e944973af2256d4d670c12dd75304c319f58f4e40df6fb18ef996cb47e063676') + b'1234567890' * 100, + bytes.fromhex('97ede832378531dd0f4c668685d166e797da27b47d8cd441e885b60abd5e0cb2'), +) diff --git a/setup.py b/setup.py index 02dd18863..92588ef7d 100644 --- a/setup.py +++ b/setup.py @@ -128,8 +128,18 @@ def detect_lz4(prefixes): return prefix +def detect_libb2(prefixes): + for prefix in prefixes: + filename = os.path.join(prefix, 'include', 'blake2.h') + if os.path.exists(filename): + with open(filename, 'r') as fd: + if 'blake2b_init' in fd.read(): + return prefix + include_dirs = [] library_dirs = [] +define_macros = [] +crypto_libraries = ['crypto'] possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local', '/opt/pkg', ] @@ -153,6 +163,18 @@ if lz4_prefix: elif not on_rtd: raise Exception('Unable to find LZ4 headers. (Looked here: {})'.format(', '.join(possible_lz4_prefixes))) +possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/local/libb2', + '/usr/local/borg', '/opt/local', '/opt/pkg', ] +if os.environ.get('BORG_LIBB2_PREFIX'): + possible_libb2_prefixes.insert(0, os.environ.get('BORG_LIBB2_PREFIX')) +libb2_prefix = detect_libb2(possible_libb2_prefixes) +if libb2_prefix: + print('Detected and preferring libb2 over bundled BLAKE2') + include_dirs.append(os.path.join(libb2_prefix, 'include')) + library_dirs.append(os.path.join(libb2_prefix, 'lib')) + crypto_libraries.append('b2') + define_macros.append(('BORG_USE_LIBB2', 'YES')) + with open('README.rst', 'r') as fd: long_description = fd.read() @@ -326,8 +348,8 @@ cmdclass = { ext_modules = [] if not on_rtd: ext_modules += [ - Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs), - Extension('borg.crypto', [crypto_source], libraries=['crypto'], include_dirs=include_dirs, library_dirs=library_dirs), + Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), + Extension('borg.crypto', [crypto_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), Extension('borg.chunker', [chunker_source]), Extension('borg.hashindex', [hashindex_source]) ] diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 785ef668f..4999440ae 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -959,7 +959,7 @@ class Archiver: else: encrypted = 'Yes (%s)' % key.NAME print('Encrypted: %s' % encrypted) - if key.NAME == 'key file': + if key.NAME.startswith('key file'): print('Key file: %s' % key.find_key()) print('Cache: %s' % cache.path) print(DASHES) @@ -1556,6 +1556,7 @@ class Archiver: 'Access to all sub-directories is granted implicitly; PATH doesn\'t need to directly point to a repository.') subparser.add_argument('--append-only', dest='append_only', action='store_true', help='only allow appending to repository segment files') + init_epilog = textwrap.dedent(""" This command initializes an empty repository. A repository is a filesystem directory containing the deduplicated data from zero or more archives. @@ -1599,8 +1600,21 @@ class Archiver: You can change your passphrase for existing repos at any time, it won't affect the encryption/decryption key or other secrets. - When encrypting, AES-CTR-256 is used for encryption, and HMAC-SHA256 for - authentication. Hardware acceleration will be used automatically. + Encryption modes + ++++++++++++++++ + + repokey and keyfile use AES-CTR-256 for encryption and HMAC-SHA256 for + authentication in an encrypt-then-MAC (EtM) construction. The chunk ID hash + is HMAC-SHA256 as well (with a separate key). + + repokey-blake2 and keyfile-blake2 use the same authenticated encryption, but + use a keyed BLAKE2b-256 hash for the chunk ID hash. + + "authenticated" mode uses no encryption, but authenticates repository contents + through the same keyed BLAKE2b-256 hash as the other blake2 modes. + The key is stored like repokey. + + Hardware acceleration will be used automatically. """) subparser = subparsers.add_parser('init', parents=[common_parser], add_help=False, description=self.do_init.__doc__, epilog=init_epilog, @@ -1611,7 +1625,8 @@ class Archiver: type=location_validator(archive=False), help='repository to create') subparser.add_argument('-e', '--encryption', dest='encryption', - choices=('none', 'keyfile', 'repokey'), default='repokey', + choices=('none', 'keyfile', 'repokey', 'keyfile-blake2', 'repokey-blake2', 'authenticated'), + default='repokey', help='select encryption key mode (default: "%(default)s")') subparser.add_argument('-a', '--append-only', dest='append_only', action='store_true', help='create an append-only mode repository') diff --git a/src/borg/blake2-libselect.h b/src/borg/blake2-libselect.h new file mode 100644 index 000000000..5486400eb --- /dev/null +++ b/src/borg/blake2-libselect.h @@ -0,0 +1,5 @@ +#ifdef BORG_USE_LIBB2 +#include +#else +#include "blake2/blake2b-ref.c" +#endif diff --git a/src/borg/blake2/COPYING b/src/borg/blake2/COPYING new file mode 100644 index 000000000..6ca207ef0 --- /dev/null +++ b/src/borg/blake2/COPYING @@ -0,0 +1,122 @@ +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. + diff --git a/src/borg/blake2/README.md b/src/borg/blake2/README.md new file mode 100644 index 000000000..696febaa2 --- /dev/null +++ b/src/borg/blake2/README.md @@ -0,0 +1,13 @@ +# BLAKE2 + +This is the reference source code package of BLAKE2. + +All code is triple-licensed under the [CC0](http://creativecommons.org/publicdomain/zero/1.0), +the [OpenSSL Licence](https://www.openssl.org/source/license.html), +or the [Apache Public License 2.0](http://www.apache.org/licenses/LICENSE-2.0), +at your choosing. + +More: [https://blake2.net](https://blake2.net). [GitHub repository](https://github.com/BLAKE2/BLAKE2). + +Contact: contact@blake2.net + diff --git a/src/borg/blake2/blake2-impl.h b/src/borg/blake2/blake2-impl.h new file mode 100644 index 000000000..ad9089eee --- /dev/null +++ b/src/borg/blake2/blake2-impl.h @@ -0,0 +1,161 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#ifndef BLAKE2_IMPL_H +#define BLAKE2_IMPL_H + +#include +#include + +#if !defined(__cplusplus) && (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L) + #if defined(_MSC_VER) + #define BLAKE2_INLINE __inline + #elif defined(__GNUC__) + #define BLAKE2_INLINE __inline__ + #else + #define BLAKE2_INLINE + #endif +#else + #define BLAKE2_INLINE inline +#endif + +static BLAKE2_INLINE uint32_t load32( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint32_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return (( uint32_t )( p[0] ) << 0) | + (( uint32_t )( p[1] ) << 8) | + (( uint32_t )( p[2] ) << 16) | + (( uint32_t )( p[3] ) << 24) ; +#endif +} + +static BLAKE2_INLINE uint64_t load64( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint64_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return (( uint64_t )( p[0] ) << 0) | + (( uint64_t )( p[1] ) << 8) | + (( uint64_t )( p[2] ) << 16) | + (( uint64_t )( p[3] ) << 24) | + (( uint64_t )( p[4] ) << 32) | + (( uint64_t )( p[5] ) << 40) | + (( uint64_t )( p[6] ) << 48) | + (( uint64_t )( p[7] ) << 56) ; +#endif +} + +static BLAKE2_INLINE uint16_t load16( const void *src ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + uint16_t w; + memcpy(&w, src, sizeof w); + return w; +#else + const uint8_t *p = ( const uint8_t * )src; + return (( uint16_t )( p[0] ) << 0) | + (( uint16_t )( p[1] ) << 8) ; +#endif +} + +static BLAKE2_INLINE void store16( void *dst, uint16_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + *p++ = ( uint8_t )w; w >>= 8; + *p++ = ( uint8_t )w; +#endif +} + +static BLAKE2_INLINE void store32( void *dst, uint32_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); +#endif +} + +static BLAKE2_INLINE void store64( void *dst, uint64_t w ) +{ +#if defined(NATIVE_LITTLE_ENDIAN) + memcpy(dst, &w, sizeof w); +#else + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); + p[4] = (uint8_t)(w >> 32); + p[5] = (uint8_t)(w >> 40); + p[6] = (uint8_t)(w >> 48); + p[7] = (uint8_t)(w >> 56); +#endif +} + +static BLAKE2_INLINE uint64_t load48( const void *src ) +{ + const uint8_t *p = ( const uint8_t * )src; + return (( uint64_t )( p[0] ) << 0) | + (( uint64_t )( p[1] ) << 8) | + (( uint64_t )( p[2] ) << 16) | + (( uint64_t )( p[3] ) << 24) | + (( uint64_t )( p[4] ) << 32) | + (( uint64_t )( p[5] ) << 40) ; +} + +static BLAKE2_INLINE void store48( void *dst, uint64_t w ) +{ + uint8_t *p = ( uint8_t * )dst; + p[0] = (uint8_t)(w >> 0); + p[1] = (uint8_t)(w >> 8); + p[2] = (uint8_t)(w >> 16); + p[3] = (uint8_t)(w >> 24); + p[4] = (uint8_t)(w >> 32); + p[5] = (uint8_t)(w >> 40); +} + +static BLAKE2_INLINE uint32_t rotr32( const uint32_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 32 - c ) ); +} + +static BLAKE2_INLINE uint64_t rotr64( const uint64_t w, const unsigned c ) +{ + return ( w >> c ) | ( w << ( 64 - c ) ); +} + +/* prevents compiler optimizing out memset() */ +static BLAKE2_INLINE void secure_zero_memory(void *v, size_t n) +{ + static void *(*const volatile memset_v)(void *, int, size_t) = &memset; + memset_v(v, 0, n); +} + +#endif + diff --git a/src/borg/blake2/blake2.h b/src/borg/blake2/blake2.h new file mode 100644 index 000000000..6420c5367 --- /dev/null +++ b/src/borg/blake2/blake2.h @@ -0,0 +1,196 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ +#ifndef BLAKE2_H +#define BLAKE2_H + +#include +#include + +#if defined(_MSC_VER) +#define BLAKE2_PACKED(x) __pragma(pack(push, 1)) x __pragma(pack(pop)) +#else +#define BLAKE2_PACKED(x) x __attribute__((packed)) +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + + enum blake2s_constant + { + BLAKE2S_BLOCKBYTES = 64, + BLAKE2S_OUTBYTES = 32, + BLAKE2S_KEYBYTES = 32, + BLAKE2S_SALTBYTES = 8, + BLAKE2S_PERSONALBYTES = 8 + }; + + enum blake2b_constant + { + BLAKE2B_BLOCKBYTES = 128, + BLAKE2B_OUTBYTES = 64, + BLAKE2B_KEYBYTES = 64, + BLAKE2B_SALTBYTES = 16, + BLAKE2B_PERSONALBYTES = 16 + }; + + typedef struct blake2s_state__ + { + uint32_t h[8]; + uint32_t t[2]; + uint32_t f[2]; + uint8_t buf[BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2s_state; + + typedef struct blake2b_state__ + { + uint64_t h[8]; + uint64_t t[2]; + uint64_t f[2]; + uint8_t buf[BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + uint8_t last_node; + } blake2b_state; + + typedef struct blake2sp_state__ + { + blake2s_state S[8][1]; + blake2s_state R[1]; + uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2sp_state; + + typedef struct blake2bp_state__ + { + blake2b_state S[4][1]; + blake2b_state R[1]; + uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; + size_t buflen; + size_t outlen; + } blake2bp_state; + + + BLAKE2_PACKED(struct blake2s_param__ + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint16_t xof_length; /* 14 */ + uint8_t node_depth; /* 15 */ + uint8_t inner_length; /* 16 */ + /* uint8_t reserved[0]; */ + uint8_t salt[BLAKE2S_SALTBYTES]; /* 24 */ + uint8_t personal[BLAKE2S_PERSONALBYTES]; /* 32 */ + }); + + typedef struct blake2s_param__ blake2s_param; + + BLAKE2_PACKED(struct blake2b_param__ + { + uint8_t digest_length; /* 1 */ + uint8_t key_length; /* 2 */ + uint8_t fanout; /* 3 */ + uint8_t depth; /* 4 */ + uint32_t leaf_length; /* 8 */ + uint32_t node_offset; /* 12 */ + uint32_t xof_length; /* 16 */ + uint8_t node_depth; /* 17 */ + uint8_t inner_length; /* 18 */ + uint8_t reserved[14]; /* 32 */ + uint8_t salt[BLAKE2B_SALTBYTES]; /* 48 */ + uint8_t personal[BLAKE2B_PERSONALBYTES]; /* 64 */ + }); + + typedef struct blake2b_param__ blake2b_param; + + typedef struct blake2xs_state__ + { + blake2s_state S[1]; + blake2s_param P[1]; + } blake2xs_state; + + typedef struct blake2xb_state__ + { + blake2b_state S[1]; + blake2b_param P[1]; + } blake2xb_state; + + /* Padded structs result in a compile-time error */ + enum { + BLAKE2_DUMMY_1 = 1/(sizeof(blake2s_param) == BLAKE2S_OUTBYTES), + BLAKE2_DUMMY_2 = 1/(sizeof(blake2b_param) == BLAKE2B_OUTBYTES) + }; + + /* Streaming API */ + int blake2s_init( blake2s_state *S, size_t outlen ); + int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); + int blake2s_update( blake2s_state *S, const void *in, size_t inlen ); + int blake2s_final( blake2s_state *S, void *out, size_t outlen ); + + int blake2b_init( blake2b_state *S, size_t outlen ); + int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); + int blake2b_update( blake2b_state *S, const void *in, size_t inlen ); + int blake2b_final( blake2b_state *S, void *out, size_t outlen ); + + int blake2sp_init( blake2sp_state *S, size_t outlen ); + int blake2sp_init_key( blake2sp_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2sp_update( blake2sp_state *S, const void *in, size_t inlen ); + int blake2sp_final( blake2sp_state *S, void *out, size_t outlen ); + + int blake2bp_init( blake2bp_state *S, size_t outlen ); + int blake2bp_init_key( blake2bp_state *S, size_t outlen, const void *key, size_t keylen ); + int blake2bp_update( blake2bp_state *S, const void *in, size_t inlen ); + int blake2bp_final( blake2bp_state *S, void *out, size_t outlen ); + + /* Variable output length API */ + int blake2xs_init( blake2xs_state *S, const size_t outlen ); + int blake2xs_init_key( blake2xs_state *S, const size_t outlen, const void *key, size_t keylen ); + int blake2xs_update( blake2xs_state *S, const void *in, size_t inlen ); + int blake2xs_final(blake2xs_state *S, void *out, size_t outlen); + + int blake2xb_init( blake2xb_state *S, const size_t outlen ); + int blake2xb_init_key( blake2xb_state *S, const size_t outlen, const void *key, size_t keylen ); + int blake2xb_update( blake2xb_state *S, const void *in, size_t inlen ); + int blake2xb_final(blake2xb_state *S, void *out, size_t outlen); + + /* Simple API */ + int blake2s( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + int blake2sp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2bp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + int blake2xs( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + int blake2xb( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + + /* This is simply an alias for blake2b */ + int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); + +#if defined(__cplusplus) +} +#endif + +#endif + diff --git a/src/borg/blake2/blake2b-ref.c b/src/borg/blake2/blake2b-ref.c new file mode 100644 index 000000000..0d36fb0de --- /dev/null +++ b/src/borg/blake2/blake2b-ref.c @@ -0,0 +1,380 @@ +/* + BLAKE2 reference source code package - reference C implementations + + Copyright 2012, Samuel Neves . You may use this under the + terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at + your option. The terms of these licenses can be found at: + + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 + - OpenSSL license : https://www.openssl.org/source/license.html + - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 + + More information about the BLAKE2 hash function can be found at + https://blake2.net. +*/ + +#include +#include +#include + +#include "blake2.h" +#include "blake2-impl.h" + +static const uint64_t blake2b_IV[8] = +{ + 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL +}; + +static const uint8_t blake2b_sigma[12][16] = +{ + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } +}; + + +static void blake2b_set_lastnode( blake2b_state *S ) +{ + S->f[1] = (uint64_t)-1; +} + +/* Some helper functions, not necessarily useful */ +static int blake2b_is_lastblock( const blake2b_state *S ) +{ + return S->f[0] != 0; +} + +static void blake2b_set_lastblock( blake2b_state *S ) +{ + if( S->last_node ) blake2b_set_lastnode( S ); + + S->f[0] = (uint64_t)-1; +} + +static void blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) +{ + S->t[0] += inc; + S->t[1] += ( S->t[0] < inc ); +} + +static void blake2b_init0( blake2b_state *S ) +{ + size_t i; + memset( S, 0, sizeof( blake2b_state ) ); + + for( i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; +} + +/* init xors IV with input parameter block */ +int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) +{ + const uint8_t *p = ( const uint8_t * )( P ); + size_t i; + + blake2b_init0( S ); + + /* IV XOR ParamBlock */ + for( i = 0; i < 8; ++i ) + S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); + + S->outlen = P->digest_length; + return 0; +} + + + +int blake2b_init( blake2b_state *S, size_t outlen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + P->digest_length = (uint8_t)outlen; + P->key_length = 0; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store32( &P->node_offset, 0 ); + store32( &P->xof_length, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + return blake2b_init_param( S, P ); +} + + +int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) +{ + blake2b_param P[1]; + + if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; + + if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; + + P->digest_length = (uint8_t)outlen; + P->key_length = (uint8_t)keylen; + P->fanout = 1; + P->depth = 1; + store32( &P->leaf_length, 0 ); + store32( &P->node_offset, 0 ); + store32( &P->xof_length, 0 ); + P->node_depth = 0; + P->inner_length = 0; + memset( P->reserved, 0, sizeof( P->reserved ) ); + memset( P->salt, 0, sizeof( P->salt ) ); + memset( P->personal, 0, sizeof( P->personal ) ); + + if( blake2b_init_param( S, P ) < 0 ) return -1; + + { + uint8_t block[BLAKE2B_BLOCKBYTES]; + memset( block, 0, BLAKE2B_BLOCKBYTES ); + memcpy( block, key, keylen ); + blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); + secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ + } + return 0; +} + +#define G(r,i,a,b,c,d) \ + do { \ + a = a + b + m[blake2b_sigma[r][2*i+0]]; \ + d = rotr64(d ^ a, 32); \ + c = c + d; \ + b = rotr64(b ^ c, 24); \ + a = a + b + m[blake2b_sigma[r][2*i+1]]; \ + d = rotr64(d ^ a, 16); \ + c = c + d; \ + b = rotr64(b ^ c, 63); \ + } while(0) + +#define ROUND(r) \ + do { \ + G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ + G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ + G(r,2,v[ 2],v[ 6],v[10],v[14]); \ + G(r,3,v[ 3],v[ 7],v[11],v[15]); \ + G(r,4,v[ 0],v[ 5],v[10],v[15]); \ + G(r,5,v[ 1],v[ 6],v[11],v[12]); \ + G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ + G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ + } while(0) + +static void blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) +{ + uint64_t m[16]; + uint64_t v[16]; + size_t i; + + for( i = 0; i < 16; ++i ) { + m[i] = load64( block + i * sizeof( m[i] ) ); + } + + for( i = 0; i < 8; ++i ) { + v[i] = S->h[i]; + } + + v[ 8] = blake2b_IV[0]; + v[ 9] = blake2b_IV[1]; + v[10] = blake2b_IV[2]; + v[11] = blake2b_IV[3]; + v[12] = blake2b_IV[4] ^ S->t[0]; + v[13] = blake2b_IV[5] ^ S->t[1]; + v[14] = blake2b_IV[6] ^ S->f[0]; + v[15] = blake2b_IV[7] ^ S->f[1]; + + ROUND( 0 ); + ROUND( 1 ); + ROUND( 2 ); + ROUND( 3 ); + ROUND( 4 ); + ROUND( 5 ); + ROUND( 6 ); + ROUND( 7 ); + ROUND( 8 ); + ROUND( 9 ); + ROUND( 10 ); + ROUND( 11 ); + + for( i = 0; i < 8; ++i ) { + S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; + } +} + +#undef G +#undef ROUND + +int blake2b_update( blake2b_state *S, const void *pin, size_t inlen ) +{ + const unsigned char * in = (const unsigned char *)pin; + if( inlen > 0 ) + { + size_t left = S->buflen; + size_t fill = BLAKE2B_BLOCKBYTES - left; + if( inlen > fill ) + { + S->buflen = 0; + memcpy( S->buf + left, in, fill ); /* Fill buffer */ + blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); + blake2b_compress( S, S->buf ); /* Compress */ + in += fill; inlen -= fill; + while(inlen > BLAKE2B_BLOCKBYTES) { + blake2b_increment_counter(S, BLAKE2B_BLOCKBYTES); + blake2b_compress( S, in ); + in += BLAKE2B_BLOCKBYTES; + inlen -= BLAKE2B_BLOCKBYTES; + } + } + memcpy( S->buf + S->buflen, in, inlen ); + S->buflen += inlen; + } + return 0; +} + +int blake2b_final( blake2b_state *S, void *out, size_t outlen ) +{ + uint8_t buffer[BLAKE2B_OUTBYTES] = {0}; + size_t i; + + if( out == NULL || outlen < S->outlen ) + return -1; + + if( blake2b_is_lastblock( S ) ) + return -1; + + blake2b_increment_counter( S, S->buflen ); + blake2b_set_lastblock( S ); + memset( S->buf + S->buflen, 0, BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ + blake2b_compress( S, S->buf ); + + for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ + store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); + + memcpy( out, buffer, S->outlen ); + secure_zero_memory(buffer, sizeof(buffer)); + return 0; +} + +/* inlen, at least, should be uint64_t. Others can be size_t. */ +int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) +{ + blake2b_state S[1]; + + /* Verify parameters */ + if ( NULL == in && inlen > 0 ) return -1; + + if ( NULL == out ) return -1; + + if( NULL == key && keylen > 0 ) return -1; + + if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; + + if( keylen > BLAKE2B_KEYBYTES ) return -1; + + if( keylen > 0 ) + { + if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; + } + else + { + if( blake2b_init( S, outlen ) < 0 ) return -1; + } + + blake2b_update( S, ( const uint8_t * )in, inlen ); + blake2b_final( S, out, outlen ); + return 0; +} + +int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) { + return blake2b(out, outlen, in, inlen, key, keylen); +} + +#if defined(SUPERCOP) +int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) +{ + return blake2b( out, BLAKE2B_OUTBYTES, in, inlen, NULL, 0 ); +} +#endif + +#if defined(BLAKE2B_SELFTEST) +#include +#include "blake2-kat.h" +int main( void ) +{ + uint8_t key[BLAKE2B_KEYBYTES]; + uint8_t buf[BLAKE2_KAT_LENGTH]; + size_t i, step; + + for( i = 0; i < BLAKE2B_KEYBYTES; ++i ) + key[i] = ( uint8_t )i; + + for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) + buf[i] = ( uint8_t )i; + + /* Test simple API */ + for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) + { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b( hash, BLAKE2B_OUTBYTES, buf, i, key, BLAKE2B_KEYBYTES ); + + if( 0 != memcmp( hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES ) ) + { + goto fail; + } + } + + /* Test streaming API */ + for(step = 1; step < BLAKE2B_BLOCKBYTES; ++step) { + for (i = 0; i < BLAKE2_KAT_LENGTH; ++i) { + uint8_t hash[BLAKE2B_OUTBYTES]; + blake2b_state S; + uint8_t * p = buf; + size_t mlen = i; + int err = 0; + + if( (err = blake2b_init_key(&S, BLAKE2B_OUTBYTES, key, BLAKE2B_KEYBYTES)) < 0 ) { + goto fail; + } + + while (mlen >= step) { + if ( (err = blake2b_update(&S, p, step)) < 0 ) { + goto fail; + } + mlen -= step; + p += step; + } + if ( (err = blake2b_update(&S, p, mlen)) < 0) { + goto fail; + } + if ( (err = blake2b_final(&S, hash, BLAKE2B_OUTBYTES)) < 0) { + goto fail; + } + + if (0 != memcmp(hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES)) { + goto fail; + } + } + } + + puts( "ok" ); + return 0; +fail: + puts("error"); + return -1; +} +#endif + diff --git a/src/borg/crypto.pyx b/src/borg/crypto.pyx index 286d596b4..7fa8b8913 100644 --- a/src/borg/crypto.pyx +++ b/src/borg/crypto.pyx @@ -6,6 +6,15 @@ from cpython.buffer cimport PyBUF_SIMPLE, PyObject_GetBuffer, PyBuffer_Release API_VERSION = 3 +cdef extern from "blake2-libselect.h": + ctypedef struct blake2b_state: + pass + + int blake2b_init(blake2b_state *S, size_t outlen) nogil + int blake2b_update(blake2b_state *S, const void *input, size_t inlen) nogil + int blake2b_final(blake2b_state *S, void *out, size_t outlen) nogil + + cdef extern from "openssl/evp.h": ctypedef struct EVP_MD: pass @@ -201,3 +210,40 @@ def hmac_sha256(key, data): finally: PyBuffer_Release(&data_buf) return md + + +cdef blake2b_update_from_buffer(blake2b_state *state, obj): + cdef Py_buffer buf = ro_buffer(obj) + try: + with nogil: + rc = blake2b_update(state, buf.buf, buf.len) + if rc == -1: + raise Exception('blake2b_update() failed') + finally: + PyBuffer_Release(&buf) + + +def blake2b_256(key, data): + cdef blake2b_state state + if blake2b_init(&state, 32) == -1: + raise Exception('blake2b_init() failed') + + md = bytes(32) + cdef unsigned char *md_ptr = md + cdef unsigned char *key_ptr = key + + # This is secure, because BLAKE2 is not vulnerable to length-extension attacks (unlike SHA-1/2, MD-5 and others). + # See the BLAKE2 paper section 2.9 "Keyed hashing (MAC and PRF)" for details. + # A nice benefit is that this simpler prefix-MAC mode has less overhead than the more complex HMAC mode. + # We don't use the BLAKE2 parameter block (via blake2s_init_key) for this to + # avoid incompatibility with the limited API of OpenSSL. + rc = blake2b_update(&state, key_ptr, len(key)) + if rc == -1: + raise Exception('blake2b_update() failed') + blake2b_update_from_buffer(&state, data) + + rc = blake2b_final(&state, md_ptr, 32) + if rc == -1: + raise Exception('blake2b_final() failed') + + return md diff --git a/src/borg/key.py b/src/borg/key.py index aa603c0cb..f5636ba38 100644 --- a/src/borg/key.py +++ b/src/borg/key.py @@ -14,7 +14,7 @@ logger = create_logger() from .constants import * # NOQA from .compress import Compressor, get_compressor -from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks, hmac_sha256 +from .crypto import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_aes_blocks, hmac_sha256, blake2b_256 from .helpers import Chunk from .helpers import Error, IntegrityError from .helpers import yes @@ -62,6 +62,12 @@ def key_creator(repository, args): return KeyfileKey.create(repository, args) elif args.encryption == 'repokey': return RepoKey.create(repository, args) + elif args.encryption == 'keyfile-blake2': + return Blake2KeyfileKey.create(repository, args) + elif args.encryption == 'repokey-blake2': + return Blake2RepoKey.create(repository, args) + elif args.encryption == 'authenticated': + return AuthenticatedKey.create(repository, args) else: return PlaintextKey.create(repository, args) @@ -78,6 +84,12 @@ def key_factory(repository, manifest_data): return RepoKey.detect(repository, manifest_data) elif key_type == PlaintextKey.TYPE: return PlaintextKey.detect(repository, manifest_data) + elif key_type == Blake2KeyfileKey.TYPE: + return Blake2KeyfileKey.detect(repository, manifest_data) + elif key_type == Blake2RepoKey.TYPE: + return Blake2RepoKey.detect(repository, manifest_data) + elif key_type == AuthenticatedKey.TYPE: + return AuthenticatedKey.detect(repository, manifest_data) else: raise UnsupportedPayloadError(key_type) @@ -149,6 +161,28 @@ class PlaintextKey(KeyBase): return Chunk(data) +class ID_BLAKE2b_256: + """ + Key mix-in class for using BLAKE2b-256 for the id key. + + The id_key length must be 32 bytes. + """ + + def id_hash(self, data): + return blake2b_256(self.id_key, data) + + +class ID_HMAC_SHA_256: + """ + Key mix-in class for using HMAC-SHA-256 for the id key. + + The id_key length must be 32 bytes. + """ + + def id_hash(self, data): + return hmac_sha256(self.id_key, data) + + class AESKeyBase(KeyBase): """Common base class shared by KeyfileKey and PassphraseKey @@ -164,11 +198,6 @@ class AESKeyBase(KeyBase): PAYLOAD_OVERHEAD = 1 + 32 + 8 # TYPE + HMAC + NONCE - def id_hash(self, data): - """Return HMAC hash using the "id" HMAC key - """ - return hmac_sha256(self.id_key, data) - def encrypt(self, chunk): chunk = self.compress(chunk) self.nonce_manager.ensure_reservation(num_aes_blocks(len(chunk.data))) @@ -272,7 +301,7 @@ class Passphrase(str): return pbkdf2_hmac('sha256', self.encode('utf-8'), salt, iterations, length) -class PassphraseKey(AESKeyBase): +class PassphraseKey(ID_HMAC_SHA_256, AESKeyBase): # This mode was killed in borg 1.0, see: https://github.com/borgbackup/borg/issues/97 # Reasons: # - you can never ever change your passphrase for existing repos. @@ -432,7 +461,7 @@ class KeyfileKeyBase(AESKeyBase): raise NotImplementedError -class KeyfileKey(KeyfileKeyBase): +class KeyfileKey(ID_HMAC_SHA_256, KeyfileKeyBase): TYPE = 0x00 NAME = 'key file' FILE_ID = 'BORG_KEY' @@ -492,7 +521,7 @@ class KeyfileKey(KeyfileKeyBase): self.target = target -class RepoKey(KeyfileKeyBase): +class RepoKey(ID_HMAC_SHA_256, KeyfileKeyBase): TYPE = 0x03 NAME = 'repokey' @@ -522,3 +551,33 @@ class RepoKey(KeyfileKeyBase): key_data = key_data.encode('utf-8') # remote repo: msgpack issue #99, giving bytes target.save_key(key_data) self.target = target + + +class Blake2KeyfileKey(ID_BLAKE2b_256, KeyfileKey): + TYPE = 0x04 + NAME = 'key file BLAKE2b' + FILE_ID = 'BORG_KEY' + + +class Blake2RepoKey(ID_BLAKE2b_256, RepoKey): + TYPE = 0x05 + NAME = 'repokey BLAKE2b' + + +class AuthenticatedKey(ID_BLAKE2b_256, RepoKey): + TYPE = 0x06 + NAME = 'authenticated BLAKE2b' + + def encrypt(self, chunk): + chunk = self.compress(chunk) + return b''.join([self.TYPE_STR, chunk.data]) + + def decrypt(self, id, data, decompress=True): + if data[0] != self.TYPE: + raise IntegrityError('Chunk %s: Invalid envelope' % bin_to_hex(id)) + payload = memoryview(data)[1:] + if not decompress: + return Chunk(payload) + data = self.compressor.decompress(payload) + self.assert_id(id, data) + return Chunk(data) diff --git a/src/borg/selftest.py b/src/borg/selftest.py index 139ed7e80..40d8d06e9 100644 --- a/src/borg/selftest.py +++ b/src/borg/selftest.py @@ -30,7 +30,7 @@ SELFTEST_CASES = [ ChunkerTestCase, ] -SELFTEST_COUNT = 29 +SELFTEST_COUNT = 30 class SelfTestResult(TestResult): diff --git a/src/borg/testsuite/crypto.py b/src/borg/testsuite/crypto.py index b79a5d838..aa138a763 100644 --- a/src/borg/testsuite/crypto.py +++ b/src/borg/testsuite/crypto.py @@ -1,6 +1,6 @@ from binascii import hexlify, unhexlify -from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes, hmac_sha256 +from ..crypto import AES, bytes_to_long, bytes_to_int, long_to_bytes, hmac_sha256, blake2b_256 from ..crypto import increment_iv, bytes16_to_int, int_to_bytes16 from . import BaseTestCase @@ -80,3 +80,19 @@ class CryptoTestCase(BaseTestCase): hmac = unhexlify('82558a389a443c0ea4cc819899f2083a' '85f0faa3e578f8077a2e3ff46729665b') assert hmac_sha256(key, data) == hmac + + def test_blake2b_256(self): + # In BLAKE2 the output length actually is part of the hashes personality - it is *not* simple truncation like in + # the SHA-2 family. Therefore we need to generate test vectors ourselves (as is true for most applications that + # are not precisely vanilla BLAKE2b-512 or BLAKE2s-256). + # + # Obtained via "b2sum" utility from the official BLAKE2 repository. It calculates the exact hash of a file's + # contents, no extras (like length) included. + assert blake2b_256(b'', b'abc') == unhexlify('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319') + assert blake2b_256(b'a', b'bc') == unhexlify('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319') + assert blake2b_256(b'ab', b'c') == unhexlify('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319') + assert blake2b_256(b'abc', b'') == unhexlify('bddd813c634239723171ef3fee98579b94964e3bb1cb3e427262c8c068d52319') + + key = unhexlify('e944973af2256d4d670c12dd75304c319f58f4e40df6fb18ef996cb47e063676') + data = memoryview(b'1234567890' * 100) + assert blake2b_256(key, data) == unhexlify('97ede832378531dd0f4c668685d166e797da27b47d8cd441e885b60abd5e0cb2') diff --git a/src/borg/testsuite/key.py b/src/borg/testsuite/key.py index 95c74e36f..5f456b0bf 100644 --- a/src/borg/testsuite/key.py +++ b/src/borg/testsuite/key.py @@ -11,7 +11,8 @@ from ..helpers import Location from ..helpers import Chunk from ..helpers import IntegrityError from ..helpers import get_nonces_dir -from ..key import PlaintextKey, PassphraseKey, KeyfileKey, Passphrase, PasswordRetriesExceeded, bin_to_hex +from ..key import PlaintextKey, PassphraseKey, KeyfileKey, RepoKey, Blake2KeyfileKey, Blake2RepoKey, AuthenticatedKey +from ..key import Passphrase, PasswordRetriesExceeded, bin_to_hex class TestKey: @@ -34,6 +35,24 @@ class TestKey: """)) keyfile2_id = unhexlify('c3fbf14bc001ebcc3cd86e696c13482ed071740927cd7cbe1b01b4bfcee49314') + keyfile_blake2_key_file = """ + BORG_KEY 0000000000000000000000000000000000000000000000000000000000000000 + hqlhbGdvcml0aG2mc2hhMjU2pGRhdGHaANAwo4EbUPF/kLQXhQnT4LxRc1advS8lUiegDa + q2Q6oOkP1Jc7MwBa7ZVMgoBG1sBeKYO6Sn6W6BBrHbMR8Dxv7xquaQIh8jIpnjLWpzyFIk + JlijFiTWI58Sxj+2D19b2ayFolnGkF9PJSARgfaieo0GkryqjcIgcXuKHO/H9NfaUDk5YJ + UqrJ9TUMohXSQzwF1pO4ak2BHPZKnbeJ7XL/8fFN8VFQZl27R0et4WlTFRBI1qQYyQaTiL + +/1ICMUpVsQM0mvyW6dc8/zGMsAlmZVApGhhc2jaACDdRF7uPv90UN3zsZy5Be89728RBl + zKvtzupDyTsfrJMqppdGVyYXRpb25zzgABhqCkc2FsdNoAIGTK3TR09UZqw1bPi17gyHOi + 7YtSp4BVK7XptWeKh6Vip3ZlcnNpb24B""".strip() + + keyfile_blake2_cdata = bytes.fromhex('04dd21cc91140ef009bc9e4dd634d075e39d39025ccce1289c' + '5536f9cb57f5f8130404040404040408ec852921309243b164') + # Verified against b2sum. Entire string passed to BLAKE2, including the 32 byte key contained in + # keyfile_blake2_key_file above is + # 037fb9b75b20d623f1d5a568050fccde4a1b7c5f5047432925e941a17c7a2d0d7061796c6f6164 + # p a y l o a d + keyfile_blake2_id = bytes.fromhex('a22d4fc81bb61c3846c334a09eaf28d22dd7df08c9a7a41e713ef28d80eebd45') + @pytest.fixture def keys_dir(self, request, monkeypatch, tmpdir): monkeypatch.setenv('BORG_KEYS_DIR', tmpdir) @@ -41,7 +60,11 @@ class TestKey: @pytest.fixture(params=( KeyfileKey, - PlaintextKey + PlaintextKey, + RepoKey, + Blake2KeyfileKey, + Blake2RepoKey, + AuthenticatedKey, )) def key(self, request, monkeypatch): monkeypatch.setenv('BORG_PASSPHRASE', 'test') @@ -61,6 +84,12 @@ class TestKey: def commit_nonce_reservation(self, next_unreserved, start_nonce): pass + def save_key(self, data): + self.key_data = data + + def load_key(self): + return self.key_data + def test_plaintext(self): key = PlaintextKey.create(None, None) chunk = Chunk(b'foo') @@ -128,6 +157,13 @@ class TestKey: key = KeyfileKey.detect(self.MockRepository(), self.keyfile2_cdata) assert key.decrypt(self.keyfile2_id, self.keyfile2_cdata).data == b'payload' + def test_keyfile_blake2(self, monkeypatch, keys_dir): + with keys_dir.join('keyfile').open('w') as fd: + fd.write(self.keyfile_blake2_key_file) + monkeypatch.setenv('BORG_PASSPHRASE', 'passphrase') + key = Blake2KeyfileKey.detect(self.MockRepository(), self.keyfile_blake2_cdata) + assert key.decrypt(self.keyfile_blake2_id, self.keyfile_blake2_cdata).data == b'payload' + def test_passphrase(self, keys_dir, monkeypatch): monkeypatch.setenv('BORG_PASSPHRASE', 'test') key = PassphraseKey.create(self.MockRepository(), None) @@ -193,6 +229,14 @@ class TestKey: with pytest.raises(IntegrityError): key.assert_id(id, plaintext_changed) + def test_authenticated_encrypt(self, monkeypatch): + monkeypatch.setenv('BORG_PASSPHRASE', 'test') + key = AuthenticatedKey.create(self.MockRepository(), self.MockArgs()) + plaintext = Chunk(b'123456789') + authenticated = key.encrypt(plaintext) + # 0x06 is the key TYPE, 0x0000 identifies CNONE compression + assert authenticated == b'\x06\x00\x00' + plaintext.data + class TestPassphrase: def test_passphrase_new_verification(self, capsys, monkeypatch):