From ba38d5199b0fae92121121b24dcbae21b74e47db Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 17 Dec 2017 02:55:41 +0100 Subject: [PATCH 1/3] move blake2 files to subdir ref/ (as seen in orig. repo) (cherry picked from commit 8924d78d44c4aaec6cee4f846560fb710c28e9ea) --- src/borg/algorithms/blake2-libselect.h | 2 +- src/borg/algorithms/blake2/{ => ref}/blake2-impl.h | 0 src/borg/algorithms/blake2/{ => ref}/blake2.h | 0 src/borg/algorithms/blake2/{ => ref}/blake2b-ref.c | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename src/borg/algorithms/blake2/{ => ref}/blake2-impl.h (100%) rename src/borg/algorithms/blake2/{ => ref}/blake2.h (100%) rename src/borg/algorithms/blake2/{ => ref}/blake2b-ref.c (100%) diff --git a/src/borg/algorithms/blake2-libselect.h b/src/borg/algorithms/blake2-libselect.h index 5486400eb..3dbfcd527 100644 --- a/src/borg/algorithms/blake2-libselect.h +++ b/src/borg/algorithms/blake2-libselect.h @@ -1,5 +1,5 @@ #ifdef BORG_USE_LIBB2 #include #else -#include "blake2/blake2b-ref.c" +#include "blake2/ref/blake2b-ref.c" #endif diff --git a/src/borg/algorithms/blake2/blake2-impl.h b/src/borg/algorithms/blake2/ref/blake2-impl.h similarity index 100% rename from src/borg/algorithms/blake2/blake2-impl.h rename to src/borg/algorithms/blake2/ref/blake2-impl.h diff --git a/src/borg/algorithms/blake2/blake2.h b/src/borg/algorithms/blake2/ref/blake2.h similarity index 100% rename from src/borg/algorithms/blake2/blake2.h rename to src/borg/algorithms/blake2/ref/blake2.h diff --git a/src/borg/algorithms/blake2/blake2b-ref.c b/src/borg/algorithms/blake2/ref/blake2b-ref.c similarity index 100% rename from src/borg/algorithms/blake2/blake2b-ref.c rename to src/borg/algorithms/blake2/ref/blake2b-ref.c From 0118b939d909f4b2ba02a96feb5fd57ec49d91f2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 17 Dec 2017 03:18:03 +0100 Subject: [PATCH 2/3] update blake2 reference impl file (cosmetic) only whitespace changes. (cherry picked from commit b56f6cdbc0263572cac932ae248580392b5cf20b) --- src/borg/algorithms/blake2/ref/blake2-impl.h | 7 +++---- src/borg/algorithms/blake2/ref/blake2.h | 1 - src/borg/algorithms/blake2/ref/blake2b-ref.c | 7 +++---- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/borg/algorithms/blake2/ref/blake2-impl.h b/src/borg/algorithms/blake2/ref/blake2-impl.h index ad9089eee..5dff7fc7a 100644 --- a/src/borg/algorithms/blake2/ref/blake2-impl.h +++ b/src/borg/algorithms/blake2/ref/blake2-impl.h @@ -1,14 +1,14 @@ /* BLAKE2 reference source code package - reference C implementations - + Copyright 2012, Samuel Neves . You may use this under the terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at your option. The terms of these licenses can be found at: - + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 - OpenSSL license : https://www.openssl.org/source/license.html - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 - + More information about the BLAKE2 hash function can be found at https://blake2.net. */ @@ -158,4 +158,3 @@ static BLAKE2_INLINE void secure_zero_memory(void *v, size_t n) } #endif - diff --git a/src/borg/algorithms/blake2/ref/blake2.h b/src/borg/algorithms/blake2/ref/blake2.h index 6420c5367..ad62f260e 100644 --- a/src/borg/algorithms/blake2/ref/blake2.h +++ b/src/borg/algorithms/blake2/ref/blake2.h @@ -193,4 +193,3 @@ extern "C" { #endif #endif - diff --git a/src/borg/algorithms/blake2/ref/blake2b-ref.c b/src/borg/algorithms/blake2/ref/blake2b-ref.c index 0d36fb0de..cd38b1ba0 100644 --- a/src/borg/algorithms/blake2/ref/blake2b-ref.c +++ b/src/borg/algorithms/blake2/ref/blake2b-ref.c @@ -1,14 +1,14 @@ /* BLAKE2 reference source code package - reference C implementations - + Copyright 2012, Samuel Neves . You may use this under the terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at your option. The terms of these licenses can be found at: - + - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 - OpenSSL license : https://www.openssl.org/source/license.html - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 - + More information about the BLAKE2 hash function can be found at https://blake2.net. */ @@ -377,4 +377,3 @@ fail: return -1; } #endif - From 1bdaffc62cce12b4abb82ca1f28495d5af85527a Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 17 Dec 2017 03:06:12 +0100 Subject: [PATCH 3/3] crypto.low_level extension: build blake2 like zstd, see #3415 (cherry picked from commit 4a2fec32b6429109e9425f1fb54998f3e903a781) --- setup.py | 31 ++++++----- setup_b2.py | 72 ++++++++++++++++++++++++++ src/borg/algorithms/blake2-libselect.h | 2 +- 3 files changed, 88 insertions(+), 17 deletions(-) create mode 100644 setup_b2.py diff --git a/setup.py b/setup.py index 8c7863d29..a8c4a415e 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ import textwrap import setup_lz4 import setup_zstd +import setup_b2 # True: use the shared liblz4 (>= 1.7.0 / r129) from the system, False: use the bundled lz4 code prefer_system_liblz4 = True @@ -21,6 +22,9 @@ prefer_system_liblz4 = True # True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code prefer_system_libzstd = True +# True: use the shared libb2 from the system, False: use the bundled blake2 code +prefer_system_libb2 = True + min_python = (3, 4) my_python = sys.version_info @@ -146,19 +150,9 @@ def detect_openssl(prefixes): return prefix -def detect_libb2(prefixes): - for prefix in prefixes: - filename = os.path.join(prefix, 'include', 'blake2.h') - if os.path.exists(filename): - with open(filename, 'r') as fd: - if 'blake2b_init' in fd.read(): - return prefix - - include_dirs = [] library_dirs = [] define_macros = [] -crypto_libraries = ['crypto'] possible_openssl_prefixes = ['/usr', '/usr/local', '/usr/local/opt/openssl', '/usr/local/ssl', '/usr/local/openssl', '/usr/local/borg', '/opt/local', '/opt/pkg', ] @@ -187,13 +181,13 @@ possible_libb2_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libb2', '/usr/l '/usr/local/borg', '/opt/local', '/opt/pkg', ] if os.environ.get('BORG_LIBB2_PREFIX'): possible_libb2_prefixes.insert(0, os.environ.get('BORG_LIBB2_PREFIX')) -libb2_prefix = detect_libb2(possible_libb2_prefixes) -if libb2_prefix: +libb2_prefix = setup_b2.b2_system_prefix(possible_libb2_prefixes) +if prefer_system_libb2 and libb2_prefix: print('Detected and preferring libb2 over bundled BLAKE2') - include_dirs.append(os.path.join(libb2_prefix, 'include')) - library_dirs.append(os.path.join(libb2_prefix, 'lib')) - crypto_libraries.append('b2') define_macros.append(('BORG_USE_LIBB2', 'YES')) + libb2_system = True +else: + libb2_system = False possible_libzstd_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libzstd', '/usr/local/libzstd', '/usr/local/borg', '/opt/local', '/opt/pkg', ] @@ -774,9 +768,14 @@ if not on_rtd: compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd', system_prefix=libzstd_prefix, system=libzstd_system, multithreaded=False, legacy=False, **compress_ext_kwargs) + crypto_ext_kwargs = dict(sources=[crypto_ll_source], libraries=['crypto'], + include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros) + crypto_ext_kwargs = setup_b2.b2_ext_kwargs(bundled_path='src/borg/algorithms/blake2', + system_prefix=libb2_prefix, system=libb2_system, + **crypto_ext_kwargs) ext_modules += [ Extension('borg.compress', **compress_ext_kwargs), - Extension('borg.crypto.low_level', [crypto_ll_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), + Extension('borg.crypto.low_level', **crypto_ext_kwargs), Extension('borg.hashindex', [hashindex_source]), Extension('borg.item', [item_source]), Extension('borg.chunker', [chunker_source]), diff --git a/setup_b2.py b/setup_b2.py new file mode 100644 index 000000000..e69e0cdf7 --- /dev/null +++ b/setup_b2.py @@ -0,0 +1,72 @@ +# Support code for building a C extension with blake2 files +# +# Copyright (c) 2016-present, Gregory Szorc (original code for zstd) +# 2017-present, Thomas Waldmann (mods to make it more generic, code for blake2) +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +import os + +# b2 files, structure as seen in BLAKE2 (reference implementation) project repository: + +b2_sources = [ + 'ref/blake2b-ref.c', +] + +b2_includes = [ + 'ref', +] + + +def b2_system_prefix(prefixes): + for prefix in prefixes: + filename = os.path.join(prefix, 'include', 'blake2.h') + if os.path.exists(filename): + with open(filename, 'r') as fd: + if 'blake2b_init' in fd.read(): + return prefix + + +def b2_ext_kwargs(bundled_path, system_prefix=None, system=False, **kwargs): + """amend kwargs with b2 stuff for a distutils.extension.Extension initialization. + + bundled_path: relative (to this file) path to the bundled library source code files + system_prefix: where the system-installed library can be found + system: True: use the system-installed shared library, False: use the bundled library code + kwargs: distutils.extension.Extension kwargs that should be amended + returns: amended kwargs + """ + def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path, ))) for path in paths] + + use_system = system and system_prefix is not None + + sources = kwargs.get('sources', []) + if not use_system: + sources += multi_join(b2_sources, bundled_path) + + include_dirs = kwargs.get('include_dirs', []) + if use_system: + include_dirs += multi_join(['include'], system_prefix) + else: + include_dirs += multi_join(b2_includes, bundled_path) + + library_dirs = kwargs.get('library_dirs', []) + if use_system: + library_dirs += multi_join(['lib'], system_prefix) + + libraries = kwargs.get('libraries', []) + if use_system: + libraries += ['b2', ] + + extra_compile_args = kwargs.get('extra_compile_args', []) + if not use_system: + extra_compile_args += [] # not used yet + + ret = dict(**kwargs) + ret.update(dict(sources=sources, extra_compile_args=extra_compile_args, + include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) + return ret diff --git a/src/borg/algorithms/blake2-libselect.h b/src/borg/algorithms/blake2-libselect.h index 3dbfcd527..5cd970d88 100644 --- a/src/borg/algorithms/blake2-libselect.h +++ b/src/borg/algorithms/blake2-libselect.h @@ -1,5 +1,5 @@ #ifdef BORG_USE_LIBB2 #include #else -#include "blake2/ref/blake2b-ref.c" +#include "blake2/ref/blake2.h" #endif