From 34b92ffdaac59f2a7fd30a7a4328062f502ff3b2 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 5 Dec 2017 04:16:25 +0100 Subject: [PATCH] support code to build bundled zstd code into the compress extension setup_zstd.py modified so it is just amending the Extension() kwargs, but the Extension is initialized by the caller. this way, amending can happend multiple times (e.g. for multiple compression algorithms). also: - move include/library dirs processing for system-library case - move system zstd prefix detection to setup_zstd module - cosmetic: setup.py whitespace fixes - prefer system zstd option, document zstd min. requirement --- setup.py | 46 +++---- setup_zstd.py | 198 +++++++++++++-------------- src/borg/algorithms/zstd-libselect.h | 2 +- 3 files changed, 123 insertions(+), 123 deletions(-) diff --git a/setup.py b/setup.py index 819432e10..136f52816 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,11 @@ from distutils.core import Command import textwrap +import setup_zstd + +# True: use the shared libzstd (>= 1.3.0) from the system, False: use the bundled zstd code +prefer_system_libzstd = True + min_python = (3, 5) my_python = sys.version_info @@ -159,15 +164,6 @@ def detect_libb2(prefixes): return prefix -def detect_libzstd(prefixes): - for prefix in prefixes: - filename = os.path.join(prefix, 'include', 'zstd.h') - if os.path.exists(filename): - with open(filename, 'r') as fd: - if 'ZSTD_getFrameContentSize' in fd.read(): - return prefix - - include_dirs = [] library_dirs = [] define_macros = [] @@ -212,13 +208,13 @@ possible_libzstd_prefixes = ['/usr', '/usr/local', '/usr/local/opt/libzstd', '/u '/usr/local/borg', '/opt/local', '/opt/pkg', ] if os.environ.get('BORG_LIBZSTD_PREFIX'): possible_libzstd_prefixes.insert(0, os.environ.get('BORG_LIBZSTD_PREFIX')) -libzstd_prefix = detect_libzstd(possible_libzstd_prefixes) -if libzstd_prefix: +libzstd_prefix = setup_zstd.zstd_system_prefix(possible_libzstd_prefixes) +if prefer_system_libzstd and libzstd_prefix: print('Detected and preferring libzstd over bundled ZSTD') - include_dirs.append(os.path.join(libzstd_prefix, 'include')) - library_dirs.append(os.path.join(libzstd_prefix, 'lib')) - compression_libraries.append('zstd') define_macros.append(('BORG_USE_LIBZSTD', 'YES')) + libzstd_system = True +else: + libzstd_system = False with open('README.rst', 'r') as fd: @@ -777,18 +773,22 @@ cmdclass = { ext_modules = [] if not on_rtd: + compress_ext_kwargs = dict(sources=[compress_source], include_dirs=include_dirs, library_dirs=library_dirs, + libraries=compression_libraries, define_macros=define_macros) + compress_ext_kwargs = setup_zstd.zstd_ext_kwargs(bundled_path='src/borg/algorithms/zstd', + system_prefix=libzstd_prefix, system=libzstd_system, + multithreaded=False, legacy=False, **compress_ext_kwargs) ext_modules += [ - Extension('borg.compress', [compress_source], libraries=compression_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), - Extension('borg.crypto.low_level', [crypto_ll_source, crypto_helpers], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), - Extension('borg.hashindex', [hashindex_source]), - Extension('borg.item', [item_source]), - Extension('borg.chunker', [chunker_source]), - Extension('borg.algorithms.checksums', [checksums_source]), - -] + Extension('borg.compress', **compress_ext_kwargs), + Extension('borg.crypto.low_level', [crypto_ll_source, crypto_helpers], libraries=crypto_libraries, + include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), + Extension('borg.hashindex', [hashindex_source]), + Extension('borg.item', [item_source]), + Extension('borg.chunker', [chunker_source]), + Extension('borg.algorithms.checksums', [checksums_source]), + ] if not sys.platform.startswith(('win32', )): ext_modules.append(Extension('borg.platform.posix', [platform_posix_source])) - if sys.platform == 'linux': ext_modules.append(Extension('borg.platform.linux', [platform_linux_source], libraries=['acl'])) elif sys.platform.startswith('freebsd'): diff --git a/setup_zstd.py b/setup_zstd.py index 8e1f86c51..bf23cb584 100644 --- a/setup_zstd.py +++ b/setup_zstd.py @@ -1,125 +1,125 @@ +# Support code for building a C extension with zstd files +# # Copyright (c) 2016-present, Gregory Szorc +# 2017-present, Thomas Waldmann (mods to make it more generic) # All rights reserved. # # This software may be modified and distributed under the terms # of the BSD license. See the LICENSE file for details. import os -from distutils.extension import Extension +# zstd files, structure as seen in zstd project repository: -zstd_sources = ['zstd/%s' % p for p in ( - 'common/entropy_common.c', - 'common/error_private.c', - 'common/fse_decompress.c', - 'common/pool.c', - 'common/threading.c', - 'common/xxhash.c', - 'common/zstd_common.c', - 'compress/fse_compress.c', - 'compress/huf_compress.c', - 'compress/zstd_compress.c', - 'compress/zstdmt_compress.c', - 'decompress/huf_decompress.c', - 'decompress/zstd_decompress.c', - 'dictBuilder/cover.c', - 'dictBuilder/divsufsort.c', - 'dictBuilder/zdict.c', -)] +zstd_sources = [ + 'lib/common/entropy_common.c', + 'lib/common/error_private.c', + 'lib/common/fse_decompress.c', + 'lib/common/pool.c', + 'lib/common/threading.c', + 'lib/common/xxhash.c', + 'lib/common/zstd_common.c', + 'lib/compress/fse_compress.c', + 'lib/compress/huf_compress.c', + 'lib/compress/zstd_compress.c', + 'lib/compress/zstd_double_fast.c', + 'lib/compress/zstd_fast.c', + 'lib/compress/zstd_lazy.c', + 'lib/compress/zstd_ldm.c', + 'lib/compress/zstd_opt.c', + 'lib/compress/zstdmt_compress.c', + 'lib/decompress/huf_decompress.c', + 'lib/decompress/zstd_decompress.c', + 'lib/dictBuilder/cover.c', + 'lib/dictBuilder/divsufsort.c', + 'lib/dictBuilder/zdict.c', +] -zstd_sources_legacy = ['zstd/%s' % p for p in ( - 'deprecated/zbuff_common.c', - 'deprecated/zbuff_compress.c', - 'deprecated/zbuff_decompress.c', - 'legacy/zstd_v01.c', - 'legacy/zstd_v02.c', - 'legacy/zstd_v03.c', - 'legacy/zstd_v04.c', - 'legacy/zstd_v05.c', - 'legacy/zstd_v06.c', - 'legacy/zstd_v07.c' -)] +zstd_sources_legacy = [ + 'lib/deprecated/zbuff_common.c', + 'lib/deprecated/zbuff_compress.c', + 'lib/deprecated/zbuff_decompress.c', + 'lib/legacy/zstd_v01.c', + 'lib/legacy/zstd_v02.c', + 'lib/legacy/zstd_v03.c', + 'lib/legacy/zstd_v04.c', + 'lib/legacy/zstd_v05.c', + 'lib/legacy/zstd_v06.c', + 'lib/legacy/zstd_v07.c', +] zstd_includes = [ - 'zstd', - 'zstd/common', - 'zstd/compress', - 'zstd/decompress', - 'zstd/dictBuilder', + 'lib', + 'lib/common', + 'lib/compress', + 'lib/decompress', + 'lib/dictBuilder', ] zstd_includes_legacy = [ - 'zstd/deprecated', - 'zstd/legacy', -] - -ext_includes = [ - 'c-ext', - 'zstd/common', -] - -ext_sources = [ - 'zstd/common/pool.c', - 'zstd/common/threading.c', - 'zstd.c', - 'c-ext/bufferutil.c', - 'c-ext/compressiondict.c', - 'c-ext/compressobj.c', - 'c-ext/compressor.c', - 'c-ext/compressoriterator.c', - 'c-ext/compressionparams.c', - 'c-ext/compressionreader.c', - 'c-ext/compressionwriter.c', - 'c-ext/constants.c', - 'c-ext/decompressobj.c', - 'c-ext/decompressor.c', - 'c-ext/decompressoriterator.c', - 'c-ext/decompressionreader.c', - 'c-ext/decompressionwriter.c', - 'c-ext/frameparams.c', -] - -zstd_depends = [ - 'c-ext/python-zstandard.h', + 'lib/deprecated', + 'lib/legacy', ] -def get_c_extension(support_legacy=False, system_zstd=False, name='zstd'): - """Obtain a distutils.extension.Extension for the C extension.""" - root = os.path.abspath(os.path.dirname(__file__)) +def zstd_system_prefix(prefixes): + for prefix in prefixes: + filename = os.path.join(prefix, 'include', 'zstd.h') + if os.path.exists(filename): + with open(filename, 'r') as fd: + if 'ZSTD_getFrameContentSize' in fd.read(): # checks for zstd >= 1.3.0 + return prefix - sources = set([os.path.join(root, p) for p in ext_sources]) - if not system_zstd: - sources.update([os.path.join(root, p) for p in zstd_sources]) - if support_legacy: - sources.update([os.path.join(root, p) for p in zstd_sources_legacy]) - sources = list(sources) - include_dirs = set([os.path.join(root, d) for d in ext_includes]) - if not system_zstd: - include_dirs.update([os.path.join(root, d) for d in zstd_includes]) - if support_legacy: - include_dirs.update([os.path.join(root, d) for d in zstd_includes_legacy]) - include_dirs = list(include_dirs) +def zstd_ext_kwargs(bundled_path, system_prefix=None, system=False, multithreaded=False, legacy=False, **kwargs): + """amend kwargs with zstd suff for a distutils.extension.Extension initialization. - depends = [os.path.join(root, p) for p in zstd_depends] + bundled_path: relative (to this file) path to the bundled library source code files + system_prefix: where the system-installed library can be found + system: True: use the system-installed shared library, False: use the bundled library code + multithreaded: True: define ZSTD_MULTITHREAD + legacy: include legacy API support + kwargs: distutils.extension.Extension kwargs that should be amended + returns: amended kwargs + """ + def multi_join(paths, *path_segments): + """apply os.path.join on a list of paths""" + return [os.path.join(*(path_segments + (path, ))) for path in paths] - extra_args = ['-DZSTD_MULTITHREAD'] + use_system = system and system_prefix is not None - if not system_zstd: - extra_args.append('-DZSTDLIB_VISIBILITY=') - extra_args.append('-DZDICTLIB_VISIBILITY=') - extra_args.append('-DZSTDERRORLIB_VISIBILITY=') - extra_args.append('-fvisibility=hidden') + sources = kwargs.get('sources', []) + if not use_system: + sources += multi_join(zstd_sources, bundled_path) + if legacy: + sources += multi_join(zstd_sources_legacy, bundled_path) - if not system_zstd and support_legacy: - extra_args.append('-DZSTD_LEGACY_SUPPORT=1') + include_dirs = kwargs.get('include_dirs', []) + if use_system: + include_dirs += multi_join(['include'], system_prefix) + else: + include_dirs += multi_join(zstd_includes, bundled_path) + if legacy: + include_dirs += multi_join(zstd_includes_legacy, bundled_path) - libraries = ['zstd'] if system_zstd else [] + library_dirs = kwargs.get('library_dirs', []) + if use_system: + library_dirs += multi_join(['lib'], system_prefix) - # TODO compile with optimizations. - return Extension(name, sources, - include_dirs=include_dirs, - depends=depends, - extra_compile_args=extra_args, - libraries=libraries) + libraries = kwargs.get('libraries', []) + if use_system: + libraries += ['zstd', ] + + extra_compile_args = kwargs.get('extra_compile_args', []) + if multithreaded: + extra_compile_args += ['-DZSTD_MULTITHREAD', ] + if not use_system: + extra_compile_args += ['-DZSTDLIB_VISIBILITY=', '-DZDICTLIB_VISIBILITY=', '-DZSTDERRORLIB_VISIBILITY=', ] + # '-fvisibility=hidden' does not work, doesn't find PyInit_compress then + if legacy: + extra_compile_args += ['-DZSTD_LEGACY_SUPPORT=1', ] + + ret = dict(**kwargs) + ret.update(dict(sources=sources, extra_compile_args=extra_compile_args, + include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)) + return ret diff --git a/src/borg/algorithms/zstd-libselect.h b/src/borg/algorithms/zstd-libselect.h index bb71553c1..9a4ded364 100644 --- a/src/borg/algorithms/zstd-libselect.h +++ b/src/borg/algorithms/zstd-libselect.h @@ -1,5 +1,5 @@ #ifdef BORG_USE_LIBZSTD #include #else -#error "TODO" +#include "zstd/lib/zstd.h" #endif