diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89c1a1187..8ef2e8f65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,7 +67,6 @@ jobs: env: # Configure pkg-config to use OpenSSL from Homebrew PKG_CONFIG_PATH: /usr/local/opt/openssl@1.1/lib/pkgconfig - BORG_LIBDEFLATE_PREFIX: /usr # on ubuntu 20.04 pkgconfig does not find libdeflate TOXENV: ${{ matrix.toxenv }} runs-on: ${{ matrix.os }} @@ -96,7 +95,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y pkg-config build-essential - sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev libdeflate-dev liblz4-dev libzstd-dev + sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev sudo apt-get install -y libfuse-dev fuse || true # Required for Python llfuse module sudo apt-get install -y libfuse3-dev fuse3 || true # Required for Python pyfuse3 module @@ -106,7 +105,6 @@ jobs: brew install pkg-config || brew upgrade pkg-config brew install zstd || brew upgrade zstd brew install lz4 || brew upgrade lz4 - brew install libdeflate || brew upgrade libdeflate brew install xxhash || brew upgrade xxhash brew install openssl@1.1 || brew upgrade openssl@1.1 brew install --cask macfuse || brew upgrade --cask macfuse # Required for Python llfuse module diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3caec9d3b..ba31216ca 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -28,7 +28,7 @@ jobs: # Learn more about CodeQL language support at https://git.io/codeql-language-support env: - BORG_LIBDEFLATE_PREFIX: /usr # on ubuntu 20.04 pkgconfig does not find libdeflate + # nothing special right now steps: - name: Checkout repository @@ -52,7 +52,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y pkg-config build-essential - sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev libdeflate-dev liblz4-dev libzstd-dev + sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v1 diff --git a/Vagrantfile b/Vagrantfile index 10d3a4cd0..37d1572e9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -15,7 +15,7 @@ def packages_debianoid(user) apt-get -y -qq update apt-get -y -qq dist-upgrade # for building borgbackup and dependencies: - apt install -y libssl-dev libacl1-dev libxxhash-dev libdeflate-dev liblz4-dev libzstd-dev pkg-config + apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev pkg-config apt install -y libfuse-dev fuse || true apt install -y libfuse3-dev fuse3 || true apt install -y locales || true @@ -29,7 +29,6 @@ def packages_debianoid(user) apt install -y zlib1g-dev libbz2-dev libncurses5-dev libreadline-dev liblzma-dev libsqlite3-dev libffi-dev # older debian / ubuntu have no .pc file for these, so we need to point at the lib/header location: echo 'export BORG_LIBXXHASH_PREFIX=/usr' >> ~vagrant/.bash_profile - echo 'export BORG_LIBDEFLATE_PREFIX=/usr' >> ~vagrant/.bash_profile EOF end @@ -40,7 +39,7 @@ def packages_freebsd # install all the (security and other) updates, base system freebsd-update --not-running-from-cron fetch install # for building borgbackup and dependencies: - pkg install -y xxhash libdeflate liblz4 zstd pkgconf + pkg install -y xxhash liblz4 zstd pkgconf pkg install -y fusefs-libs || true pkg install -y fusefs-libs3 || true pkg install -y git bash # fakeroot causes lots of troubles on freebsd @@ -74,7 +73,6 @@ def packages_openbsd pkg_add bash chsh -s bash vagrant pkg_add xxhash - pkg_add libdeflate pkg_add lz4 pkg_add zstd pkg_add git # no fakeroot @@ -119,7 +117,7 @@ def packages_darwin sudo softwareupdate --install --all which brew || CI=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" brew update > /dev/null - brew install pkg-config readline openssl@1.1 xxhash libdeflate zstd lz4 xz + brew install pkg-config readline openssl@1.1 xxhash zstd lz4 xz brew install --cask macfuse # brew upgrade # upgrade everything (takes rather long) echo 'export PKG_CONFIG_PATH=/usr/local/opt/openssl@1.1/lib/pkgconfig' >> ~vagrant/.bash_profile diff --git a/docs/installation.rst b/docs/installation.rst index 60b05e45d..ce28b52f3 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -167,7 +167,6 @@ following dependencies first: - liblz4_ >= 1.7.0 (r129) - libzstd_ >= 1.3.0 - libxxhash >= 0.8.1 (0.8.0 might work also) - - libdeflate >= 1.5 * pkg-config (cli tool) and pkgconfig python package (borg uses these to discover header and library location - if it can't import pkgconfig and is not pointed to header/library locations via env vars [see setup.py], @@ -202,7 +201,7 @@ Install the dependencies with development headers:: sudo apt-get install python3 python3-dev python3-pip python3-virtualenv \ libacl1-dev libacl1 \ libssl-dev \ - liblz4-dev libzstd-dev libxxhash-dev libdeflate-dev \ + liblz4-dev libzstd-dev libxxhash-dev \ build-essential \ pkg-config python3-pkgconfig sudo apt-get install libfuse-dev fuse # needed for llfuse @@ -220,7 +219,7 @@ Install the dependencies with development headers:: sudo dnf install python3 python3-devel python3-pip python3-virtualenv \ libacl-devel libacl \ openssl-devel \ - lz4-devel libzstd-devel xxhash-devel libdeflate-devel \ + lz4-devel libzstd-devel xxhash-devel \ pkgconf python3-pkgconfig sudo dnf install gcc gcc-c++ redhat-rpm-config sudo dnf install fuse-devel fuse # needed for llfuse @@ -236,8 +235,7 @@ Install the dependencies automatically using zypper:: Alternatively, you can enumerate all build dependencies in the command line:: sudo zypper install python3 python3-devel \ - libacl-devel openssl-devel \ - libxxhash-devel libdeflate-devel \ + libacl-devel openssl-devel libxxhash-devel \ python3-Cython python3-Sphinx python3-msgpack-python python3-pkgconfig pkgconf \ python3-pytest python3-setuptools python3-setuptools_scm \ python3-sphinx_rtd_theme gcc gcc-c++ @@ -249,7 +247,7 @@ macOS When installing via Homebrew_, dependencies are installed automatically. To install dependencies manually:: - brew install python3 openssl zstd lz4 xxhash libdeflate + brew install python3 openssl zstd lz4 xxhash brew install pkg-config pip3 install virtualenv pkgconfig @@ -286,7 +284,7 @@ and commands to make FUSE work for using the mount command. pkg install -y python3 pkgconf pkg install openssl - pkg install liblz4 zstd xxhash libdeflate + pkg install liblz4 zstd xxhash pkg install fusefs-libs # needed for llfuse pkg install -y git python3 -m ensurepip # to install pip for Python3 @@ -316,7 +314,7 @@ Use the Cygwin installer to install the dependencies:: python39 python39-devel python39-pkgconfig python39-setuptools python39-pip python39-wheel python39-virtualenv - libssl-devel libxxhash-devel libdeflate-devel liblz4-devel libzstd-devel + libssl-devel libxxhash-devel liblz4-devel libzstd-devel binutils gcc-g++ git make openssh diff --git a/setup.py b/setup.py index 2d91a6a47..f962e2c49 100644 --- a/setup.py +++ b/setup.py @@ -186,7 +186,6 @@ if not on_rtd: checksums_ext_kwargs = members_appended( dict(sources=[checksums_source]), lib_ext_kwargs(pc, 'BORG_LIBXXHASH_PREFIX', 'xxhash', 'libxxhash', '>= 0.7.3'), - lib_ext_kwargs(pc, 'BORG_LIBDEFLATE_PREFIX', 'deflate', 'libdeflate', '>= 1.5'), dict(extra_compile_args=cflags), ) diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 3c141234f..f4f2ef355 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -715,21 +715,13 @@ class Archiver: ]: print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s") - import zlib - from .checksums import crc32, deflate_crc32, xxh64 + from .checksums import crc32, xxh64 print("Non-cryptographic checksums / hashes ===========================") size = "1GB" tests = [ ("xxh64", lambda: xxh64(random_10M)), + ("crc32 (zlib)", lambda: crc32(random_10M)), ] - if crc32 is zlib.crc32: - tests.insert(0, ("crc32 (zlib, used)", lambda: crc32(random_10M))) - tests.insert(1, ("crc32 (libdeflate)", lambda: deflate_crc32(random_10M))) - elif crc32 is deflate_crc32: - tests.insert(0, ("crc32 (libdeflate, used)", lambda: crc32(random_10M))) - tests.insert(1, ("crc32 (zlib)", lambda: zlib.crc32(random_10M))) - else: - raise Error("crc32 benchmarking code missing") for spec, func in tests: print(f"{spec:<24} {size:<10} {timeit(func, number=100):.3f}s") diff --git a/src/borg/checksums.pyx b/src/borg/checksums.pyx index b3372b8c2..32a21770b 100644 --- a/src/borg/checksums.pyx +++ b/src/borg/checksums.pyx @@ -33,31 +33,15 @@ cdef extern from "xxhash.h": XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) -cdef extern from "libdeflate.h": - uint32_t libdeflate_crc32(uint32_t crc, const void *buffer, size_t len) - - cdef Py_buffer ro_buffer(object data) except *: cdef Py_buffer view PyObject_GetBuffer(data, &view, PyBUF_SIMPLE) return view -def deflate_crc32(data, value=0): - cdef Py_buffer data_buf = ro_buffer(data) - cdef uint32_t val = value - try: - return libdeflate_crc32(val, data_buf.buf, data_buf.len) - finally: - PyBuffer_Release(&data_buf) - - -if is_darwin: - # macOS (darwin) has a highly optimized zlib.crc32 (Intel as well as Apple Silicon M1) - crc32 = zlib.crc32 -else: - # on Linux x64 (and maybe others), libdeflate_crc32 is faster than zlib.crc32 - crc32 = deflate_crc32 +# borg 2.0's new repos do not compute crc32 over big amounts of data, +# so speed does not matter much any more and we can just use zlib.crc32. +crc32 = zlib.crc32 def xxh64(data, seed=0): diff --git a/src/borg/testsuite/checksums.py b/src/borg/testsuite/checksums.py index 2513d7a50..2567fda8a 100644 --- a/src/borg/testsuite/checksums.py +++ b/src/borg/testsuite/checksums.py @@ -1,30 +1,8 @@ -import os -import zlib from binascii import unhexlify -import pytest - from .. import checksums from ..helpers import bin_to_hex -crc32_implementations = [checksums.deflate_crc32] - - -@pytest.mark.parametrize('implementation', crc32_implementations) -def test_crc32(implementation): - # This includes many critical values, like misc. length and misc. aligned start addresses. - data = os.urandom(300) - mv = memoryview(data) - initial_crc = 0x12345678 - for start in range(0, 4): # 4B / int32 alignment, head processing - for length in [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, - 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, - 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, - 127, 128, 129, 130, 131, 132, 133, 134, 135, - 255, 256, 257, ]: - d = mv[start:start+length] - assert zlib.crc32(d, initial_crc) == implementation(d, initial_crc) - def test_xxh64(): assert bin_to_hex(checksums.xxh64(b'test', 123)) == '2b81b9401bef86cf'