upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-06-14 20:13:21 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #9397 from defnvary/use_zstd_python_stdlib

Browse source 
use zstd from python lib or backports.zstd (python<'3.14'), closes #9261
This commit is contained in:
TW 2026-02-23 12:10:52 +01:00 committed by GitHub
commit 23060e3943
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 39 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
.github/workflows/ci.yml vendored
View file

@ -79,7 +79,7 @@ jobs:
run: |
sudo apt-get update
sudo apt-get install -y pkg-config build-essential
sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev
- name: Install Python dependencies
run: |
@ -200,7 +200,7 @@ jobs:
run: |
sudo apt-get update
sudo apt-get install -y pkg-config build-essential
sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev
sudo apt-get install -y bash zsh fish # for shell completion tests
sudo apt-get install -y rclone openssh-server curl
if [[ "$TOXENV" == *"llfuse"* ]]; then
@ -435,7 +435,7 @@ jobs:
freebsd)
export IGNORE_OSVERSION=yes
sudo -E pkg update -f
sudo -E pkg install -y xxhash liblz4 zstd pkgconf
sudo -E pkg install -y xxhash liblz4 pkgconf
sudo -E pkg install -y fusefs-libs
sudo -E kldload fusefs
sudo -E sysctl vfs.usermount=1
@ -491,7 +491,7 @@ jobs:
echo "https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null
sudo -E pkgin update
sudo -E pkgin -y upgrade
sudo -E pkgin -y install zstd lz4 xxhash git
sudo -E pkgin -y install lz4 xxhash git
sudo -E pkgin -y install rust
sudo -E pkgin -y install pkg-config
sudo -E pkgin -y install py311-pip py311-virtualenv py311-tox
@ -525,7 +525,7 @@ jobs:
;;
openbsd)
sudo -E pkg_add xxhash lz4 zstd git
sudo -E pkg_add xxhash lz4 git
sudo -E pkg_add rust
sudo -E pkg_add openssl%3.4
sudo -E pkg_add py3-pip py3-virtualenv py3-tox
@ -563,12 +563,12 @@ jobs:
haiku)
pkgman refresh
pkgman install -y git pkgconfig zstd lz4 xxhash
pkgman install -y git pkgconfig lz4 xxhash
pkgman install -y openssl3
pkgman install -y rust_bin
pkgman install -y python3.10
pkgman install -y cffi
pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel
pkgman install -y lz4_devel xxhash_devel openssl3_devel libffi_devel
# there is no pkgman package for tox, so we install it into a venv
python3 -m ensurepip --upgrade
@ -578,7 +578,6 @@ jobs:
export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
export BORG_LIBLZ4_PREFIX=/system/develop
export BORG_LIBZSTD_PREFIX=/system/develop
export BORG_LIBXXHASH_PREFIX=/system/develop
export BORG_OPENSSL_PREFIX=/system/develop
pip install -r requirements.d/development.txt

2
.github/workflows/codeql-analysis.yml vendored
View file

@ -66,7 +66,7 @@ jobs:
run: |
sudo apt-get update
sudo apt-get install -y pkg-config build-essential
sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev
sudo apt-get install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4

1
.readthedocs.yaml
View file

@ -16,7 +16,6 @@ build:
- libacl1-dev
- libssl-dev
- liblz4-dev
- libzstd-dev
- libxxhash-dev
python:

1
Brewfile
View file

@ -1,5 +1,4 @@
brew 'pkgconf'
brew 'zstd'
brew 'lz4'
brew 'xxhash'
brew 'openssl@3'

7
Vagrantfile vendored
View file

@ -16,7 +16,7 @@ def packages_debianoid(user)
apt-get -y -qq dist-upgrade
# for building borgbackup and dependencies:
apt install -y pkg-config
apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev libzstd-dev || true
apt install -y libssl-dev libacl1-dev libxxhash-dev liblz4-dev || true
apt install -y libfuse-dev fuse || true
apt install -y libfuse3-dev fuse3 || true
apt install -y locales || true
@ -38,7 +38,7 @@ def packages_freebsd
# install all the (security and other) updates, base system
freebsd-update --not-running-from-cron fetch install
# for building borgbackup and dependencies:
pkg install -y xxhash liblz4 zstd pkgconf
pkg install -y xxhash liblz4 pkgconf
pkg install -y fusefs-libs || true
pkg install -y fusefs-libs3 || true
pkg install -y rust
@ -85,7 +85,6 @@ def packages_openbsd
chsh -s bash vagrant
pkg_add xxhash
pkg_add lz4
pkg_add zstd
pkg_add git # no fakeroot
pkg_add rust
pkg_add openssl%3.4
@ -100,7 +99,7 @@ def packages_netbsd
echo 'https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/$arch/9.3/All' > /usr/pkg/etc/pkgin/repositories.conf
pkgin update
pkgin -y upgrade
pkg_add zstd lz4 xxhash git
pkg_add lz4 xxhash git
pkg_add rust
pkg_add bash
chsh -s bash vagrant

13
docs/installation.rst
View file

@ -166,7 +166,6 @@ development header files (sometimes in a separate `-dev` or `-devel` package).
* libacl_ (which depends on libattr_)
* libxxhash_ >= 0.8.1
* liblz4_ >= 1.7.0 (r129)
* libzstd_ >= 1.3.0
* libffi (required for argon2-cffi-bindings)
* pkg-config (cli tool) - Borg uses this to discover header and library
locations automatically. Alternatively, you can also point to them via some
@ -201,7 +200,7 @@ Arch Linux
Install the runtime and build dependencies::
pacman -S python python-pip python-virtualenv openssl acl xxhash lz4 zstd base-devel
pacman -S python python-pip python-virtualenv openssl acl xxhash lz4 base-devel
pacman -S fuse2 # needed for llfuse
pacman -S fuse3 # needed for pyfuse3
@ -217,7 +216,7 @@ Install the dependencies with development headers::
sudo apt-get install python3 python3-dev python3-pip python3-virtualenv \
libacl1-dev \
libssl-dev \
liblz4-dev libzstd-dev libxxhash-dev \
liblz4-dev libxxhash-dev \
libffi-dev \
build-essential pkg-config
sudo apt-get install libfuse-dev fuse # needed for llfuse
@ -235,7 +234,7 @@ Install the dependencies with development headers::
sudo dnf install python3 python3-devel python3-pip python3-virtualenv \
libacl-devel \
openssl-devel \
lz4-devel libzstd-devel xxhash-devel \
lz4-devel xxhash-devel \
libffi-devel \
pkgconf
sudo dnf install gcc gcc-c++ redhat-rpm-config
@ -252,7 +251,7 @@ Install the dependencies automatically using zypper::
Alternatively, you can enumerate all build dependencies in the command line::
sudo zypper install python3 python3-devel \
libacl-devel openssl-devel xxhash-devel libzstd-devel liblz4-devel \
libacl-devel openssl-devel xxhash-devel liblz4-devel \
libffi-devel \
python3-Cython python3-Sphinx python3-msgpack-python python3-pkgconfig pkgconf \
python3-pytest python3-setuptools python3-setuptools_scm \
@ -303,7 +302,7 @@ and commands to make FUSE work for using the mount command.
pkg install -y python3 pkgconf
pkg install openssl
pkg install liblz4 zstd xxhash
pkg install liblz4 xxhash
pkg install fusefs-libs # needed for llfuse
pkg install -y git
python3 -m ensurepip # to install pip for Python3
@ -347,7 +346,7 @@ Use the Cygwin installer to install the dependencies::
python39 python39-devel
python39-setuptools python39-pip python39-wheel python39-virtualenv
libssl-devel libxxhash-devel liblz4-devel libzstd-devel
libssl-devel libxxhash-devel liblz4-devel
binutils gcc-g++ git make openssh
Make sure to use a virtual environment to avoid confusions with any Python installed on Windows.

3
docs/usage/general/environment.rst.inc
View file

@ -218,9 +218,6 @@ Building:
BORG_LIBLZ4_PREFIX
Adds given prefix directory to the default locations. If a 'include/lz4.h' is found Borg
will be linked against the system liblz4 instead of a bundled implementation. (setup.py)
BORG_LIBZSTD_PREFIX
Adds given prefix directory to the default locations. If a 'include/zstd.h' is found Borg
will be linked against the system libzstd instead of a bundled implementation. (setup.py)
Please note:

1
pyproject.toml
View file

@ -38,6 +38,7 @@ dependencies = [
"platformdirs >=2.6.0, <5.0.0; sys_platform != 'darwin'", # for others: 2.6+ works consistently.
"argon2-cffi",
"shtab>=1.8.0",
"backports-zstd; python_version < '3.14'", # for python < 3.14.
]
[project.optional-dependencies]

1
scripts/Dockerfile.linux-run
View file

@ -10,7 +10,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
libacl1-dev \
libxxhash-dev \
liblz4-dev \
libzstd-dev \
libfuse3-dev \
fuse3 \
python3-dev \

2
scripts/msys2-install-deps
View file

@ -1,6 +1,6 @@
#!/bin/bash
pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,zstd,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko}
pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko}
if [ "$1" = "development" ]; then
pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-python-{pytest,pytest-benchmark,pytest-cov,pytest-xdist}

1
setup.py
View file

@ -160,7 +160,6 @@ if not on_rtd:
compress_ext_kwargs = members_appended(
dict(sources=[compress_source]),
lib_ext_kwargs(pc, "BORG_LIBLZ4_PREFIX", "lz4", "liblz4", ">= 1.7.0"),
lib_ext_kwargs(pc, "BORG_LIBZSTD_PREFIX", "zstd", "libzstd", ">= 1.3.0"),
dict(extra_compile_args=cflags),
)

89
src/borg/compress.pyx
View file

@ -29,6 +29,12 @@ except ImportError:
from .constants import MAX_DATA_SIZE, ROBJ_FILE_STREAM
from .helpers import Buffer, DecompressionError
import sys
if sys.version_info >= (3, 14):
from compression import zstd
else:
from backports import zstd
cdef extern from "lz4.h":
@ -36,21 +42,8 @@ cdef extern from "lz4.h":
int LZ4_decompress_safe(const char* source, char* dest, int inputSize, int maxOutputSize) nogil
int LZ4_compressBound(int inputSize) nogil
cdef extern from "zstd.h":
size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) nogil
size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t compressedSize) nogil
size_t ZSTD_compressBound(size_t srcSize) nogil
unsigned long long ZSTD_CONTENTSIZE_UNKNOWN
unsigned long long ZSTD_CONTENTSIZE_ERROR
unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) nogil
unsigned ZSTD_isError(size_t code) nogil
const char* ZSTD_getErrorName(size_t code) nogil
buffer = Buffer(bytearray, size=0)
cdef class CompressorBase:
"""
base class for all (de)compression classes,
@ -303,12 +296,10 @@ class LZMA(DecidingCompressor):
except lzma.LZMAError as e:
raise DecompressionError(str(e)) from None
class ZSTD(DecidingCompressor):
"""zstd compression / decompression (pypi: zstandard, gh: python-zstandard)"""
# This is a NOT THREAD SAFE implementation.
# Only ONE python context must be created at a time.
# It should work flawlessly as long as borg will call ONLY ONE compression job at time.
"""
zstd compression / decompression (python stdlib (python >= 3.14))
"""
ID = 0x03
name = 'zstd'
@ -316,61 +307,21 @@ class ZSTD(DecidingCompressor):
super().__init__(level=level, legacy_mode=legacy_mode, **kwargs)
self.level = level
def _decide(self, meta, idata):
"""
Decides what to do with *data*. Returns (compressor, zstd_data).
*zstd_data* is the ZSTD result if *compressor* is ZSTD as well, otherwise it is None.
"""
if not isinstance(idata, bytes):
idata = bytes(idata) # code below does not work with memoryview
cdef int isize = len(idata)
cdef int osize
cdef char *source = idata
cdef char *dest
cdef int level = self.level
osize = ZSTD_compressBound(isize)
buf = buffer.get(osize)
dest = <char *> buf
with nogil:
osize = ZSTD_compress(dest, osize, source, isize, level)
if ZSTD_isError(osize):
raise Exception('zstd compress failed: %s' % ZSTD_getErrorName(osize))
# only compress if the result actually is smaller
if osize < isize:
return self, (meta, dest[:osize])
def _decide(self, meta, data):
zstd_data = zstd.compress(data, self.level)
if len(zstd_data) < len(data):
return self, (meta, zstd_data)
else:
return NONE_COMPRESSOR, (meta, None)
def decompress(self, meta, data):
meta, idata = super().decompress(meta, data)
if not isinstance(idata, bytes):
idata = bytes(idata) # code below does not work with memoryview
cdef int isize = len(idata)
cdef unsigned long long osize
cdef unsigned long long rsize
cdef char *source = idata
cdef char *dest
osize = ZSTD_getFrameContentSize(source, isize)
if osize == ZSTD_CONTENTSIZE_ERROR:
raise DecompressionError('zstd get size failed: data was not compressed by zstd')
if osize == ZSTD_CONTENTSIZE_UNKNOWN:
raise DecompressionError('zstd get size failed: original size unknown')
meta, data = super().decompress(meta, data)
try:
buf = buffer.get(osize)
except MemoryError:
raise DecompressionError('MemoryError')
dest = <char *> buf
with nogil:
rsize = ZSTD_decompress(dest, osize, source, isize)
if ZSTD_isError(rsize):
raise DecompressionError('zstd decompress failed: %s' % ZSTD_getErrorName(rsize))
if rsize != osize:
raise DecompressionError('zstd decompress failed: size mismatch')
data = dest[:osize]
self.check_fix_size(meta, data)
return meta, data
data = zstd.decompress(data)
self.check_fix_size(meta, data)
return meta, data
except zstd.ZstdError as e:
raise DecompressionError(str(e)) from None
class ZLIB(DecidingCompressor):
"""