upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-06-11 01:41:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

Merge 252cf06eb9 into e016a51cf2

Browse source
This commit is contained in:
TW 2026-05-22 11:08:24 +02:00 committed by GitHub
commit a946e20eb9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 128 additions and 226 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

154
.github/workflows/ci.yml vendored
View file

@ -29,107 +29,8 @@ permissions:
contents: read
jobs:
lint:
runs-on: ubuntu-22.04
timeout-minutes: 5
steps:
- uses: actions/checkout@v6
- uses: astral-sh/ruff-action@v3
security:
runs-on: ubuntu-24.04
timeout-minutes: 5
steps:
- uses: actions/checkout@v6
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.10'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install bandit[toml]
- name: Run Bandit
run: |
bandit -r src/borg -c pyproject.toml
asan_ubsan:
runs-on: ubuntu-24.04
timeout-minutes: 25
needs: [lint]
steps:
- uses: actions/checkout@v6
with:
# Just fetching one commit is not enough for setuptools-scm, so we fetch all.
fetch-depth: 0
fetch-tags: true
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.12'
- name: Install system packages
run: |
sudo apt-get update
sudo apt-get install -y pkg-config build-essential
sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev
- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.d/development.lock.txt
- name: Build Borg with ASan/UBSan
# Build the C/Cython extensions with AddressSanitizer and UndefinedBehaviorSanitizer enabled.
# How this works:
# - The -fsanitize=address,undefined flags inject runtime checks into our native code. If a bug is hit
# (e.g., buffer overflow, use-after-free, out-of-bounds, or undefined behavior), the sanitizer prints
# a detailed error report to stderr, including a stack trace, and forces the process to exit with
# non-zero status. In CI, this will fail the step/job so you will notice.
# - ASAN_OPTIONS/UBSAN_OPTIONS configure the sanitizers' runtime behavior (see below for meanings).
env:
CFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined"
CXXFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined"
LDFLAGS: "-fsanitize=address,undefined"
# ASAN_OPTIONS controls AddressSanitizer runtime tweaks:
# - detect_leaks=0: Disable LeakSanitizer to avoid false positives with CPython/pymalloc in short-lived tests.
# - strict_string_checks=1: Make invalid string operations (e.g., over-reads) more likely to be detected.
# - check_initialization_order=1: Catch uses that depend on static initialization order (C++).
# - detect_stack_use_after_return=1: Detect stack-use-after-return via stack poisoning (may increase overhead).
ASAN_OPTIONS: "detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1"
# UBSAN_OPTIONS controls UndefinedBehaviorSanitizer runtime:
# - print_stacktrace=1: Include a stack trace for UB reports to ease debugging.
# Note: UBSan is recoverable by default (process may continue after reporting). If you want CI to
# abort immediately and fail on the first UB, add `halt_on_error=1` (e.g., UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1").
UBSAN_OPTIONS: "print_stacktrace=1"
# PYTHONDEVMODE enables additional Python runtime checks and warnings.
PYTHONDEVMODE: "1"
run: pip install -e .
- name: Run tests under sanitizers
env:
ASAN_OPTIONS: "detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1"
UBSAN_OPTIONS: "print_stacktrace=1"
PYTHONDEVMODE: "1"
# Ensure the ASan runtime is loaded first to avoid "ASan runtime does not come first" warnings.
# We discover libasan/libubsan paths via gcc and preload them for the Python test process.
# the remote tests are slow and likely won't find anything useful
run: |
set -euo pipefail
export LD_PRELOAD="$(gcc -print-file-name=libasan.so):$(gcc -print-file-name=libubsan.so)"
echo "Using LD_PRELOAD=$LD_PRELOAD"
pytest -v --benchmark-skip -k "not remote"
native_tests:
needs: [lint]
permissions:
contents: read
id-token: write
@ -141,21 +42,10 @@ jobs:
${{ fromJSON(
github.event_name == 'pull_request' && '{
"include": [
{"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "mypy"},
{"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "docs"},
{"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-llfuse"},
{"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-pyfuse3"},
{"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-mfusepy"}
{"os": "macos-15", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-15-arm64-gh"},
]
}' || '{
"include": [
{"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "py311-pyfuse3", "binary": "borg-linux-glibc235-x86_64-gh"},
{"os": "ubuntu-22.04-arm", "python-version": "3.11", "toxenv": "py311-pyfuse3", "binary": "borg-linux-glibc235-arm64-gh"},
{"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-llfuse"},
{"os": "ubuntu-24.04", "python-version": "3.13", "toxenv": "py313-pyfuse3"},
{"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-mfusepy"},
{"os": "macos-15", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-15-arm64-gh"},
{"os": "macos-15-intel", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-15-x86_64-gh"}
]
}'
) }}
@ -380,7 +270,6 @@ jobs:
attestations: write
runs-on: ubuntu-24.04
timeout-minutes: 180
needs: [lint]
continue-on-error: true
strategy:
@ -388,11 +277,11 @@ jobs:
matrix:
include:
- os: freebsd
version: '14.3'
version: '15.0'
display_name: FreeBSD
# Controls binary build and provenance attestation on tags
do_binaries: true
artifact_prefix: borg-freebsd-14-x86_64-gh
artifact_prefix: borg-freebsd-15-x86_64-gh
- os: netbsd
version: '10.1'
@ -523,6 +412,30 @@ jobs:
touch ${TMPDIR}/testfile
lsextattr user ${TMPDIR}/testfile && echo "[xattr] *** xattrs SUPPORTED on ${TMPDIR}! ***"
# NetBSD 10 has a too old OpenSSL, build a fresher one.
VERSION="3.5.6"
echo "--- Building OpenSSL ${VERSION} ---"
PREFIX="/usr/local"
JOBS=$(sysctl -n hw.ncpu)
pushd /tmp
ftp -o "openssl-${VERSION}.tar.gz" "https://www.openssl.org/source/openssl-${VERSION}.tar.gz"
tar xzf "openssl-${VERSION}.tar.gz"
pushd "openssl-${VERSION}"
./Configure --prefix="${PREFIX}" --openssldir="${PREFIX}/etc/ssl" --libdir=lib \
-Wl,-rpath,${PREFIX}/lib shared threads no-tests no-docs
export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
make -j"${JOBS}"
sudo -E make install
popd
rm -rf "/tmp/openssl-${VERSION}" "/tmp/openssl-${VERSION}.tar.gz"
popd
"${PREFIX}/bin/openssl" version
export BORG_OPENSSL_PREFIX="${PREFIX}"
export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}"
tox3 -e py311-none
;;
@ -558,11 +471,12 @@ jobs:
haiku)
pkgman refresh
pkgman install -y git pkgconfig lz4
pkgman install -y openssl3
pkgman install -y rust_bin
pkgman install -y python3.10
pkgman install -y cffi
pkgman install -y lz4_devel openssl3_devel libffi_devel
pkgman install -y python3.10 lz4_devel
# haiku r1beta5 has OpenSSL 3.0.14, so we manually pull 3.5.6 from current master:
curl -L -O https://eu.hpkg.haiku-os.org/haikuports/master/x86_64/current/packages/openssl3-3.5.6-1-x86_64.hpkg
curl -L -O https://eu.hpkg.haiku-os.org/haikuports/master/x86_64/current/packages/openssl3_devel-3.5.6-1-x86_64.hpkg
pkgman install -y openssl3-3.5.6-1-x86_64.hpkg openssl3_devel-3.5.6-1-x86_64.hpkg
# there is no pkgman package for tox, so we install it into a venv
python3 -m ensurepip --upgrade
@ -572,7 +486,6 @@ jobs:
export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
export BORG_LIBLZ4_PREFIX=/system/develop
export BORG_OPENSSL_PREFIX=/system/develop
pip install -r requirements.d/development.lock.txt
pip install -e .
@ -622,7 +535,6 @@ jobs:
if: true # can be used to temporarily disable the build
runs-on: windows-latest
timeout-minutes: 90
needs: [lint]
env:
PY_COLORS: 1
@ -648,7 +560,7 @@ jobs:
- name: Build python venv
run: |
# building cffi / argon2-cffi in the venv fails, so we try to use the system packages
# building native extensions in the venv fails, so we try to use the system packages
python -m venv --system-site-packages env
. env/bin/activate
# python -m pip install --upgrade pip

86
.github/workflows/codeql-analysis.yml vendored
View file

@ -1,86 +0,0 @@
# CodeQL semantic code analysis engine
name: "CodeQL"
on:
push:
branches: [ master ]
paths:
- '**.py'
- '**.pyx'
- '**.c'
- '**.h'
- '.github/workflows/codeql-analysis.yml'
pull_request:
# The branches below must be a subset of the branches above
branches: [ master ]
paths:
- '**.py'
- '**.pyx'
- '**.c'
- '**.h'
- '.github/workflows/codeql-analysis.yml'
schedule:
- cron: '39 2 * * 5'
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
name: Analyze
runs-on: ubuntu-24.04
timeout-minutes: 20
permissions:
actions: read
contents: read
security-events: write
strategy:
fail-fast: false
matrix:
language: [ 'cpp', 'python' ]
# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
# Learn more about CodeQL language support at https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/
steps:
- name: Checkout repository
uses: actions/checkout@v6
with:
# Just fetching one commit is not enough for setuptools-scm, so we fetch all.
fetch-depth: 0
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: 3.11
- name: Cache pip
uses: actions/cache@v5
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }}
restore-keys: |
${{ runner.os }}-pip-
${{ runner.os }}-
- name: Install requirements
run: |
sudo apt-get update
sudo apt-get install -y pkg-config build-essential
sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v4
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
- name: Build and install Borg
run: |
python3 -m venv ../borg-env
source ../borg-env/bin/activate
pip3 install -r requirements.d/development.txt
pip3 install -ve .
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v4

4
docs/installation.rst
View file

@ -162,10 +162,10 @@ following dependencies first. For the libraries you will also need their
development header files (sometimes in a separate `-dev` or `-devel` package).
* `Python 3`_ >= 3.10.0
* OpenSSL_ >= 1.1.1 (LibreSSL will not work)
* OpenSSL_ >= 3.2.0 (LibreSSL will not work)
* libacl_ (which depends on libattr_)
* liblz4_ >= 1.7.0 (r129)
* libffi (required for argon2-cffi-bindings)
* pkg-config (cli tool) - Borg uses this to discover header and library
locations automatically. Alternatively, you can also point to them via some
environment variables, see setup.py.

2
docs/internals/security.rst
View file

@ -241,7 +241,7 @@ on widely used libraries providing them:
primitives implemented in libcrypto.
- SHA-256, SHA-512 and BLAKE2b from Python's hashlib_ standard library module are used.
- HMAC and a constant-time comparison from Python's hmac_ standard library module are used.
- argon2 is used via argon2-cffi.
- argon2 is used from OpenSSL (>= 3.2).
.. _Horton principle: https://en.wikipedia.org/wiki/Horton_Principle
.. _length extension: https://en.wikipedia.org/wiki/Length_extension_attack

1
pyproject.toml
View file

@ -36,7 +36,6 @@ dependencies = [
"packaging",
"platformdirs >=3.0.0, <5.0.0; sys_platform == 'darwin'", # for macOS: breaking changes in 3.0.0.
"platformdirs >=2.6.0, <5.0.0; sys_platform != 'darwin'", # for others: 2.6+ works consistently.
"argon2-cffi",
"shtab>=1.8.0",
"backports-zstd; python_version < '3.14'", # for python < 3.14.
"xxhash>=2.0.0",

2
scripts/msys2-install-deps
View file

@ -1,6 +1,6 @@
#!/bin/bash
pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko}
pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko}
if [ "$1" = "development" ]; then
pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-python-{pytest,pytest-benchmark,pytest-cov,pytest-xdist}

13
setup.py
View file

@ -29,6 +29,7 @@ sys.path += [os.path.dirname(__file__)]
is_win32 = sys.platform.startswith("win32")
is_openbsd = sys.platform.startswith("openbsd")
is_netbsd = sys.platform.startswith("netbsd")
# Number of threads to use for cythonize, not used on Windows
cpu_threads = multiprocessing.cpu_count() if multiprocessing and multiprocessing.get_start_method() != "spawn" else None
@ -137,7 +138,7 @@ if not on_rtd:
)
if is_win32:
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "libcrypto", "libcrypto", ">=1.1.1", lib_subdir="")
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "libcrypto", "libcrypto", ">=3.2.0", lib_subdir="")
elif is_openbsd:
# Use OpenSSL (not LibreSSL) because we need AES-OCB via the EVP API. Link
# it statically to avoid conflicting with shared libcrypto from the base
@ -148,8 +149,16 @@ if not on_rtd:
include_dirs=[os.path.join(openssl_prefix, "include", openssl_name)],
extra_objects=[os.path.join(openssl_prefix, "lib", openssl_name, "libcrypto.a")],
)
elif is_netbsd and os.environ.get("BORG_OPENSSL_PREFIX"):
# Similarly for NetBSD, if we built a custom OpenSSL, link it statically
# to avoid dynamic linker conflicts with the system OpenSSL loaded by Python.
openssl_prefix = os.environ.get("BORG_OPENSSL_PREFIX")
crypto_ext_lib = dict(
include_dirs=[os.path.join(openssl_prefix, "include")],
extra_objects=[os.path.join(openssl_prefix, "lib", "libcrypto.a")],
)
else:
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libcrypto", ">=1.1.1")
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libcrypto", ">=3.2.0")
crypto_ext_kwargs = members_appended(
dict(sources=[crypto_ll_source]), crypto_ext_lib, dict(extra_compile_args=cflags)

14
src/borg/crypto/key.py
View file

@ -11,8 +11,6 @@ from ..logger import create_logger
logger = create_logger()
import argon2.low_level
from ..constants import * # NOQA
from ..helpers import StableDict
from ..helpers import Error, IntegrityError
@ -466,17 +464,9 @@ class FlexiKey:
parallelism = 1
# 8 is the smallest value that avoids the "Memory cost is too small" exception
memory_cost = 8
type_map = {"i": argon2.low_level.Type.I, "d": argon2.low_level.Type.D, "id": argon2.low_level.Type.ID}
key = argon2.low_level.hash_secret_raw(
secret=passphrase.encode("utf-8"),
hash_len=output_len_in_bytes,
salt=salt,
time_cost=time_cost,
memory_cost=memory_cost,
parallelism=parallelism,
type=type_map[type],
return low_level.argon2_hash(
passphrase.encode("utf-8"), salt, time_cost, memory_cost, parallelism, output_len_in_bytes, type
)
return key
def decrypt_key_file_pbkdf2(self, encrypted_key, passphrase):
key = self.pbkdf2(passphrase, encrypted_key.salt, encrypted_key.iterations, 32)

78
src/borg/crypto/low_level.pyx
View file

@ -88,6 +88,34 @@ cdef extern from "openssl/evp.h":
int EVP_CTRL_AEAD_SET_TAG
int EVP_CTRL_AEAD_SET_IVLEN
cdef extern from "openssl/kdf.h":
ctypedef struct EVP_KDF:
pass
ctypedef struct EVP_KDF_CTX:
pass
EVP_KDF *EVP_KDF_fetch(void *ctx, const char *algorithm, const char *properties)
void EVP_KDF_free(EVP_KDF *kdf)
EVP_KDF_CTX *EVP_KDF_CTX_new(EVP_KDF *kdf)
void EVP_KDF_CTX_free(EVP_KDF_CTX *ctx)
cdef extern from "openssl/params.h":
ctypedef struct OSSL_PARAM:
pass
OSSL_PARAM OSSL_PARAM_construct_uint32(const char *key, uint32_t *buf)
OSSL_PARAM OSSL_PARAM_construct_octet_string(const char *key, void *buf, size_t bsize)
OSSL_PARAM OSSL_PARAM_construct_end()
cdef extern from "openssl/core_names.h":
const char *OSSL_KDF_PARAM_THREADS
const char *OSSL_KDF_PARAM_ARGON2_LANES
const char *OSSL_KDF_PARAM_ITER
const char *OSSL_KDF_PARAM_ARGON2_MEMCOST
const char *OSSL_KDF_PARAM_SALT
const char *OSSL_KDF_PARAM_PASSWORD
cdef extern from "openssl/kdf.h":
int EVP_KDF_derive(EVP_KDF_CTX *ctx, unsigned char *key, size_t keylen, const OSSL_PARAM params[])
import struct
@ -944,3 +972,53 @@ cdef class CSPRNG:
# Swap items[i] and items[j]
items[i], items[j] = items[j], items[i]
def argon2_hash(bytes secret, bytes salt, uint32_t time_cost, uint32_t memory_cost,
uint32_t parallelism, uint32_t hash_len, type):
cdef EVP_KDF *kdf = NULL
cdef EVP_KDF_CTX *kctx = NULL
cdef OSSL_PARAM params[8]
cdef OSSL_PARAM *p = params
cdef uint32_t threads = 1
cdef bytes result
cdef const char *alg_name
cdef const unsigned char *secret_c = secret
cdef const unsigned char *salt_c = salt
if type == "i" or type == b"i":
alg_name = b"ARGON2I"
elif type == "d" or type == b"d":
alg_name = b"ARGON2D"
elif type == "id" or type == b"id":
alg_name = b"ARGON2ID"
else:
raise ValueError("Invalid argon2 type")
kdf = EVP_KDF_fetch(NULL, alg_name, NULL)
if kdf == NULL:
raise CryptoError("Argon2 KDF not found in OpenSSL (requires >= 3.2)")
kctx = EVP_KDF_CTX_new(kdf)
if kctx == NULL:
EVP_KDF_free(kdf)
raise MemoryError("Failed to create KDF context")
p[0] = OSSL_PARAM_construct_uint32(OSSL_KDF_PARAM_THREADS, &threads)
p[1] = OSSL_PARAM_construct_uint32(OSSL_KDF_PARAM_ARGON2_LANES, &parallelism)
p[2] = OSSL_PARAM_construct_uint32(OSSL_KDF_PARAM_ITER, &time_cost)
p[3] = OSSL_PARAM_construct_uint32(OSSL_KDF_PARAM_ARGON2_MEMCOST, &memory_cost)
p[4] = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_SALT, <void *>salt_c, len(salt))
p[5] = OSSL_PARAM_construct_octet_string(OSSL_KDF_PARAM_PASSWORD, <void *>secret_c, len(secret))
p[6] = OSSL_PARAM_construct_end()
result = PyBytes_FromStringAndSize(NULL, hash_len)
if EVP_KDF_derive(kctx, <unsigned char *>result, hash_len, params) <= 0:
EVP_KDF_CTX_free(kctx)
EVP_KDF_free(kdf)
raise CryptoError("EVP_KDF_derive failed")
EVP_KDF_CTX_free(kctx)
EVP_KDF_free(kdf)
return result