Merge pull request #9154 from ThomasWaldmann/unify-ci-master
Some checks failed
Lint / lint (push) Has been cancelled
CI / lint (push) Has been cancelled
CI / security (push) Has been cancelled
CodeQL / Analyze (push) Has been cancelled
CI / asan_ubsan (push) Has been cancelled
CI / native_tests (push) Has been cancelled
CI / vm_tests (Haiku, false, haiku, r1beta5) (push) Has been cancelled
CI / vm_tests (NetBSD, false, netbsd, 10.1) (push) Has been cancelled
CI / vm_tests (OpenBSD, false, openbsd, 7.7) (push) Has been cancelled
CI / vm_tests (borg-freebsd-14-x86_64-gh, FreeBSD, true, freebsd, 14.3) (push) Has been cancelled
CI / windows_tests (push) Has been cancelled

Integrate CI (master)
This commit is contained in:
TW 2025-11-07 19:45:44 +01:00 committed by GitHub
commit 11c880bd76
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 188 additions and 357 deletions

View file

@ -54,7 +54,7 @@ jobs:
runs-on: ubuntu-24.04
timeout-minutes: 25
needs: [security]
needs: [lint, security]
steps:
- uses: actions/checkout@v4
@ -120,7 +120,7 @@ jobs:
echo "Using LD_PRELOAD=$LD_PRELOAD"
pytest -v --benchmark-skip -k "not remote"
posix_tests:
native_tests:
needs: [lint, security]
permissions:
@ -328,12 +328,197 @@ jobs:
path: artifacts/*
if-no-files-found: error
vm_tests:
permissions:
contents: read
id-token: write
attestations: write
runs-on: ubuntu-24.04
timeout-minutes: 90
needs: [lint, security]
continue-on-error: true
strategy:
fail-fast: false
matrix:
include:
- os: freebsd
version: '14.3'
display_name: FreeBSD
# Controls binary build and provenance attestation on tags
do_binaries: true
artifact_prefix: borg-freebsd-14-x86_64-gh
- os: netbsd
version: '10.1'
display_name: NetBSD
do_binaries: false
- os: openbsd
version: '7.7'
display_name: OpenBSD
do_binaries: false
- os: haiku
version: 'r1beta5'
display_name: Haiku
do_binaries: false
steps:
- name: Check out repository
uses: actions/checkout@v4
with:
fetch-depth: 0
fetch-tags: true
- name: Test on ${{ matrix.display_name }}
id: cross_os
uses: cross-platform-actions/action@v0.29.0
env:
DO_BINARIES: ${{ matrix.do_binaries }}
with:
operating_system: ${{ matrix.os }}
version: ${{ matrix.version }}
shell: bash
run: |
set -euxo pipefail
case "${{ matrix.os }}" in
freebsd)
export IGNORE_OSVERSION=yes
sudo -E pkg update -f
sudo -E pkg install -y xxhash liblz4 zstd pkgconf
# Install one of the FUSE libraries; fail if neither is available
sudo -E pkg install -y fusefs-libs || sudo -E pkg install -y fusefs-libs3
sudo -E pkg install -y rust
sudo -E pkg install -y git
sudo -E pkg install -y python310 py310-sqlite3
sudo -E pkg install -y python311 py311-sqlite3 py311-pip py311-virtualenv
sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python3
sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python
sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip3
sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip
python -m venv .venv
. .venv/bin/activate
python -V
pip -V
python -m pip install --upgrade pip wheel
pip install -r requirements.d/development.txt
pip install -e .
pytest -v -n auto --benchmark-skip
if [[ "${DO_BINARIES}" == "true" && "${GITHUB_REF:-}" == refs/tags/* ]]; then
python -m pip install 'pyinstaller==6.14.2'
mkdir -p dist/binary
pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec
pushd dist/binary
echo "single-file binary"
chmod +x borg.exe
./borg.exe -V
echo "single-directory binary"
chmod +x borg-dir/borg.exe
./borg-dir/borg.exe -V
tar czf borg.tgz borg-dir
popd
mkdir -p artifacts
if [ -f dist/binary/borg.exe ]; then
cp -v dist/binary/borg.exe artifacts/${{ matrix.artifact_prefix }}
fi
if [ -f dist/binary/borg.tgz ]; then
cp -v dist/binary/borg.tgz artifacts/${{ matrix.artifact_prefix }}.tgz
fi
fi
;;
netbsd)
arch="$(uname -m)"
sudo -E mkdir -p /usr/pkg/etc/pkgin
echo "http://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null
sudo -E pkgin update || true
sudo -E pkgin -y upgrade || true
sudo -E pkgin -y install zstd lz4 xxhash git || true
sudo -E pkgin -y install rust || true
sudo -E pkgin -y install pkg-config || true
sudo -E pkgin -y install py311-pip py311-virtualenv || true
sudo -E ln -sf /usr/pkg/bin/python3.11 /usr/pkg/bin/python3 || true
sudo -E ln -sf /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3 || true
sudo -E ln -sf /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3 || true
python3 -m virtualenv .venv
. .venv/bin/activate
python -m pip install --upgrade pip
pip install -r requirements.d/development.txt
pip install -e .
pytest -v -n auto --benchmark-skip
;;
openbsd)
# echo "https://ftp.eu.openbsd.org/pub/OpenBSD" | sudo tee /etc/installurl > /dev/null || true
sudo -E pkg_add xxhash || true
sudo -E pkg_add lz4 || true
sudo -E pkg_add zstd || true
sudo -E pkg_add git || true
sudo -E pkg_add rust || true
sudo -E pkg_add openssl%3.4 || true
sudo -E pkg_add py3-pip || true
sudo -E pkg_add py3-virtualenv || true
export BORG_OPENSSL_NAME=eopenssl34
python3 -m virtualenv .venv
. .venv/bin/activate
python -m pip install --upgrade pip
pip install -r requirements.d/development.txt
pip install -e .
pytest -v -n auto --benchmark-skip
;;
haiku)
pkgman refresh || true
pkgman install -y git pkgconfig zstd lz4 xxhash || true
pkgman install -y openssl3 || true
pkgman install -y rust_bin || true
pkgman install -y python3.10 || true
pkgman install -y cffi || true
python3 -V || true
pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel || true
export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
export BORG_LIBLZ4_PREFIX=/system/develop
export BORG_LIBZSTD_PREFIX=/system/develop
export BORG_LIBXXHASH_PREFIX=/system/develop
export BORG_OPENSSL_PREFIX=/system/develop
python3 -m ensurepip --upgrade || true
python3 -m pip install --upgrade pip wheel
python3 -m venv .venv
. .venv/bin/activate
python -V
pip install -r requirements.d/development.txt
pip install -e .
pytest -v --benchmark-skip -k "not remote and not socket"
;;
esac
- name: Upload artifacts
if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.os }}-${{ matrix.version }}-dist
path: artifacts/*
if-no-files-found: ignore
- name: Attest provenance
if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries
uses: actions/attest-build-provenance@v3
with:
subject-path: 'artifacts/*'
windows_tests:
if: false # can be used to temporarily disable the build
runs-on: windows-latest
timeout-minutes: 120
needs: posix_tests
needs: native_tests
env:
PY_COLORS: 1

View file

@ -1,119 +0,0 @@
name: CI FreeBSD
on:
push:
branches: [ master ]
tags:
- '2.*'
pull_request:
branches: [ master ]
paths:
- '**.py'
- '**.pyx'
- '**.c'
- '**.h'
- '**.yml'
- '**.toml'
- '**.cfg'
- '**.ini'
- 'requirements.d/*'
- '!docs/**'
jobs:
freebsd_tests:
name: FreeBSD tests and build
permissions:
contents: read
id-token: write
attestations: write
runs-on: ubuntu-24.04
continue-on-error: true
steps:
- name: Check out repository
uses: actions/checkout@v4
with:
fetch-depth: 0
fetch-tags: true
- name: Run on FreeBSD
id: freebsd
uses: cross-platform-actions/action@v0.29.0
with:
operating_system: freebsd
version: '14.3'
shell: bash
# Increase resources a bit for building Cython extensions
cpu_count: 3
memory: 8G
run: |
set -euxo pipefail
# Update package catalog and install dependencies
export IGNORE_OSVERSION=yes
sudo -E pkg update -f
sudo -E pkg install -y xxhash liblz4 zstd pkgconf
sudo -E pkg install -y fusefs-libs || true
sudo -E pkg install -y fusefs-libs3 || true
sudo -E pkg install -y rust
sudo -E pkg install -y git # fakeroot causes lots of troubles on freebsd
sudo -E pkg install -y python310 py310-sqlite3
sudo -E pkg install -y python311 py311-sqlite3 py311-pip py311-virtualenv
# Ensure python3 and pip point to the chosen version
sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python3 || true
sudo ln -sf /usr/local/bin/python3.11 /usr/local/bin/python || true
sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip3 || true
sudo ln -sf /usr/local/bin/pip3.11 /usr/local/bin/pip || true
# Create and activate venv
python -m venv .venv
. .venv/bin/activate
# Build prerequisites
python -V
pip -V
python -m pip install --upgrade pip wheel
# Install dev requirements and project in editable mode
pip install -r requirements.d/development.txt
pip install -e .
# Run tests (skip benchmarks which are slow on CI)
pytest -v -n auto --benchmark-skip
# Only build PyInstaller binaries for tags
if [[ "${GITHUB_REF:-}" == refs/tags/* ]]; then
python -m pip install 'pyinstaller==6.14.2'
mkdir -p dist/binary
pyinstaller --clean --distpath=dist/binary scripts/borg.exe.spec
# Smoke-test the built binaries and package dir-mode as tarball
pushd dist/binary
echo "single-file binary"
chmod +x borg.exe
./borg.exe -V
echo "single-directory binary"
chmod +x borg-dir/borg.exe
./borg-dir/borg.exe -V
tar czf borg.tgz borg-dir
popd
# Prepare artifacts
mkdir -p artifacts
if [ -f dist/binary/borg.exe ]; then
cp -v dist/binary/borg.exe artifacts/borg-freebsd-14-x86_64-gh
fi
if [ -f dist/binary/borg.tgz ]; then
cp -v dist/binary/borg.tgz artifacts/borg-freebsd-14-x86_64-gh.tgz
fi
fi
- name: Upload FreeBSD artifacts
if: startsWith(github.ref, 'refs/tags/')
uses: actions/upload-artifact@v4
with:
name: freebsd14-dist
path: artifacts/*
if-no-files-found: ignore
- name: Attest provenance for FreeBSD artifacts
if: startsWith(github.ref, 'refs/tags/')
uses: actions/attest-build-provenance@v3
with:
subject-path: 'artifacts/*'

View file

@ -1,86 +0,0 @@
name: CI Haiku
on:
push:
branches: [ master ]
tags:
- '2.*'
pull_request:
branches: [ master ]
paths:
- '**.py'
- '**.pyx'
- '**.c'
- '**.h'
- '**.yml'
- '**.toml'
- '**.cfg'
- '**.ini'
- 'requirements.d/*'
- '!docs/**'
jobs:
haiku_tests:
name: Haiku tests
runs-on: ubuntu-24.04
timeout-minutes: 90
continue-on-error: true
steps:
- name: Check out repository
uses: actions/checkout@v4
with:
# setuptools-scm needs full history for versioning
fetch-depth: 0
fetch-tags: true
- name: Run on Haiku
id: haiku
uses: cross-platform-actions/action@v0.29.0
with:
operating_system: haiku
version: 'r1beta5'
shell: bash
cpu_count: 2
memory: 6G
run: |
set -euxo pipefail
# Refresh repositories
pkgman refresh || true
# pkgman update -y || true
# Base tools and libraries
pkgman install -y git pkgconfig zstd lz4 xxhash || true
pkgman install -y openssl3 || true
pkgman install -y rust_bin || true
pkgman install -y python3.10 || true
pkgman install -y cffi || true
python3 -V || true
# Development headers and pkg-config files needed for building C extensions
pkgman install -y lz4_devel zstd_devel xxhash_devel openssl3_devel libffi_devel || true
# Ensure pkg-config can find .pc files
export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
# Point Borg build to library/include prefixes (avoids pkg-config name mismatches)
export BORG_LIBLZ4_PREFIX=/system/develop
export BORG_LIBZSTD_PREFIX=/system/develop
export BORG_LIBXXHASH_PREFIX=/system/develop
export BORG_OPENSSL_PREFIX=/system/develop
# Ensure pip is available
python3 -m ensurepip --upgrade || true
python3 -m pip install --upgrade pip wheel
# Create and activate a virtualenv
python3 -m venv .venv
. .venv/bin/activate
python -V
# Install development/test dependencies
pip install -r requirements.d/development.txt
# Build Borg (editable)
pip install -e .
# Run tests; skip benchmarks
# remote archiver tests are broken on Haiku
pytest -v --benchmark-skip -k "not remote and not socket"

View file

@ -1,76 +0,0 @@
name: CI NetBSD
on:
push:
branches: [ master ]
tags:
- '2.*'
pull_request:
branches: [ master ]
paths:
- '**.py'
- '**.pyx'
- '**.c'
- '**.h'
- '**.yml'
- '**.toml'
- '**.cfg'
- '**.ini'
- 'requirements.d/*'
- '!docs/**'
jobs:
netbsd_tests:
name: NetBSD tests
runs-on: ubuntu-24.04
timeout-minutes: 90
continue-on-error: true
steps:
- name: Check out repository
uses: actions/checkout@v4
with:
# setuptools-scm needs full history for versioning
fetch-depth: 0
fetch-tags: true
- name: Run on NetBSD
id: netbsd
uses: cross-platform-actions/action@v0.29.0
with:
operating_system: netbsd
version: '10.1'
shell: bash
cpu_count: 3
memory: 6G
run: |
set -euxo pipefail
# Configure pkgin repository based on architecture.
arch="$(uname -m)"
sudo -E mkdir -p /usr/pkg/etc/pkgin
echo "http://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null
sudo -E pkgin update || true
sudo -E pkgin -y upgrade || true
# Base tools and libraries
sudo -E pkgin -y install zstd lz4 xxhash git || true
sudo -E pkgin -y install rust || true
# Work around NetBSD 9.3 bug with .bash_profile in screen sessions
# echo "export PROMPT_COMMAND=" >> ~/.bash_profile || true
sudo -E pkgin -y install pkg-config || true
# Python 3.11 and tools
sudo -E pkgin -y install py311-pip py311-virtualenv || true
sudo -E ln -sf /usr/pkg/bin/python3.11 /usr/pkg/bin/python3 || true
sudo -E ln -sf /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3 || true
sudo -E ln -sf /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3 || true
# Create and activate a virtualenv
python3 -m virtualenv .venv
. .venv/bin/activate
python -m pip install --upgrade pip
# Install development/test dependencies
pip install -r requirements.d/development.txt
# Build Borg (editable)
pip install -e .
# Run tests; skip benchmarks
pytest -v -n auto --benchmark-skip

View file

@ -1,73 +0,0 @@
name: CI OpenBSD
on:
push:
branches: [ master ]
tags:
- '2.*'
pull_request:
branches: [ master ]
paths:
- '**.py'
- '**.pyx'
- '**.c'
- '**.h'
- '**.yml'
- '**.toml'
- '**.cfg'
- '**.ini'
- 'requirements.d/*'
- '!docs/**'
jobs:
openbsd_tests:
name: OpenBSD tests
runs-on: ubuntu-24.04
timeout-minutes: 90
continue-on-error: true
steps:
- name: Check out repository
uses: actions/checkout@v4
with:
# setuptools-scm needs full history for versioning
fetch-depth: 0
fetch-tags: true
- name: Run on OpenBSD
id: openbsd
uses: cross-platform-actions/action@v0.29.0
with:
operating_system: openbsd
version: '7.7'
shell: bash
cpu_count: 3
memory: 6G
run: |
set -euxo pipefail
# On OpenBSD, use pkg_add and set installurl. Use sudo for privileged operations.
# echo "https://ftp.eu.openbsd.org/pub/OpenBSD" | sudo tee /etc/installurl > /dev/null
# crypto / compression libs
sudo -E pkg_add xxhash || true
sudo -E pkg_add lz4 || true
sudo -E pkg_add zstd || true
# toolchain / VCS
sudo -E pkg_add git || true
sudo -E pkg_add rust || true
# OpenSSL 3.4
sudo -E pkg_add openssl%3.4 || true
# Python tooling
sudo -E pkg_add py3-pip || true
sudo -E pkg_add py3-virtualenv || true
export BORG_OPENSSL_NAME=eopenssl34
# Create and activate a virtualenv
python3 -m virtualenv .venv
. .venv/bin/activate
python -m pip install --upgrade pip
# Install development/test dependencies
pip install -r requirements.d/development.txt
# Build Borg (editable)
pip install -e .
# Run tests; skip benchmarks
pytest -v -n auto --benchmark-skip