From 61609f60a791c18e70d2d7c85c5c3ac2fdb59ed4 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 13 Feb 2026 22:58:44 +0100 Subject: [PATCH] linux-run: run commands (e.g. tox) in a podman linux container this is useful to run the linux tests on a mac, e.g. to test fuse stuff. (cherry picked from commit 40a2bb7032a7010f0ba3dc7a5850a4f988e423ca) --- docs/development.rst | 44 ++++++++++++++ scripts/Dockerfile.linux-tox | 30 ++++++++++ scripts/linux-run | 110 +++++++++++++++++++++++++++++++++++ 3 files changed, 184 insertions(+) create mode 100644 scripts/Dockerfile.linux-tox create mode 100755 scripts/linux-run diff --git a/docs/development.rst b/docs/development.rst index 8584b8a81..509494bd3 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -304,6 +304,50 @@ Usage:: # To copy files from the VM (in this case, the generated binary): vagrant scp OS:/vagrant/borg/borg.exe . +Using Podman +------------ + +macOS-based developers (and others who prefer containers) can run the Linux test suite locally using Podman. + +Prerequisites: + +- Install Podman (e.g., ``brew install podman``). +- Initialize the Podman machine, only once: ``podman machine init``. +- Start the Podman machine, before using it: ``podman machine start``. + +Usage:: + + # Open an interactive shell in the container (default if no command given): + ./scripts/linux-run + + # Run the default tox environment: + ./scripts/linux-run tox + + # Run a specific tox environment: + ./scripts/linux-run tox -e py311-pyfuse3 + + # Pass arguments to pytest (e.g., run specific tests): + ./scripts/linux-run tox -e py313-pyfuse3 -- -k mount + + # Switch base image (temporarily): + ./scripts/linux-run --image python:3.11-bookworm tox + +Resource Usage +~~~~~~~~~~~~~~ + +The default Podman VM uses 2GB RAM and half your CPUs. +For heavy tests (parallel execution), this might be tight. + +- **Check usage:** Run ``podman stats`` in another terminal while tests are running. +- **Increase resources:** + + :: + + podman machine stop + podman machine set --cpus 6 --memory 4096 + podman machine start + + Creating standalone binaries ---------------------------- diff --git a/scripts/Dockerfile.linux-tox b/scripts/Dockerfile.linux-tox new file mode 100644 index 000000000..7d4379245 --- /dev/null +++ b/scripts/Dockerfile.linux-tox @@ -0,0 +1,30 @@ +ARG BASE_IMAGE=python:3.13 +FROM ${BASE_IMAGE} + +# Install system dependencies +# These match the dependencies installed in CI for Linux (Debian/Ubuntu) +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + pkg-config \ + libssl-dev \ + libacl1-dev \ + libxxhash-dev \ + liblz4-dev \ + libzstd-dev \ + libfuse3-dev \ + fuse3 \ + python3-dev \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Install tox +RUN pip install tox + +# Set working directory +WORKDIR /app + +# Check that we have the expected python version +RUN python3 --version + +# Default command (can be overridden) +CMD ["tox"] diff --git a/scripts/linux-run b/scripts/linux-run new file mode 100755 index 000000000..c56f8db65 --- /dev/null +++ b/scripts/linux-run @@ -0,0 +1,110 @@ +#!/bin/bash +# run commands in a linux container, e.g. for testing - for more info, see docs/development.rst. + +set -euo pipefail +# set -x # Uncomment for debugging + +# Default configuration +BASE_IMAGE="python:3.13" +IMAGE_NAME="borg-test-env" +CONTAINER_NAME="borg-test-runner" + +usage() { + echo "Usage: $0 [options] [command [args...]]" + echo "" + echo "Options:" + echo " --image IMAGE Base Docker image to use (default: $BASE_IMAGE)" + echo " --rebuild Force rebuild of the container image" + echo " --help Show this help message" + echo "" + echo "Examples:" + echo " $0 # Open interactive shell (default)" + echo " $0 tox -e py313-pyfuse3 # Run tox in the container" + echo " $0 --image python:3.11 tox -e py311-none # Run with specific image" + echo " $0 ls -la # Run arbitrary command" + exit 1 +} + +# Parse specific arguments +REBUILD=false + +# We use an array to store the command and its arguments. +COMMAND=() + +while [[ $# -gt 0 ]]; do + case $1 in + --image) + BASE_IMAGE="$2" + shift 2 + ;; + --rebuild) + REBUILD=true + shift + ;; + --help) + usage + ;; + --) + # Stop option parsing and treat the rest as command + shift + COMMAND+=("$@") + break + ;; + *) + # Stop option parsing and treat this and the rest as command + COMMAND+=("$@") + break + ;; + esac +done + +# Ensure Podman is available +if ! command -v podman &> /dev/null; then + echo "Error: podman is not installed. Please run 'brew install podman' and initialize it." + exit 1 +fi + +# Build the image if needed or requested. +# We tag the image with the base image name to allow multiple versions. +TAG_SUFFIX=$(echo "$BASE_IMAGE" | tr ':' '-') +FULL_IMAGE_NAME="${IMAGE_NAME}:${TAG_SUFFIX}" + +if [[ "$REBUILD" == "true" ]] || ! podman image exists "$FULL_IMAGE_NAME"; then + echo "Building test image based on $BASE_IMAGE..." + podman build \ + --build-arg BASE_IMAGE="$BASE_IMAGE" \ + -t "$FULL_IMAGE_NAME" \ + -f scripts/Dockerfile.linux-tox . +fi + +echo "Running in container..." +echo "Base Image: $BASE_IMAGE" +if [[ ${#COMMAND[@]} -gt 0 ]]; then + echo "Command: ${COMMAND[*]}" +else + echo "Command: /bin/bash (default)" +fi + +# Run the container +# --userns=keep-id: Maps the container user to the host user (Crucial for macOS volume mounts) +# --security-opt label=disable: Disables SELinux separation if active (harmless on macOS) +# --device /dev/fuse: Required for FUSE filesystem support (borg mount tests) +# --volume $(pwd):/app: Mounts current directory to /app +# --mount type=volume,dst=/tmp: Using a volume for /tmp to prevent issues with tmp files from different users + +CMD=(podman run --rm -it \ + --name "$CONTAINER_NAME" \ + --userns=keep-id \ + --cap-add SYS_ADMIN \ + --security-opt label=disable \ + --device /dev/fuse \ + --volume "$(pwd):/app" \ + --mount type=volume,dst=/tmp \ + --workdir /app \ + "$FULL_IMAGE_NAME") + +if [[ ${#COMMAND[@]} -gt 0 ]]; then + "${CMD[@]}" "${COMMAND[@]}" +else + "${CMD[@]}" /bin/bash +fi