Merge pull request #9469 from ThomasWaldmann/ci-locked-and-canary

CI: use locked requirements, add canary job, fixes #9361

.github/workflows/canary.yml

@@ -0,0 +1,94 @@
+name: Canary (Unlocked Requirements)
+
+on:
+  schedule:
+    - cron: '0 9 * * 0'  # Run weekly on Sunday at 09:00 UTC
+  workflow_dispatch:      # Allow manual trigger
+
+permissions:
+  contents: read
+
+jobs:
+  canary_tests:
+    name: Canary (${{ matrix.os }}, ${{ matrix.python-version }}, ${{ matrix.toxenv }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 360
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # A representative subset of environments
+          - os: ubuntu-22.04
+            python-version: '3.10'
+            toxenv: py310-llfuse
+          - os: ubuntu-24.04
+            python-version: '3.12'
+            toxenv: py312-pyfuse3
+          - os: ubuntu-24.04
+            python-version: '3.14'
+            toxenv: py314-mfusepy
+          - os: macos-15
+            python-version: '3.14'
+            toxenv: py314-none
+          - os: windows-latest
+            python-version: '3.14'
+            toxenv: py314-windows
+
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 0
+        fetch-tags: true
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Linux packages
+      if: ${{ runner.os == 'Linux' }}
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y pkg-config build-essential
+        sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev
+        if [[ "${{ matrix.toxenv }}" == *"llfuse"* ]]; then
+          sudo apt-get install -y libfuse-dev fuse
+        elif [[ "${{ matrix.toxenv }}" == *"pyfuse3"* || "${{ matrix.toxenv }}" == *"mfusepy"* ]]; then
+          sudo apt-get install -y libfuse3-dev fuse3
+        fi
+
+    - name: Install macOS packages
+      if: ${{ runner.os == 'macOS' }}
+      run: |
+        brew bundle install || true
+
+    - name: Install Python requirements (UNLOCKED)
+      run: |
+        python -m pip install --upgrade pip setuptools wheel
+        # Use UNLOCKED requirements to catch upstream breakages
+        pip install -r requirements.d/development.txt
+
+    - name: Install borgbackup
+      run: |
+        if [[ "${{ matrix.toxenv }}" == *"llfuse"* ]]; then
+          pip install -e ".[llfuse,cockpit]"
+        elif [[ "${{ matrix.toxenv }}" == *"pyfuse3"* ]]; then
+          pip install -e ".[pyfuse3,cockpit]"
+        elif [[ "${{ matrix.toxenv }}" == *"mfusepy"* ]]; then
+          pip install -e ".[mfusepy,cockpit]"
+        else
+          pip install -e ".[cockpit]"
+        fi
+
+    - name: Run tests (Canary)
+      shell: bash
+      run: |
+        if [[ "${{ matrix.toxenv }}" == *"-windows" ]]; then
+          python -m pytest -n4 --benchmark-skip -vv -rs -k "not remote" --cov=borg --cov-config=pyproject.toml --cov-report=xml --junitxml=test-results.xml
+        else
+          # Force tox to use the unlocked requirements in its environment creation
+          # by overriding the deps if possible, or just trusting it uses development.txt 
+          # which we already installed in the root. Actually tox creates its own venv.
+          # We need to tell tox to use the unlocked file.
+          tox -e ${{ matrix.toxenv }} --override "env_run_base.deps=[-rrequirements.d/development.txt]"
+        fi

.github/workflows/ci.yml

@@ -84,7 +84,7 @@ jobs:
     - name: Install Python dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r requirements.d/development.txt
+        pip install -r requirements.d/development.lock.txt
 
     - name: Build Borg with ASan/UBSan
       # Build the C/Cython extensions with AddressSanitizer and UndefinedBehaviorSanitizer enabled.
@@ -182,7 +182,7 @@ jobs:
       uses: actions/cache@v5
       with:
         path: ~/.cache/pip
-        key: ${{ runner.os }}-${{ runner.arch }}-pip-${{ hashFiles('requirements.d/development.txt') }}
+        key: ${{ runner.os }}-${{ runner.arch }}-pip-${{ hashFiles('requirements.d/development.lock.txt') }}
         restore-keys: |
             ${{ runner.os }}-${{ runner.arch }}-pip-
             ${{ runner.os }}-${{ runner.arch }}-
@@ -191,7 +191,7 @@ jobs:
       uses: actions/cache@v5
       with:
         path: .tox
-        key: ${{ runner.os }}-${{ runner.arch }}-tox-${{ matrix.toxenv }}-${{ hashFiles('requirements.d/development.txt', 'pyproject.toml') }}
+        key: ${{ runner.os }}-${{ runner.arch }}-tox-${{ matrix.toxenv }}-${{ hashFiles('requirements.d/development.lock.txt', 'pyproject.toml') }}
         restore-keys: |
           ${{ runner.os }}-${{ runner.arch }}-tox-${{ matrix.toxenv }}-
           ${{ runner.os }}-${{ runner.arch }}-tox-
@@ -276,7 +276,7 @@ jobs:
     - name: Install Python requirements
       run: |
         python -m pip install --upgrade pip setuptools wheel
-        pip install -r requirements.d/development.txt
+        pip install -r requirements.d/development.lock.txt
 
     - name: Install borgbackup
       run: |
@@ -460,7 +460,7 @@ jobs:
                 python -V
                 pip -V
                 python -m pip install --upgrade pip wheel
-                pip install -r requirements.d/development.txt
+                pip install -r requirements.d/development.lock.txt
                 pip install -e ".[mfusepy,cockpit,s3,sftp]"
                 tox -e py311-mfusepy
 
@@ -548,7 +548,7 @@ jobs:
                 python -V
                 pip -V
                 python -m pip install --upgrade pip wheel
-                pip install -r requirements.d/development.txt
+                pip install -r requirements.d/development.lock.txt
                 # no fuse support on omnios in our tests usually
                 pip install -e .
 
@@ -573,7 +573,7 @@ jobs:
                 export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
                 export BORG_LIBLZ4_PREFIX=/system/develop
                 export BORG_OPENSSL_PREFIX=/system/develop
-                pip install -r requirements.d/development.txt
+                pip install -r requirements.d/development.lock.txt
                 pip install -e .
 
                 # troubles with either tox or pytest xdist, so we run pytest manually:

docs/development.rst

@@ -271,7 +271,7 @@ First, just install borg into a virtual env :ref:`as described before <git-insta
 To install some additional packages needed for running the tests, activate your
 virtual env and run::
 
-  pip install -r requirements.d/development.txt
+  pip install -r requirements.d/development.lock.txt
 
 
 This project utilizes pre-commit to format and lint code before it is committed.

docs/installation.rst

@@ -446,7 +446,7 @@ manually using the ``SETUPTOOLS_SCM_PRETEND_VERSION`` environment variable.
 
     # install borg dependencies into virtualenv
     cd borg
-    pip install -r requirements.d/development.txt
+    pip install -r requirements.d/development.lock.txt
     pip install -r requirements.d/docs.txt  # optional, to build the docs
 
     # set a borg version if setuptools-scm fails to do so automatically

pyproject.toml

@@ -177,7 +177,7 @@ env_list = ["py{310,311,312,313,314}-{none,llfuse,pyfuse3,mfusepy}", "docs", "ru
 
 [tool.tox.env_run_base]
 package = "editable-legacy"  # without this it does not find setup_docs when running under fakeroot
-deps = ["-rrequirements.d/development.txt"]
+deps = ["-rrequirements.d/development.lock.txt"]
 commands = [["python", "-m", "pytest", "-v", "-n", "{env:XDISTN:auto}", "-rs", "--cov=borg", "--cov-config=pyproject.toml", "--cov-report=xml", "--junitxml=test-results.xml", "--benchmark-skip", "--pyargs", "{posargs:borg.testsuite}"]]
 pass_env = ["*"]  # fakeroot -u needs some env vars