upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-04-20 21:57:03 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

speed up prune/list/repo-list tests, see #9324

Browse source 
there are a lot of files in src_dir (due to the __pycache__ subdir).
for tests that do not need that, we can use a much smaller set of files,
now provided by the backup_files fixture.
This commit is contained in:
Thomas Waldmann 2026-03-07 21:47:05 +01:00
parent b5471bbe36
commit 93cd407ff5
No known key found for this signature in database
GPG key ID: 243ACFA951F78E01
5 changed files with 86 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
src/borg/conftest.py
View file

@ -66,6 +66,18 @@ def set_env_variables():
os.environ["BORG_SELFTEST"] = "disabled"
@pytest.fixture(scope="session")
def backup_files(tmp_path_factory):
# create a relatively simple / minimal set of test files
path = tmp_path_factory.mktemp("backup")
(path / "empty").write_bytes(b"")
(path / "dir1").mkdir()
(path / "dir1" / "text.txt").write_text("text content")
(path / "dir2").mkdir()
(path / "dir2" / "binary.bin").write_bytes(b"\x00\x01\x02\x03")
return str(path)
class ArchiverSetup:
EXE: str = None # python source based
FORK_DEFAULT = False

2
src/borg/testsuite/archiver/__init__.py
View file

@ -36,6 +36,8 @@ RK_ENCRYPTION = "--encryption=repokey-aes-ocb"
KF_ENCRYPTION = "--encryption=keyfile-chacha20-poly1305"
# This points to the ``src/borg/archiver`` directory (small, with only a few files).
# There are quite a lot of files in there, because there is a __pycache__ subdirectory.
# Consider rather using the backup_files fixtures, which only has a few small files/dirs.
src_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "archiver"))
src_file = "archiver/__init__.py" # relative path of one file in src_dir

6
src/borg/testsuite/archiver/list_cmd_test.py
View file

@ -3,15 +3,15 @@ import os
import pytest
from ...constants import * # NOQA
from . import src_dir, cmd, create_regular_file, generate_archiver_tests, RK_ENCRYPTION, requires_hardlinks
from . import cmd, create_regular_file, generate_archiver_tests, RK_ENCRYPTION, requires_hardlinks
pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA
def test_list_format(archivers, request):
def test_list_format(archivers, request, backup_files):
archiver = request.getfixturevalue(archivers)
cmd(archiver, "repo-create", RK_ENCRYPTION)
cmd(archiver, "create", "test", src_dir)
cmd(archiver, "create", "test", backup_files)
output_1 = cmd(archiver, "list", "test")
output_2 = cmd(
archiver, "list", "test", "--format", "{mode} {user:6} {group:6} {size:8d} {mtime} {path}{extra}{NEWLINE}"

102
src/borg/testsuite/archiver/prune_cmd_test.py
View file

@ -5,28 +5,28 @@ import pytest
from ...constants import * # NOQA
from ...archiver.prune_cmd import prune_split, prune_within
from . import cmd, RK_ENCRYPTION, src_dir, generate_archiver_tests
from . import cmd, RK_ENCRYPTION, generate_archiver_tests
from ...helpers import interval
pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA
def _create_archive_ts(archiver, name, y, m, d, H=0, M=0, S=0):
def _create_archive_ts(archiver, backup_files, name, y, m, d, H=0, M=0, S=0):
cmd(
archiver,
"create",
"--timestamp",
datetime(y, m, d, H, M, S, 0).strftime(ISO_FORMAT_NO_USECS), # naive == local time / local tz
name,
src_dir,
backup_files,
)
def test_prune_repository(archivers, request):
def test_prune_repository(archivers, request, backup_files):
archiver = request.getfixturevalue(archivers)
cmd(archiver, "repo-create", RK_ENCRYPTION)
cmd(archiver, "create", "test1", src_dir)
cmd(archiver, "create", "test2", src_dir)
cmd(archiver, "create", "test1", backup_files)
cmd(archiver, "create", "test2", backup_files)
output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1")
assert re.search(r"Would prune:\s+test1", output)
# Must keep the latest archive:
@ -42,41 +42,41 @@ def test_prune_repository(archivers, request):
# This test must match docs/misc/prune-example.txt
def test_prune_repository_example(archivers, request):
def test_prune_repository_example(archivers, request, backup_files):
archiver = request.getfixturevalue(archivers)
cmd(archiver, "repo-create", RK_ENCRYPTION)
# Archives that will be kept, per the example
# Oldest archive
_create_archive_ts(archiver, "test01", 2015, 1, 1)
_create_archive_ts(archiver, backup_files, "test01", 2015, 1, 1)
# 6 monthly archives
_create_archive_ts(archiver, "test02", 2015, 6, 30)
_create_archive_ts(archiver, "test03", 2015, 7, 31)
_create_archive_ts(archiver, "test04", 2015, 8, 31)
_create_archive_ts(archiver, "test05", 2015, 9, 30)
_create_archive_ts(archiver, "test06", 2015, 10, 31)
_create_archive_ts(archiver, "test07", 2015, 11, 30)
_create_archive_ts(archiver, backup_files, "test02", 2015, 6, 30)
_create_archive_ts(archiver, backup_files, "test03", 2015, 7, 31)
_create_archive_ts(archiver, backup_files, "test04", 2015, 8, 31)
_create_archive_ts(archiver, backup_files, "test05", 2015, 9, 30)
_create_archive_ts(archiver, backup_files, "test06", 2015, 10, 31)
_create_archive_ts(archiver, backup_files, "test07", 2015, 11, 30)
# 14 daily archives
_create_archive_ts(archiver, "test08", 2015, 12, 17)
_create_archive_ts(archiver, "test09", 2015, 12, 18)
_create_archive_ts(archiver, "test10", 2015, 12, 20)
_create_archive_ts(archiver, "test11", 2015, 12, 21)
_create_archive_ts(archiver, "test12", 2015, 12, 22)
_create_archive_ts(archiver, "test13", 2015, 12, 23)
_create_archive_ts(archiver, "test14", 2015, 12, 24)
_create_archive_ts(archiver, "test15", 2015, 12, 25)
_create_archive_ts(archiver, "test16", 2015, 12, 26)
_create_archive_ts(archiver, "test17", 2015, 12, 27)
_create_archive_ts(archiver, "test18", 2015, 12, 28)
_create_archive_ts(archiver, "test19", 2015, 12, 29)
_create_archive_ts(archiver, "test20", 2015, 12, 30)
_create_archive_ts(archiver, "test21", 2015, 12, 31)
_create_archive_ts(archiver, backup_files, "test08", 2015, 12, 17)
_create_archive_ts(archiver, backup_files, "test09", 2015, 12, 18)
_create_archive_ts(archiver, backup_files, "test10", 2015, 12, 20)
_create_archive_ts(archiver, backup_files, "test11", 2015, 12, 21)
_create_archive_ts(archiver, backup_files, "test12", 2015, 12, 22)
_create_archive_ts(archiver, backup_files, "test13", 2015, 12, 23)
_create_archive_ts(archiver, backup_files, "test14", 2015, 12, 24)
_create_archive_ts(archiver, backup_files, "test15", 2015, 12, 25)
_create_archive_ts(archiver, backup_files, "test16", 2015, 12, 26)
_create_archive_ts(archiver, backup_files, "test17", 2015, 12, 27)
_create_archive_ts(archiver, backup_files, "test18", 2015, 12, 28)
_create_archive_ts(archiver, backup_files, "test19", 2015, 12, 29)
_create_archive_ts(archiver, backup_files, "test20", 2015, 12, 30)
_create_archive_ts(archiver, backup_files, "test21", 2015, 12, 31)
# Additional archives that would be pruned
# The second backup of the year
_create_archive_ts(archiver, "test22", 2015, 1, 2)
_create_archive_ts(archiver, backup_files, "test22", 2015, 1, 2)
# The next older monthly backup
_create_archive_ts(archiver, "test23", 2015, 5, 31)
_create_archive_ts(archiver, backup_files, "test23", 2015, 5, 31)
# The next older daily backup
_create_archive_ts(archiver, "test24", 2015, 12, 16)
_create_archive_ts(archiver, backup_files, "test24", 2015, 12, 16)
output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=14", "--keep-monthly=6", "--keep-yearly=1")
# Prune second backup of the year
assert re.search(r"Would prune:\s+test22", output)
@ -104,7 +104,7 @@ def test_prune_repository_example(archivers, request):
assert "test%02d" % i not in output
def test_prune_quarterly(archivers, request):
def test_prune_quarterly(archivers, request, backup_files):
# Example worked through by hand when developing the quarterly
# strategy, based on existing backups where the quarterly strategy
# is desired. Weekly/monthly backups that do not affect results were
@ -143,7 +143,7 @@ def test_prune_quarterly(archivers, request):
# Initialize our repo.
cmd(archiver, "repo-create", RK_ENCRYPTION)
for a, (y, m, d) in zip(map(mk_name, test_dates), test_dates):
_create_archive_ts(archiver, a, y, m, d)
_create_archive_ts(archiver, backup_files, a, y, m, d)
to_prune = list(set(test_dates) - set(to_keep))
@ -176,24 +176,24 @@ def test_prune_quarterly(archivers, request):
# With an initial and daily backup, prune daily until oldest is replaced by a monthly backup
def test_prune_retain_and_expire_oldest(archivers, request):
def test_prune_retain_and_expire_oldest(archivers, request, backup_files):
archiver = request.getfixturevalue(archivers)
cmd(archiver, "repo-create", RK_ENCRYPTION)
# Initial backup
_create_archive_ts(archiver, "original_archive", 2020, 9, 1, 11, 15)
_create_archive_ts(archiver, backup_files, "original_archive", 2020, 9, 1, 11, 15)
# Archive and prune daily for 30 days
for i in range(1, 31):
_create_archive_ts(archiver, "september%02d" % i, 2020, 9, i, 12)
_create_archive_ts(archiver, backup_files, "september%02d" % i, 2020, 9, i, 12)
cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1")
# Archive and prune 6 days into the next month
for i in range(1, 7):
_create_archive_ts(archiver, "october%02d" % i, 2020, 10, i, 12)
_create_archive_ts(archiver, backup_files, "october%02d" % i, 2020, 10, i, 12)
cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1")
# Oldest backup is still retained
output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1")
assert re.search(r"Keeping archive \(rule: monthly\[oldest\] #1" + r"\):\s+original_archive", output)
# Archive one more day and prune.
_create_archive_ts(archiver, "october07", 2020, 10, 7, 12)
_create_archive_ts(archiver, backup_files, "october07", 2020, 10, 7, 12)
cmd(archiver, "prune", "--keep-daily=7", "--keep-monthly=1")
# Last day of previous month is retained as monthly, and oldest is expired.
output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=7", "--keep-monthly=1")
@ -201,13 +201,13 @@ def test_prune_retain_and_expire_oldest(archivers, request):
assert "original_archive" not in output
def test_prune_repository_prefix(archivers, request):
def test_prune_repository_prefix(archivers, request, backup_files):
archiver = request.getfixturevalue(archivers)
cmd(archiver, "repo-create", RK_ENCRYPTION)
cmd(archiver, "create", "foo-2015-08-12-10:00", src_dir)
cmd(archiver, "create", "foo-2015-08-12-20:00", src_dir)
cmd(archiver, "create", "bar-2015-08-12-10:00", src_dir)
cmd(archiver, "create", "bar-2015-08-12-20:00", src_dir)
cmd(archiver, "create", "foo-2015-08-12-10:00", backup_files)
cmd(archiver, "create", "foo-2015-08-12-20:00", backup_files)
cmd(archiver, "create", "bar-2015-08-12-10:00", backup_files)
cmd(archiver, "create", "bar-2015-08-12-20:00", backup_files)
output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:foo-*")
assert re.search(r"Keeping archive \(rule: daily #1\):\s+foo-2015-08-12-20:00", output)
assert re.search(r"Would prune:\s+foo-2015-08-12-10:00", output)
@ -224,13 +224,13 @@ def test_prune_repository_prefix(archivers, request):
assert "bar-2015-08-12-20:00" in output
def test_prune_repository_glob(archivers, request):
def test_prune_repository_glob(archivers, request, backup_files):
archiver = request.getfixturevalue(archivers)
cmd(archiver, "repo-create", RK_ENCRYPTION)
cmd(archiver, "create", "2015-08-12-10:00-foo", src_dir)
cmd(archiver, "create", "2015-08-12-20:00-foo", src_dir)
cmd(archiver, "create", "2015-08-12-10:00-bar", src_dir)
cmd(archiver, "create", "2015-08-12-20:00-bar", src_dir)
cmd(archiver, "create", "2015-08-12-10:00-foo", backup_files)
cmd(archiver, "create", "2015-08-12-20:00-foo", backup_files)
cmd(archiver, "create", "2015-08-12-10:00-bar", backup_files)
cmd(archiver, "create", "2015-08-12-20:00-bar", backup_files)
output = cmd(archiver, "prune", "--list", "--dry-run", "--keep-daily=1", "--match-archives=sh:2015-*-foo")
assert re.search(r"Keeping archive \(rule: daily #1\):\s+2015-08-12-20:00-foo", output)
assert re.search(r"Would prune:\s+2015-08-12-10:00-foo", output)
@ -402,15 +402,15 @@ def test_prune_split_no_archives():
assert kept_because == {}
def test_prune_list_with_metadata_format(archivers, request):
def test_prune_list_with_metadata_format(archivers, request, backup_files):
# Regression test for: prune --list with a format string that requires loading
# archive metadata (e.g. {hostname}) must not fail when archives are deleted.
# The bug was that format_item() was called after archive.delete(), causing
# Archive.DoesNotExist when the formatter tried to lazy-load the archive.
archiver = request.getfixturevalue(archivers)
cmd(archiver, "repo-create", RK_ENCRYPTION)
cmd(archiver, "create", "test1", src_dir)
cmd(archiver, "create", "test2", src_dir)
cmd(archiver, "create", "test1", backup_files)
cmd(archiver, "create", "test2", backup_files)
# {hostname} is a "call key" that triggers lazy loading of the archive from the repo.
# With the buggy code this would raise Archive.DoesNotExist for the pruned archive.
output = cmd(archiver, "prune", "--list", "--keep-daily=1", "--format={name} {hostname}{NL}")

35
src/borg/testsuite/archiver/repo_list_cmd_test.py
View file

@ -2,28 +2,29 @@ import json
import os
from ...constants import * # NOQA
from . import cmd, checkts, create_src_archive, create_regular_file, src_dir, generate_archiver_tests, RK_ENCRYPTION
from . import cmd, checkts, create_regular_file, generate_archiver_tests, RK_ENCRYPTION
from .prune_cmd_test import _create_archive_ts
pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA
def test_repo_list_glob(archivers, request):
def test_repo_list_glob(archivers, request, backup_files):
archiver = request.getfixturevalue(archivers)
cmd(archiver, "repo-create", RK_ENCRYPTION)
cmd(archiver, "create", "test-1", src_dir)
cmd(archiver, "create", "something-else-than-test-1", src_dir)
cmd(archiver, "create", "test-2", src_dir)
cmd(archiver, "create", "test-1", backup_files)
cmd(archiver, "create", "something-else-than-test-1", backup_files)
cmd(archiver, "create", "test-2", backup_files)
output = cmd(archiver, "repo-list", "--match-archives=sh:test-*")
assert "test-1" in output
assert "test-2" in output
assert "something-else" not in output
def test_archives_format(archivers, request):
def test_archives_format(archivers, request, backup_files):
archiver = request.getfixturevalue(archivers)
cmd(archiver, "repo-create", RK_ENCRYPTION)
cmd(archiver, "create", "--comment", "comment 1", "test-1", src_dir)
cmd(archiver, "create", "--comment", "comment 2", "test-2", src_dir)
cmd(archiver, "create", "--comment", "comment 1", "test-1", backup_files)
cmd(archiver, "create", "--comment", "comment 2", "test-2", backup_files)
output_1 = cmd(archiver, "repo-list")
output_2 = cmd(
archiver,
@ -54,13 +55,13 @@ def test_size_nfiles(archivers, request):
assert 123456 <= int(o_t[2]) < 123999 # There is some metadata overhead
def test_date_matching(archivers, request):
def test_date_matching(archivers, request, backup_files):
archiver = request.getfixturevalue(archivers)
cmd(archiver, "repo-create", RK_ENCRYPTION)
create_src_archive(archiver, "archive-2022-11-20", ts="2022-11-20T23:59:59")
create_src_archive(archiver, "archive-2022-12-18", ts="2022-12-18T23:59:59")
create_src_archive(archiver, "archive-now")
_create_archive_ts(archiver, backup_files, "archive-2022-11-20", 2022, 11, 20, 23, 59, 59)
_create_archive_ts(archiver, backup_files, "archive-2022-12-18", 2022, 12, 18, 23, 59, 59)
cmd(archiver, "create", "archive-now", backup_files)
cmd(archiver, "check", "-v", "--oldest=23e", exit_code=2)
@ -150,13 +151,13 @@ def test_repo_list_json(archivers, request):
checkts(archive0["time"])
def test_repo_list_deleted(archivers, request):
def test_repo_list_deleted(archivers, request, backup_files):
archiver = request.getfixturevalue(archivers)
cmd(archiver, "repo-create", RK_ENCRYPTION)
cmd(archiver, "create", "normal1", src_dir)
cmd(archiver, "create", "deleted1", src_dir)
cmd(archiver, "create", "normal2", src_dir)
cmd(archiver, "create", "deleted2", src_dir)
cmd(archiver, "create", "normal1", backup_files)
cmd(archiver, "create", "deleted1", backup_files)
cmd(archiver, "create", "normal2", backup_files)
cmd(archiver, "create", "deleted2", backup_files)
cmd(archiver, "delete", "-a", "sh:deleted*")
output = cmd(archiver, "repo-list")
assert "normal1" in output