From 72e72dc43106662ee8b8029a139ddd3d7a229df1 Mon Sep 17 00:00:00 2001 From: Hugo Wallenburg Date: Mon, 25 May 2026 22:11:18 +0200 Subject: [PATCH] Adds 'prune --since', base timestamp to prune from A flag to explicitly "save" all archives older than a certain timestamp, and to function as a base timestamp from which to base interval timedelta calculations. Allows for precise time interval manipulation for superusers and as a bonus simplifies time-based testing and alleviates the need for an external dependency to freeze time in test. Includes some refactoring of do_prune for logical flow & naming that came up while iterating on these changes. --- requirements.d/development.lock.txt | 1 - requirements.d/development.txt | 1 - src/borg/archiver/prune_cmd.py | 119 ++++--- src/borg/testsuite/archiver/prune_cmd_test.py | 318 +++++++++--------- 4 files changed, 244 insertions(+), 195 deletions(-) diff --git a/requirements.d/development.lock.txt b/requirements.d/development.lock.txt index 08dbde72a..f2cc2f490 100644 --- a/requirements.d/development.lock.txt +++ b/requirements.d/development.lock.txt @@ -11,7 +11,6 @@ pytest-xdist==3.8.0 coverage[toml]==7.14.0 pytest-cov==7.1.0 pytest-benchmark==5.2.3 -freezegun==1.5.5 Cython==3.2.5 pre-commit==4.6.0 types-PyYAML==6.0.12.20260518 diff --git a/requirements.d/development.txt b/requirements.d/development.txt index 522c3e637..d46a28e42 100644 --- a/requirements.d/development.txt +++ b/requirements.d/development.txt @@ -11,7 +11,6 @@ pytest-xdist coverage[toml] pytest-cov pytest-benchmark -freezegun Cython pre-commit bandit[toml] diff --git a/src/borg/archiver/prune_cmd.py b/src/borg/archiver/prune_cmd.py index 134d43f3b..70654f078 100644 --- a/src/borg/archiver/prune_cmd.py +++ b/src/borg/archiver/prune_cmd.py @@ -8,7 +8,7 @@ from itertools import count, combinations from ._common import with_repository, Highlander from ..constants import * # NOQA from ..helpers import ArchiveFormatter, ProgressIndicatorPercent, CommandError, Error -from ..helpers import archivename_validator, interval, int_or_interval, sig_int +from ..helpers import archivename_validator, interval, int_or_interval, sig_int, timestamp from ..helpers import json_print, basic_json_data from ..helpers.argparsing import ArgumentParser from ..manifest import ArchiveInfo, Manifest @@ -108,6 +108,9 @@ PRUNE_QUARTERLY_13WEEKLY = PruningRule("quarterly_13weekly", quarterly_13weekly_ PRUNE_QUARTERLY_3MONTHLY = PruningRule("quarterly_3monthly", quarterly_3monthly_period_func) PRUNE_YEARLY = PruningRule("yearly", pattern_period_func("%Y")) +# Fake rule used to indicate archives skipped by --since +PRUNE_SINCE = PruningRule("skip", unique_period_func()) + PRUNING_RULES = [ PRUNE_WITHIN, PRUNE_LAST, @@ -128,7 +131,7 @@ def prune( archives: list[ArchiveInfo], rule: PruningRule, n_or_interval: int | timedelta, - base_timestamp: datetime | None, + since_timestamp: datetime | None, keep_oldest: bool, previously_kept: frozenset[ArchiveInfo] = frozenset(), ) -> dict[ArchiveInfo, KeepResult]: @@ -138,9 +141,9 @@ def prune( if isinstance(n_or_interval, int): n, earliest_timestamp = n_or_interval, None else: - if base_timestamp is None: - raise ValueError("base_timestamp is required when using interval-based pruning") - n, earliest_timestamp = None, base_timestamp - n_or_interval + if since_timestamp is None: + raise ValueError("since_timestamp is required when using interval-based pruning") + n, earliest_timestamp = None, since_timestamp - n_or_interval keep: dict[ArchiveInfo, KeepResult] = {} @@ -175,6 +178,44 @@ class PruneMixIn: """Prune archives according to specified rules.""" self._validate_prune_args(args) + match = [args.name] if args.name else args.match_archives + archives = manifest.archives.list(match=match, sort_by=["ts"], reverse=True) + archives = [ai for ai in archives if "@PROT" not in ai.tags] + + # Archives to keep along with the rule that ensured them being kept + keep = {} + + since = getattr(args, PRUNE_SINCE.key) + candidate_archives = archives + + if since is not None: + # `--since` is a prefilter: Archives from after this time are kept by default. They are not considered for + # pruning at all. They won't falsely occupy an active retention period. + for archive in archives: + if archive.ts <= since: + break + keep[archive] = KeepResult(rule=PRUNE_SINCE, idx=len(keep)) + candidate_archives = archives[len(keep) :] + + # Apply each retention rule to all candidate archives. The + # `previously_kept` parameter prevents later (coarser-grained) rules + # from double-counting archives already retained by earlier rules. + active_rules = [ + (rule, getattr(args, rule.key)) for rule in PRUNING_RULES if getattr(args, rule.key) is not None + ] + for rule, n_or_interval in active_rules: + keep |= prune( + archives=candidate_archives, + rule=rule, + n_or_interval=n_or_interval, + since_timestamp=(since if since is not None else datetime.now().astimezone()), + keep_oldest=( + rule == active_rules[-1][0] + ), # Activate keep_oldest rule only for the largest active interval + previously_kept=frozenset(keep), + ) + + archives_to_prune = set(archives) - set(keep) if args.format is not None: format = args.format @@ -184,37 +225,17 @@ class PruneMixIn: format = os.environ.get("BORG_PRUNE_FORMAT", "{archive:<36} {time} [{id}]") formatter = ArchiveFormatter(format, repository, manifest, manifest.key, iec=args.iec) - match = [args.name] if args.name else args.match_archives - archives = manifest.archives.list(match=match, sort_by=["ts"], reverse=True) - archives = [ai for ai in archives if "@PROT" not in ai.tags] - - # Archives to keep along with the rule that ensured them being kept - keep = {} - - base_timestamp = datetime.now().astimezone() - active_rules = {rule: getattr(args, rule.key) for rule in PRUNING_RULES if getattr(args, rule.key) is not None} - for i, (rule, n_or_interval) in enumerate(active_rules.items(), 1): - keep |= prune( - archives=archives, - rule=rule, - n_or_interval=n_or_interval, - base_timestamp=base_timestamp, - keep_oldest=i == len(active_rules), # Activate keep_oldest rule only for the largest active interval - previously_kept=frozenset(keep), - ) - - to_delete = set(archives) - set(keep) - if not args.json: - logger.info("Repository contains %d archives.", manifest.archives.count()) - logger.info("Applying rules to the matching %d archives...", len(archives)) - logger.info("Keeping %d archives, pruning %d archives.", len(keep), len(to_delete)) if args.json: output_data = [] + else: + logger.info("Repository contains %d archives.", manifest.archives.count()) + logger.info("Applying rules to the matching %d archives...", len(archives)) + logger.info("Keeping %d archives, pruning %d archives.", len(keep), len(archives_to_prune)) + list_logger = logging.getLogger("borg.output.list") # set up counters for the progress display - to_delete_len = len(to_delete) - archives_deleted = 0 - pi = ProgressIndicatorPercent(total=len(to_delete), msg="Pruning archives %3.0f%%", msgid="prune") + num_archives_deleted = 0 + pi = ProgressIndicatorPercent(total=len(archives_to_prune), msg="Pruning archives %3.0f%%", msgid="prune") for archive_info in archives: if sig_int and sig_int.action_done(): break @@ -224,18 +245,18 @@ class PruneMixIn: archive_data = formatter.get_item_data(archive_info, jsonline=True) else: archive_formatted = formatter.format_item(archive_info, jsonline=False) - if archive_info in to_delete: + if archive_info in archives_to_prune: if not args.json: pi.show() - archives_deleted += 1 + num_archives_deleted += 1 if args.dry_run: log_message = "Would prune:" else: - log_message = "Pruning archive (%d/%d):" % (archives_deleted, to_delete_len) + log_message = f"Pruning archive ({num_archives_deleted}/{len(archives_to_prune)}):" manifest.archives.delete_by_id(archive_info.id) if args.json: archive_data["kept"] = False - archive_data["deleted_archive_number"] = archives_deleted + archive_data["deleted_archive_number"] = num_archives_deleted else: result = keep[archive_info] result_message = f"{result.rule.key}{'[oldest]' if result.oldest else ''} #{result.idx + 1}" @@ -249,21 +270,21 @@ class PruneMixIn: if ( args.output_list or not (args.list_pruned or args.list_kept) - or (args.list_pruned and archive_info in to_delete) - or (args.list_kept and archive_info not in to_delete) + or (args.list_pruned and archive_info in archives_to_prune) + or (args.list_kept and archive_info not in archives_to_prune) ): output_data.append(archive_data) elif ( args.output_list - or (args.list_pruned and archive_info in to_delete) - or (args.list_kept and archive_info not in to_delete) + or (args.list_pruned and archive_info in archives_to_prune) + or (args.list_kept and archive_info not in archives_to_prune) ): list_logger.info(f"{log_message:<44} {archive_formatted}") if not args.json: pi.finish() if args.json: json_print(basic_json_data(manifest, extra={"archives": output_data})) - if archives_deleted > 0 and not args.dry_run: + if num_archives_deleted > 0 and not args.dry_run: manifest.write() self.print_warning('Done. Run "borg compact" to free space.', wc=None) if sig_int: @@ -371,6 +392,14 @@ class PruneMixIn: timezone of the system where borg prune runs, and weeks go from Monday to Sunday. Specifying a negative number of archives to keep means that there is no limit. + The ``--since`` option restricts pruning to archives older than the given + TIMESTAMP. Archives newer than this timestamp are kept unconditionally + as a pre-filter. When ``--since`` is used together with interval-based + ``--keep-*`` options (e.g. ``--keep-daily 7d``), the interval is + measured backwards from the given timestamp rather than from the + current time. Count-based retention does not count the unconditionally + kept archives. + Borg will retain the oldest archive if any of the secondly, minutely, hourly, daily, weekly, monthly, quarterly, or yearly rules was not otherwise able to meet its retention target. This enables the first chronological archive to @@ -419,6 +448,14 @@ class PruneMixIn: "but keys used in it are added to the JSON output. " "Some keys are always present. Note: JSON can only represent text.", ) + subparser.add_argument( + "--since", + metavar="TIMESTAMP", + dest=PRUNE_SINCE.key, + type=timestamp, + action=Highlander, + help="only consider archives older than this for pruning", + ) subparser.add_argument( "--keep-within", metavar="INTERVAL", diff --git a/src/borg/testsuite/archiver/prune_cmd_test.py b/src/borg/testsuite/archiver/prune_cmd_test.py index a1b0dc05d..94cdbf023 100644 --- a/src/borg/testsuite/archiver/prune_cmd_test.py +++ b/src/borg/testsuite/archiver/prune_cmd_test.py @@ -4,8 +4,6 @@ import pytest import re from operator import attrgetter from datetime import datetime, timezone, timedelta -from freezegun import freeze_time - from ...constants import * # NOQA from ...archiver.prune_cmd import ( PRUNING_RULES, @@ -26,15 +24,14 @@ from . import cmd, RK_ENCRYPTION, generate_archiver_tests pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA -def _create_archive_ts(archiver, backup_files, name, y, m, d, H=0, M=0, S=0, us=0, tzinfo=None): - cmd( - archiver, - "create", - "--timestamp", - datetime(y, m, d, H, M, S, us, tzinfo=tzinfo).strftime(ISO_FORMAT_ZONE), - name, - backup_files, - ) +def _create_archive_dt(archiver, backup_files, name, dt, tzinfo=timezone.utc): + if dt.tzinfo is None: + dt = dt.replace(tzinfo=tzinfo) + cmd(archiver, "create", "--timestamp", dt.strftime(ISO_FORMAT_ZONE), name, backup_files) + + +def _create_archive_ts(archiver, backup_files, name, y, m, d, H=0, M=0, S=0, us=0, tzinfo=timezone.utc): + _create_archive_dt(archiver, backup_files, name, datetime(y, m, d, H, M, S, us, tzinfo=tzinfo)) def test_prune_repository(archivers, request, backup_files): @@ -165,7 +162,6 @@ def test_prune_quarterly(archivers, request, backup_files): # Use 99 instead of -1 to test that oldest backup is kept. output = cmd(archiver, "prune", "--list", "--dry-run", f"--keep-{strat}=99") - print(output) for a in map(mk_name, to_prune): assert re.search(rf"Would prune:\s+{a}", output) @@ -474,195 +470,194 @@ def test_prune_keep_last_same_second(archivers, request, backup_files): assert re.search(r"Keeping archive \(rule: last #\d\):\s+test2", output) -@freeze_time(datetime(2023, 12, 31, 23, 59, 59, tzinfo=None)) # Non-leap year ending on a Sunday -def test_prune_keep_int_or_interval(archivers, request, backup_files): +@pytest.mark.parametrize("keep_arg", ["--keep=2", "--keep=1S"]) +def test_prune_keep_int_or_interval(archivers, request, backup_files, keep_arg): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - _create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 31, 23, 59, 59) - _create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 31, 23, 59, 59) - _create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 31, 23, 59, 58) - for keep_arg in ["--keep=2", "--keep=1S"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) - assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test-1", output) - assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test-2", output) - assert re.search(r"Would prune:\s+test-3", output) + dt = datetime(2023, 12, 31, 23, 59, 59, tzinfo=timezone.utc) + _create_archive_dt(archiver, backup_files, "test-1", dt) + _create_archive_dt(archiver, backup_files, "test-2", dt) + _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(seconds=1)) + output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg) + assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) @pytest.mark.parametrize("keep_arg", ["--keep-daily=-1", "--keep-daily=all"]) def test_prune_keep_all(archivers, request, backup_files, keep_arg): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - _create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 30, 23, 59, 59, tzinfo=timezone.utc) - _create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 29, 23, 59, 59, tzinfo=timezone.utc) - _create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 28, 23, 59, 59, tzinfo=timezone.utc) - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg) + dt = datetime(2023, 12, 31, 23, 59, 59, tzinfo=timezone.utc) + _create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=1)) + _create_archive_dt(archiver, backup_files, "test-2", dt - timedelta(days=2)) + _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=3)) + output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg) assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output) assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output) assert re.search(r"Keeping archive \(rule: daily #3\):\s+test-3", output) -@freeze_time(datetime(2023, 12, 31, 23, 59, 59, tzinfo=None)) +@pytest.mark.parametrize("keep_arg", ["--keep-secondly=2", "--keep-secondly=2S"]) def test_prune_keep_secondly_int_or_interval(archivers, request, backup_files, keep_arg): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - _create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 31, 23, 59, 58) - _create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 31, 23, 59, 57, 1) - _create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 31, 23, 59, 57) - _create_archive_ts(archiver, backup_files, "test-4", 2023, 12, 31, 23, 59, 56, 999999) - for keep_arg in ["--keep-secondly=2", "--keep-secondly=2S"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() - assert re.search(r"Keeping archive \(rule: secondly #1\):\s+test-1", output.pop(0)) - assert re.search(r"Keeping archive \(rule: secondly #2\):\s+test-2", output.pop(0)) - assert re.search(r"Would prune:\s+test-3", output.pop(0)) - assert re.search(r"Would prune:\s+test-4", output.pop(0)) + dt = datetime(2023, 12, 31, 23, 59, 59, tzinfo=timezone.utc) + _create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(seconds=1)) + _create_archive_dt(archiver, backup_files, "test-2", dt - timedelta(seconds=1, microseconds=999999)) + _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(seconds=2)) + _create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(seconds=2, microseconds=1)) + output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg) + assert re.search(r"Keeping archive \(rule: secondly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: secondly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Would prune:\s+test-4", output) -@freeze_time(datetime(2023, 12, 31, 23, 59, 0, tzinfo=None)) -def test_prune_keep_minutely_int_or_interval(archivers, request, backup_files): +@pytest.mark.parametrize("keep_arg", ["--keep-minutely=3", "--keep-minutely=3M"]) +def test_prune_keep_minutely_int_or_interval(archivers, request, backup_files, keep_arg): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - _create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 31, 23, 58) - _create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 31, 23, 57, 1) - _create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 31, 23, 57) - _create_archive_ts(archiver, backup_files, "test-4", 2023, 12, 31, 23, 56, 0, 1) # Last possible microsecond - _create_archive_ts(archiver, backup_files, "test-5", 2023, 12, 31, 23, 56) - for keep_arg in ["--keep-minutely=3", "--keep-minutely=3M"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() - assert re.search(r"Keeping archive \(rule: minutely #1\):\s+test-1", output.pop(0)) - assert re.search(r"Keeping archive \(rule: minutely #2\):\s+test-2", output.pop(0)) - assert re.search(r"Would prune:\s+test-3", output.pop(0)) - assert re.search(r"Keeping archive \(rule: minutely #3\):\s+test-4", output.pop(0)) - assert re.search(r"Would prune:\s+test-5", output.pop(0)) + dt = datetime(2023, 12, 31, 23, 59, 0, tzinfo=timezone.utc) + _create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(minutes=1)) + _create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(minutes=2)).replace(second=1)) + _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(minutes=2)) + _create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(minutes=3)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(minutes=3)) + output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg) + assert re.search(r"Keeping archive \(rule: minutely #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: minutely #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: minutely #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) -@freeze_time(datetime(2023, 12, 31, 23, 0, 0, tzinfo=None)) -def test_prune_keep_hourly_int_or_interval(archivers, request, backup_files): +@pytest.mark.parametrize("keep_arg", ["--keep-hourly=3", "--keep-hourly=3H"]) +def test_prune_keep_hourly_int_or_interval(archivers, request, backup_files, keep_arg): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - _create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 31, 22) - _create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 31, 21, us=1) - _create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 31, 21) - _create_archive_ts(archiver, backup_files, "test-4", 2023, 12, 31, 20, us=1) # Last possible microsecond - _create_archive_ts(archiver, backup_files, "test-5", 2023, 12, 31, 20) - for keep_arg in ["--keep-hourly=3", "--keep-hourly=3H"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() - assert re.search(r"Keeping archive \(rule: hourly #1\):\s+test-1", output.pop(0)) - assert re.search(r"Keeping archive \(rule: hourly #2\):\s+test-2", output.pop(0)) - assert re.search(r"Would prune:\s+test-3", output.pop(0)) - assert re.search(r"Keeping archive \(rule: hourly #3\):\s+test-4", output.pop(0)) - assert re.search(r"Would prune:\s+test-5", output.pop(0)) + dt = datetime(2023, 12, 31, 23, 0, 0, tzinfo=timezone.utc) + _create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(hours=1)) + _create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(hours=2)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(hours=2)) + _create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(hours=3)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(hours=3)) + output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg) + assert re.search(r"Keeping archive \(rule: hourly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: hourly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: hourly #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) -@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) -def test_prune_keep_daily_int_or_interval(archivers, request, backup_files): +@pytest.mark.parametrize("keep_arg", ["--keep-daily=3", "--keep-daily=3d"]) +def test_prune_keep_daily_int_or_interval(archivers, request, backup_files, keep_arg): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - _create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 30) - _create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 29, S=1) - _create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 29) - _create_archive_ts(archiver, backup_files, "test-4", 2023, 12, 28, us=1) # Last possible microsecond - _create_archive_ts(archiver, backup_files, "test-5", 2023, 12, 28) - for keep_arg in ["--keep-daily=3", "--keep-daily=3d"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() - assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output.pop(0)) - assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output.pop(0)) - assert re.search(r"Would prune:\s+test-3", output.pop(0)) - assert re.search(r"Keeping archive \(rule: daily #3\):\s+test-4", output.pop(0)) - assert re.search(r"Would prune:\s+test-5", output.pop(0)) + dt = datetime(2023, 12, 31, 0, 0, 0, tzinfo=timezone.utc) + _create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=1)) + _create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(days=2)).replace(second=1)) + _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=2)) + _create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(days=3)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=3)) + output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg) + assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: daily #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) -@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) -def test_prune_keep_weekly_int_or_interval(archivers, request, backup_files): +@pytest.mark.parametrize("keep_arg", ["--keep-weekly=3", "--keep-weekly=3w"]) +def test_prune_keep_weekly_int_or_interval(archivers, request, backup_files, keep_arg): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - _create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 24) - _create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 17, us=1) - _create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 17) - _create_archive_ts(archiver, backup_files, "test-4", 2023, 12, 10, us=1) # Last possible microsecond - _create_archive_ts(archiver, backup_files, "test-5", 2023, 12, 10) - for keep_arg in ["--keep-weekly=3", "--keep-weekly=3w"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() - assert re.search(r"Keeping archive \(rule: weekly #1\):\s+test-1", output.pop(0)) - assert re.search(r"Keeping archive \(rule: weekly #2\):\s+test-2", output.pop(0)) - assert re.search(r"Would prune:\s+test-3", output.pop(0)) - assert re.search(r"Keeping archive \(rule: weekly #3\):\s+test-4", output.pop(0)) - assert re.search(r"Would prune:\s+test-5", output.pop(0)) + dt = datetime(2023, 12, 31, 0, 0, 0, tzinfo=timezone.utc) + _create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=7)) + _create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(days=14)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=14)) + _create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(days=21)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=21)) + output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg) + assert re.search(r"Keeping archive \(rule: weekly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: weekly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: weekly #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) -@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) -def test_prune_keep_monthly_int_or_interval(archivers, request, backup_files): +@pytest.mark.parametrize("keep_arg", ["--keep-monthly=3", "--keep-monthly=3m"]) +def test_prune_keep_monthly_int_or_interval(archivers, request, backup_files, keep_arg): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - _create_archive_ts(archiver, backup_files, "test-1", 2023, 11, 30) - _create_archive_ts( - archiver, backup_files, "test-2", 2023, 10, 30, us=1 - ) # Month defined as 31 days, so not Oct 31st - _create_archive_ts(archiver, backup_files, "test-3", 2023, 10, 30) - _create_archive_ts(archiver, backup_files, "test-4", 2023, 9, 29, us=1) # Last possible microsecond - _create_archive_ts(archiver, backup_files, "test-5", 2023, 9, 29) - for keep_arg in ["--keep-monthly=3", "--keep-monthly=3m"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() - assert re.search(r"Keeping archive \(rule: monthly #1\):\s+test-1", output.pop(0)) - assert re.search(r"Keeping archive \(rule: monthly #2\):\s+test-2", output.pop(0)) - assert re.search(r"Would prune:\s+test-3", output.pop(0)) - assert re.search(r"Keeping archive \(rule: monthly #3\):\s+test-4", output.pop(0)) - assert re.search(r"Would prune:\s+test-5", output.pop(0)) + dt = datetime(2023, 12, 31, 0, 0, 0, tzinfo=timezone.utc) + _create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=31)) + _create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(days=62)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=62)) + _create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(days=93)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=93)) + output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg) + assert re.search(r"Keeping archive \(rule: monthly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: monthly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: monthly #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) # 2023-12-31 is Sunday, week 52. Makes these week calculations a little easier. -@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) -def test_prune_keep_13weekly_int_or_interval(archivers, request, backup_files): +@pytest.mark.parametrize("keep_arg", ["--keep-13weekly=3", "--keep-13weekly=39w"]) +def test_prune_keep_13weekly_int_or_interval(archivers, request, backup_files, keep_arg): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - _create_archive_ts(archiver, backup_files, "test-1", 2023, 10, 1) - _create_archive_ts(archiver, backup_files, "test-2", 2023, 7, 2, us=1) - _create_archive_ts(archiver, backup_files, "test-3", 2023, 7, 2) - _create_archive_ts(archiver, backup_files, "test-4", 2023, 4, 2, us=1) # Last possible microsecond - _create_archive_ts(archiver, backup_files, "test-5", 2023, 4, 2) - for keep_arg in ["--keep-13weekly=3", "--keep-13weekly=39w"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() - assert re.search(r"Keeping archive \(rule: quarterly_13weekly #1\):\s+test-1", output.pop(0)) - assert re.search(r"Keeping archive \(rule: quarterly_13weekly #2\):\s+test-2", output.pop(0)) - assert re.search(r"Would prune:\s+test-3", output.pop(0)) - assert re.search(r"Keeping archive \(rule: quarterly_13weekly #3\):\s+test-4", output.pop(0)) - assert re.search(r"Would prune:\s+test-5", output.pop(0)) + dt = datetime(2023, 12, 31, 0, 0, 0, tzinfo=timezone.utc) + _create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=91)) + _create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(days=182)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=182)) + _create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(days=273)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=273)) + output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg) + assert re.search(r"Keeping archive \(rule: quarterly_13weekly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: quarterly_13weekly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: quarterly_13weekly #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) -@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) -def test_prune_keep_3monthly_int_or_interval(archivers, request, backup_files): +@pytest.mark.parametrize("keep_arg", ["--keep-3monthly=3", "--keep-3monthly=275d"]) +def test_prune_keep_3monthly_int_or_interval(archivers, request, backup_files, keep_arg): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - _create_archive_ts(archiver, backup_files, "test-1", 2023, 9, 30) - _create_archive_ts(archiver, backup_files, "test-2", 2023, 6, 30, us=1) - _create_archive_ts(archiver, backup_files, "test-3", 2023, 6, 30) - _create_archive_ts(archiver, backup_files, "test-4", 2023, 3, 31, us=1) # Last possible microsecond - _create_archive_ts(archiver, backup_files, "test-5", 2023, 3, 31) - # 275d is the interval from now to 2023-03-31 - for keep_arg in ["--keep-3monthly=3", "--keep-3monthly=275d"]: - output = cmd(archiver, "prune", "--list", "--short", "--dry-run", keep_arg).splitlines() - assert re.search(r"Keeping archive \(rule: quarterly_3monthly #1\):\s+test-1", output.pop(0)) - assert re.search(r"Keeping archive \(rule: quarterly_3monthly #2\):\s+test-2", output.pop(0)) - assert re.search(r"Would prune:\s+test-3", output.pop(0)) - assert re.search(r"Keeping archive \(rule: quarterly_3monthly #3\):\s+test-4", output.pop(0)) - assert re.search(r"Would prune:\s+test-5", output.pop(0)) + dt = datetime(2023, 12, 31, 0, 0, 0, tzinfo=timezone.utc) + _create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=92)) + _create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(days=184)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=184)) + _create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(days=275)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=275)) + # 275d is the interval from dt to the oldest kept monthly archive + output = cmd(archiver, "prune", "--list", "--short", "--dry-run", "--since", dt.isoformat(), keep_arg) + assert re.search(r"Keeping archive \(rule: quarterly_3monthly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: quarterly_3monthly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: quarterly_3monthly #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) -@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None)) -def test_prune_keep_yearly_int_or_interval(archivers, request, backup_files): +@pytest.mark.parametrize("keep_arg", ["--keep-yearly=3", "--keep-yearly=3y"]) +def test_prune_keep_yearly_int_or_interval(archivers, request, backup_files, keep_arg): archiver = request.getfixturevalue(archivers) cmd(archiver, "repo-create", RK_ENCRYPTION) - _create_archive_ts(archiver, backup_files, "test-1", 2022, 12, 31) - _create_archive_ts(archiver, backup_files, "test-2", 2021, 12, 31, us=1) - _create_archive_ts(archiver, backup_files, "test-3", 2021, 12, 31) - _create_archive_ts(archiver, backup_files, "test-4", 2020, 12, 31, us=1) # Last possible microsecond - _create_archive_ts(archiver, backup_files, "test-5", 2020, 12, 31) - for keep_arg in ["--keep-yearly=3", "--keep-yearly=3y"]: - output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines() - assert re.search(r"Keeping archive \(rule: yearly #1\):\s+test-1", output.pop(0)) - assert re.search(r"Keeping archive \(rule: yearly #2\):\s+test-2", output.pop(0)) - assert re.search(r"Would prune:\s+test-3", output.pop(0)) - assert re.search(r"Keeping archive \(rule: yearly #3\):\s+test-4", output.pop(0)) - assert re.search(r"Would prune:\s+test-5", output.pop(0)) + dt = datetime(2023, 12, 31, 0, 0, 0, tzinfo=timezone.utc) + _create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=365)) + _create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(days=730)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=730)) + _create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(days=1095)).replace(microsecond=1)) + _create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=1095)) + output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg) + assert re.search(r"Keeping archive \(rule: yearly #1\):\s+test-1", output) + assert re.search(r"Keeping archive \(rule: yearly #2\):\s+test-2", output) + assert re.search(r"Would prune:\s+test-3", output) + assert re.search(r"Keeping archive \(rule: yearly #3\):\s+test-4", output) + assert re.search(r"Would prune:\s+test-5", output) def test_prune_no_args(archivers, request): @@ -773,6 +768,25 @@ def test_prune_int_rolling_schedule_oldest_retention(): assert archives[-1].ts.strftime("%m-%d") == "01-31" +def test_prune_since_prefiltered_archives_ignored_in_pruning(archivers, request, backup_files): + archiver = request.getfixturevalue(archivers) + cmd(archiver, "repo-create", RK_ENCRYPTION) + dt = datetime(2024, 6, 6, 12, 0, 0, tzinfo=timezone.utc) + + _create_archive_dt(archiver, backup_files, "test-a", dt + timedelta(hours=1)) + _create_archive_dt(archiver, backup_files, "test-b", dt - timedelta(hours=1)) + _create_archive_dt(archiver, backup_files, "test-c", dt - timedelta(days=1)) + + output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), "--keep-daily=1") + + # 'test-b' is kept, meaning 'test-a' was entirely skipped for pruning consideration. + # They would otherwise have occupied the same period. + assert re.search(r"Keeping archive \(rule: skip #1\):\s+test-a", output) + assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-b", output) + + assert re.search(r"Would prune:\s+test-c", output) + + def test_prune_interval_rolling_schedule_oldest_retention(): daily_interval = timedelta(days=6) monthly_interval = timedelta(days=31 * 3) # Matching --keep-monthly=3m after argument parsing