mirror of
https://github.com/borgbackup/borg.git
synced 2026-07-14 20:41:04 -04:00
Adds 'prune --since', base timestamp to prune from
A flag to explicitly "save" all archives older than a certain timestamp, and to function as a base timestamp from which to base interval timedelta calculations. Allows for precise time interval manipulation for superusers and as a bonus simplifies time-based testing and alleviates the need for an external dependency to freeze time in test. Includes some refactoring of do_prune for logical flow & naming that came up while iterating on these changes.
This commit is contained in:
Hugo Wallenburg
parent
eca719aeec
commit
72e72dc431
4 changed files with 244 additions and 195 deletions
|
|
@ -11,7 +11,6 @@ pytest-xdist==3.8.0
|
|||
coverage[toml]==7.14.0
|
||||
pytest-cov==7.1.0
|
||||
pytest-benchmark==5.2.3
|
||||
freezegun==1.5.5
|
||||
Cython==3.2.5
|
||||
pre-commit==4.6.0
|
||||
types-PyYAML==6.0.12.20260518
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ pytest-xdist
|
|||
coverage[toml]
|
||||
pytest-cov
|
||||
pytest-benchmark
|
||||
freezegun
|
||||
Cython
|
||||
pre-commit
|
||||
bandit[toml]
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from itertools import count, combinations
|
|||
from ._common import with_repository, Highlander
|
||||
from ..constants import * # NOQA
|
||||
from ..helpers import ArchiveFormatter, ProgressIndicatorPercent, CommandError, Error
|
||||
from ..helpers import archivename_validator, interval, int_or_interval, sig_int
|
||||
from ..helpers import archivename_validator, interval, int_or_interval, sig_int, timestamp
|
||||
from ..helpers import json_print, basic_json_data
|
||||
from ..helpers.argparsing import ArgumentParser
|
||||
from ..manifest import ArchiveInfo, Manifest
|
||||
|
|
@ -108,6 +108,9 @@ PRUNE_QUARTERLY_13WEEKLY = PruningRule("quarterly_13weekly", quarterly_13weekly_
|
|||
PRUNE_QUARTERLY_3MONTHLY = PruningRule("quarterly_3monthly", quarterly_3monthly_period_func)
|
||||
PRUNE_YEARLY = PruningRule("yearly", pattern_period_func("%Y"))
|
||||
|
||||
# Fake rule used to indicate archives skipped by --since
|
||||
PRUNE_SINCE = PruningRule("skip", unique_period_func())
|
||||
|
||||
PRUNING_RULES = [
|
||||
PRUNE_WITHIN,
|
||||
PRUNE_LAST,
|
||||
|
|
@ -128,7 +131,7 @@ def prune(
|
|||
archives: list[ArchiveInfo],
|
||||
rule: PruningRule,
|
||||
n_or_interval: int | timedelta,
|
||||
base_timestamp: datetime | None,
|
||||
since_timestamp: datetime | None,
|
||||
keep_oldest: bool,
|
||||
previously_kept: frozenset[ArchiveInfo] = frozenset(),
|
||||
) -> dict[ArchiveInfo, KeepResult]:
|
||||
|
|
@ -138,9 +141,9 @@ def prune(
|
|||
if isinstance(n_or_interval, int):
|
||||
n, earliest_timestamp = n_or_interval, None
|
||||
else:
|
||||
if base_timestamp is None:
|
||||
raise ValueError("base_timestamp is required when using interval-based pruning")
|
||||
n, earliest_timestamp = None, base_timestamp - n_or_interval
|
||||
if since_timestamp is None:
|
||||
raise ValueError("since_timestamp is required when using interval-based pruning")
|
||||
n, earliest_timestamp = None, since_timestamp - n_or_interval
|
||||
|
||||
keep: dict[ArchiveInfo, KeepResult] = {}
|
||||
|
||||
|
|
@ -175,6 +178,44 @@ class PruneMixIn:
|
|||
"""Prune archives according to specified rules."""
|
||||
self._validate_prune_args(args)
|
||||
|
||||
match = [args.name] if args.name else args.match_archives
|
||||
archives = manifest.archives.list(match=match, sort_by=["ts"], reverse=True)
|
||||
archives = [ai for ai in archives if "@PROT" not in ai.tags]
|
||||
|
||||
# Archives to keep along with the rule that ensured them being kept
|
||||
keep = {}
|
||||
|
||||
since = getattr(args, PRUNE_SINCE.key)
|
||||
candidate_archives = archives
|
||||
|
||||
if since is not None:
|
||||
# `--since` is a prefilter: Archives from after this time are kept by default. They are not considered for
|
||||
# pruning at all. They won't falsely occupy an active retention period.
|
||||
for archive in archives:
|
||||
if archive.ts <= since:
|
||||
break
|
||||
keep[archive] = KeepResult(rule=PRUNE_SINCE, idx=len(keep))
|
||||
candidate_archives = archives[len(keep) :]
|
||||
|
||||
# Apply each retention rule to all candidate archives. The
|
||||
# `previously_kept` parameter prevents later (coarser-grained) rules
|
||||
# from double-counting archives already retained by earlier rules.
|
||||
active_rules = [
|
||||
(rule, getattr(args, rule.key)) for rule in PRUNING_RULES if getattr(args, rule.key) is not None
|
||||
]
|
||||
for rule, n_or_interval in active_rules:
|
||||
keep |= prune(
|
||||
archives=candidate_archives,
|
||||
rule=rule,
|
||||
n_or_interval=n_or_interval,
|
||||
since_timestamp=(since if since is not None else datetime.now().astimezone()),
|
||||
keep_oldest=(
|
||||
rule == active_rules[-1][0]
|
||||
), # Activate keep_oldest rule only for the largest active interval
|
||||
previously_kept=frozenset(keep),
|
||||
)
|
||||
|
||||
archives_to_prune = set(archives) - set(keep)
|
||||
|
||||
if args.format is not None:
|
||||
format = args.format
|
||||
|
|
@ -184,37 +225,17 @@ class PruneMixIn:
|
|||
format = os.environ.get("BORG_PRUNE_FORMAT", "{archive:<36} {time} [{id}]")
|
||||
formatter = ArchiveFormatter(format, repository, manifest, manifest.key, iec=args.iec)
|
||||
|
||||
match = [args.name] if args.name else args.match_archives
|
||||
archives = manifest.archives.list(match=match, sort_by=["ts"], reverse=True)
|
||||
archives = [ai for ai in archives if "@PROT" not in ai.tags]
|
||||
|
||||
# Archives to keep along with the rule that ensured them being kept
|
||||
keep = {}
|
||||
|
||||
base_timestamp = datetime.now().astimezone()
|
||||
active_rules = {rule: getattr(args, rule.key) for rule in PRUNING_RULES if getattr(args, rule.key) is not None}
|
||||
for i, (rule, n_or_interval) in enumerate(active_rules.items(), 1):
|
||||
keep |= prune(
|
||||
archives=archives,
|
||||
rule=rule,
|
||||
n_or_interval=n_or_interval,
|
||||
base_timestamp=base_timestamp,
|
||||
keep_oldest=i == len(active_rules), # Activate keep_oldest rule only for the largest active interval
|
||||
previously_kept=frozenset(keep),
|
||||
)
|
||||
|
||||
to_delete = set(archives) - set(keep)
|
||||
if not args.json:
|
||||
logger.info("Repository contains %d archives.", manifest.archives.count())
|
||||
logger.info("Applying rules to the matching %d archives...", len(archives))
|
||||
logger.info("Keeping %d archives, pruning %d archives.", len(keep), len(to_delete))
|
||||
if args.json:
|
||||
output_data = []
|
||||
else:
|
||||
logger.info("Repository contains %d archives.", manifest.archives.count())
|
||||
logger.info("Applying rules to the matching %d archives...", len(archives))
|
||||
logger.info("Keeping %d archives, pruning %d archives.", len(keep), len(archives_to_prune))
|
||||
|
||||
list_logger = logging.getLogger("borg.output.list")
|
||||
# set up counters for the progress display
|
||||
to_delete_len = len(to_delete)
|
||||
archives_deleted = 0
|
||||
pi = ProgressIndicatorPercent(total=len(to_delete), msg="Pruning archives %3.0f%%", msgid="prune")
|
||||
num_archives_deleted = 0
|
||||
pi = ProgressIndicatorPercent(total=len(archives_to_prune), msg="Pruning archives %3.0f%%", msgid="prune")
|
||||
for archive_info in archives:
|
||||
if sig_int and sig_int.action_done():
|
||||
break
|
||||
|
|
@ -224,18 +245,18 @@ class PruneMixIn:
|
|||
archive_data = formatter.get_item_data(archive_info, jsonline=True)
|
||||
else:
|
||||
archive_formatted = formatter.format_item(archive_info, jsonline=False)
|
||||
if archive_info in to_delete:
|
||||
if archive_info in archives_to_prune:
|
||||
if not args.json:
|
||||
pi.show()
|
||||
archives_deleted += 1
|
||||
num_archives_deleted += 1
|
||||
if args.dry_run:
|
||||
log_message = "Would prune:"
|
||||
else:
|
||||
log_message = "Pruning archive (%d/%d):" % (archives_deleted, to_delete_len)
|
||||
log_message = f"Pruning archive ({num_archives_deleted}/{len(archives_to_prune)}):"
|
||||
manifest.archives.delete_by_id(archive_info.id)
|
||||
if args.json:
|
||||
archive_data["kept"] = False
|
||||
archive_data["deleted_archive_number"] = archives_deleted
|
||||
archive_data["deleted_archive_number"] = num_archives_deleted
|
||||
else:
|
||||
result = keep[archive_info]
|
||||
result_message = f"{result.rule.key}{'[oldest]' if result.oldest else ''} #{result.idx + 1}"
|
||||
|
|
@ -249,21 +270,21 @@ class PruneMixIn:
|
|||
if (
|
||||
args.output_list
|
||||
or not (args.list_pruned or args.list_kept)
|
||||
or (args.list_pruned and archive_info in to_delete)
|
||||
or (args.list_kept and archive_info not in to_delete)
|
||||
or (args.list_pruned and archive_info in archives_to_prune)
|
||||
or (args.list_kept and archive_info not in archives_to_prune)
|
||||
):
|
||||
output_data.append(archive_data)
|
||||
elif (
|
||||
args.output_list
|
||||
or (args.list_pruned and archive_info in to_delete)
|
||||
or (args.list_kept and archive_info not in to_delete)
|
||||
or (args.list_pruned and archive_info in archives_to_prune)
|
||||
or (args.list_kept and archive_info not in archives_to_prune)
|
||||
):
|
||||
list_logger.info(f"{log_message:<44} {archive_formatted}")
|
||||
if not args.json:
|
||||
pi.finish()
|
||||
if args.json:
|
||||
json_print(basic_json_data(manifest, extra={"archives": output_data}))
|
||||
if archives_deleted > 0 and not args.dry_run:
|
||||
if num_archives_deleted > 0 and not args.dry_run:
|
||||
manifest.write()
|
||||
self.print_warning('Done. Run "borg compact" to free space.', wc=None)
|
||||
if sig_int:
|
||||
|
|
@ -371,6 +392,14 @@ class PruneMixIn:
|
|||
timezone of the system where borg prune runs, and weeks go from Monday to Sunday.
|
||||
Specifying a negative number of archives to keep means that there is no limit.
|
||||
|
||||
The ``--since`` option restricts pruning to archives older than the given
|
||||
TIMESTAMP. Archives newer than this timestamp are kept unconditionally
|
||||
as a pre-filter. When ``--since`` is used together with interval-based
|
||||
``--keep-*`` options (e.g. ``--keep-daily 7d``), the interval is
|
||||
measured backwards from the given timestamp rather than from the
|
||||
current time. Count-based retention does not count the unconditionally
|
||||
kept archives.
|
||||
|
||||
Borg will retain the oldest archive if any of the secondly, minutely, hourly,
|
||||
daily, weekly, monthly, quarterly, or yearly rules was not otherwise able to
|
||||
meet its retention target. This enables the first chronological archive to
|
||||
|
|
@ -419,6 +448,14 @@ class PruneMixIn:
|
|||
"but keys used in it are added to the JSON output. "
|
||||
"Some keys are always present. Note: JSON can only represent text.",
|
||||
)
|
||||
subparser.add_argument(
|
||||
"--since",
|
||||
metavar="TIMESTAMP",
|
||||
dest=PRUNE_SINCE.key,
|
||||
type=timestamp,
|
||||
action=Highlander,
|
||||
help="only consider archives older than this for pruning",
|
||||
)
|
||||
subparser.add_argument(
|
||||
"--keep-within",
|
||||
metavar="INTERVAL",
|
||||
|
|
|
|||
|
|
@ -4,8 +4,6 @@ import pytest
|
|||
import re
|
||||
from operator import attrgetter
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from freezegun import freeze_time
|
||||
|
||||
from ...constants import * # NOQA
|
||||
from ...archiver.prune_cmd import (
|
||||
PRUNING_RULES,
|
||||
|
|
@ -26,15 +24,14 @@ from . import cmd, RK_ENCRYPTION, generate_archiver_tests
|
|||
pytest_generate_tests = lambda metafunc: generate_archiver_tests(metafunc, kinds="local,remote,binary") # NOQA
|
||||
|
||||
|
||||
def _create_archive_ts(archiver, backup_files, name, y, m, d, H=0, M=0, S=0, us=0, tzinfo=None):
|
||||
cmd(
|
||||
archiver,
|
||||
"create",
|
||||
"--timestamp",
|
||||
datetime(y, m, d, H, M, S, us, tzinfo=tzinfo).strftime(ISO_FORMAT_ZONE),
|
||||
name,
|
||||
backup_files,
|
||||
)
|
||||
def _create_archive_dt(archiver, backup_files, name, dt, tzinfo=timezone.utc):
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=tzinfo)
|
||||
cmd(archiver, "create", "--timestamp", dt.strftime(ISO_FORMAT_ZONE), name, backup_files)
|
||||
|
||||
|
||||
def _create_archive_ts(archiver, backup_files, name, y, m, d, H=0, M=0, S=0, us=0, tzinfo=timezone.utc):
|
||||
_create_archive_dt(archiver, backup_files, name, datetime(y, m, d, H, M, S, us, tzinfo=tzinfo))
|
||||
|
||||
|
||||
def test_prune_repository(archivers, request, backup_files):
|
||||
|
|
@ -165,7 +162,6 @@ def test_prune_quarterly(archivers, request, backup_files):
|
|||
|
||||
# Use 99 instead of -1 to test that oldest backup is kept.
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", f"--keep-{strat}=99")
|
||||
print(output)
|
||||
for a in map(mk_name, to_prune):
|
||||
assert re.search(rf"Would prune:\s+{a}", output)
|
||||
|
||||
|
|
@ -474,195 +470,194 @@ def test_prune_keep_last_same_second(archivers, request, backup_files):
|
|||
assert re.search(r"Keeping archive \(rule: last #\d\):\s+test2", output)
|
||||
|
||||
|
||||
@freeze_time(datetime(2023, 12, 31, 23, 59, 59, tzinfo=None)) # Non-leap year ending on a Sunday
|
||||
def test_prune_keep_int_or_interval(archivers, request, backup_files):
|
||||
@pytest.mark.parametrize("keep_arg", ["--keep=2", "--keep=1S"])
|
||||
def test_prune_keep_int_or_interval(archivers, request, backup_files, keep_arg):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", RK_ENCRYPTION)
|
||||
_create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 31, 23, 59, 59)
|
||||
_create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 31, 23, 59, 59)
|
||||
_create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 31, 23, 59, 58)
|
||||
for keep_arg in ["--keep=2", "--keep=1S"]:
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg)
|
||||
assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test-1", output)
|
||||
assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test-2", output)
|
||||
assert re.search(r"Would prune:\s+test-3", output)
|
||||
dt = datetime(2023, 12, 31, 23, 59, 59, tzinfo=timezone.utc)
|
||||
_create_archive_dt(archiver, backup_files, "test-1", dt)
|
||||
_create_archive_dt(archiver, backup_files, "test-2", dt)
|
||||
_create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(seconds=1))
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg)
|
||||
assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test-1", output)
|
||||
assert re.search(r"Keeping archive \(rule: keep #\d\):\s+test-2", output)
|
||||
assert re.search(r"Would prune:\s+test-3", output)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("keep_arg", ["--keep-daily=-1", "--keep-daily=all"])
|
||||
def test_prune_keep_all(archivers, request, backup_files, keep_arg):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", RK_ENCRYPTION)
|
||||
_create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 30, 23, 59, 59, tzinfo=timezone.utc)
|
||||
_create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 29, 23, 59, 59, tzinfo=timezone.utc)
|
||||
_create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 28, 23, 59, 59, tzinfo=timezone.utc)
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg)
|
||||
dt = datetime(2023, 12, 31, 23, 59, 59, tzinfo=timezone.utc)
|
||||
_create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-2", dt - timedelta(days=2))
|
||||
_create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=3))
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg)
|
||||
assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output)
|
||||
assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output)
|
||||
assert re.search(r"Keeping archive \(rule: daily #3\):\s+test-3", output)
|
||||
|
||||
|
||||
@freeze_time(datetime(2023, 12, 31, 23, 59, 59, tzinfo=None))
|
||||
@pytest.mark.parametrize("keep_arg", ["--keep-secondly=2", "--keep-secondly=2S"])
|
||||
def test_prune_keep_secondly_int_or_interval(archivers, request, backup_files, keep_arg):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", RK_ENCRYPTION)
|
||||
_create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 31, 23, 59, 58)
|
||||
_create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 31, 23, 59, 57, 1)
|
||||
_create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 31, 23, 59, 57)
|
||||
_create_archive_ts(archiver, backup_files, "test-4", 2023, 12, 31, 23, 59, 56, 999999)
|
||||
for keep_arg in ["--keep-secondly=2", "--keep-secondly=2S"]:
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines()
|
||||
assert re.search(r"Keeping archive \(rule: secondly #1\):\s+test-1", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: secondly #2\):\s+test-2", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-3", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-4", output.pop(0))
|
||||
dt = datetime(2023, 12, 31, 23, 59, 59, tzinfo=timezone.utc)
|
||||
_create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(seconds=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-2", dt - timedelta(seconds=1, microseconds=999999))
|
||||
_create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(seconds=2))
|
||||
_create_archive_dt(archiver, backup_files, "test-4", dt - timedelta(seconds=2, microseconds=1))
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg)
|
||||
assert re.search(r"Keeping archive \(rule: secondly #1\):\s+test-1", output)
|
||||
assert re.search(r"Keeping archive \(rule: secondly #2\):\s+test-2", output)
|
||||
assert re.search(r"Would prune:\s+test-3", output)
|
||||
assert re.search(r"Would prune:\s+test-4", output)
|
||||
|
||||
|
||||
@freeze_time(datetime(2023, 12, 31, 23, 59, 0, tzinfo=None))
|
||||
def test_prune_keep_minutely_int_or_interval(archivers, request, backup_files):
|
||||
@pytest.mark.parametrize("keep_arg", ["--keep-minutely=3", "--keep-minutely=3M"])
|
||||
def test_prune_keep_minutely_int_or_interval(archivers, request, backup_files, keep_arg):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", RK_ENCRYPTION)
|
||||
_create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 31, 23, 58)
|
||||
_create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 31, 23, 57, 1)
|
||||
_create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 31, 23, 57)
|
||||
_create_archive_ts(archiver, backup_files, "test-4", 2023, 12, 31, 23, 56, 0, 1) # Last possible microsecond
|
||||
_create_archive_ts(archiver, backup_files, "test-5", 2023, 12, 31, 23, 56)
|
||||
for keep_arg in ["--keep-minutely=3", "--keep-minutely=3M"]:
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines()
|
||||
assert re.search(r"Keeping archive \(rule: minutely #1\):\s+test-1", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: minutely #2\):\s+test-2", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-3", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: minutely #3\):\s+test-4", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-5", output.pop(0))
|
||||
dt = datetime(2023, 12, 31, 23, 59, 0, tzinfo=timezone.utc)
|
||||
_create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(minutes=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(minutes=2)).replace(second=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(minutes=2))
|
||||
_create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(minutes=3)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(minutes=3))
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg)
|
||||
assert re.search(r"Keeping archive \(rule: minutely #1\):\s+test-1", output)
|
||||
assert re.search(r"Keeping archive \(rule: minutely #2\):\s+test-2", output)
|
||||
assert re.search(r"Would prune:\s+test-3", output)
|
||||
assert re.search(r"Keeping archive \(rule: minutely #3\):\s+test-4", output)
|
||||
assert re.search(r"Would prune:\s+test-5", output)
|
||||
|
||||
|
||||
@freeze_time(datetime(2023, 12, 31, 23, 0, 0, tzinfo=None))
|
||||
def test_prune_keep_hourly_int_or_interval(archivers, request, backup_files):
|
||||
@pytest.mark.parametrize("keep_arg", ["--keep-hourly=3", "--keep-hourly=3H"])
|
||||
def test_prune_keep_hourly_int_or_interval(archivers, request, backup_files, keep_arg):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", RK_ENCRYPTION)
|
||||
_create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 31, 22)
|
||||
_create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 31, 21, us=1)
|
||||
_create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 31, 21)
|
||||
_create_archive_ts(archiver, backup_files, "test-4", 2023, 12, 31, 20, us=1) # Last possible microsecond
|
||||
_create_archive_ts(archiver, backup_files, "test-5", 2023, 12, 31, 20)
|
||||
for keep_arg in ["--keep-hourly=3", "--keep-hourly=3H"]:
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines()
|
||||
assert re.search(r"Keeping archive \(rule: hourly #1\):\s+test-1", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: hourly #2\):\s+test-2", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-3", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: hourly #3\):\s+test-4", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-5", output.pop(0))
|
||||
dt = datetime(2023, 12, 31, 23, 0, 0, tzinfo=timezone.utc)
|
||||
_create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(hours=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(hours=2)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(hours=2))
|
||||
_create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(hours=3)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(hours=3))
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg)
|
||||
assert re.search(r"Keeping archive \(rule: hourly #1\):\s+test-1", output)
|
||||
assert re.search(r"Keeping archive \(rule: hourly #2\):\s+test-2", output)
|
||||
assert re.search(r"Would prune:\s+test-3", output)
|
||||
assert re.search(r"Keeping archive \(rule: hourly #3\):\s+test-4", output)
|
||||
assert re.search(r"Would prune:\s+test-5", output)
|
||||
|
||||
|
||||
@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None))
|
||||
def test_prune_keep_daily_int_or_interval(archivers, request, backup_files):
|
||||
@pytest.mark.parametrize("keep_arg", ["--keep-daily=3", "--keep-daily=3d"])
|
||||
def test_prune_keep_daily_int_or_interval(archivers, request, backup_files, keep_arg):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", RK_ENCRYPTION)
|
||||
_create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 30)
|
||||
_create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 29, S=1)
|
||||
_create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 29)
|
||||
_create_archive_ts(archiver, backup_files, "test-4", 2023, 12, 28, us=1) # Last possible microsecond
|
||||
_create_archive_ts(archiver, backup_files, "test-5", 2023, 12, 28)
|
||||
for keep_arg in ["--keep-daily=3", "--keep-daily=3d"]:
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines()
|
||||
assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-3", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: daily #3\):\s+test-4", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-5", output.pop(0))
|
||||
dt = datetime(2023, 12, 31, 0, 0, 0, tzinfo=timezone.utc)
|
||||
_create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(days=2)).replace(second=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=2))
|
||||
_create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(days=3)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=3))
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg)
|
||||
assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-1", output)
|
||||
assert re.search(r"Keeping archive \(rule: daily #2\):\s+test-2", output)
|
||||
assert re.search(r"Would prune:\s+test-3", output)
|
||||
assert re.search(r"Keeping archive \(rule: daily #3\):\s+test-4", output)
|
||||
assert re.search(r"Would prune:\s+test-5", output)
|
||||
|
||||
|
||||
@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None))
|
||||
def test_prune_keep_weekly_int_or_interval(archivers, request, backup_files):
|
||||
@pytest.mark.parametrize("keep_arg", ["--keep-weekly=3", "--keep-weekly=3w"])
|
||||
def test_prune_keep_weekly_int_or_interval(archivers, request, backup_files, keep_arg):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", RK_ENCRYPTION)
|
||||
_create_archive_ts(archiver, backup_files, "test-1", 2023, 12, 24)
|
||||
_create_archive_ts(archiver, backup_files, "test-2", 2023, 12, 17, us=1)
|
||||
_create_archive_ts(archiver, backup_files, "test-3", 2023, 12, 17)
|
||||
_create_archive_ts(archiver, backup_files, "test-4", 2023, 12, 10, us=1) # Last possible microsecond
|
||||
_create_archive_ts(archiver, backup_files, "test-5", 2023, 12, 10)
|
||||
for keep_arg in ["--keep-weekly=3", "--keep-weekly=3w"]:
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines()
|
||||
assert re.search(r"Keeping archive \(rule: weekly #1\):\s+test-1", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: weekly #2\):\s+test-2", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-3", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: weekly #3\):\s+test-4", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-5", output.pop(0))
|
||||
dt = datetime(2023, 12, 31, 0, 0, 0, tzinfo=timezone.utc)
|
||||
_create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=7))
|
||||
_create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(days=14)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=14))
|
||||
_create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(days=21)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=21))
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg)
|
||||
assert re.search(r"Keeping archive \(rule: weekly #1\):\s+test-1", output)
|
||||
assert re.search(r"Keeping archive \(rule: weekly #2\):\s+test-2", output)
|
||||
assert re.search(r"Would prune:\s+test-3", output)
|
||||
assert re.search(r"Keeping archive \(rule: weekly #3\):\s+test-4", output)
|
||||
assert re.search(r"Would prune:\s+test-5", output)
|
||||
|
||||
|
||||
@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None))
|
||||
def test_prune_keep_monthly_int_or_interval(archivers, request, backup_files):
|
||||
@pytest.mark.parametrize("keep_arg", ["--keep-monthly=3", "--keep-monthly=3m"])
|
||||
def test_prune_keep_monthly_int_or_interval(archivers, request, backup_files, keep_arg):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", RK_ENCRYPTION)
|
||||
_create_archive_ts(archiver, backup_files, "test-1", 2023, 11, 30)
|
||||
_create_archive_ts(
|
||||
archiver, backup_files, "test-2", 2023, 10, 30, us=1
|
||||
) # Month defined as 31 days, so not Oct 31st
|
||||
_create_archive_ts(archiver, backup_files, "test-3", 2023, 10, 30)
|
||||
_create_archive_ts(archiver, backup_files, "test-4", 2023, 9, 29, us=1) # Last possible microsecond
|
||||
_create_archive_ts(archiver, backup_files, "test-5", 2023, 9, 29)
|
||||
for keep_arg in ["--keep-monthly=3", "--keep-monthly=3m"]:
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines()
|
||||
assert re.search(r"Keeping archive \(rule: monthly #1\):\s+test-1", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: monthly #2\):\s+test-2", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-3", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: monthly #3\):\s+test-4", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-5", output.pop(0))
|
||||
dt = datetime(2023, 12, 31, 0, 0, 0, tzinfo=timezone.utc)
|
||||
_create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=31))
|
||||
_create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(days=62)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=62))
|
||||
_create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(days=93)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=93))
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg)
|
||||
assert re.search(r"Keeping archive \(rule: monthly #1\):\s+test-1", output)
|
||||
assert re.search(r"Keeping archive \(rule: monthly #2\):\s+test-2", output)
|
||||
assert re.search(r"Would prune:\s+test-3", output)
|
||||
assert re.search(r"Keeping archive \(rule: monthly #3\):\s+test-4", output)
|
||||
assert re.search(r"Would prune:\s+test-5", output)
|
||||
|
||||
|
||||
# 2023-12-31 is Sunday, week 52. Makes these week calculations a little easier.
|
||||
@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None))
|
||||
def test_prune_keep_13weekly_int_or_interval(archivers, request, backup_files):
|
||||
@pytest.mark.parametrize("keep_arg", ["--keep-13weekly=3", "--keep-13weekly=39w"])
|
||||
def test_prune_keep_13weekly_int_or_interval(archivers, request, backup_files, keep_arg):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", RK_ENCRYPTION)
|
||||
_create_archive_ts(archiver, backup_files, "test-1", 2023, 10, 1)
|
||||
_create_archive_ts(archiver, backup_files, "test-2", 2023, 7, 2, us=1)
|
||||
_create_archive_ts(archiver, backup_files, "test-3", 2023, 7, 2)
|
||||
_create_archive_ts(archiver, backup_files, "test-4", 2023, 4, 2, us=1) # Last possible microsecond
|
||||
_create_archive_ts(archiver, backup_files, "test-5", 2023, 4, 2)
|
||||
for keep_arg in ["--keep-13weekly=3", "--keep-13weekly=39w"]:
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines()
|
||||
assert re.search(r"Keeping archive \(rule: quarterly_13weekly #1\):\s+test-1", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: quarterly_13weekly #2\):\s+test-2", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-3", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: quarterly_13weekly #3\):\s+test-4", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-5", output.pop(0))
|
||||
dt = datetime(2023, 12, 31, 0, 0, 0, tzinfo=timezone.utc)
|
||||
_create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=91))
|
||||
_create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(days=182)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=182))
|
||||
_create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(days=273)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=273))
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg)
|
||||
assert re.search(r"Keeping archive \(rule: quarterly_13weekly #1\):\s+test-1", output)
|
||||
assert re.search(r"Keeping archive \(rule: quarterly_13weekly #2\):\s+test-2", output)
|
||||
assert re.search(r"Would prune:\s+test-3", output)
|
||||
assert re.search(r"Keeping archive \(rule: quarterly_13weekly #3\):\s+test-4", output)
|
||||
assert re.search(r"Would prune:\s+test-5", output)
|
||||
|
||||
|
||||
@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None))
|
||||
def test_prune_keep_3monthly_int_or_interval(archivers, request, backup_files):
|
||||
@pytest.mark.parametrize("keep_arg", ["--keep-3monthly=3", "--keep-3monthly=275d"])
|
||||
def test_prune_keep_3monthly_int_or_interval(archivers, request, backup_files, keep_arg):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", RK_ENCRYPTION)
|
||||
_create_archive_ts(archiver, backup_files, "test-1", 2023, 9, 30)
|
||||
_create_archive_ts(archiver, backup_files, "test-2", 2023, 6, 30, us=1)
|
||||
_create_archive_ts(archiver, backup_files, "test-3", 2023, 6, 30)
|
||||
_create_archive_ts(archiver, backup_files, "test-4", 2023, 3, 31, us=1) # Last possible microsecond
|
||||
_create_archive_ts(archiver, backup_files, "test-5", 2023, 3, 31)
|
||||
# 275d is the interval from now to 2023-03-31
|
||||
for keep_arg in ["--keep-3monthly=3", "--keep-3monthly=275d"]:
|
||||
output = cmd(archiver, "prune", "--list", "--short", "--dry-run", keep_arg).splitlines()
|
||||
assert re.search(r"Keeping archive \(rule: quarterly_3monthly #1\):\s+test-1", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: quarterly_3monthly #2\):\s+test-2", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-3", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: quarterly_3monthly #3\):\s+test-4", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-5", output.pop(0))
|
||||
dt = datetime(2023, 12, 31, 0, 0, 0, tzinfo=timezone.utc)
|
||||
_create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=92))
|
||||
_create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(days=184)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=184))
|
||||
_create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(days=275)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=275))
|
||||
# 275d is the interval from dt to the oldest kept monthly archive
|
||||
output = cmd(archiver, "prune", "--list", "--short", "--dry-run", "--since", dt.isoformat(), keep_arg)
|
||||
assert re.search(r"Keeping archive \(rule: quarterly_3monthly #1\):\s+test-1", output)
|
||||
assert re.search(r"Keeping archive \(rule: quarterly_3monthly #2\):\s+test-2", output)
|
||||
assert re.search(r"Would prune:\s+test-3", output)
|
||||
assert re.search(r"Keeping archive \(rule: quarterly_3monthly #3\):\s+test-4", output)
|
||||
assert re.search(r"Would prune:\s+test-5", output)
|
||||
|
||||
|
||||
@freeze_time(datetime(2023, 12, 31, 0, 0, 0, tzinfo=None))
|
||||
def test_prune_keep_yearly_int_or_interval(archivers, request, backup_files):
|
||||
@pytest.mark.parametrize("keep_arg", ["--keep-yearly=3", "--keep-yearly=3y"])
|
||||
def test_prune_keep_yearly_int_or_interval(archivers, request, backup_files, keep_arg):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", RK_ENCRYPTION)
|
||||
_create_archive_ts(archiver, backup_files, "test-1", 2022, 12, 31)
|
||||
_create_archive_ts(archiver, backup_files, "test-2", 2021, 12, 31, us=1)
|
||||
_create_archive_ts(archiver, backup_files, "test-3", 2021, 12, 31)
|
||||
_create_archive_ts(archiver, backup_files, "test-4", 2020, 12, 31, us=1) # Last possible microsecond
|
||||
_create_archive_ts(archiver, backup_files, "test-5", 2020, 12, 31)
|
||||
for keep_arg in ["--keep-yearly=3", "--keep-yearly=3y"]:
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", keep_arg).splitlines()
|
||||
assert re.search(r"Keeping archive \(rule: yearly #1\):\s+test-1", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: yearly #2\):\s+test-2", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-3", output.pop(0))
|
||||
assert re.search(r"Keeping archive \(rule: yearly #3\):\s+test-4", output.pop(0))
|
||||
assert re.search(r"Would prune:\s+test-5", output.pop(0))
|
||||
dt = datetime(2023, 12, 31, 0, 0, 0, tzinfo=timezone.utc)
|
||||
_create_archive_dt(archiver, backup_files, "test-1", dt - timedelta(days=365))
|
||||
_create_archive_dt(archiver, backup_files, "test-2", (dt - timedelta(days=730)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-3", dt - timedelta(days=730))
|
||||
_create_archive_dt(archiver, backup_files, "test-4", (dt - timedelta(days=1095)).replace(microsecond=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-5", dt - timedelta(days=1095))
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), keep_arg)
|
||||
assert re.search(r"Keeping archive \(rule: yearly #1\):\s+test-1", output)
|
||||
assert re.search(r"Keeping archive \(rule: yearly #2\):\s+test-2", output)
|
||||
assert re.search(r"Would prune:\s+test-3", output)
|
||||
assert re.search(r"Keeping archive \(rule: yearly #3\):\s+test-4", output)
|
||||
assert re.search(r"Would prune:\s+test-5", output)
|
||||
|
||||
|
||||
def test_prune_no_args(archivers, request):
|
||||
|
|
@ -773,6 +768,25 @@ def test_prune_int_rolling_schedule_oldest_retention():
|
|||
assert archives[-1].ts.strftime("%m-%d") == "01-31"
|
||||
|
||||
|
||||
def test_prune_since_prefiltered_archives_ignored_in_pruning(archivers, request, backup_files):
|
||||
archiver = request.getfixturevalue(archivers)
|
||||
cmd(archiver, "repo-create", RK_ENCRYPTION)
|
||||
dt = datetime(2024, 6, 6, 12, 0, 0, tzinfo=timezone.utc)
|
||||
|
||||
_create_archive_dt(archiver, backup_files, "test-a", dt + timedelta(hours=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-b", dt - timedelta(hours=1))
|
||||
_create_archive_dt(archiver, backup_files, "test-c", dt - timedelta(days=1))
|
||||
|
||||
output = cmd(archiver, "prune", "--list", "--dry-run", "--since", dt.isoformat(), "--keep-daily=1")
|
||||
|
||||
# 'test-b' is kept, meaning 'test-a' was entirely skipped for pruning consideration.
|
||||
# They would otherwise have occupied the same period.
|
||||
assert re.search(r"Keeping archive \(rule: skip #1\):\s+test-a", output)
|
||||
assert re.search(r"Keeping archive \(rule: daily #1\):\s+test-b", output)
|
||||
|
||||
assert re.search(r"Would prune:\s+test-c", output)
|
||||
|
||||
|
||||
def test_prune_interval_rolling_schedule_oldest_retention():
|
||||
daily_interval = timedelta(days=6)
|
||||
monthly_interval = timedelta(days=31 * 3) # Matching --keep-monthly=3m after argument parsing
|
||||
|
|
|
|||
Loading…
Reference in a new issue