From d3a2d831b75ada56bfacc9a822e419148871cc0e Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 9 Sep 2022 23:23:20 +0200 Subject: [PATCH] refactor replace_placeholders, fixes #6966 fix replacing placeholders in archive name, --comment and --glob-archives values (even if overridden by other options like --timestamp). add test. --- src/borg/archiver/__init__.py | 13 ++++++++++++- src/borg/archiver/_common.py | 3 +-- src/borg/archiver/create_cmd.py | 9 ++------- src/borg/archiver/recreate_cmd.py | 8 +------- src/borg/helpers/__init__.py | 2 +- src/borg/helpers/parseformat.py | 23 ++++++++++++++++++----- src/borg/testsuite/archiver.py | 10 ++++++++++ src/borg/testsuite/helpers.py | 1 + 8 files changed, 46 insertions(+), 23 deletions(-) diff --git a/src/borg/archiver/__init__.py b/src/borg/archiver/__init__.py index 5ebf54604..201b32cec 100644 --- a/src/borg/archiver/__init__.py +++ b/src/borg/archiver/__init__.py @@ -15,7 +15,7 @@ try: import shlex import signal import time - from datetime import datetime + from datetime import datetime, timezone from ..logger import create_logger, setup_logging @@ -27,6 +27,7 @@ try: from ..helpers import Error, set_ec from ..helpers import format_file_size from ..helpers import remove_surrogates + from ..helpers import DatetimeWrapper, replace_placeholders from ..helpers import check_python, check_extension_modules from ..helpers import is_slow_msgpack, is_supported_msgpack, sysinfo from ..helpers import signal_handler, raising_signal_handler, SigHup, SigTerm @@ -402,8 +403,18 @@ class Archiver( } if func not in bypass_allowed: raise Error("Not allowed to bypass locking mechanism for chosen command") + # we can only have a complete knowledge of placeholder replacements we should do **after** arg parsing, + # e.g. due to options like --timestamp that override the current time. + # thus we have to initialize replace_placeholders here and process all args that need placeholder replacement. if getattr(args, "timestamp", None): + replace_placeholders.override("now", DatetimeWrapper(args.timestamp)) + replace_placeholders.override("utcnow", DatetimeWrapper(args.timestamp.astimezone(timezone.utc))) args.location = args.location.with_timestamp(args.timestamp) + for name in "name", "other_name", "newname", "glob_archives", "comment": + value = getattr(args, name, None) + if value is not None: + setattr(args, name, replace_placeholders(value)) + return args def prerun_checks(self, logger, is_serve): diff --git a/src/borg/archiver/_common.py b/src/borg/archiver/_common.py index 29da8d1cd..a3af90638 100644 --- a/src/borg/archiver/_common.py +++ b/src/borg/archiver/_common.py @@ -8,7 +8,7 @@ from ..archive import Archive from ..constants import * # NOQA from ..cache import Cache, assert_secure from ..helpers import Error -from ..helpers import GlobSpec, SortBySpec, positive_int_validator, location_validator, Location +from ..helpers import SortBySpec, positive_int_validator, location_validator, Location from ..helpers.nanorst import rst_to_terminal from ..manifest import Manifest, AI_HUMAN_SORT_KEYS from ..patterns import PatternMatcher @@ -363,7 +363,6 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True): "--glob-archives", metavar="GLOB", dest="glob_archives", - type=GlobSpec, action=Highlander, help="only consider archive names matching the glob. " 'sh: rules apply, see "borg help patterns".', ) diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index cf5272a1d..af0461113 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -17,7 +17,7 @@ from ..cache import Cache from ..constants import * # NOQA from ..compress import CompressionSpec from ..helpers import ChunkerParams -from ..helpers import NameSpec, CommentSpec, FilesCacheMode +from ..helpers import NameSpec, FilesCacheMode from ..helpers import eval_escapes from ..helpers import timestamp from ..helpers import get_cache_dir, os_stat @@ -806,12 +806,7 @@ class CreateMixIn: archive_group = subparser.add_argument_group("Archive options") archive_group.add_argument( - "--comment", - dest="comment", - metavar="COMMENT", - type=CommentSpec, - default="", - help="add a comment text to the archive", + "--comment", dest="comment", metavar="COMMENT", default="", help="add a comment text to the archive" ) archive_group.add_argument( "--timestamp", diff --git a/src/borg/archiver/recreate_cmd.py b/src/borg/archiver/recreate_cmd.py index 544588d6c..40f2d6f26 100644 --- a/src/borg/archiver/recreate_cmd.py +++ b/src/borg/archiver/recreate_cmd.py @@ -6,7 +6,6 @@ from ..archive import ArchiveRecreater from ..constants import * # NOQA from ..compress import CompressionSpec from ..helpers import archivename_validator, ChunkerParams -from ..helpers import CommentSpec from ..helpers import timestamp from ..manifest import Manifest @@ -162,12 +161,7 @@ class RecreateMixIn: help="write checkpoint every SECONDS seconds (Default: 1800)", ) archive_group.add_argument( - "--comment", - dest="comment", - metavar="COMMENT", - type=CommentSpec, - default=None, - help="add a comment text to the archive", + "--comment", dest="comment", metavar="COMMENT", default=None, help="add a comment text to the archive" ) archive_group.add_argument( "--timestamp", diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index fc05b2b47..ba9bdb37a 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -24,7 +24,7 @@ from .parseformat import ChunkerParams, FilesCacheMode, partial_format, Datetime from .parseformat import format_file_size, parse_file_size, FileSize, parse_storage_quota from .parseformat import sizeof_fmt, sizeof_fmt_iec, sizeof_fmt_decimal from .parseformat import format_line, replace_placeholders, PlaceholderError -from .parseformat import PrefixSpec, GlobSpec, CommentSpec, SortBySpec, NameSpec +from .parseformat import SortBySpec, NameSpec from .parseformat import format_archive, parse_stringified_list, clean_lines from .parseformat import Location, location_validator, archivename_validator from .parseformat import BaseFormatter, ArchiveFormatter, ItemFormatter, file_status diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index d1508aef7..ced8e77ef 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -185,7 +185,7 @@ def format_line(format, data): raise PlaceholderError(format, data, e.__class__.__name__, str(e)) -def replace_placeholders(text, overrides={}): +def _replace_placeholders(text, overrides={}): """Replace placeholders in text with their values.""" from ..platform import fqdn, hostname, getosusername @@ -208,13 +208,26 @@ def replace_placeholders(text, overrides={}): return format_line(text, data) -PrefixSpec = replace_placeholders +class PlaceholderReplacer: + def __init__(self): + self.reset() -GlobSpec = replace_placeholders + def override(self, key, value): + self.overrides[key] = value -NameSpec = replace_placeholders + def reset(self): + self.overrides = {} -CommentSpec = replace_placeholders + def __call__(self, text, overrides=None): + ovr = {} + ovr.update(self.overrides) + ovr.update(overrides or {}) + return _replace_placeholders(text, overrides=ovr) + + +replace_placeholders = PlaceholderReplacer() + +NameSpec = str def SortBySpec(text): diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 414d32e15..e6b3d88d9 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -1393,6 +1393,16 @@ class ArchiverTestCase(ArchiverTestCaseBase): self.cmd(f"--repo={self.repository_location}", "rinfo") self.cmd(f"--repo={self.repository_location}", "check") + def test_create_archivename_with_placeholder(self): + self.create_test_files() + self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) + ts = "1999-12-31T23:59:59" + name_given = "test-{now}" # placeholder in archive name gets replaced by borg + name_expected = f"test-{ts}" # placeholder in f-string gets replaced by python + self.cmd(f"--repo={self.repository_location}", "create", f"--timestamp={ts}", name_given, "input") + list_output = self.cmd(f"--repo={self.repository_location}", "rlist", "--short") + assert name_expected in list_output + def test_extract_pattern_opt(self): self.cmd(f"--repo={self.repository_location}", "rcreate", RK_ENCRYPTION) self.create_regular_file("file1", size=1024 * 80) diff --git a/src/borg/testsuite/helpers.py b/src/borg/testsuite/helpers.py index 9222c7d96..7bb9f814f 100644 --- a/src/borg/testsuite/helpers.py +++ b/src/borg/testsuite/helpers.py @@ -920,6 +920,7 @@ def test_format_line_erroneous(): def test_replace_placeholders(): + replace_placeholders.reset() # avoid overrides are spoiled by previous tests now = datetime.now() assert " " not in replace_placeholders("{now}") assert int(replace_placeholders("{now:%Y}")) == now.year