diff --git a/src/borg/archive.py b/src/borg/archive.py index 509c46ddb..61f1f0bfa 100644 --- a/src/borg/archive.py +++ b/src/borg/archive.py @@ -25,7 +25,6 @@ from . import xattr from .chunkers import get_chunker, Chunk from .cache import ChunkListEntry, build_chunkindex_from_repo, delete_chunkindex_cache from .crypto.key import key_factory, UnsupportedPayloadError -from .compress import CompressionSpec from .constants import * # NOQA from .crypto.low_level import IntegrityError as IntegrityErrorBase from .helpers import BackupError, BackupRaceConditionError, BackupItemExcluded @@ -35,7 +34,7 @@ from .helpers import HardLinkManager from .helpers import ChunkIteratorFileWrapper, open_item from .helpers import Error, IntegrityError, set_ec from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns -from .helpers import parse_timestamp, archive_ts_now +from .helpers import parse_timestamp, archive_ts_now, CompressionSpec from .helpers import OutputTimestamp, format_timedelta, format_file_size, file_status, FileSize from .helpers import safe_encode, make_path_safe, remove_surrogates, text_to_json, join_cmd, remove_dotdot_prefixes from .helpers import StableDict diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py index d152ff834..c5fac73e3 100644 --- a/src/borg/archiver/benchmark_cmd.py +++ b/src/borg/archiver/benchmark_cmd.py @@ -7,7 +7,7 @@ import time from ..constants import * # NOQA from ..crypto.key import FlexiKey -from ..helpers import format_file_size +from ..helpers import format_file_size, CompressionSpec from ..helpers import json_print from ..helpers import msgpack from ..helpers import get_reset_ec @@ -302,8 +302,6 @@ class BenchmarkMixIn: else: print(f"{spec:<24} {number_kdf:<10} {dt:.3f}s") - from ..compress import CompressionSpec - if not args.json: print("Compression ====================================================") else: diff --git a/src/borg/archiver/completion_cmd.py b/src/borg/archiver/completion_cmd.py index 22b7208bf..1268abf84 100644 --- a/src/borg/archiver/completion_cmd.py +++ b/src/borg/archiver/completion_cmd.py @@ -63,13 +63,13 @@ from ..helpers import ( FilesCacheMode, PathSpec, ChunkerParams, + CompressionSpec, tag_validator, relative_time_marker_validator, parse_file_size, ) from ..helpers.argparsing import ArgumentParser, RawDescriptionHelpFormatter from ..helpers.time import timestamp -from ..compress import CompressionSpec from ..helpers.parseformat import partial_format from ..manifest import AI_HUMAN_SORT_KEYS diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index e888ff975..b5c3bf2c6 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -15,8 +15,7 @@ from ..archive import BackupError, BackupOSError, BackupItemExcluded, backup_io, from ..archive import FilesystemObjectProcessors, MetadataCollector, ChunksProcessor from ..cache import Cache from ..constants import * # NOQA -from ..compress import CompressionSpec -from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec +from ..helpers import comment_validator, ChunkerParams, FilesystemPathSpec, CompressionSpec from ..helpers import archivename_validator, FilesCacheMode from ..helpers import eval_escapes from ..helpers import timestamp, archive_ts_now diff --git a/src/borg/archiver/debug_cmd.py b/src/borg/archiver/debug_cmd.py index 4e47f1e5c..366c20aa0 100644 --- a/src/borg/archiver/debug_cmd.py +++ b/src/borg/archiver/debug_cmd.py @@ -2,14 +2,13 @@ import json import textwrap from ..archive import Archive -from ..compress import CompressionSpec from ..constants import * # NOQA from ..helpers import msgpack from ..helpers import sysinfo from ..helpers import bin_to_hex, hex_to_bin, prepare_dump_dict from ..helpers import dash_open from ..helpers import StableDict -from ..helpers import archivename_validator +from ..helpers import archivename_validator, CompressionSpec from ..helpers import CommandError, RTError from ..helpers.argparsing import ArgumentParser, RawDescriptionHelpFormatter from ..manifest import Manifest diff --git a/src/borg/archiver/recreate_cmd.py b/src/borg/archiver/recreate_cmd.py index 5a26e5557..effb6d3d9 100644 --- a/src/borg/archiver/recreate_cmd.py +++ b/src/borg/archiver/recreate_cmd.py @@ -2,8 +2,7 @@ from ._common import with_repository, Highlander from ._common import build_matcher from ..archive import ArchiveRecreater from ..constants import * # NOQA -from ..compress import CompressionSpec -from ..helpers import archivename_validator, comment_validator, PathSpec, ChunkerParams, bin_to_hex +from ..helpers import archivename_validator, comment_validator, PathSpec, ChunkerParams, bin_to_hex, CompressionSpec from ..helpers import timestamp from ..helpers.argparsing import ArgumentParser, RawDescriptionHelpFormatter from ..manifest import Manifest diff --git a/src/borg/archiver/repo_compress_cmd.py b/src/borg/archiver/repo_compress_cmd.py index 769dd8ac1..58f3ea402 100644 --- a/src/borg/archiver/repo_compress_cmd.py +++ b/src/borg/archiver/repo_compress_cmd.py @@ -2,9 +2,9 @@ from collections import defaultdict from ._common import with_repository, Highlander from ..constants import * # NOQA -from ..compress import CompressionSpec, ObfuscateSize, Auto, COMPRESSOR_TABLE +from ..compress import ObfuscateSize, Auto, COMPRESSOR_TABLE from ..hashindex import ChunkIndex -from ..helpers import sig_int, ProgressIndicatorPercent, Error +from ..helpers import sig_int, ProgressIndicatorPercent, Error, CompressionSpec from ..helpers.argparsing import ArgumentParser, RawDescriptionHelpFormatter from ..repository import Repository from ..remote import RemoteRepository diff --git a/src/borg/archiver/tar_cmds.py b/src/borg/archiver/tar_cmds.py index 7702c2449..c3718c0ef 100644 --- a/src/borg/archiver/tar_cmds.py +++ b/src/borg/archiver/tar_cmds.py @@ -5,7 +5,6 @@ import stat import tarfile from ..archive import Archive, TarfileObjectProcessors, ChunksProcessor -from ..compress import CompressionSpec from ..constants import * # NOQA from ..helpers import HardLinkManager, IncludePatternNeverMatchedWarning from ..helpers import ProgressIndicatorPercent @@ -13,7 +12,7 @@ from ..helpers import dash_open from ..helpers import msgpack from ..helpers import create_filter_process from ..helpers import ChunkIteratorFileWrapper -from ..helpers import archivename_validator, comment_validator, PathSpec, ChunkerParams +from ..helpers import archivename_validator, comment_validator, PathSpec, ChunkerParams, CompressionSpec from ..helpers import remove_surrogates from ..helpers import timestamp, archive_ts_now from ..helpers import basic_json_data, json_print diff --git a/src/borg/archiver/transfer_cmd.py b/src/borg/archiver/transfer_cmd.py index dab3abc66..8e868c88a 100644 --- a/src/borg/archiver/transfer_cmd.py +++ b/src/borg/archiver/transfer_cmd.py @@ -1,13 +1,12 @@ from ._common import with_repository, with_other_repository, Highlander from ..archive import Archive, cached_hash, DownloadPipeline from ..chunkers import get_chunker -from ..compress import CompressionSpec from ..constants import * # NOQA from ..crypto.key import uses_same_id_hash, uses_same_chunker_secret from ..helpers import Error from ..helpers import location_validator, Location, archivename_validator, comment_validator from ..helpers import format_file_size, bin_to_hex -from ..helpers import ChunkerParams, ChunkIteratorFileWrapper +from ..helpers import ChunkerParams, ChunkIteratorFileWrapper, CompressionSpec from ..helpers.argparsing import ArgumentParser, ArgumentTypeError, RawDescriptionHelpFormatter from ..item import ChunkListEntry from ..manifest import Manifest diff --git a/src/borg/compress.pyi b/src/borg/compress.pyi index c8a271a1c..d627e6e20 100644 --- a/src/borg/compress.pyi +++ b/src/borg/compress.pyi @@ -2,12 +2,6 @@ from typing import Any, Type, Dict, Tuple def get_compressor(name: str, **kwargs) -> Any: ... -class CompressionSpec: - def __init__(self, spec: str) -> None: ... - @property - def compressor(self) -> Any: ... - inner: CompressionSpec - class Compressor: def __init__(self, name: Any = ..., **kwargs) -> None: ... def compress(self, meta: Dict, data: bytes) -> Tuple[Dict, bytes]: ... diff --git a/src/borg/compress.pyx b/src/borg/compress.pyx index 733828720..faf303127 100644 --- a/src/borg/compress.pyx +++ b/src/borg/compress.pyx @@ -614,66 +614,3 @@ class Compressor: return cls, (255 if cls.name == 'zlib_legacy' else level) else: raise ValueError('No decompressor for this data found: %r.', data[:2]) - -class CompressionSpec: - def __init__(self, s): - if isinstance(s, CompressionSpec): - self.__dict__.update(s.__dict__) - return - values = s.split(',') - count = len(values) - if count < 1: - raise ArgumentTypeError("not enough arguments") - # --compression algo[,level] - self.name = values[0] - if self.name in ('none', 'lz4', ): - return - elif self.name in ('zlib', 'lzma', 'zlib_legacy'): # zlib_legacy just for testing - if count < 2: - level = 6 # default compression level in py stdlib - elif count == 2: - level = int(values[1]) - if not 0 <= level <= 9: - raise ArgumentTypeError("level must be >= 0 and <= 9") - else: - raise ArgumentTypeError("too many arguments") - self.level = level - elif self.name in ('zstd', ): - if count < 2: - level = 3 # default compression level in zstd - elif count == 2: - level = int(values[1]) - if not 1 <= level <= 22: - raise ArgumentTypeError("level must be >= 1 and <= 22") - else: - raise ArgumentTypeError("too many arguments") - self.level = level - elif self.name == 'auto': - if 2 <= count <= 3: - compression = ','.join(values[1:]) - else: - raise ArgumentTypeError("bad arguments") - self.inner = CompressionSpec(compression) - elif self.name == 'obfuscate': - if 3 <= count <= 5: - level = int(values[1]) - if not ((1 <= level <= 6) or (110 <= level <= 123) or (level == 250)): - raise ArgumentTypeError("level must be (inclusively) within 1...6, 110...123 or equal to 250") - self.level = level - compression = ','.join(values[2:]) - else: - raise ArgumentTypeError("bad arguments") - self.inner = CompressionSpec(compression) - else: - raise ArgumentTypeError("unsupported compression type") - - @property - def compressor(self): - if self.name in ('none', 'lz4', ): - return get_compressor(self.name) - elif self.name in ('zlib', 'lzma', 'zstd', 'zlib_legacy'): - return get_compressor(self.name, level=self.level) - elif self.name == 'auto': - return get_compressor(self.name, compressor=self.inner.compressor) - elif self.name == 'obfuscate': - return get_compressor(self.name, level=self.level, compressor=self.inner.compressor) diff --git a/src/borg/helpers/__init__.py b/src/borg/helpers/__init__.py index 7902d5bb6..d9e90f5ce 100644 --- a/src/borg/helpers/__init__.py +++ b/src/borg/helpers/__init__.py @@ -32,6 +32,7 @@ from .parseformat import ( PathSpec, FilesystemPathSpec, SortBySpec, + CompressionSpec, ChunkerParams, FilesCacheMode, partial_format, diff --git a/src/borg/helpers/parseformat.py b/src/borg/helpers/parseformat.py index 844cd3da4..c84326941 100644 --- a/src/borg/helpers/parseformat.py +++ b/src/borg/helpers/parseformat.py @@ -165,6 +165,72 @@ def interval(s): return seconds +class CompressionSpec: + def __init__(self, s): + if isinstance(s, CompressionSpec): + self.__dict__.update(s.__dict__) + return + values = s.split(",") + count = len(values) + if count < 1: + raise ArgumentTypeError("not enough arguments") + # --compression algo[,level] + self.name = values[0] + if self.name in ("none", "lz4"): + return + elif self.name in ("zlib", "lzma", "zlib_legacy"): # zlib_legacy just for testing + if count < 2: + level = 6 # default compression level in py stdlib + elif count == 2: + level = int(values[1]) + if not 0 <= level <= 9: + raise ArgumentTypeError("level must be >= 0 and <= 9") + else: + raise ArgumentTypeError("too many arguments") + self.level = level + elif self.name in ("zstd",): + if count < 2: + level = 3 # default compression level in zstd + elif count == 2: + level = int(values[1]) + if not 1 <= level <= 22: + raise ArgumentTypeError("level must be >= 1 and <= 22") + else: + raise ArgumentTypeError("too many arguments") + self.level = level + elif self.name == "auto": + if 2 <= count <= 3: + compression = ",".join(values[1:]) + else: + raise ArgumentTypeError("bad arguments") + self.inner = CompressionSpec(compression) + elif self.name == "obfuscate": + if 3 <= count <= 5: + level = int(values[1]) + if not ((1 <= level <= 6) or (110 <= level <= 123) or (level == 250)): + raise ArgumentTypeError("level must be (inclusively) within 1...6, 110...123 or equal to 250") + self.level = level + compression = ",".join(values[2:]) + else: + raise ArgumentTypeError("bad arguments") + self.inner = CompressionSpec(compression) + else: + raise ArgumentTypeError("unsupported compression type") + + @property + def compressor(self): + from ..compress import get_compressor + + if self.name in ("none", "lz4"): + return get_compressor(self.name) + elif self.name in ("zlib", "lzma", "zstd", "zlib_legacy"): + return get_compressor(self.name, level=self.level) + elif self.name == "auto": + return get_compressor(self.name, compressor=self.inner.compressor) + elif self.name == "obfuscate": + return get_compressor(self.name, level=self.level, compressor=self.inner.compressor) + + def ChunkerParams(s): if isinstance(s, tuple): return s diff --git a/src/borg/testsuite/compress_test.py b/src/borg/testsuite/compress_test.py index 7dbc7574d..62ef59f51 100644 --- a/src/borg/testsuite/compress_test.py +++ b/src/borg/testsuite/compress_test.py @@ -3,7 +3,8 @@ import zlib import pytest -from ..compress import get_compressor, Compressor, CompressionSpec, CNONE, ZLIB, LZ4, LZMA, ZSTD, Auto +from ..compress import get_compressor, Compressor, CNONE, ZLIB, LZ4, LZMA, ZSTD, Auto +from ..helpers import CompressionSpec from ..constants import ROBJ_FILE_STREAM, ROBJ_ARCHIVE_META from ..helpers.argparsing import ArgumentTypeError