From 4c0012bddfc91e45167f65293369814511695de9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 3 Aug 2015 00:31:33 +0200 Subject: [PATCH] add lzma compression needs python 3.3+, on 3.2 it won't be available. --- borg/archiver.py | 6 ++++-- borg/compress.pyx | 30 +++++++++++++++++++++++++++++- borg/helpers.py | 6 ++++-- borg/testsuite/compress.py | 27 ++++++++++++++++++++++++--- 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/borg/archiver.py b/borg/archiver.py index 032313dbf..fb6db8a19 100644 --- a/borg/archiver.py +++ b/borg/archiver.py @@ -641,12 +641,14 @@ Type "Yes I am sure" if you understand this and want to continue.\n""") help='select compression algorithm and level, by giving a number: ' '0 == no compression [default], ' '1..9 == zlib level 1..9, ' - '10 == lz4. ' + '10 == lz4, ' + '20-29 == lzma level 0..9.' 'Alternatively, you can also give a name and optionally additional args: ' 'null == no compression, ' 'zlib == zlib (default level 6), ' 'zlib,0 .. zlib,9 == zlib (with level 0..9), ' - 'lz4 == lz4.') + 'lz4 == lz4, ' + 'lzma,0 .. lzma,9 == lzma (with level 0..9).') subparser.add_argument('archive', metavar='ARCHIVE', type=location_validator(archive=True), help='archive to create') diff --git a/borg/compress.pyx b/borg/compress.pyx index 03815b3a5..c1bdeff82 100644 --- a/borg/compress.pyx +++ b/borg/compress.pyx @@ -1,4 +1,8 @@ import zlib +try: + import lzma +except ImportError: + lzma = None cdef extern from "lz4.h": int LZ4_compress_limitedOutput(const char* source, char* dest, int inputSize, int maxOutputSize) nogil @@ -104,6 +108,29 @@ cdef class LZ4(CompressorBase): return dest[:osize] +class LZMA(CompressorBase): + """ + lzma compression / decompression (python 3.3+ stdlib) + """ + ID = b'\x02\x00' + name = 'lzma' + + def __init__(self, level=6, **kwargs): + super().__init__(**kwargs) + self.level = level + if lzma is None: + raise ValueError('No lzma support found.') + + def compress(self, data): + # we do not need integrity checks in lzma, we do that already + data = lzma.compress(data, preset=self.level, check=lzma.CHECK_NONE) + return super().compress(data) + + def decompress(self, data): + data = super().decompress(data) + return lzma.decompress(data) + + class ZLIB(CompressorBase): """ zlib compression / decompression (python stdlib) @@ -137,8 +164,9 @@ COMPRESSOR_TABLE = { CNULL.name: CNULL, LZ4.name: LZ4, ZLIB.name: ZLIB, + LZMA.name: LZMA, } -COMPRESSOR_LIST = [LZ4, CNULL, ZLIB, ] # check fast stuff first +COMPRESSOR_LIST = [LZ4, CNULL, ZLIB, LZMA, ] # check fast stuff first def get_compressor(name, **kwargs): cls = COMPRESSOR_TABLE[name] diff --git a/borg/helpers.py b/borg/helpers.py index 69a6db0db..020c263e7 100644 --- a/borg/helpers.py +++ b/borg/helpers.py @@ -302,13 +302,15 @@ def CompressionSpec(s): return dict(name='zlib', level=compression) if compression == 10: return dict(name='lz4') + if 20 <= compression <= 29: + return dict(name='lzma', level=compression-20) raise ValueError except ValueError: # --compression algo[,...] name = compression if name in ('null', 'lz4', ): return dict(name=name) - if name == 'zlib': + if name in ('zlib', 'lzma', ): if count < 2: level = 6 # default compression level in py stdlib elif count == 2: @@ -317,7 +319,7 @@ def CompressionSpec(s): raise ValueError else: raise ValueError - return dict(name='zlib', level=level) + return dict(name=name, level=level) raise ValueError diff --git a/borg/testsuite/compress.py b/borg/testsuite/compress.py index 441214e7b..6d7319c1b 100644 --- a/borg/testsuite/compress.py +++ b/borg/testsuite/compress.py @@ -1,4 +1,8 @@ import zlib +try: + import lzma +except ImportError: + lzma = None import pytest @@ -6,7 +10,7 @@ from ..compress import get_compressor, Compressor, CNULL, ZLIB, LZ4 buffer = bytes(2**16) -data = b'fooooooooobaaaaaaaar' +data = b'fooooooooobaaaaaaaar' * 10 params = dict(name='zlib', level=6, buffer=buffer) @@ -46,6 +50,16 @@ def test_zlib(): assert data == Compressor(**params).decompress(cdata) # autodetect +def test_lzma(): + if lzma is None: + pytest.skip("No lzma support found.") + c = get_compressor(name='lzma') + cdata = c.compress(data) + assert len(cdata) < len(data) + assert data == c.decompress(cdata) + assert data == Compressor(**params).decompress(cdata) # autodetect + + def test_autodetect_invalid(): with pytest.raises(ValueError): Compressor(**params).decompress(b'\xff\xfftotalcrap') @@ -68,13 +82,20 @@ def test_zlib_compat(): def test_compressor(): - for params in [ + params_list = [ dict(name='null', buffer=buffer), dict(name='lz4', buffer=buffer), dict(name='zlib', level=0, buffer=buffer), dict(name='zlib', level=6, buffer=buffer), dict(name='zlib', level=9, buffer=buffer), - ]: + ] + if lzma: + params_list += [ + dict(name='lzma', level=0, buffer=buffer), + dict(name='lzma', level=6, buffer=buffer), + dict(name='lzma', level=9, buffer=buffer), + ] + for params in params_list: c = Compressor(**params) assert data == c.decompress(c.compress(data))