mirror of
https://github.com/borgbackup/borg.git
synced 2026-06-11 09:59:19 -04:00
tests: only warn about "invalid" chunker params, fixes #7590
we previously allowed some weird chunker params, so we just warn about them instead of rejecting them. e.g. one could use super small chunk sizes. borg can do that, but you'll end up with a huge amount of chunks and very large hash tables (RAM usage) to manage them. also, one can violate the min <= mask <= max chunker param condition as in #7590 and it will somehow work, but will likely not dedup as good as when not violating that. borg2 **will** reject such strange chunker params, see #7586 for some ideas how to "fix" your repos.
This commit is contained in:
parent
965e8a957f
commit
5cc4e3af6a
2 changed files with 22 additions and 18 deletions
|
|
@ -95,10 +95,16 @@ def interval(s):
|
|||
|
||||
|
||||
def ChunkerParams(s):
|
||||
def reject_or_warn(msg, reject):
|
||||
if reject:
|
||||
raise argparse.ArgumentTypeError(msg)
|
||||
else:
|
||||
logger.warning(msg)
|
||||
|
||||
params = s.strip().split(',')
|
||||
count = len(params)
|
||||
if count == 0:
|
||||
raise argparse.ArgumentTypeError('no chunker params given')
|
||||
reject_or_warn('no chunker params given', True)
|
||||
algo = params[0].lower()
|
||||
if algo == CH_FIXED and 2 <= count <= 3: # fixed, block_size[, header_size]
|
||||
block_size = int(params[1])
|
||||
|
|
@ -109,11 +115,9 @@ def ChunkerParams(s):
|
|||
# or in-memory chunk management.
|
||||
# choose the block (chunk) size wisely: if you have a lot of data and you cut
|
||||
# it into very small chunks, you are asking for trouble!
|
||||
raise argparse.ArgumentTypeError('block_size must not be less than 64 Bytes')
|
||||
reject_or_warn('block_size must not be less than 64 Bytes', False)
|
||||
if block_size > MAX_DATA_SIZE or header_size > MAX_DATA_SIZE:
|
||||
raise argparse.ArgumentTypeError(
|
||||
'block_size and header_size must not exceed MAX_DATA_SIZE [%d]' % MAX_DATA_SIZE
|
||||
)
|
||||
reject_or_warn('block_size and header_size must not exceed MAX_DATA_SIZE [%d]' % MAX_DATA_SIZE, True)
|
||||
return algo, block_size, header_size
|
||||
if algo == 'default' and count == 1: # default
|
||||
return CHUNKER_PARAMS
|
||||
|
|
@ -121,16 +125,12 @@ def ChunkerParams(s):
|
|||
if algo == CH_BUZHASH and count == 5 or count == 4: # [buzhash, ]chunk_min, chunk_max, chunk_mask, window_size
|
||||
chunk_min, chunk_max, chunk_mask, window_size = (int(p) for p in params[count - 4:])
|
||||
if not (chunk_min <= chunk_mask <= chunk_max):
|
||||
raise argparse.ArgumentTypeError('required: chunk_min <= chunk_mask <= chunk_max')
|
||||
reject_or_warn('required: chunk_min <= chunk_mask <= chunk_max', False)
|
||||
if chunk_min < 6:
|
||||
# see comment in 'fixed' algo check
|
||||
raise argparse.ArgumentTypeError(
|
||||
'min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)'
|
||||
)
|
||||
reject_or_warn('min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)', False)
|
||||
if chunk_max > 23:
|
||||
raise argparse.ArgumentTypeError(
|
||||
'max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)'
|
||||
)
|
||||
reject_or_warn('max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)', True)
|
||||
return CH_BUZHASH, chunk_min, chunk_max, chunk_mask, window_size
|
||||
raise argparse.ArgumentTypeError('invalid chunker params')
|
||||
|
||||
|
|
|
|||
|
|
@ -349,16 +349,20 @@ def test_chunkerparams():
|
|||
assert ChunkerParams('fixed,4096') == ('fixed', 4096, 0)
|
||||
assert ChunkerParams('fixed,4096,200') == ('fixed', 4096, 200)
|
||||
# invalid values checking
|
||||
borg2 = False # for borg < 2, we only emit a warning, but do not raise ArgumentTypeError for some cases
|
||||
with pytest.raises(ArgumentTypeError):
|
||||
ChunkerParams('crap,1,2,3,4') # invalid algo
|
||||
with pytest.raises(ArgumentTypeError):
|
||||
ChunkerParams('buzhash,5,7,6,4095') # too small min. size
|
||||
if borg2:
|
||||
with pytest.raises(ArgumentTypeError):
|
||||
ChunkerParams('buzhash,5,7,6,4095') # too small min. size
|
||||
with pytest.raises(ArgumentTypeError):
|
||||
ChunkerParams('buzhash,19,24,21,4095') # too big max. size
|
||||
with pytest.raises(ArgumentTypeError):
|
||||
ChunkerParams('buzhash,23,19,21,4095') # violates min <= mask <= max
|
||||
with pytest.raises(ArgumentTypeError):
|
||||
ChunkerParams('fixed,63') # too small block size
|
||||
if borg2:
|
||||
with pytest.raises(ArgumentTypeError):
|
||||
ChunkerParams('buzhash,23,19,21,4095') # violates min <= mask <= max
|
||||
if borg2:
|
||||
with pytest.raises(ArgumentTypeError):
|
||||
ChunkerParams('fixed,63') # too small block size
|
||||
with pytest.raises(ArgumentTypeError):
|
||||
ChunkerParams('fixed,%d,%d' % (MAX_DATA_SIZE + 1, 4096)) # too big block size
|
||||
with pytest.raises(ArgumentTypeError):
|
||||
|
|
|
|||
Loading…
Reference in a new issue