tests: only warn about "invalid" chunker params, fixes #7590

we previously allowed some weird chunker params, so we just warn
about them instead of rejecting them.

e.g. one could use super small chunk sizes. borg can do that,
but you'll end up with a huge amount of chunks and very large
hash tables (RAM usage) to manage them.

also, one can violate the min <= mask <= max chunker param condition
as in #7590 and it will somehow work, but will likely not dedup as good
as when not violating that.

borg2 **will** reject such strange chunker params, see #7586 for
some ideas how to "fix" your repos.
This commit is contained in:
Thomas Waldmann 2023-05-19 17:46:29 +02:00
parent 965e8a957f
commit 5cc4e3af6a
No known key found for this signature in database
GPG key ID: 243ACFA951F78E01
2 changed files with 22 additions and 18 deletions

View file

@ -95,10 +95,16 @@ def interval(s):
def ChunkerParams(s):
def reject_or_warn(msg, reject):
if reject:
raise argparse.ArgumentTypeError(msg)
else:
logger.warning(msg)
params = s.strip().split(',')
count = len(params)
if count == 0:
raise argparse.ArgumentTypeError('no chunker params given')
reject_or_warn('no chunker params given', True)
algo = params[0].lower()
if algo == CH_FIXED and 2 <= count <= 3: # fixed, block_size[, header_size]
block_size = int(params[1])
@ -109,11 +115,9 @@ def ChunkerParams(s):
# or in-memory chunk management.
# choose the block (chunk) size wisely: if you have a lot of data and you cut
# it into very small chunks, you are asking for trouble!
raise argparse.ArgumentTypeError('block_size must not be less than 64 Bytes')
reject_or_warn('block_size must not be less than 64 Bytes', False)
if block_size > MAX_DATA_SIZE or header_size > MAX_DATA_SIZE:
raise argparse.ArgumentTypeError(
'block_size and header_size must not exceed MAX_DATA_SIZE [%d]' % MAX_DATA_SIZE
)
reject_or_warn('block_size and header_size must not exceed MAX_DATA_SIZE [%d]' % MAX_DATA_SIZE, True)
return algo, block_size, header_size
if algo == 'default' and count == 1: # default
return CHUNKER_PARAMS
@ -121,16 +125,12 @@ def ChunkerParams(s):
if algo == CH_BUZHASH and count == 5 or count == 4: # [buzhash, ]chunk_min, chunk_max, chunk_mask, window_size
chunk_min, chunk_max, chunk_mask, window_size = (int(p) for p in params[count - 4:])
if not (chunk_min <= chunk_mask <= chunk_max):
raise argparse.ArgumentTypeError('required: chunk_min <= chunk_mask <= chunk_max')
reject_or_warn('required: chunk_min <= chunk_mask <= chunk_max', False)
if chunk_min < 6:
# see comment in 'fixed' algo check
raise argparse.ArgumentTypeError(
'min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)'
)
reject_or_warn('min. chunk size exponent must not be less than 6 (2^6 = 64B min. chunk size)', False)
if chunk_max > 23:
raise argparse.ArgumentTypeError(
'max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)'
)
reject_or_warn('max. chunk size exponent must not be more than 23 (2^23 = 8MiB max. chunk size)', True)
return CH_BUZHASH, chunk_min, chunk_max, chunk_mask, window_size
raise argparse.ArgumentTypeError('invalid chunker params')

View file

@ -349,16 +349,20 @@ def test_chunkerparams():
assert ChunkerParams('fixed,4096') == ('fixed', 4096, 0)
assert ChunkerParams('fixed,4096,200') == ('fixed', 4096, 200)
# invalid values checking
borg2 = False # for borg < 2, we only emit a warning, but do not raise ArgumentTypeError for some cases
with pytest.raises(ArgumentTypeError):
ChunkerParams('crap,1,2,3,4') # invalid algo
with pytest.raises(ArgumentTypeError):
ChunkerParams('buzhash,5,7,6,4095') # too small min. size
if borg2:
with pytest.raises(ArgumentTypeError):
ChunkerParams('buzhash,5,7,6,4095') # too small min. size
with pytest.raises(ArgumentTypeError):
ChunkerParams('buzhash,19,24,21,4095') # too big max. size
with pytest.raises(ArgumentTypeError):
ChunkerParams('buzhash,23,19,21,4095') # violates min <= mask <= max
with pytest.raises(ArgumentTypeError):
ChunkerParams('fixed,63') # too small block size
if borg2:
with pytest.raises(ArgumentTypeError):
ChunkerParams('buzhash,23,19,21,4095') # violates min <= mask <= max
if borg2:
with pytest.raises(ArgumentTypeError):
ChunkerParams('fixed,63') # too small block size
with pytest.raises(ArgumentTypeError):
ChunkerParams('fixed,%d,%d' % (MAX_DATA_SIZE + 1, 4096)) # too big block size
with pytest.raises(ArgumentTypeError):