diff --git a/src/borg/helpers/msgpack.py b/src/borg/helpers/msgpack.py index 5c30cbb78..ee3f47685 100644 --- a/src/borg/helpers/msgpack.py +++ b/src/borg/helpers/msgpack.py @@ -3,8 +3,9 @@ from ..constants import * # NOQA # wrapping msgpack --------------------------------------------------------------------------------------------------- # -# due to the planned breaking api changes in upstream msgpack, we wrap it the way we need it - +# due to the breaking api changes in upstream msgpack (from 0.x to 1.0), we wrapped it the way we need it - # to avoid having lots of clutter in the calling code. see tickets #968 and #3632. +# as borg 1.4 now requires msgpack > 1.0 anyway, the wrapper has reduced functionality, but was kept. # # Packing # ------- @@ -30,8 +31,6 @@ from msgpack import OutOfData version = mp_version -_post_100 = version >= (1, 0, 0) - class PackException(Exception): """Exception while msgpack packing""" @@ -73,35 +72,19 @@ def pack(o, stream, *, use_bin_type=False, unicode_errors=None, **kwargs): raise PackException(e) -# Note: after requiring msgpack >= 0.6.1 we can remove the max_*_len args and -# rely on msgpack auto-computing DoS-safe max values from len(data) for -# unpack(data) or from max_buffer_len for Unpacker(max_buffer_len=N). -# maybe we can also use that to simplify get_limited_unpacker(). - class Unpacker(mp_Unpacker): def __init__(self, file_like=None, *, read_size=0, use_list=True, raw=True, object_hook=None, object_pairs_hook=None, list_hook=None, unicode_errors=None, max_buffer_size=0, ext_hook=ExtType, - strict_map_key=False, - max_str_len=2147483647, # 2**32-1 - max_bin_len=2147483647, - max_array_len=2147483647, - max_map_len=2147483647, - max_ext_len=2147483647): + strict_map_key=False): assert raw is True assert unicode_errors is None kw = dict(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw, object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook, unicode_errors=unicode_errors, max_buffer_size=max_buffer_size, ext_hook=ext_hook, - max_str_len=max_str_len, - max_bin_len=max_bin_len, - max_array_len=max_array_len, - max_map_len=max_map_len, - max_ext_len=max_ext_len) - if _post_100: - kw['strict_map_key'] = strict_map_key + strict_map_key=strict_map_key) super().__init__(**kw) def unpack(self): @@ -123,49 +106,21 @@ class Unpacker(mp_Unpacker): next = __next__ -def unpackb(packed, *, raw=True, unicode_errors=None, - strict_map_key=False, - max_str_len=2147483647, # 2**32-1 - max_bin_len=2147483647, - max_array_len=2147483647, - max_map_len=2147483647, - max_ext_len=2147483647, - **kwargs): +def unpackb(packed, *, raw=True, unicode_errors=None, strict_map_key=False, **kwargs): assert unicode_errors is None try: - kw = dict(raw=raw, unicode_errors=unicode_errors, - max_str_len=max_str_len, - max_bin_len=max_bin_len, - max_array_len=max_array_len, - max_map_len=max_map_len, - max_ext_len=max_ext_len) + kw = dict(raw=raw, unicode_errors=unicode_errors, strict_map_key=strict_map_key) kw.update(kwargs) - if _post_100: - kw['strict_map_key'] = strict_map_key return mp_unpackb(packed, **kw) except Exception as e: raise UnpackException(e) -def unpack(stream, *, raw=True, unicode_errors=None, - strict_map_key=False, - max_str_len=2147483647, # 2**32-1 - max_bin_len=2147483647, - max_array_len=2147483647, - max_map_len=2147483647, - max_ext_len=2147483647, - **kwargs): +def unpack(stream, *, raw=True, unicode_errors=None, strict_map_key=False, **kwargs): assert unicode_errors is None try: - kw = dict(raw=raw, unicode_errors=unicode_errors, - max_str_len=max_str_len, - max_bin_len=max_bin_len, - max_array_len=max_array_len, - max_map_len=max_map_len, - max_ext_len=max_ext_len) + kw = dict(raw=raw, unicode_errors=unicode_errors, strict_map_key=strict_map_key) kw.update(kwargs) - if _post_100: - kw['strict_map_key'] = strict_map_key return mp_unpack(stream, **kw) except Exception as e: raise UnpackException(e) @@ -188,42 +143,15 @@ def is_supported_msgpack(): def get_limited_unpacker(kind): """return a limited Unpacker because we should not trust msgpack data received from remote""" - args = dict(use_list=False, # return tuples, not lists - max_bin_len=0, # not used - max_ext_len=0, # not used - max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE), - max_str_len=MAX_OBJECT_SIZE, # a chunk or other repo object - ) - if kind == 'server': - args.update(dict(max_array_len=100, # misc. cmd tuples - max_map_len=100, # misc. cmd dicts - )) - elif kind == 'client': - args.update(dict(max_array_len=LIST_SCAN_LIMIT, # result list from repo.list() / .scan() - max_map_len=100, # misc. result dicts - )) - elif kind == 'manifest': - args.update(dict(use_list=True, # default value - max_array_len=100, # ITEM_KEYS ~= 22 - max_map_len=MAX_ARCHIVES, # list of archives - max_str_len=255, # archive name - object_hook=StableDict, - )) - elif kind == 'archive': - args.update(dict(use_list=True, # default value - max_map_len=100, # ARCHIVE_KEYS ~= 20 - max_str_len=10000, # comment - object_hook=StableDict, - )) - elif kind == 'key': - args.update(dict(use_list=True, # default value - max_array_len=0, # not used - max_map_len=10, # EncryptedKey dict - max_str_len=4000, # inner key data - object_hook=StableDict, - )) + # Note: msgpack >= 0.6.1 auto-computes DoS-safe max values from len(data) for + # unpack(data) or from max_buffer_size for Unpacker(max_buffer_size=N). + args = dict(use_list=False, max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE)) # return tuples, not lists + if kind in ('server', 'client'): + pass # nothing special + elif kind in ('manifest', 'archive', 'key'): + args.update(dict(use_list=True, object_hook=StableDict)) # default value else: - raise ValueError('kind must be "server", "client", "manifest" or "key"') + raise ValueError('kind must be "server", "client", "manifest", "archive" or "key"') return Unpacker(**args)