upstream/borgbackup
1
0
Fork 0
mirror of https://github.com/borgbackup/borg.git synced 2026-06-14 20:13:21 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge pull request #8028 from ThomasWaldmann/modernize-msgpack-1.4

Browse source 
modernize msgpack wrapper
This commit is contained in:
TW 2024-01-07 05:39:12 +01:00 committed by GitHub
commit 499e07d842
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 16 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

104
src/borg/helpers/msgpack.py
View file

@ -3,8 +3,9 @@ from ..constants import * # NOQA
# wrapping msgpack ---------------------------------------------------------------------------------------------------
#
# due to the planned breaking api changes in upstream msgpack, we wrap it the way we need it -
# due to the breaking api changes in upstream msgpack (from 0.x to 1.0), we wrapped it the way we need it -
# to avoid having lots of clutter in the calling code. see tickets #968 and #3632.
# as borg 1.4 now requires msgpack > 1.0 anyway, the wrapper has reduced functionality, but was kept.
#
# Packing
# -------
@ -30,8 +31,6 @@ from msgpack import OutOfData
version = mp_version
_post_100 = version >= (1, 0, 0)
class PackException(Exception):
"""Exception while msgpack packing"""
@ -73,35 +72,19 @@ def pack(o, stream, *, use_bin_type=False, unicode_errors=None, **kwargs):
raise PackException(e)
# Note: after requiring msgpack >= 0.6.1 we can remove the max_*_len args and
# rely on msgpack auto-computing DoS-safe max values from len(data) for
# unpack(data) or from max_buffer_len for Unpacker(max_buffer_len=N).
# maybe we can also use that to simplify get_limited_unpacker().
class Unpacker(mp_Unpacker):
def __init__(self, file_like=None, *, read_size=0, use_list=True, raw=True,
object_hook=None, object_pairs_hook=None, list_hook=None,
unicode_errors=None, max_buffer_size=0,
ext_hook=ExtType,
strict_map_key=False,
max_str_len=2147483647, # 2**32-1
max_bin_len=2147483647,
max_array_len=2147483647,
max_map_len=2147483647,
max_ext_len=2147483647):
strict_map_key=False):
assert raw is True
assert unicode_errors is None
kw = dict(file_like=file_like, read_size=read_size, use_list=use_list, raw=raw,
object_hook=object_hook, object_pairs_hook=object_pairs_hook, list_hook=list_hook,
unicode_errors=unicode_errors, max_buffer_size=max_buffer_size,
ext_hook=ext_hook,
max_str_len=max_str_len,
max_bin_len=max_bin_len,
max_array_len=max_array_len,
max_map_len=max_map_len,
max_ext_len=max_ext_len)
if _post_100:
kw['strict_map_key'] = strict_map_key
strict_map_key=strict_map_key)
super().__init__(**kw)
def unpack(self):
@ -123,49 +106,21 @@ class Unpacker(mp_Unpacker):
next = __next__
def unpackb(packed, *, raw=True, unicode_errors=None,
strict_map_key=False,
max_str_len=2147483647, # 2**32-1
max_bin_len=2147483647,
max_array_len=2147483647,
max_map_len=2147483647,
max_ext_len=2147483647,
**kwargs):
def unpackb(packed, *, raw=True, unicode_errors=None, strict_map_key=False, **kwargs):
assert unicode_errors is None
try:
kw = dict(raw=raw, unicode_errors=unicode_errors,
max_str_len=max_str_len,
max_bin_len=max_bin_len,
max_array_len=max_array_len,
max_map_len=max_map_len,
max_ext_len=max_ext_len)
kw = dict(raw=raw, unicode_errors=unicode_errors, strict_map_key=strict_map_key)
kw.update(kwargs)
if _post_100:
kw['strict_map_key'] = strict_map_key
return mp_unpackb(packed, **kw)
except Exception as e:
raise UnpackException(e)
def unpack(stream, *, raw=True, unicode_errors=None,
strict_map_key=False,
max_str_len=2147483647, # 2**32-1
max_bin_len=2147483647,
max_array_len=2147483647,
max_map_len=2147483647,
max_ext_len=2147483647,
**kwargs):
def unpack(stream, *, raw=True, unicode_errors=None, strict_map_key=False, **kwargs):
assert unicode_errors is None
try:
kw = dict(raw=raw, unicode_errors=unicode_errors,
max_str_len=max_str_len,
max_bin_len=max_bin_len,
max_array_len=max_array_len,
max_map_len=max_map_len,
max_ext_len=max_ext_len)
kw = dict(raw=raw, unicode_errors=unicode_errors, strict_map_key=strict_map_key)
kw.update(kwargs)
if _post_100:
kw['strict_map_key'] = strict_map_key
return mp_unpack(stream, **kw)
except Exception as e:
raise UnpackException(e)
@ -188,42 +143,15 @@ def is_supported_msgpack():
def get_limited_unpacker(kind):
"""return a limited Unpacker because we should not trust msgpack data received from remote"""
args = dict(use_list=False, # return tuples, not lists
max_bin_len=0, # not used
max_ext_len=0, # not used
max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE),
max_str_len=MAX_OBJECT_SIZE, # a chunk or other repo object
)
if kind == 'server':
args.update(dict(max_array_len=100, # misc. cmd tuples
max_map_len=100, # misc. cmd dicts
))
elif kind == 'client':
args.update(dict(max_array_len=LIST_SCAN_LIMIT, # result list from repo.list() / .scan()
max_map_len=100, # misc. result dicts
))
elif kind == 'manifest':
args.update(dict(use_list=True, # default value
max_array_len=100, # ITEM_KEYS ~= 22
max_map_len=MAX_ARCHIVES, # list of archives
max_str_len=255, # archive name
object_hook=StableDict,
))
elif kind == 'archive':
args.update(dict(use_list=True, # default value
max_map_len=100, # ARCHIVE_KEYS ~= 20
max_str_len=10000, # comment
object_hook=StableDict,
))
elif kind == 'key':
args.update(dict(use_list=True, # default value
max_array_len=0, # not used
max_map_len=10, # EncryptedKey dict
max_str_len=4000, # inner key data
object_hook=StableDict,
))
# Note: msgpack >= 0.6.1 auto-computes DoS-safe max values from len(data) for
# unpack(data) or from max_buffer_size for Unpacker(max_buffer_size=N).
args = dict(use_list=False, max_buffer_size=3 * max(BUFSIZE, MAX_OBJECT_SIZE)) # return tuples, not lists
if kind in ('server', 'client'):
pass # nothing special
elif kind in ('manifest', 'archive', 'key'):
args.update(dict(use_list=True, object_hook=StableDict)) # default value
else:
raise ValueError('kind must be "server", "client", "manifest" or "key"')
raise ValueError('kind must be "server", "client", "manifest", "archive" or "key"')
return Unpacker(**args)