From b275d18c6be480e2444703ee227aab6dd8c2f35d Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 15 Mar 2015 04:45:52 +0100 Subject: [PATCH] msgpack: limit unpacker, use bin type for bytes we only need a little metadata and 1 medium sized piece of data, so avoid memory allocation issues that could be caused by tampered input data. bin type is more appropriate for binary data than str type (which could be also encoded text). --- attic/key.py | 21 ++++++++++++++------- attic/testsuite/key.py | 20 ++++++++++---------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/attic/key.py b/attic/key.py index 0a9bad461..5c26e1661 100644 --- a/attic/key.py +++ b/attic/key.py @@ -658,12 +658,19 @@ def parser03(all_data): # new & flexible meta is a Meta namedtuple and contains all required information about data. data is maybe compressed (see meta) and maybe encrypted (see meta). """ - # TODO use Unpacker(..., max_*_len=NOTMORETHANNEEDED) to avoid any memory - # allocation issues on untrusted and potentially tampered input data. - # Problem: we currently must use older msgpack because pure python impl. - # is broken in 0.4.2 < version <= 0.4.5, but this api is only offered by - # more recent ones, not by 0.4.2. So, fix here when 0.4.6 is out. :-( - meta_tuple, data = msgpack.unpackb(all_data[1:]) + max_len = 2000000 # XXX formula? + unpacker = msgpack.Unpacker( + use_list=False, + # avoid memory allocation issues causes by tampered input data. + max_buffer_size=max_len, # does not work in 0.4.6 unpackb C implementation + max_array_len=10, # meta_tuple + max_bin_len=max_len, # data + max_str_len=0, # not used yet + max_map_len=0, # not used yet + max_ext_len=0, # not used yet + ) + unpacker.feed(all_data[1:]) + meta_tuple, data = unpacker.unpack() meta = Meta(*meta_tuple) compressor, keyer, maccer, cipher = get_implementations(meta) return meta, data, compressor, keyer, maccer, cipher @@ -688,7 +695,7 @@ def key_factory(repository, manifest_data): def generate(meta, data): # always create new-style 0x03 format - return b'\x03' + msgpack.packb((meta, data)) + return b'\x03' + msgpack.packb((meta, data), use_bin_type=True) def compressor_creator(args): diff --git a/attic/testsuite/key.py b/attic/testsuite/key.py index b70939d61..76129402d 100644 --- a/attic/testsuite/key.py +++ b/attic/testsuite/key.py @@ -19,19 +19,19 @@ class KeyTestCase(AtticTestCase): keyfile2_key_file = """ ATTIC KEY 0000000000000000000000000000000000000000000000000000000000000000 -hqppdGVyYXRpb25zzgABhqCpYWxnb3JpdGhtpGdtYWOkaGFzaNoAII1CqUnJzgKISX3lwR -+wWqMAAAAAAAAAAAAAAAAAAAAApGRhdGHaANBGe/oYLxHbAq72vjwEpgNMV73dTMkZkYh4 -0WtFC65DwZmqvwbwBBaq1g+fiym+khRtrn9hZvF6rpjk0RrAURSxCXIt/XUNQzQlcQjYbb -kTT0aFk3DkKbwA/pgx10s/nWBmz9xv4yT5uoewOdPV009nJnrLdIz1zJTPvy2ylejHF3Na -Sy/B/tWA9PIeRZzrDe/lVY6YBs8lKz1jtT/3vCJFCa+LOSSJHV+tExnpgO0NBTxDmTckRe -vk3IRPVUml5VXHoUYEUEj6QpBA2F4NKdSzpHNhbHTaACDh3gxO3vgi+K/KMmBebec6RhBy -QQWJNlInT3+yKnQpdqd2ZXJzaW9uAQ==""".strip() +hqlhbGdvcml0aG2kZ21hY6d2ZXJzaW9uAaRoYXNo2gAgeXkW700i+1t5mroRI9YQuAAAAA +AAAAAAAAAAAAAAAACkZGF0YdoA0FVh2YsC4Nd5Pd+9wm6m/HbXnfy7ahBQNUp/grFY/LN7 +CPZYHM9tblJ40Kklnn6pktJhgEizgOzK435wbRWeuYiLO4+W0AEX74i0GcFafOhN7DyLYA +jE1qQMTm7tK2LlapnKVOOiH3KV67pdSMtRYDrHbx0Gud3jBtfMGU39nuwEFfWwIzQ8b4Tm +SWlG6orGwmvRJn8a5H+JtOY90e+tM7s2M4VF6p8grtUyighYxJrO4Y78/fsDpSHbYAh+en +6GrpcESLKYoDtgqiyjle0LpQ6kc2FsdNoAIKhlgtF1As4InTAsR3bCQif78vGjYYMKerJQ +ge5ZaKvpqml0ZXJhdGlvbnPOAAGGoA==""".strip() keyfile2_cdata = unhexlify(re.sub('\W', '', """ - 03929606001402da002046c635e7ce41b65c5c075fa6afb97f5100000000000000000000000000000000 - a80000000000000000affb14944408753093ba2860edb49220 + 03929600001402c4207f9b12b337e123e322ca2af795788ee100000000000000000000000000000000 + c4080000000000000000c407624711de25ab38 """)) - keyfile2_id = unhexlify('94899966ce3eaad825f37500c8c87ef100000000000000000000000000000000') + keyfile2_id = unhexlify('4d532cec0eb8ec34d65c5491b5158b1400000000000000000000000000000000') def setUp(self): self.tmppath = tempfile.mkdtemp()