diff --git a/pkg/icingadb/objectpacker/objectpacker.go b/pkg/icingadb/objectpacker/objectpacker.go index dcd86ae8..44c6b92b 100644 --- a/pkg/icingadb/objectpacker/objectpacker.go +++ b/pkg/icingadb/objectpacker/objectpacker.go @@ -12,14 +12,32 @@ import ( // PackAny packs any JSON-encodable value (ex. structs, also ignores interfaces like encoding.TextMarshaler) // to a BSON-similar format suitable for consistent hashing. Spec: -// https://github.com/Icinga/icinga2/blob/2cb995e/lib/base/object-packer.cpp#L222-L231 +// +// PackAny(nil) => 0x0 +// PackAny(false) => 0x1 +// PackAny(true) => 0x2 +// PackAny(float64(42)) => 0x3 ieee754_binary64_bigendian(42) +// PackAny("exämple") => 0x4 uint64_bigendian(len([]byte("exämple"))) []byte("exämple") +// PackAny([]uint8{0x42}) => 0x4 uint64_bigendian(len([]uint8{0x42})) []uint8{0x42} +// PackAny([1]uint8{0x42}) => 0x4 uint64_bigendian(len([1]uint8{0x42})) [1]uint8{0x42} +// PackAny([]T{x,y}) => 0x5 uint64_bigendian(len([]T{x,y})) PackAny(x) PackAny(y) +// PackAny(map[K]V{x:y}) => 0x6 uint64_bigendian(len(map[K]V{x:y})) len(map_key(x)) map_key(x) PackAny(y) +// PackAny((*T)(nil)) => 0x0 +// PackAny((*T)(0x42)) => PackAny(*(*T)(0x42)) +// PackAny(x) => panic() +// +// map_key([1]uint8{0x42}) => [1]uint8{0x42} +// map_key(x) => []byte(fmt.Sprint(x)) func PackAny(in interface{}, out io.Writer) error { return packValue(reflect.ValueOf(in), out) } +var tByte = reflect.TypeOf(byte(0)) +var tBytes = reflect.TypeOf([]uint8(nil)) + // packValue does the actual job of packAny and just exists for recursion w/o unneccessary reflect.ValueOf calls. func packValue(in reflect.Value, out io.Writer) error { - switch in.Kind() { + switch kind := in.Kind(); kind { case reflect.Invalid: // nil _, err := out.Write([]byte{0}) return err @@ -31,13 +49,29 @@ func packValue(in reflect.Value, out io.Writer) error { _, err := out.Write([]byte{1}) return err } - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return packFloat64(float64(in.Int()), out) - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - return packFloat64(float64(in.Uint()), out) - case reflect.Float32, reflect.Float64: - return packFloat64(in.Float(), out) + case reflect.Float64: + if _, err := out.Write([]byte{3}); err != nil { + return err + } + + return binary.Write(out, binary.BigEndian, in.Float()) case reflect.Array, reflect.Slice: + if typ := in.Type(); typ.Elem() == tByte { + if kind == reflect.Array { + if !in.CanAddr() { + vNewElem := reflect.New(typ).Elem() + vNewElem.Set(in) + in = vNewElem + } + + in = in.Slice(0, in.Len()) + } + + // Pack []byte as string, not array of numbers. + return packString(in.Convert(tBytes). // Support types.Binary + Interface().([]uint8), out) + } + if _, err := out.Write([]byte{5}); err != nil { return err } @@ -53,6 +87,12 @@ func packValue(in reflect.Value, out io.Writer) error { } } + // If there aren't any values to pack, ... + if l < 1 { + // ... create one and pack it - panics on disallowed type. + _ = packValue(reflect.Zero(in.Type().Elem()), ioutil.Discard) + } + return nil case reflect.Interface: return packValue(in.Elem(), out) @@ -76,10 +116,30 @@ func packValue(in reflect.Value, out io.Writer) error { { iter := in.MapRange() for iter.Next() { - // Disallow (panic) some types in map keys (recursively), too - _ = packValue(iter.Key(), ioutil.Discard) + var packedKey []byte + if key := iter.Key(); key.Kind() == reflect.Array { + if typ := key.Type(); typ.Elem() == tByte { + if !key.CanAddr() { + vNewElem := reflect.New(typ).Elem() + vNewElem.Set(key) + key = vNewElem + } - sorted = append(sorted, kv{[]byte(fmt.Sprint(iter.Key().Interface())), iter.Value()}) + packedKey = key.Slice(0, key.Len()).Interface().([]byte) + } else { + // Not just stringify the key (below), but also pack it (here) - panics on disallowed type. + _ = packValue(iter.Key(), ioutil.Discard) + + packedKey = []byte(fmt.Sprint(key.Interface())) + } + } else { + // Not just stringify the key (below), but also pack it (here) - panics on disallowed type. + _ = packValue(iter.Key(), ioutil.Discard) + + packedKey = []byte(fmt.Sprint(key.Interface())) + } + + sorted = append(sorted, kv{packedKey, iter.Value()}) } } @@ -99,35 +159,44 @@ func packValue(in reflect.Value, out io.Writer) error { } } + // If there aren't any key-value pairs to pack, ... + if l < 1 { + typ := in.Type() + + // ... create one and pack it - panics on disallowed type. + _ = packValue(reflect.Zero(typ.Key()), ioutil.Discard) + _ = packValue(reflect.Zero(typ.Elem()), ioutil.Discard) + } + return nil case reflect.Ptr: if in.IsNil() { - return packValue(reflect.Value{}, out) + err := packValue(reflect.Value{}, out) + + // Create a fictive referenced value and pack it - panics on disallowed type. + _ = packValue(reflect.Zero(in.Type().Elem()), ioutil.Discard) + + return err } else { return packValue(in.Elem(), out) } case reflect.String: - if _, err := out.Write([]byte{4}); err != nil { - return err - } - - b := []byte(in.String()) - if err := binary.Write(out, binary.BigEndian, uint64(len(b))); err != nil { - return err - } - - _, err := out.Write(b) - return err + return packString([]byte(in.String()), out) default: panic("bad type: " + in.Kind().String()) } } -// packFloat64 deduplicates float packing of multiple locations in packValue. -func packFloat64(in float64, out io.Writer) error { - if _, errWr := out.Write([]byte{3}); errWr != nil { - return errWr +// packString deduplicates string packing of multiple locations in packValue. +func packString(in []byte, out io.Writer) error { + if _, err := out.Write([]byte{4}); err != nil { + return err } - return binary.Write(out, binary.BigEndian, in) + if err := binary.Write(out, binary.BigEndian, uint64(len(in))); err != nil { + return err + } + + _, err := out.Write(in) + return err } diff --git a/pkg/icingadb/objectpacker/objectpacker_test.go b/pkg/icingadb/objectpacker/objectpacker_test.go index 5d36cecb..6afb5882 100644 --- a/pkg/icingadb/objectpacker/objectpacker_test.go +++ b/pkg/icingadb/objectpacker/objectpacker_test.go @@ -2,6 +2,7 @@ package objectpacker import ( "bytes" + "github.com/icinga/icingadb/pkg/types" "io" "testing" "unsafe" @@ -61,24 +62,24 @@ func TestPackAny(t *testing.T) { assertPackAny(t, false, []byte{1}) assertPackAny(t, true, []byte{2}) - assertPackAny(t, -42, []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0}) - assertPackAny(t, int8(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0}) - assertPackAny(t, int16(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0}) - assertPackAny(t, int32(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0}) - assertPackAny(t, int64(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0}) + assertPackAnyPanic(t, -42, 0) + assertPackAnyPanic(t, int8(-42), 0) + assertPackAnyPanic(t, int16(-42), 0) + assertPackAnyPanic(t, int32(-42), 0) + assertPackAnyPanic(t, int64(-42), 0) - assertPackAny(t, uint(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0}) - assertPackAny(t, uint8(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0}) - assertPackAny(t, uint16(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0}) - assertPackAny(t, uint32(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0}) - assertPackAny(t, uint64(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0}) - assertPackAny(t, uintptr(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0}) + assertPackAnyPanic(t, uint(42), 0) + assertPackAnyPanic(t, uint8(42), 0) + assertPackAnyPanic(t, uint16(42), 0) + assertPackAnyPanic(t, uint32(42), 0) + assertPackAnyPanic(t, uint64(42), 0) + assertPackAnyPanic(t, uintptr(42), 0) - assertPackAny(t, float32(-42.5), []byte{3, 0xc0, 0x45, 0x40, 0, 0, 0, 0, 0}) + assertPackAnyPanic(t, float32(-42.5), 0) assertPackAny(t, -42.5, []byte{3, 0xc0, 0x45, 0x40, 0, 0, 0, 0, 0}) - assertPackAny(t, []struct{}(nil), []byte{5, 0, 0, 0, 0, 0, 0, 0, 0}) - assertPackAny(t, []struct{}{}, []byte{5, 0, 0, 0, 0, 0, 0, 0, 0}) + assertPackAnyPanic(t, []struct{}(nil), 9) + assertPackAnyPanic(t, []struct{}{}, 9) assertPackAny(t, []interface{}{nil, true, -42.5}, []byte{ 5, 0, 0, 0, 0, 0, 0, 0, 3, @@ -95,8 +96,8 @@ func TestPackAny(t *testing.T) { assertPackAnyPanic(t, []interface{}{0 + 0i}, 9) - assertPackAny(t, map[struct{}]struct{}(nil), []byte{6, 0, 0, 0, 0, 0, 0, 0, 0}) - assertPackAny(t, map[struct{}]struct{}{}, []byte{6, 0, 0, 0, 0, 0, 0, 0, 0}) + assertPackAnyPanic(t, map[struct{}]struct{}(nil), 9) + assertPackAnyPanic(t, map[struct{}]struct{}{}, 9) assertPackAny(t, map[interface{}]interface{}{true: "", "nil": -42.5}, []byte{ 6, 0, 0, 0, 0, 0, 0, 0, 2, @@ -106,21 +107,44 @@ func TestPackAny(t *testing.T) { 4, 0, 0, 0, 0, 0, 0, 0, 0, }) - assertPackAny(t, map[string]uint8{"": 42}, []byte{ + assertPackAny(t, map[string]float64{"": 42}, []byte{ 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0x40, 0x45, 0, 0, 0, 0, 0, 0, }) + assertPackAny(t, map[[1]byte]bool{[1]byte{42}: true}, []byte{ + 6, 0, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 1, 42, + 2, + }) + assertPackAnyPanic(t, map[struct{}]struct{}{{}: {}}, 9) - assertPackAny(t, (*int)(nil), []byte{0}) - assertPackAny(t, new(int), []byte{3, 0, 0, 0, 0, 0, 0, 0, 0}) + assertPackAny(t, (*string)(nil), []byte{0}) + assertPackAnyPanic(t, (*int)(nil), 0) + assertPackAny(t, new(float64), []byte{3, 0, 0, 0, 0, 0, 0, 0, 0}) assertPackAny(t, "", []byte{4, 0, 0, 0, 0, 0, 0, 0, 0}) assertPackAny(t, "a", []byte{4, 0, 0, 0, 0, 0, 0, 0, 1, 'a'}) assertPackAny(t, "ä", []byte{4, 0, 0, 0, 0, 0, 0, 0, 2, 0xc3, 0xa4}) + { + var binary [256]byte + for i := range binary { + binary[i] = byte(i) + } + + assertPackAny(t, binary, append([]byte{4, 0, 0, 0, 0, 0, 0, 1, 0}, binary[:]...)) + assertPackAny(t, binary[:], append([]byte{4, 0, 0, 0, 0, 0, 0, 1, 0}, binary[:]...)) + assertPackAny(t, types.Binary(binary[:]), append([]byte{4, 0, 0, 0, 0, 0, 0, 1, 0}, binary[:]...)) + } + + { + type myByte byte + assertPackAnyPanic(t, []myByte(nil), 9) + } + assertPackAnyPanic(t, complex64(0+0i), 0) assertPackAnyPanic(t, 0+0i, 0) assertPackAnyPanic(t, make(chan struct{}, 0), 0)