upstream/icingadb
1
0
Fork 0
mirror of https://github.com/Icinga/icingadb.git synced 2026-05-28 04:35:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #50 from lippserd/bugfix/obj-pack-bytes

Browse source 
PackAny(): support types.Binary
This commit is contained in:
Eric Lippmann 2021-05-11 13:53:54 +02:00 committed by GitHub
commit 420193f228
2 changed files with 140 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

125
pkg/icingadb/objectpacker/objectpacker.go
View file

@ -12,14 +12,32 @@ import (
// PackAny packs any JSON-encodable value (ex. structs, also ignores interfaces like encoding.TextMarshaler)
// to a BSON-similar format suitable for consistent hashing. Spec:
// https://github.com/Icinga/icinga2/blob/2cb995e/lib/base/object-packer.cpp#L222-L231
//
// PackAny(nil) => 0x0
// PackAny(false) => 0x1
// PackAny(true) => 0x2
// PackAny(float64(42)) => 0x3 ieee754_binary64_bigendian(42)
// PackAny("exämple") => 0x4 uint64_bigendian(len([]byte("exämple"))) []byte("exämple")
// PackAny([]uint8{0x42}) => 0x4 uint64_bigendian(len([]uint8{0x42})) []uint8{0x42}
// PackAny([1]uint8{0x42}) => 0x4 uint64_bigendian(len([1]uint8{0x42})) [1]uint8{0x42}
// PackAny([]T{x,y}) => 0x5 uint64_bigendian(len([]T{x,y})) PackAny(x) PackAny(y)
// PackAny(map[K]V{x:y}) => 0x6 uint64_bigendian(len(map[K]V{x:y})) len(map_key(x)) map_key(x) PackAny(y)
// PackAny((*T)(nil)) => 0x0
// PackAny((*T)(0x42)) => PackAny(*(*T)(0x42))
// PackAny(x) => panic()
//
// map_key([1]uint8{0x42}) => [1]uint8{0x42}
// map_key(x) => []byte(fmt.Sprint(x))
func PackAny(in interface{}, out io.Writer) error {
return packValue(reflect.ValueOf(in), out)
}
var tByte = reflect.TypeOf(byte(0))
var tBytes = reflect.TypeOf([]uint8(nil))
// packValue does the actual job of packAny and just exists for recursion w/o unneccessary reflect.ValueOf calls.
func packValue(in reflect.Value, out io.Writer) error {
switch in.Kind() {
switch kind := in.Kind(); kind {
case reflect.Invalid: // nil
_, err := out.Write([]byte{0})
return err
@ -31,13 +49,29 @@ func packValue(in reflect.Value, out io.Writer) error {
_, err := out.Write([]byte{1})
return err
}
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return packFloat64(float64(in.Int()), out)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return packFloat64(float64(in.Uint()), out)
case reflect.Float32, reflect.Float64:
return packFloat64(in.Float(), out)
case reflect.Float64:
if _, err := out.Write([]byte{3}); err != nil {
return err
}
return binary.Write(out, binary.BigEndian, in.Float())
case reflect.Array, reflect.Slice:
if typ := in.Type(); typ.Elem() == tByte {
if kind == reflect.Array {
if !in.CanAddr() {
vNewElem := reflect.New(typ).Elem()
vNewElem.Set(in)
in = vNewElem
}
in = in.Slice(0, in.Len())
}
// Pack []byte as string, not array of numbers.
return packString(in.Convert(tBytes). // Support types.Binary
Interface().([]uint8), out)
}
if _, err := out.Write([]byte{5}); err != nil {
return err
}
@ -53,6 +87,12 @@ func packValue(in reflect.Value, out io.Writer) error {
}
}
// If there aren't any values to pack, ...
if l < 1 {
// ... create one and pack it - panics on disallowed type.
_ = packValue(reflect.Zero(in.Type().Elem()), ioutil.Discard)
}
return nil
case reflect.Interface:
return packValue(in.Elem(), out)
@ -76,10 +116,30 @@ func packValue(in reflect.Value, out io.Writer) error {
{
iter := in.MapRange()
for iter.Next() {
// Disallow (panic) some types in map keys (recursively), too
_ = packValue(iter.Key(), ioutil.Discard)
var packedKey []byte
if key := iter.Key(); key.Kind() == reflect.Array {
if typ := key.Type(); typ.Elem() == tByte {
if !key.CanAddr() {
vNewElem := reflect.New(typ).Elem()
vNewElem.Set(key)
key = vNewElem
}
sorted = append(sorted, kv{[]byte(fmt.Sprint(iter.Key().Interface())), iter.Value()})
packedKey = key.Slice(0, key.Len()).Interface().([]byte)
} else {
// Not just stringify the key (below), but also pack it (here) - panics on disallowed type.
_ = packValue(iter.Key(), ioutil.Discard)
packedKey = []byte(fmt.Sprint(key.Interface()))
}
} else {
// Not just stringify the key (below), but also pack it (here) - panics on disallowed type.
_ = packValue(iter.Key(), ioutil.Discard)
packedKey = []byte(fmt.Sprint(key.Interface()))
}
sorted = append(sorted, kv{packedKey, iter.Value()})
}
}
@ -99,35 +159,44 @@ func packValue(in reflect.Value, out io.Writer) error {
}
}
// If there aren't any key-value pairs to pack, ...
if l < 1 {
typ := in.Type()
// ... create one and pack it - panics on disallowed type.
_ = packValue(reflect.Zero(typ.Key()), ioutil.Discard)
_ = packValue(reflect.Zero(typ.Elem()), ioutil.Discard)
}
return nil
case reflect.Ptr:
if in.IsNil() {
return packValue(reflect.Value{}, out)
err := packValue(reflect.Value{}, out)
// Create a fictive referenced value and pack it - panics on disallowed type.
_ = packValue(reflect.Zero(in.Type().Elem()), ioutil.Discard)
return err
} else {
return packValue(in.Elem(), out)
}
case reflect.String:
if _, err := out.Write([]byte{4}); err != nil {
return err
}
b := []byte(in.String())
if err := binary.Write(out, binary.BigEndian, uint64(len(b))); err != nil {
return err
}
_, err := out.Write(b)
return err
return packString([]byte(in.String()), out)
default:
panic("bad type: " + in.Kind().String())
}
}
// packFloat64 deduplicates float packing of multiple locations in packValue.
func packFloat64(in float64, out io.Writer) error {
if _, errWr := out.Write([]byte{3}); errWr != nil {
return errWr
// packString deduplicates string packing of multiple locations in packValue.
func packString(in []byte, out io.Writer) error {
if _, err := out.Write([]byte{4}); err != nil {
return err
}
return binary.Write(out, binary.BigEndian, in)
if err := binary.Write(out, binary.BigEndian, uint64(len(in))); err != nil {
return err
}
_, err := out.Write(in)
return err
}

62
pkg/icingadb/objectpacker/objectpacker_test.go
View file

@ -2,6 +2,7 @@ package objectpacker
import (
"bytes"
"github.com/icinga/icingadb/pkg/types"
"io"
"testing"
"unsafe"
@ -61,24 +62,24 @@ func TestPackAny(t *testing.T) {
assertPackAny(t, false, []byte{1})
assertPackAny(t, true, []byte{2})
assertPackAny(t, -42, []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0})
assertPackAny(t, int8(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0})
assertPackAny(t, int16(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0})
assertPackAny(t, int32(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0})
assertPackAny(t, int64(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0})
assertPackAnyPanic(t, -42, 0)
assertPackAnyPanic(t, int8(-42), 0)
assertPackAnyPanic(t, int16(-42), 0)
assertPackAnyPanic(t, int32(-42), 0)
assertPackAnyPanic(t, int64(-42), 0)
assertPackAny(t, uint(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0})
assertPackAny(t, uint8(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0})
assertPackAny(t, uint16(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0})
assertPackAny(t, uint32(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0})
assertPackAny(t, uint64(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0})
assertPackAny(t, uintptr(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0})
assertPackAnyPanic(t, uint(42), 0)
assertPackAnyPanic(t, uint8(42), 0)
assertPackAnyPanic(t, uint16(42), 0)
assertPackAnyPanic(t, uint32(42), 0)
assertPackAnyPanic(t, uint64(42), 0)
assertPackAnyPanic(t, uintptr(42), 0)
assertPackAny(t, float32(-42.5), []byte{3, 0xc0, 0x45, 0x40, 0, 0, 0, 0, 0})
assertPackAnyPanic(t, float32(-42.5), 0)
assertPackAny(t, -42.5, []byte{3, 0xc0, 0x45, 0x40, 0, 0, 0, 0, 0})
assertPackAny(t, []struct{}(nil), []byte{5, 0, 0, 0, 0, 0, 0, 0, 0})
assertPackAny(t, []struct{}{}, []byte{5, 0, 0, 0, 0, 0, 0, 0, 0})
assertPackAnyPanic(t, []struct{}(nil), 9)
assertPackAnyPanic(t, []struct{}{}, 9)
assertPackAny(t, []interface{}{nil, true, -42.5}, []byte{
5, 0, 0, 0, 0, 0, 0, 0, 3,
@ -95,8 +96,8 @@ func TestPackAny(t *testing.T) {
assertPackAnyPanic(t, []interface{}{0 + 0i}, 9)
assertPackAny(t, map[struct{}]struct{}(nil), []byte{6, 0, 0, 0, 0, 0, 0, 0, 0})
assertPackAny(t, map[struct{}]struct{}{}, []byte{6, 0, 0, 0, 0, 0, 0, 0, 0})
assertPackAnyPanic(t, map[struct{}]struct{}(nil), 9)
assertPackAnyPanic(t, map[struct{}]struct{}{}, 9)
assertPackAny(t, map[interface{}]interface{}{true: "", "nil": -42.5}, []byte{
6, 0, 0, 0, 0, 0, 0, 0, 2,
@ -106,21 +107,44 @@ func TestPackAny(t *testing.T) {
4, 0, 0, 0, 0, 0, 0, 0, 0,
})
assertPackAny(t, map[string]uint8{"": 42}, []byte{
assertPackAny(t, map[string]float64{"": 42}, []byte{
6, 0, 0, 0, 0, 0, 0, 0, 1,
0, 0, 0, 0, 0, 0, 0, 0,
3, 0x40, 0x45, 0, 0, 0, 0, 0, 0,
})
assertPackAny(t, map[[1]byte]bool{[1]byte{42}: true}, []byte{
6, 0, 0, 0, 0, 0, 0, 0, 1,
0, 0, 0, 0, 0, 0, 0, 1, 42,
2,
})
assertPackAnyPanic(t, map[struct{}]struct{}{{}: {}}, 9)
assertPackAny(t, (*int)(nil), []byte{0})
assertPackAny(t, new(int), []byte{3, 0, 0, 0, 0, 0, 0, 0, 0})
assertPackAny(t, (*string)(nil), []byte{0})
assertPackAnyPanic(t, (*int)(nil), 0)
assertPackAny(t, new(float64), []byte{3, 0, 0, 0, 0, 0, 0, 0, 0})
assertPackAny(t, "", []byte{4, 0, 0, 0, 0, 0, 0, 0, 0})
assertPackAny(t, "a", []byte{4, 0, 0, 0, 0, 0, 0, 0, 1, 'a'})
assertPackAny(t, "ä", []byte{4, 0, 0, 0, 0, 0, 0, 0, 2, 0xc3, 0xa4})
{
var binary [256]byte
for i := range binary {
binary[i] = byte(i)
}
assertPackAny(t, binary, append([]byte{4, 0, 0, 0, 0, 0, 0, 1, 0}, binary[:]...))
assertPackAny(t, binary[:], append([]byte{4, 0, 0, 0, 0, 0, 0, 1, 0}, binary[:]...))
assertPackAny(t, types.Binary(binary[:]), append([]byte{4, 0, 0, 0, 0, 0, 0, 1, 0}, binary[:]...))
}
{
type myByte byte
assertPackAnyPanic(t, []myByte(nil), 9)
}
assertPackAnyPanic(t, complex64(0+0i), 0)
assertPackAnyPanic(t, 0+0i, 0)
assertPackAnyPanic(t, make(chan struct{}, 0), 0)