mirror of
https://github.com/Icinga/icingadb.git
synced 2026-05-28 04:35:54 -04:00
Merge pull request #47 from lippserd/feature/object-packer
Introduce PackAny()
This commit is contained in:
Eric Lippmann
GitHub
commit
a5d446bc76
2 changed files with 304 additions and 0 deletions
133
pkg/icingadb/objectpacker/objectpacker.go
Normal file
133
pkg/icingadb/objectpacker/objectpacker.go
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
package objectpacker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"reflect"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// PackAny packs any JSON-encodable value (ex. structs, also ignores interfaces like encoding.TextMarshaler)
|
||||
// to a BSON-similar format suitable for consistent hashing. Spec:
|
||||
// https://github.com/Icinga/icinga2/blob/2cb995e/lib/base/object-packer.cpp#L222-L231
|
||||
func PackAny(in interface{}, out io.Writer) error {
|
||||
return packValue(reflect.ValueOf(in), out)
|
||||
}
|
||||
|
||||
// packValue does the actual job of packAny and just exists for recursion w/o unneccessary reflect.ValueOf calls.
|
||||
func packValue(in reflect.Value, out io.Writer) error {
|
||||
switch in.Kind() {
|
||||
case reflect.Invalid: // nil
|
||||
_, err := out.Write([]byte{0})
|
||||
return err
|
||||
case reflect.Bool:
|
||||
if in.Bool() {
|
||||
_, err := out.Write([]byte{2})
|
||||
return err
|
||||
} else {
|
||||
_, err := out.Write([]byte{1})
|
||||
return err
|
||||
}
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
return packFloat64(float64(in.Int()), out)
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
|
||||
return packFloat64(float64(in.Uint()), out)
|
||||
case reflect.Float32, reflect.Float64:
|
||||
return packFloat64(in.Float(), out)
|
||||
case reflect.Array, reflect.Slice:
|
||||
if _, err := out.Write([]byte{5}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
l := in.Len()
|
||||
if err := binary.Write(out, binary.BigEndian, uint64(l)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for i := 0; i < l; i++ {
|
||||
if err := packValue(in.Index(i), out); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
case reflect.Interface:
|
||||
return packValue(in.Elem(), out)
|
||||
case reflect.Map:
|
||||
type kv struct {
|
||||
key []byte
|
||||
value reflect.Value
|
||||
}
|
||||
|
||||
if _, err := out.Write([]byte{6}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
l := in.Len()
|
||||
if err := binary.Write(out, binary.BigEndian, uint64(l)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sorted := make([]kv, 0, l)
|
||||
|
||||
{
|
||||
iter := in.MapRange()
|
||||
for iter.Next() {
|
||||
// Disallow (panic) some types in map keys (recursively), too
|
||||
_ = packValue(iter.Key(), ioutil.Discard)
|
||||
|
||||
sorted = append(sorted, kv{[]byte(fmt.Sprint(iter.Key().Interface())), iter.Value()})
|
||||
}
|
||||
}
|
||||
|
||||
sort.Slice(sorted, func(i, j int) bool { return bytes.Compare(sorted[i].key, sorted[j].key) < 0 })
|
||||
|
||||
for _, kv := range sorted {
|
||||
if err := binary.Write(out, binary.BigEndian, uint64(len(kv.key))); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := out.Write(kv.key); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := packValue(kv.value, out); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
case reflect.Ptr:
|
||||
if in.IsNil() {
|
||||
return packValue(reflect.Value{}, out)
|
||||
} else {
|
||||
return packValue(in.Elem(), out)
|
||||
}
|
||||
case reflect.String:
|
||||
if _, err := out.Write([]byte{4}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b := []byte(in.String())
|
||||
if err := binary.Write(out, binary.BigEndian, uint64(len(b))); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err := out.Write(b)
|
||||
return err
|
||||
default:
|
||||
panic("bad type: " + in.Kind().String())
|
||||
}
|
||||
}
|
||||
|
||||
// packFloat64 deduplicates float packing of multiple locations in packValue.
|
||||
func packFloat64(in float64, out io.Writer) error {
|
||||
if _, errWr := out.Write([]byte{3}); errWr != nil {
|
||||
return errWr
|
||||
}
|
||||
|
||||
return binary.Write(out, binary.BigEndian, in)
|
||||
}
|
||||
171
pkg/icingadb/objectpacker/objectpacker_test.go
Normal file
171
pkg/icingadb/objectpacker/objectpacker_test.go
Normal file
|
|
@ -0,0 +1,171 @@
|
|||
package objectpacker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// limitedWriter allows writing a specific amount of data.
|
||||
type limitedWriter struct {
|
||||
// limit specifies how many bytes to allow to write.
|
||||
limit int
|
||||
}
|
||||
|
||||
var _ io.Writer = (*limitedWriter)(nil)
|
||||
|
||||
// Write returns io.EOF once lw.limit is exceeded, nil otherwise.
|
||||
func (lw *limitedWriter) Write(p []byte) (n int, err error) {
|
||||
if len(p) <= lw.limit {
|
||||
lw.limit -= len(p)
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
n = lw.limit
|
||||
err = io.EOF
|
||||
|
||||
lw.limit = 0
|
||||
return
|
||||
}
|
||||
|
||||
func TestLimitedWriter_Write(t *testing.T) {
|
||||
assertLimitedWriter_Write(t, 3, []byte{1, 2}, 2, nil, 1)
|
||||
assertLimitedWriter_Write(t, 3, []byte{1, 2, 3}, 3, nil, 0)
|
||||
assertLimitedWriter_Write(t, 3, []byte{1, 2, 3, 4}, 3, io.EOF, 0)
|
||||
assertLimitedWriter_Write(t, 0, []byte{1}, 0, io.EOF, 0)
|
||||
assertLimitedWriter_Write(t, 0, nil, 0, nil, 0)
|
||||
}
|
||||
|
||||
func assertLimitedWriter_Write(t *testing.T, limitBefore int, p []byte, n int, err error, limitAfter int) {
|
||||
t.Helper()
|
||||
|
||||
lw := limitedWriter{limitBefore}
|
||||
actualN, actualErr := lw.Write(p)
|
||||
|
||||
if actualErr != err {
|
||||
t.Errorf("_, err := (&limitedWriter{%d}).Write(%#v); err != %#v", limitBefore, p, err)
|
||||
}
|
||||
|
||||
if actualN != n {
|
||||
t.Errorf("n, _ := (&limitedWriter{%d}).Write(%#v); n != %d", limitBefore, p, n)
|
||||
}
|
||||
|
||||
if lw.limit != limitAfter {
|
||||
t.Errorf("lw := limitedWriter{%d}; lw.Write(%#v); lw.limit != %d", limitBefore, p, limitAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPackAny(t *testing.T) {
|
||||
assertPackAny(t, nil, []byte{0})
|
||||
assertPackAny(t, false, []byte{1})
|
||||
assertPackAny(t, true, []byte{2})
|
||||
|
||||
assertPackAny(t, -42, []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0})
|
||||
assertPackAny(t, int8(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0})
|
||||
assertPackAny(t, int16(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0})
|
||||
assertPackAny(t, int32(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0})
|
||||
assertPackAny(t, int64(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0})
|
||||
|
||||
assertPackAny(t, uint(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0})
|
||||
assertPackAny(t, uint8(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0})
|
||||
assertPackAny(t, uint16(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0})
|
||||
assertPackAny(t, uint32(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0})
|
||||
assertPackAny(t, uint64(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0})
|
||||
assertPackAny(t, uintptr(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0})
|
||||
|
||||
assertPackAny(t, float32(-42.5), []byte{3, 0xc0, 0x45, 0x40, 0, 0, 0, 0, 0})
|
||||
assertPackAny(t, -42.5, []byte{3, 0xc0, 0x45, 0x40, 0, 0, 0, 0, 0})
|
||||
|
||||
assertPackAny(t, []struct{}(nil), []byte{5, 0, 0, 0, 0, 0, 0, 0, 0})
|
||||
assertPackAny(t, []struct{}{}, []byte{5, 0, 0, 0, 0, 0, 0, 0, 0})
|
||||
|
||||
assertPackAny(t, []interface{}{nil, true, -42.5}, []byte{
|
||||
5, 0, 0, 0, 0, 0, 0, 0, 3,
|
||||
0,
|
||||
2,
|
||||
3, 0xc0, 0x45, 0x40, 0, 0, 0, 0, 0,
|
||||
})
|
||||
|
||||
assertPackAny(t, []string{"", "a"}, []byte{
|
||||
5, 0, 0, 0, 0, 0, 0, 0, 2,
|
||||
4, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
4, 0, 0, 0, 0, 0, 0, 0, 1, 'a',
|
||||
})
|
||||
|
||||
assertPackAnyPanic(t, []interface{}{0 + 0i}, 9)
|
||||
|
||||
assertPackAny(t, map[struct{}]struct{}(nil), []byte{6, 0, 0, 0, 0, 0, 0, 0, 0})
|
||||
assertPackAny(t, map[struct{}]struct{}{}, []byte{6, 0, 0, 0, 0, 0, 0, 0, 0})
|
||||
|
||||
assertPackAny(t, map[interface{}]interface{}{true: "", "nil": -42.5}, []byte{
|
||||
6, 0, 0, 0, 0, 0, 0, 0, 2,
|
||||
0, 0, 0, 0, 0, 0, 0, 3, 'n', 'i', 'l',
|
||||
3, 0xc0, 0x45, 0x40, 0, 0, 0, 0, 0,
|
||||
0, 0, 0, 0, 0, 0, 0, 4, 't', 'r', 'u', 'e',
|
||||
4, 0, 0, 0, 0, 0, 0, 0, 0,
|
||||
})
|
||||
|
||||
assertPackAny(t, map[string]uint8{"": 42}, []byte{
|
||||
6, 0, 0, 0, 0, 0, 0, 0, 1,
|
||||
0, 0, 0, 0, 0, 0, 0, 0,
|
||||
3, 0x40, 0x45, 0, 0, 0, 0, 0, 0,
|
||||
})
|
||||
|
||||
assertPackAnyPanic(t, map[struct{}]struct{}{{}: {}}, 9)
|
||||
|
||||
assertPackAny(t, (*int)(nil), []byte{0})
|
||||
assertPackAny(t, new(int), []byte{3, 0, 0, 0, 0, 0, 0, 0, 0})
|
||||
|
||||
assertPackAny(t, "", []byte{4, 0, 0, 0, 0, 0, 0, 0, 0})
|
||||
assertPackAny(t, "a", []byte{4, 0, 0, 0, 0, 0, 0, 0, 1, 'a'})
|
||||
assertPackAny(t, "ä", []byte{4, 0, 0, 0, 0, 0, 0, 0, 2, 0xc3, 0xa4})
|
||||
|
||||
assertPackAnyPanic(t, complex64(0+0i), 0)
|
||||
assertPackAnyPanic(t, 0+0i, 0)
|
||||
assertPackAnyPanic(t, make(chan struct{}, 0), 0)
|
||||
assertPackAnyPanic(t, func() {}, 0)
|
||||
assertPackAnyPanic(t, struct{}{}, 0)
|
||||
assertPackAnyPanic(t, unsafe.Pointer(uintptr(0)), 0)
|
||||
}
|
||||
|
||||
func assertPackAny(t *testing.T, in interface{}, out []byte) {
|
||||
t.Helper()
|
||||
|
||||
{
|
||||
buf := &bytes.Buffer{}
|
||||
if err := PackAny(in, buf); err == nil {
|
||||
if bytes.Compare(buf.Bytes(), out) != 0 {
|
||||
t.Errorf("buf := &bytes.Buffer{}; packAny(%#v, buf); bytes.Compare(buf.Bytes(), %#v) != 0", in, out)
|
||||
}
|
||||
} else {
|
||||
t.Errorf("packAny(%#v, &bytes.Buffer{}) != nil", in)
|
||||
}
|
||||
}
|
||||
|
||||
for i := 0; i < len(out); i++ {
|
||||
if PackAny(in, &limitedWriter{i}) != io.EOF {
|
||||
t.Errorf("packAny(%#v, &limitedWriter{%d}) != io.EOF", in, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func assertPackAnyPanic(t *testing.T, in interface{}, allowToWrite int) {
|
||||
t.Helper()
|
||||
|
||||
for i := 0; i < allowToWrite; i++ {
|
||||
if PackAny(in, &limitedWriter{i}) != io.EOF {
|
||||
t.Errorf("packAny(%#v, &limitedWriter{%d}) != io.EOF", in, i)
|
||||
}
|
||||
}
|
||||
|
||||
defer func() {
|
||||
t.Helper()
|
||||
|
||||
if r := recover(); r == nil {
|
||||
t.Errorf("packAny(%#v, &limitedWriter{%d}) didn't panic", in, allowToWrite)
|
||||
}
|
||||
}()
|
||||
|
||||
_ = PackAny(in, &limitedWriter{allowToWrite})
|
||||
}
|
||||
Loading…
Reference in a new issue