From 429ea1ca4872de0d61b40edc645b2e13f6fe508e Mon Sep 17 00:00:00 2001 From: "Alexander A. Klimov" Date: Tue, 4 May 2021 11:09:08 +0200 Subject: [PATCH] Introduce PackAny() --- pkg/icingadb/objectpacker/objectpacker.go | 133 ++++++++++++++ .../objectpacker/objectpacker_test.go | 171 ++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 pkg/icingadb/objectpacker/objectpacker.go create mode 100644 pkg/icingadb/objectpacker/objectpacker_test.go diff --git a/pkg/icingadb/objectpacker/objectpacker.go b/pkg/icingadb/objectpacker/objectpacker.go new file mode 100644 index 00000000..dcd86ae8 --- /dev/null +++ b/pkg/icingadb/objectpacker/objectpacker.go @@ -0,0 +1,133 @@ +package objectpacker + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "io/ioutil" + "reflect" + "sort" +) + +// PackAny packs any JSON-encodable value (ex. structs, also ignores interfaces like encoding.TextMarshaler) +// to a BSON-similar format suitable for consistent hashing. Spec: +// https://github.com/Icinga/icinga2/blob/2cb995e/lib/base/object-packer.cpp#L222-L231 +func PackAny(in interface{}, out io.Writer) error { + return packValue(reflect.ValueOf(in), out) +} + +// packValue does the actual job of packAny and just exists for recursion w/o unneccessary reflect.ValueOf calls. +func packValue(in reflect.Value, out io.Writer) error { + switch in.Kind() { + case reflect.Invalid: // nil + _, err := out.Write([]byte{0}) + return err + case reflect.Bool: + if in.Bool() { + _, err := out.Write([]byte{2}) + return err + } else { + _, err := out.Write([]byte{1}) + return err + } + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return packFloat64(float64(in.Int()), out) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return packFloat64(float64(in.Uint()), out) + case reflect.Float32, reflect.Float64: + return packFloat64(in.Float(), out) + case reflect.Array, reflect.Slice: + if _, err := out.Write([]byte{5}); err != nil { + return err + } + + l := in.Len() + if err := binary.Write(out, binary.BigEndian, uint64(l)); err != nil { + return err + } + + for i := 0; i < l; i++ { + if err := packValue(in.Index(i), out); err != nil { + return err + } + } + + return nil + case reflect.Interface: + return packValue(in.Elem(), out) + case reflect.Map: + type kv struct { + key []byte + value reflect.Value + } + + if _, err := out.Write([]byte{6}); err != nil { + return err + } + + l := in.Len() + if err := binary.Write(out, binary.BigEndian, uint64(l)); err != nil { + return err + } + + sorted := make([]kv, 0, l) + + { + iter := in.MapRange() + for iter.Next() { + // Disallow (panic) some types in map keys (recursively), too + _ = packValue(iter.Key(), ioutil.Discard) + + sorted = append(sorted, kv{[]byte(fmt.Sprint(iter.Key().Interface())), iter.Value()}) + } + } + + sort.Slice(sorted, func(i, j int) bool { return bytes.Compare(sorted[i].key, sorted[j].key) < 0 }) + + for _, kv := range sorted { + if err := binary.Write(out, binary.BigEndian, uint64(len(kv.key))); err != nil { + return err + } + + if _, err := out.Write(kv.key); err != nil { + return err + } + + if err := packValue(kv.value, out); err != nil { + return err + } + } + + return nil + case reflect.Ptr: + if in.IsNil() { + return packValue(reflect.Value{}, out) + } else { + return packValue(in.Elem(), out) + } + case reflect.String: + if _, err := out.Write([]byte{4}); err != nil { + return err + } + + b := []byte(in.String()) + if err := binary.Write(out, binary.BigEndian, uint64(len(b))); err != nil { + return err + } + + _, err := out.Write(b) + return err + default: + panic("bad type: " + in.Kind().String()) + } +} + +// packFloat64 deduplicates float packing of multiple locations in packValue. +func packFloat64(in float64, out io.Writer) error { + if _, errWr := out.Write([]byte{3}); errWr != nil { + return errWr + } + + return binary.Write(out, binary.BigEndian, in) +} diff --git a/pkg/icingadb/objectpacker/objectpacker_test.go b/pkg/icingadb/objectpacker/objectpacker_test.go new file mode 100644 index 00000000..5d36cecb --- /dev/null +++ b/pkg/icingadb/objectpacker/objectpacker_test.go @@ -0,0 +1,171 @@ +package objectpacker + +import ( + "bytes" + "io" + "testing" + "unsafe" +) + +// limitedWriter allows writing a specific amount of data. +type limitedWriter struct { + // limit specifies how many bytes to allow to write. + limit int +} + +var _ io.Writer = (*limitedWriter)(nil) + +// Write returns io.EOF once lw.limit is exceeded, nil otherwise. +func (lw *limitedWriter) Write(p []byte) (n int, err error) { + if len(p) <= lw.limit { + lw.limit -= len(p) + return len(p), nil + } + + n = lw.limit + err = io.EOF + + lw.limit = 0 + return +} + +func TestLimitedWriter_Write(t *testing.T) { + assertLimitedWriter_Write(t, 3, []byte{1, 2}, 2, nil, 1) + assertLimitedWriter_Write(t, 3, []byte{1, 2, 3}, 3, nil, 0) + assertLimitedWriter_Write(t, 3, []byte{1, 2, 3, 4}, 3, io.EOF, 0) + assertLimitedWriter_Write(t, 0, []byte{1}, 0, io.EOF, 0) + assertLimitedWriter_Write(t, 0, nil, 0, nil, 0) +} + +func assertLimitedWriter_Write(t *testing.T, limitBefore int, p []byte, n int, err error, limitAfter int) { + t.Helper() + + lw := limitedWriter{limitBefore} + actualN, actualErr := lw.Write(p) + + if actualErr != err { + t.Errorf("_, err := (&limitedWriter{%d}).Write(%#v); err != %#v", limitBefore, p, err) + } + + if actualN != n { + t.Errorf("n, _ := (&limitedWriter{%d}).Write(%#v); n != %d", limitBefore, p, n) + } + + if lw.limit != limitAfter { + t.Errorf("lw := limitedWriter{%d}; lw.Write(%#v); lw.limit != %d", limitBefore, p, limitAfter) + } +} + +func TestPackAny(t *testing.T) { + assertPackAny(t, nil, []byte{0}) + assertPackAny(t, false, []byte{1}) + assertPackAny(t, true, []byte{2}) + + assertPackAny(t, -42, []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0}) + assertPackAny(t, int8(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0}) + assertPackAny(t, int16(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0}) + assertPackAny(t, int32(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0}) + assertPackAny(t, int64(-42), []byte{3, 0xc0, 0x45, 0, 0, 0, 0, 0, 0}) + + assertPackAny(t, uint(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0}) + assertPackAny(t, uint8(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0}) + assertPackAny(t, uint16(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0}) + assertPackAny(t, uint32(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0}) + assertPackAny(t, uint64(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0}) + assertPackAny(t, uintptr(42), []byte{3, 0x40, 0x45, 0, 0, 0, 0, 0, 0}) + + assertPackAny(t, float32(-42.5), []byte{3, 0xc0, 0x45, 0x40, 0, 0, 0, 0, 0}) + assertPackAny(t, -42.5, []byte{3, 0xc0, 0x45, 0x40, 0, 0, 0, 0, 0}) + + assertPackAny(t, []struct{}(nil), []byte{5, 0, 0, 0, 0, 0, 0, 0, 0}) + assertPackAny(t, []struct{}{}, []byte{5, 0, 0, 0, 0, 0, 0, 0, 0}) + + assertPackAny(t, []interface{}{nil, true, -42.5}, []byte{ + 5, 0, 0, 0, 0, 0, 0, 0, 3, + 0, + 2, + 3, 0xc0, 0x45, 0x40, 0, 0, 0, 0, 0, + }) + + assertPackAny(t, []string{"", "a"}, []byte{ + 5, 0, 0, 0, 0, 0, 0, 0, 2, + 4, 0, 0, 0, 0, 0, 0, 0, 0, + 4, 0, 0, 0, 0, 0, 0, 0, 1, 'a', + }) + + assertPackAnyPanic(t, []interface{}{0 + 0i}, 9) + + assertPackAny(t, map[struct{}]struct{}(nil), []byte{6, 0, 0, 0, 0, 0, 0, 0, 0}) + assertPackAny(t, map[struct{}]struct{}{}, []byte{6, 0, 0, 0, 0, 0, 0, 0, 0}) + + assertPackAny(t, map[interface{}]interface{}{true: "", "nil": -42.5}, []byte{ + 6, 0, 0, 0, 0, 0, 0, 0, 2, + 0, 0, 0, 0, 0, 0, 0, 3, 'n', 'i', 'l', + 3, 0xc0, 0x45, 0x40, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 4, 't', 'r', 'u', 'e', + 4, 0, 0, 0, 0, 0, 0, 0, 0, + }) + + assertPackAny(t, map[string]uint8{"": 42}, []byte{ + 6, 0, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 0, + 3, 0x40, 0x45, 0, 0, 0, 0, 0, 0, + }) + + assertPackAnyPanic(t, map[struct{}]struct{}{{}: {}}, 9) + + assertPackAny(t, (*int)(nil), []byte{0}) + assertPackAny(t, new(int), []byte{3, 0, 0, 0, 0, 0, 0, 0, 0}) + + assertPackAny(t, "", []byte{4, 0, 0, 0, 0, 0, 0, 0, 0}) + assertPackAny(t, "a", []byte{4, 0, 0, 0, 0, 0, 0, 0, 1, 'a'}) + assertPackAny(t, "รค", []byte{4, 0, 0, 0, 0, 0, 0, 0, 2, 0xc3, 0xa4}) + + assertPackAnyPanic(t, complex64(0+0i), 0) + assertPackAnyPanic(t, 0+0i, 0) + assertPackAnyPanic(t, make(chan struct{}, 0), 0) + assertPackAnyPanic(t, func() {}, 0) + assertPackAnyPanic(t, struct{}{}, 0) + assertPackAnyPanic(t, unsafe.Pointer(uintptr(0)), 0) +} + +func assertPackAny(t *testing.T, in interface{}, out []byte) { + t.Helper() + + { + buf := &bytes.Buffer{} + if err := PackAny(in, buf); err == nil { + if bytes.Compare(buf.Bytes(), out) != 0 { + t.Errorf("buf := &bytes.Buffer{}; packAny(%#v, buf); bytes.Compare(buf.Bytes(), %#v) != 0", in, out) + } + } else { + t.Errorf("packAny(%#v, &bytes.Buffer{}) != nil", in) + } + } + + for i := 0; i < len(out); i++ { + if PackAny(in, &limitedWriter{i}) != io.EOF { + t.Errorf("packAny(%#v, &limitedWriter{%d}) != io.EOF", in, i) + } + } +} + +func assertPackAnyPanic(t *testing.T, in interface{}, allowToWrite int) { + t.Helper() + + for i := 0; i < allowToWrite; i++ { + if PackAny(in, &limitedWriter{i}) != io.EOF { + t.Errorf("packAny(%#v, &limitedWriter{%d}) != io.EOF", in, i) + } + } + + defer func() { + t.Helper() + + if r := recover(); r == nil { + t.Errorf("packAny(%#v, &limitedWriter{%d}) didn't panic", in, allowToWrite) + } + }() + + _ = PackAny(in, &limitedWriter{allowToWrite}) +}